/* * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for * license information. */ const gulp = require('gulp'); const args = require('yargs').argv; const colors = require('colors'); const fs = require('fs'); const util = require('util'); const path = require('path'); const glob = require('glob'); const execSync = require('child_process').execSync; var mappings = require('./codegen_mappings.json'); const defaultAutoRestVersion = '1.2.2'; var usingAutoRestVersion; const specRepoDir = args['spec-repo-dir']; const specRoot = args['spec-root'] || "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/specification"; const project = args['project']; var language = 'Azure.NodeJS'; var modeler = 'Swagger'; const regexForExcludedServices = /\/(intune|documentdbManagement|insightsManagement|insights|search)\//i; function getAutorestVersion(version) { if (!version) version = 'latest'; let getVersion, execHelp; let result = true; try { let getVersionCmd = `autorest `; let execHelpCmd = `autorest --help`; console.log(getVersionCmd); getVersion = execSync(getVersionCmd, { encoding: 'utf8' }); //console.debug(getVersion); console.log(execHelpCmd); execHelp = execSync(execHelpCmd, { encoding: 'utf8' }); //console.debug(execHelp); } catch (err) { result = false; console.log(`An error occurred while getting the "${version}" of autorest and executing "autorest --help":\n ${util.inspect(err, { depth: null })}.`); } return result; } function deleteFolderRecursive(path) { if (fs.existsSync(path)) { fs.readdirSync(path).forEach(function (file, index) { var curPath = path + "/" + file; if (fs.lstatSync(curPath).isDirectory()) { // recurse deleteFolderRecursive(curPath); } else { // delete file fs.unlinkSync(curPath); } }); fs.rmdirSync(path); } }; function clearProjectBeforeGenerating(projectDir) { let modelsDir = `${projectDir}/models`; let operationsDir = `${projectDir}/operations`; let clientTypedefFile = path.basename(glob.sync(`${projectDir}/*.d.ts`)[0] || ''); let clientJSFile = `${clientTypedefFile.split('.')[0]}.js`; let directoriesToBeDeleted = [modelsDir, operationsDir]; let filesToBeDeleted = [clientTypedefFile, clientJSFile]; directoriesToBeDeleted.forEach((dir) => { if (fs.existsSync(dir)) { deleteFolderRecursive(dir); } }); filesToBeDeleted.forEach((file) => { if (fs.existsSync(file)) { fs.unlinkSync(file); } }); return; } function generateProject(projectObj, specRoot, autoRestVersion) { let specPath = specRoot + '/' + projectObj.source; let isInputJson = projectObj.source.endsWith("json"); let result; const azureTemplate = 'Azure.NodeJs'; language = azureTemplate; //servicefabric wants to generate using generic NodeJS. if (projectObj.language && projectObj.language.match(/^NodeJS$/ig) !== null) { language = projectObj.language; } let packageName = projectObj.packageName; console.log(`\n>>>>>>>>>>>>>>>>>>>Start: "${packageName}" >>>>>>>>>>>>>>>>>>>>>>>>>`); let outputDir = `lib/services/${projectObj.dir}`; let cmd = 'autorest '; if (projectObj.batchGeneration) { cmd += `--nodejs-sdks-folder=${__dirname}/${outputDir} --package-name=${packageName} --nodejs --license-header=MICROSOFT_MIT_NO_VERSION`; } else { cmd += `--output-folder=${__dirname}/${outputDir} --package-name=${packageName} --nodejs --license-header=MICROSOFT_MIT_NO_VERSION`; } // if using azure template, pass in azure-arm argument. otherwise, get the generic template by not passing in anything. if (language === azureTemplate) cmd += ' --azure-arm '; if (isInputJson) { cmd += ` --input-file=${specPath} `; } else { cmd += ` ${specPath} `; } if (projectObj.ft !== null && projectObj.ft !== undefined) cmd += ' --payload-flattening-threshold=' + projectObj.ft; if (projectObj.clientName !== null && projectObj.clientName !== undefined) cmd += ' --override-client-name=' + projectObj.clientName; if (projectObj.tag !== null && projectObj.tag !== undefined) cmd += `--tag=${projectObj.tag}`; if (projectObj.args !== undefined) { cmd = cmd + ' ' + args; } try { //console.log(`Cleaning the output directory: "${outputDir}".`); //clearProjectBeforeGenerating(outputDir); console.log('Executing command:'); console.log('------------------------------------------------------------'); console.log(cmd); console.log('------------------------------------------------------------'); result = execSync(cmd, { encoding: 'utf8' }); console.log('Output:'); console.log(result); } catch (err) { console.log('Error:'); console.log(`An error occurred while generating client for package: "${packageName}":\n ${err.stderr}`); } console.log(`>>>>>>>>>>>>>>>>>>>>>End: "${packageName}" >>>>>>>>>>>>>>>>>>>>>>>>>\n`); return; } function installAutorest() { let installation; let isSuccessful = true; let autorestAlreadyInstalled = true; try { execSync(`autorest --help`); } catch (error) { autorestAlreadyInstalled = false; } try { if (!autorestAlreadyInstalled) { console.log('Looks like autorest is not installed on your machine. Installing autorest . . .'); let installCmd = 'npm install -g autorest'; console.log(installCmd); installation = execSync(installCmd, { encoding: 'utf8' }); //console.debug('installation'); } isSuccessful = getAutorestVersion(); } catch (err) { isSuccessful = false; console.log(`An error occurred while installing autorest via npm:\n ${util.inspect(err, { depth: null })}.`); } return isSuccessful; } function codegen(projectObj, index) { let versionSuccessfullyFound = true; let usingAutoRestVersion = defaultAutoRestVersion; function checkAutorestVersion(actualProj) { if (actualProj.autoRestVersion) { usingAutoRestVersion = actualProj.autoRestVersion; } if (index === 0) { versionSuccessfullyFound = getAutorestVersion(usingAutoRestVersion); if (!versionSuccessfullyFound) { process.exit(1); } } } function iterateProject(proj, specRoot, usingAutoRestVersion) { for (key in proj) { if (proj[key]['packageName']) { if (!versionSuccessfullyFound) { checkAutorestVersion(proj[key], index); } generateProject(proj[key], specRoot, usingAutoRestVersion); } else { iterateProject(proj[key], specRoot, usingAutoRestVersion); } } } return iterateProject(projectObj, specRoot, usingAutoRestVersion); } gulp.task('default', function () { console.log("Usage: gulp codegen [--spec-root ] [--project ]\n"); console.log("--spec-root"); console.log("\tRoot location of Swagger API specs, default value is \"https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master\""); console.log("--project\n\tProject to regenerate, default is all. List of available project names:"); Object.keys(mappings).forEach(function (i) { console.log('\t' + i.magenta); }); }); //This task is used to generate libraries based on the mappings specified above. gulp.task('codegen', function (cb) { if (project === undefined) { let arr = Object.keys(mappings); for (let i = 0; i < arr.length; i++) { codegen(mappings[arr[i]], i); } } else { if (mappings[project] === undefined) { console.error('Invalid project name "' + project + '"!'); process.exit(1); } codegen(mappings[project], null); } }); //This task validates that the entry in "main" and "types" in package.json points to a file that exists on the disk. // for best results run on mac or linux. Windows is case insenstive for file paths. Hence it will not catch those issues. //If not tested this will cause "module not found" errors for customers when they try to use the package. gulp.task('validate-each-packagejson', (cb) => { let packagePaths = glob.sync(path.join(__dirname, '/lib/services', '/**/package.json'), { ignore:  '**/node_modules/**' }); packagePaths.forEach((packagePath) => { const package = require(packagePath); //console.log(package); if (!package.name.startsWith('azure-asm-')) { console.log(`Validating package: ${package.name}`); if (package.main) { let mainPath = path.resolve(path.dirname(packagePath), package.main); if (!fs.existsSync(mainPath)) console.log(`\t>${mainPath} does not exist.`); } else { console.log(`\t>Could not find "main" entry in package.json for ${packagePath}.`); } if (package.types) { let typesPath = path.resolve(path.dirname(packagePath), package.types); if (!fs.existsSync(typesPath)) console.log(`\t>${typesPath} does not exist.`); } else { console.log(`\t>Could not find "types" entry in package.json for ${packagePath}.`); } } }); }); //This task updates the dependencies in package.json to the relative service libraries inside lib/services directory. gulp.task('update-deps-rollup', (cb) => { let packagePaths = glob.sync(path.join(__dirname, './lib/services', '/**/package.json')).filter((packagePath) => { return packagePath.match(regexForExcludedServices) === null; }); let rollupPackage = require('./package.json'); let rollupDependencies = rollupPackage.dependencies; rollupDependencies['ms-rest'] = './runtime/ms-rest'; rollupDependencies['ms-rest-azure'] = './runtime/ms-rest-azure'; packagePaths.forEach((packagePath) => { const package = require(packagePath); //console.log(package); let packageName = package.name packageDir = path.dirname(packagePath); if (rollupDependencies[packageName]) { rollupDependencies[packageName] = packageDir; } else { console.log(`Could not find ${packageName} as a dependecy in rollup package.json file..`); } }); fs.writeFileSync('./package.json', JSON.stringify(rollupPackage, null, 2), { 'encoding': 'utf8' }); }); //This task ensures that all the exposed createSomeClient() methods, can correctly instantiate clients. By doing this we test, //that the "main" entry in package.json points to a file at the correct location. We test the signature of the client constructor //is as expected. As of now HD Isnight is expected to fail as it is still using the Hyak generator. Once it moves to Autorest, it should //not fail. Before executing this task, execute `gulp update-deps-rollup`, `rm -rf node_modules` and `npm install` so that the changes inside the sdks in lib/services //are installed inside the node_modules folder. gulp.task('test-create-rollup', (cb) => { const azure = require('./lib/azure'); const keys = Object.keys(azure).filter((key) => { return key.startsWith('create') && !key.startsWith('createASM') && key.endsWith('Client') && key !== 'createSchedulerClient' }); //console.dir(keys); //console.log(keys.length); const creds = { signRequest: {} }; const subId = '1234556'; keys.forEach((key) => { console.log(key); const Client = azure[key]; var c; try { if (key === 'createKeyVaultClient' || key === 'createSubscriptionManagementClient' || key === 'createDataLakeAnalyticsJobManagementClient' || key === 'createDataLakeStoreFileSystemManagementClient' || key === 'createDataLakeAnalyticsCatalogManagementClient') { c = new Client(creds); } else if (key === 'createServiceFabricClient') { c = new Client(); } else { c = new Client(creds, subId); } //console.dir(Object.keys(c)); } catch (err) { console.dir(err); } }); }); // This task updates the codegen_mappings.json file in sync with the azure-rest-api-specs public repo. gulp.task('sync-mappings-with-repo', (cb) => { if (!specRepoDir) { return cb(new Error('Please provide --spec-repo-dir ')); } let specDir = `${specRepoDir}/specification`; const dirs = fs.readdirSync(specDir).filter(f => fs.statSync(`${specDir}/${f}`).isDirectory()); let newlyAdded = []; let originalProjectCount = Object.keys(mappings).length; for (let rp of dirs) { if (rp.toLowerCase() === 'intune' || rp.toLowerCase() === 'azsadmin' || rp.toLowerCase() === 'timeseriesinsights') continue; let rm = `${specRepoDir}/specification/${rp}/resource-manager`; let dp = `${specRepoDir}/specification/${rp}/data-plane`; if (!mappings[rp]) { mappings[rp] = {}; if (fs.existsSync(rm)) { mappings[rp]['resource-manager'] = { "packageName": `azure-arm-${rp.toLowerCase()}`, "dir": `${rp}Management/lib`, "source": `${rp}/resource-manager/readme.md` } newlyAdded.push(`${rp}['resource-manager']`); console.log(`Updating RP: ${rp}, "resource-manager".`); console.dir(mappings[rp]['resource-manager'], { depth: null, colors: true }); } if (fs.existsSync(dp)) { mappings[rp]['data-plane'] = { "packageName": `azure-${rp.toLowerCase()}`, "dir": `${rp}/lib`, "source": `${rp}/data-plane/readme.md` } newlyAdded.push(`${rp}['data-plane']`); console.log(`Updating RP: ${rp}, "data-plane".`); console.dir(mappings[rp]['data-plane'], { depth: null, colors: true }); } } else { if (fs.existsSync(rm) && !mappings[rp]['resource-manager']) { mappings[rp]['resource-manager'] = { "packageName": `azure-arm-${rp.toLowerCase()}`, "dir": `${rp}Management/lib`, "source": `${rp}/resource-manager/readme.md` } newlyAdded.push(`${rp}['resource-manager']`); console.log(`Updating RP: ${rp}, "resource-manager".`); console.dir(mappings[rp]['resource-manager'], { depth: null, colors: true }); } if (fs.existsSync(dp) && !mappings[rp]['data-plane']) { mappings[rp]['data-plane'] = { "packageName": `azure-${rp.toLowerCase()}`, "dir": `${rp}/lib`, "source": `${rp}/data-plane/readme.md` } newlyAdded.push(`${rp}['data-plane']`); console.log(`Updating RP: ${rp}, "data-plane".`); console.dir(mappings[rp]['data-plane'], { depth: null, colors: true }); } } } if (!newlyAdded.length) { console.log('\n\n> Mappings in ./codegen_mappings.json are already in sync...'); } else { console.log(`\n\n> Basic properties like "packageName", "dir" and "source" have been added to ` + `the newly added projects "${newlyAdded.join()}" in the mappings.\n\n> Please ensure that other properties ` + `like: "ft", "clientName", etc. are correctly added as deemed necessary.\n\n> If the specs repo had multiple ` + `specs in data-plane or resource-manager (for example: "datalake-analytics.data-plane" has "catalog" ` + `and "job" in it), then please update the project mappings yourself.`) } console.log(`\n\n>>>>> Total projects in the mappings before sync: ${originalProjectCount}`); console.log(`\n>>>>> Total projects in the mappings after sync: ${Object.keys(mappings).length}`); fs.writeFileSync('./codegen_mappings.json', JSON.stringify(mappings, null, 2)); }); // This task synchronizes the dependencies in package.json to the versions of relative service libraries inside lib/services directory. // This should be done in the end to ensure that all the package dependencies have the correct version. gulp.task('sync-deps-rollup', (cb) => { let packagePaths = glob.sync(path.join(__dirname, './lib/services', '/**/package.json')).filter((packagePath) => { return packagePath.match(regexForExcludedServices) === null; }); //console.log(packagePaths); console.log(`Total packages found under lib/services: ${packagePaths.length}`); let rollupPackage = require('./package.json'); let rollupDependencies = rollupPackage.dependencies; rollupDependencies['ms-rest'] = '^2.2.2'; rollupDependencies['ms-rest-azure'] = '^2.3.4'; packagePaths.forEach((packagePath) => { const package = require(packagePath); //console.log(package); let packageName = package.name; let packageVersion = package.version; rollupDependencies[packageName] = packageVersion; }); rollupPackage.dependencies = Object.keys(rollupDependencies).sort().reduce((r, k) => (r[k] = rollupDependencies[k], r), {}); console.log(`Total number of dependencies in the rollup package: ${Object.keys(rollupPackage.dependencies).length}`); fs.writeFileSync('./package.json', JSON.stringify(rollupPackage, null, 2), { 'encoding': 'utf8' }); }); gulp.task('sync-package-service-mapping', (cb) => { let packageMapping = require('./package_service_mapping'); for (let serviceName in mappings) { let serviceObj = mappings[serviceName]; let resourceMgr = serviceObj['resource-manager']; let Dataplane = serviceObj['data-plane']; if (resourceMgr) { if (resourceMgr.packageName) { if (!packageMapping[resourceMgr.packageName]) { packageMapping[resourceMgr.packageName] = { category: 'Management', 'service_name': resourceMgr.dir.split('/')[0] }; } } else { for (let service in resourceMgr) { if (resourceMgr[service].packageName) { if (!packageMapping[resourceMgr[service].packageName]) { packageMapping[resourceMgr[service].packageName] = { category: 'Management', 'service_name': resourceMgr[service].dir.split('/')[0] }; } } } } } if (Dataplane) { if (Dataplane.packageName) { if (!packageMapping[Dataplane.packageName]) { packageMapping[Dataplane.packageName] = { category: 'Client', 'service_name': Dataplane.dir.split('/')[0] }; } } else { for (let service in Dataplane) { if (Dataplane[service].packageName) { if (!packageMapping[Dataplane[service].packageName]) { packageMapping[Dataplane[service].packageName] = { category: 'Client', 'service_name': Dataplane[service].dir.split('/')[0] }; } } } } } } packageMapping = Object.keys(packageMapping).sort().reduce((r, k) => (r[k] = packageMapping[k], r), {}); fs.writeFileSync('./package_service_mapping.json', JSON.stringify(packageMapping, null, 2), { 'encoding': 'utf8' }); });