Add generating readme files and SDKs (#39)

* Refactor Gulp structure

* Install nodegit

* Fix missing imports

* Refactor structure #2

* Refactor codegen

* Fix common.ts merge conflicts

* Add basic git operations

* Add updating main readme.md

* Fix branching

* Add GitHub integration

* Add draft of SDK generation

* Add automated PR creation for SDK creation

* Refactor file to use logger

* Add passing token as a command line argument

* Add capturing printed text

* Refactor logging

* Fix incorrect path bug

* Add committing and creating PRs in transactions

* Fix hard coded token

* Remove debug token printing

* Add step skipping

* Add @azure prefix to package names

* Add additional logging

* Change generated name of data-plane SDKs

* Remove unnecessary b option
This commit is contained in:
Kamil Pajdzik 2018-10-12 12:24:28 -07:00 коммит произвёл GitHub
Родитель afbf2ce04a
Коммит a6a86ac55c
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 1099 добавлений и 518 удалений

74
.scripts/commandLine.ts Normal file
Просмотреть файл

@ -0,0 +1,74 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import * as minimist from "minimist";
import { arrayContains } from "./common";
export interface CommandLineOptions extends minimist.ParsedArgs {
"azure-sdk-for-js-repo-root": string;
"azure-rest-api-specs-root": string;
debugger: boolean;
"logging-level": string;
package: string;
"skip-sdk": boolean;
"skip-spec": boolean;
type: string;
use: boolean;
verbose: boolean;
whatif: boolean;
getSdkType(): SdkType;
}
export const commandLineConfiguration = {
string: ["azure-sdk-for-js-repo-root", "azure-rest-api-specs-root", "logging-level", "package", "type"],
boolean: ["debugger", "use", "skip-sdk", "skip-spec", "verbose", "whatif"],
alias: {
l: "logging-level",
log: "logging-level",
package: "packageName",
u: "use",
v: "version",
},
default: {
"logging-level": "info",
type: "arm"
}
};
export enum SdkType {
ResourceManager = "resource-manager",
DataPlane = "data-plane",
ControlPlane = "control-plane"
}
let _options: CommandLineOptions;
export function getCommandLineOptions() {
if (!_options) {
_options = createCommandLineParameters();
}
return _options;
}
function createCommandLineParameters() {
const args = minimist(process.argv.slice(2), commandLineConfiguration) as CommandLineOptions;
args.getSdkType = getSdkType;
return args;
}
export function getSdkType() {
const resourceManagerStrings = ["arm", "rm", "resourcemanager"]
const dataPlaneStrings = ["dp", "data", "dataplane"]
const type = this.type.toLowerCase().replace("-", "");
if (arrayContains(resourceManagerStrings, type)) {
return SdkType.ResourceManager;
} else if (arrayContains(dataPlaneStrings, type)) {
return SdkType.DataPlane;
} else {
throw new Error("Unknown SDK type");
}
}

Просмотреть файл

@ -1,22 +0,0 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import * as minimist from "minimist";
export interface CommandLineOptions extends minimist.ParsedArgs {
package: string,
type: string,
debug: boolean,
d: boolean,
verbose: boolean,
b: boolean,
getSdkType(): SdkType;
}
export enum SdkType {
ResourceManager,
DataPlane
}

57
.scripts/common.ts Normal file
Просмотреть файл

@ -0,0 +1,57 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import * as fssync from "fs";
import { promises as fs } from "fs";
import { execSync } from "child_process";
import { getLogger } from "./logger";
const _logger = getLogger();
export function arrayContains<T>(array: T[], el: T): boolean {
return array.indexOf(el) != -1
}
export async function isDirectory(directoryPath: string): Promise<boolean> {
const stats = await fs.lstat(directoryPath);
return stats.isDirectory();
}
export async function pathExists(path: string): Promise<boolean> {
return new Promise<boolean>((resolve, reject) => {
fssync.exists(path, exists => {
resolve(exists);
})
});
}
export function startsWith(value: string, prefix: string): boolean {
return value && prefix && value.indexOf(prefix) === 0;
}
export function endsWith(value: string, suffix: string): boolean {
return value && suffix && value.length >= suffix.length && value.lastIndexOf(suffix) === value.length - suffix.length;
}
export function contains(values: string[], searchString: string): boolean {
return arrayContains(values, searchString);
}
export function execute(command: string, packageFolderPath: string): void {
if (!fssync.existsSync(packageFolderPath)) {
_logger.logWithPath(packageFolderPath, "Folder not found.");
} else {
execSync(command, { cwd: packageFolderPath, stdio: "inherit" });
}
}
export function npmRunBuild(packageFolderPath: string): void {
execute("npm run build", packageFolderPath);
}
export function npmInstall(packageFolderPath: string): void {
execute("npm install", packageFolderPath);
}

Просмотреть файл

@ -1,256 +0,0 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import * as fssync from "fs";
import { promises as fs } from "fs";
import * as path from "path";
import * as minimist from "minimist";
import * as yaml from "js-yaml";
import { CommandLineOptions, SdkType } from "./commandLineOptions";
import { Logger } from "./logger";
interface readmeSettings {
"nodejs": {
"azure-arm": boolean;
"license-header": string;
"payload-flattening-threshold": number;
"package-name": string;
"output-folder": string;
"generate-license-txt": boolean | undefined;
"generate-package-json": boolean | undefined;
"generate-readme-md": boolean | undefined;
"generate-metadata": boolean | undefined;
} | undefined;
}
const repositoryName = "azure-rest-api-specs";
const specificationsSegment = "specification";
const args = minimist(process.argv.slice(2), {
string: ["package", "type"],
boolean: ["debug", "verbose"],
alias: {
d: "debug",
package: "packageName",
v: "version",
},
default: {
type: "arm"
}
}) as CommandLineOptions;
const _logger = new Logger(args);
if (!fs) {
throw new Error("This script has to be run on Node.js 10.0+");
}
function contains<T>(array: T[], el: T): boolean {
return array.indexOf(el) != -1
}
async function isDirectory(directoryPath: string): Promise<boolean> {
const stats = await fs.lstat(directoryPath);
return stats.isDirectory();
}
async function exists(path: string): Promise<boolean> {
return new Promise<boolean>((resolve, reject) => {
fssync.exists(path, exists => {
resolve(exists);
})
});
}
args.getSdkType = function () {
const resourceManagerStrings = ["arm", "rm", "resourcemanager"]
const dataPlaneStrings = ["dp", "data", "dataplane"]
const type = this.type.toLowerCase().replace("-", "");
if (contains(resourceManagerStrings, type)) {
return SdkType.ResourceManager;
} else if (contains(dataPlaneStrings, type)) {
return SdkType.DataPlane;
} else {
throw new Error("Unknown SDK type");
}
}
export async function findAzureRestApiSpecsRepository(): Promise<string> {
let currentDirectory = __dirname;
const pathData = path.parse(currentDirectory);
const rootDirectory = pathData.root;
do {
currentDirectory = path.resolve(currentDirectory, "..");
if (await containsDirectory(repositoryName, currentDirectory)) {
return path.resolve(currentDirectory, repositoryName);
}
} while (currentDirectory != rootDirectory);
throw new Error(`${repositoryName} not found!`)
}
async function containsDirectory(directoryName: string, parentPath: string): Promise<boolean> {
return await exists(path.resolve(parentPath, directoryName));
}
export async function findSdkDirectory(azureRestApiSpecsRepository: string): Promise<string> {
const sdkSegment = args.getSdkType() === SdkType.ResourceManager ? "resource-manager" : "data-plane";
const sdkPath = path.resolve(azureRestApiSpecsRepository, specificationsSegment, args.packageName, sdkSegment);
if (await !exists(sdkPath)) {
throw new Error(`${sdkPath} SDK specs don't exist`);
}
return sdkPath;
}
export async function findMissingSdks(azureRestApiSpecsRepository: string): Promise<string[]> {
const specsDirectory = path.resolve(azureRestApiSpecsRepository, specificationsSegment);
const serviceSpecs = await fs.readdir(specsDirectory);
const missingSdks = [];
for (const serviceDirectory of serviceSpecs) {
const fullServicePath = path.resolve(specsDirectory, serviceDirectory);
if (!(await isDirectory(fullServicePath))) {
continue;
}
const sdkTypeDirectories = await fs.readdir(fullServicePath);
for (const sdkTypeDirectory of sdkTypeDirectories) {
const fullSdkPath = path.resolve(fullServicePath, sdkTypeDirectory);
if (!(await isDirectory(fullSdkPath))) {
continue;
}
const readmeFiles = (await fs.readdir(fullSdkPath)).filter(file => /^readme/.test(file));
const fullSpecName = `${serviceDirectory} [${sdkTypeDirectory}]`
if (readmeFiles.length <= 0) {
// No readme.md
continue;
} else if (readmeFiles.length == 1) {
const readmeMdPath = readmeFiles[0];
if (await doesReadmeMdFileSpecifiesTypescriptSdk(readmeMdPath)) {
missingSdks.push(fullSdkPath);
_logger.logRed(`${fullSpecName}`);
} else if (args.debug) {
_logger.logGreen(fullSpecName);
}
} else if (contains(readmeFiles, "readme.nodejs.md")) {
if (!contains(readmeFiles, "readme.typescript.md")) {
missingSdks.push(fullSdkPath);
_logger.logRed(`${fullSpecName}`);
} else if (args.debug) {
_logger.logGreen(fullSpecName);
}
}
}
}
return missingSdks;
}
async function getYamlSection(buffer: Buffer, sectionBeginning: string, sectionEnd: string): Promise<Buffer> {
const beginningIndex = buffer.indexOf(sectionBeginning);
const trimmedBuffer = buffer.slice(beginningIndex + (sectionBeginning.length));
const endIndex = trimmedBuffer.indexOf(sectionEnd, 3);
const sectionBuffer = trimmedBuffer.slice(0, endIndex);
return sectionBuffer;
}
async function doesReadmeMdFileSpecifiesTypescriptSdk(readmeMdPath: string): Promise<boolean> {
const readmeMdBuffer = await fs.readFile(readmeMdPath);
const sectionBuffer = await getYamlSection(readmeMdBuffer, "``` yaml $(swagger-to-sdk)", "```");
if (sectionBuffer.includes("azure-sdk-for-js")) {
return true;
}
return false;
}
export async function copyExistingNodeJsReadme(sdkPath: string): Promise<string> {
const nodeJsReadmePath = path.resolve(sdkPath, "readme.nodejs.md");
const typescriptReadmePath = path.resolve(sdkPath, "readme.typescript.md");
if (args.verbose) {
_logger.log(`Copying ${nodeJsReadmePath} to ${typescriptReadmePath}`)
}
if (await exists(typescriptReadmePath)) {
throw new Error(`${typescriptReadmePath} file already exists`)
}
await fs.copyFile(nodeJsReadmePath, typescriptReadmePath);
return typescriptReadmePath;
}
async function updatePackageName(settings: readmeSettings): Promise<readmeSettings> {
const packageName = settings.nodejs["package-name"]
if (packageName.startsWith("arm") || !packageName.startsWith("azure-")) {
return settings;
}
settings.nodejs["package-name"] = packageName.replace("azure-", "");
return settings;
}
async function updateMetadataFields(settings: readmeSettings): Promise<readmeSettings> {
settings.nodejs["generate-metadata"] = true;
delete settings.nodejs["generate-license-txt"]
delete settings.nodejs["generate-package-json"]
delete settings.nodejs["generate-readme-md"];
return settings;
}
async function updateOutputFolder(settings: readmeSettings): Promise<readmeSettings> {
settings.nodejs["output-folder"] = `$(typescript-sdks-folder)/packages/${settings.nodejs["package-name"]}`;
return settings;
}
async function updateYamlSection(sectionText: string): Promise<string> {
const section = yaml.safeLoad(sectionText);
await updatePackageName(section);
await updateMetadataFields(section);
await updateOutputFolder(section);
section["typescript"] = section.nodejs;
delete section.nodejs;
return yaml.safeDump(section).trim();
}
export async function updateTypeScriptReadmeFile(typescriptReadmePath: string): Promise<string> {
const readmeBuffer: Buffer = await fs.readFile(typescriptReadmePath);
const readme: string = readmeBuffer.toString();
let outputReadme = readme;
const yamlSection = await getYamlSection(readmeBuffer, "``` yaml $(nodejs)", "```");
const sectionText = yamlSection.toString().trim();
const updatedYamlSection = await updateYamlSection(sectionText);
outputReadme = outputReadme.replace(sectionText, updatedYamlSection);
outputReadme = outputReadme.replace("azure-sdk-for-node", "azure-sdk-for-js");
outputReadme = outputReadme.replace("Node.js", "TypeScript");
outputReadme = outputReadme.replace("$(nodejs)", "$(typescript)");
outputReadme = outputReadme.replace("nodejs", "typescript");
outputReadme = outputReadme.replace("Node", "TypeScript");
outputReadme = outputReadme.replace("node", "typescript");
return outputReadme;
}
export async function saveContentToFile(filePath: string, content: string): Promise<void> {
await fs.writeFile(filePath, content);
}

118
.scripts/generateSdks.ts Normal file
Просмотреть файл

@ -0,0 +1,118 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import { promises as fs } from "fs";
import * as path from "path";
import { SdkType } from "./commandLine";
import { pathExists, isDirectory, arrayContains } from "./common";
import { getLogger } from "./logger";
import { doesReadmeMdFileSpecifiesTypescriptSdk } from "./readme";
const repositoryName = "azure-rest-api-specs";
const specificationsSegment = "specification";
const _logger = getLogger();
if (!fs) {
throw new Error("This script has to be run on Node.js 10.0+");
}
export async function findAzureRestApiSpecsRepositoryPath(): Promise<string> {
let currentDirectory = __dirname;
const pathData = path.parse(currentDirectory);
const rootDirectory = pathData.root;
do {
currentDirectory = path.resolve(currentDirectory, "..");
if (await containsDirectory(repositoryName, currentDirectory)) {
return path.resolve(currentDirectory, repositoryName);
}
} while (currentDirectory != rootDirectory);
return Promise.reject(`${repositoryName} not found!`)
}
async function containsDirectory(directoryName: string, parentPath: string): Promise<boolean> {
return await pathExists(path.resolve(parentPath, directoryName));
}
export async function findSdkDirectory(azureRestApiSpecsRepository: string, packageName: string, sdkType: SdkType): Promise<string> {
const sdkPath = path.resolve(azureRestApiSpecsRepository, specificationsSegment, packageName, sdkType);
if (await !pathExists(sdkPath)) {
return Promise.reject(`${sdkPath} SDK specs don't exist`);
}
return sdkPath;
}
export async function findMissingSdks(azureRestApiSpecsRepository: string): Promise<{ sdkName: string; sdkType: SdkType }[]> {
_logger.logTrace(`Finding missing SDKS in ${azureRestApiSpecsRepository}`);
const specsDirectory = path.resolve(azureRestApiSpecsRepository, specificationsSegment);
_logger.logTrace(`Reading "${azureRestApiSpecsRepository}" directory`);
const serviceSpecs = await fs.readdir(specsDirectory);
_logger.logTrace(`Found ${serviceSpecs.length} specification folders`);
const missingSdks = [];
for (const serviceDirectory of serviceSpecs) {
const fullServicePath = path.resolve(specsDirectory, serviceDirectory);
_logger.logTrace(`Analyzing ${serviceDirectory} in ${fullServicePath}`);
if (!(await isDirectory(fullServicePath))) {
_logger.logWarn(`"${fullServicePath}" is not a directory. Skipping`);
continue;
}
const sdkTypeDirectories = await fs.readdir(fullServicePath);
_logger.logTrace(`Found ${sdkTypeDirectories.length} specification type folders: [${sdkTypeDirectories}]`);
for (const sdkTypeDirectory of sdkTypeDirectories) {
const fullSdkPath = path.resolve(fullServicePath, sdkTypeDirectory);
_logger.logTrace(`Analyzing ${sdkTypeDirectory} in ${fullSdkPath}`);
if (!(await isDirectory(fullSdkPath))) {
_logger.logWarn(`"${fullServicePath}" is not a directory. Skipping`);
continue;
}
const readmeFiles = (await fs.readdir(fullSdkPath)).filter(file => /^readme/.test(file));
const fullSpecName = `${serviceDirectory} [${sdkTypeDirectory}]`
const sdk = { sdkName: serviceDirectory, sdkType: sdkTypeDirectory };
if (readmeFiles.length <= 0) {
// No readme.md
continue;
} else if (arrayContains(readmeFiles, "readme.nodejs.md")) {
if (!arrayContains(readmeFiles, "readme.typescript.md")) {
missingSdks.push(sdk);
_logger.logWithDebugDetails(`${fullSpecName}`.negative, "readme.nodejs.md exists but no matching readme.typescript.md");
} else {
_logger.logDebug(fullSpecName.positive);
}
} else if (arrayContains(readmeFiles, "readme.md")) {
const readmeMdPath = path.resolve(fullSdkPath, "readme.md");
if (await doesReadmeMdFileSpecifiesTypescriptSdk(readmeMdPath)) {
missingSdks.push(sdk);
_logger.logWithDebugDetails(`${fullSpecName}`.negative, "typescript mentioned in readme.md but no readme.typescript.md exists");
} else {
_logger.logDebug(fullSpecName.positive);
}
}
}
}
return missingSdks;
}
export async function saveContentToFile(filePath: string, content: string): Promise<void> {
await fs.writeFile(filePath, content);
}

196
.scripts/git.ts Normal file
Просмотреть файл

@ -0,0 +1,196 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import { Repository, Signature, Merge, Oid, Reference, Cred, StatusFile } from "nodegit";
import { getLogger } from "./logger";
import { getCommandLineOptions } from "./commandLine";
export type ValidateFunction = (statuses: StatusFile[]) => boolean;
export type ValidateEachFunction = (value: StatusFile, index: number, array: StatusFile[]) => boolean;
const _args = getCommandLineOptions();
const _logger = getLogger();
const _lockMap = { }
function isLocked(repositoryPath: string) {
const isLocked = _lockMap[repositoryPath];
return isLocked || false;
}
function lock(repositoryPath: string) {
_lockMap[repositoryPath] = true;
}
function unlock(repositoryPath: string) {
_lockMap[repositoryPath] = true;
}
async function waitUntilUnlocked(repositoryPath: string): Promise<void> {
_logger.logTrace("Waiting for the repository to be unlocked");
return new Promise<void>((resolve, reject) => {
const wait = () => {
setTimeout(() => {
_logger.logTrace(`Repository is ${isLocked(repositoryPath) ? "locked" : "unlocked"}`);
if (isLocked(repositoryPath)) {
wait();
} else {
resolve();
}
}, 50);
}
wait();
});
}
export async function waitAndLockGitRepository(repository: Repository): Promise<boolean> {
_logger.logTrace("Waiting to lock the repository");
const repositoryPath = repository.path();
await waitUntilUnlocked(repositoryPath);
if (!isLocked(repositoryPath)) {
lock(repositoryPath);
return isLocked(repositoryPath);
}
return waitAndLockGitRepository(repository);
}
export function unlockGitRepository(repository: Repository) {
_logger.logTrace("Unlocking the repository");
unlock(repository.path());
}
export async function openRepository(repositoryPath: string): Promise<Repository> {
_logger.logTrace(`Opening Git repository located in ${repositoryPath}`);
return Repository.open(repositoryPath)
}
export async function validateRepositoryStatus(repository: Repository): Promise<void> {
const status = await repository.getStatus();
_logger.logTrace(`Current repository status: ${JSON.stringify(status)}`);
if (status && status.length > 0) {
return Promise.reject(`Not committed changes exist in ${repository.path()} repository`);
}
_logger.logTrace(`Status of the repository validated successfully`);
}
export async function getValidatedRepository(repositoryPath: string): Promise<Repository> {
const repository = await openRepository(repositoryPath);
await validateRepositoryStatus(repository);
return repository;
}
export async function pull(repository: Repository, branchName: string, origin: string = "origin"): Promise<Oid> {
_logger.logTrace(`Pulling "${branchName}" branch from ${origin} origin in ${repository.path()} repository`);
await repository.fetchAll();
_logger.logTrace(`Fetched all successfully`);
const oid = await repository.mergeBranches(branchName, `${origin}/${branchName}`, Signature.default(repository), Merge.PREFERENCE.NONE);
const index = await repository.index();
if (index.hasConflicts()) {
throw new Error(`Conflict while pulling ${branchName} from origin.`);
}
_logger.logTrace(`Merged "${origin}/${branchName}" to "${branchName}" successfully without any conflicts`);
return oid;
}
export async function pullMaster(repository: Repository): Promise<Oid> {
return pull(repository, "master");
}
export async function createNewBranch(repository: Repository, branchName: string, checkout?: boolean): Promise<Reference> {
_logger.logTrace(`Create new branch "${branchName}" in ${repository.path()} repository`);
const headCommit = await repository.getHeadCommit();
const branchPromise = repository.createBranch(branchName, headCommit, false);
_logger.logTrace(`Created new branch "${branchName}" successfully`);
if (!checkout) {
return branchPromise;
} else {
const branch = await branchPromise;
return checkoutBranch(repository, branch.name());
}
}
function getCurrentDateSuffix(): string {
const now = new Date();
return `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}-${now.getMilliseconds()}`;
}
export async function createNewUniqueBranch(repository: Repository, branchPrefix: string, checkout?: boolean): Promise<Reference> {
return createNewBranch(repository, `${branchPrefix}-${getCurrentDateSuffix()}`, checkout);
}
export async function checkoutBranch(repository: Repository, branchName: string | Reference): Promise<Reference> {
_logger.logTrace(`Checking out ${branchName} branch`);
return repository.checkoutBranch(branchName);
}
export async function checkoutMaster(repository: Repository): Promise<Reference> {
return checkoutBranch(repository, "master");
}
export async function refreshRepository(repository: Repository) {
await pullMaster(repository);
return checkoutMaster(repository);
}
export async function commitSpecificationChanges(repository: Repository, commitMessage: string, validate?: ValidateFunction, validateEach?: ValidateEachFunction): Promise<Oid> {
_logger.logTrace(`Committing changes in "${repository.path()}" repository`);
const emptyValidate = () => true;
validate = validate || emptyValidate;
validateEach = validateEach || emptyValidate;
const status = await repository.getStatus();
if (validate(status) && status.every(validateEach)) {
var author = Signature.default(repository);
return repository.createCommitOnHead(status.map(el => el.path()), author, author, commitMessage);
} else {
return Promise.reject("Unknown changes present in the repository");
}
}
export async function pushToNewBranch(repository: Repository, branchName: string): Promise<number> {
const remote = await repository.getRemote("origin");
return remote.push([`${branchName}:${branchName}`], {
callbacks: {
credentials: function (url, userName) {
return Cred.userpassPlaintextNew(getToken(), "x-oauth-basic");
}
}
});
}
export function getToken(): string {
const token = _args.token || process.env.SDK_GEN_GITHUB_TOKEN;
_validatePersonalAccessToken(token);
return token;
}
function _validatePersonalAccessToken(token: string): void {
if (!token) {
const text =
`Github personal access token was not found as a script parameter or as an
environmental variable. Please visit https://github.com/settings/tokens,
generate new token with "repo" scope and pass it with -token switch or set
it as environmental variable named SDK_GEN_GITHUB_TOKEN.`
_logger.logError(text);
}
}

89
.scripts/github.ts Normal file
Просмотреть файл

@ -0,0 +1,89 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import * as Octokit from '@octokit/rest'
import { PullRequestsCreateParams, Response, PullRequestsCreateReviewRequestParams, PullRequestsCreateReviewRequestResponse } from '@octokit/rest';
import { getToken, createNewUniqueBranch, commitSpecificationChanges, pushToNewBranch, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction } from './git';
import { getLogger } from './logger';
import { Repository } from 'nodegit';
const _repositoryOwner = "Azure";
const _logger = getLogger();
function getAuthenticatedClient(): Octokit {
const octokit = new Octokit();
octokit.authenticate({ type: "token", token: getToken() });
return octokit;
}
export async function createPullRequest(repositoryName: string, pullRequestTitle: string, body: string, sourceBranchName: string, destinationBranchName: string = "master"): Promise<Response<Octokit.PullRequestsCreateResponse>> {
const octokit = getAuthenticatedClient();
const prOptions: PullRequestsCreateParams = {
owner: _repositoryOwner,
repo: repositoryName,
title: pullRequestTitle,
head: sourceBranchName,
base: destinationBranchName,
body: body
};
return new Promise<Response<Octokit.PullRequestsCreateResponse>>((resolve, reject) => {
octokit.pullRequests.create(prOptions, (error, response) => {
if (error) {
reject(error);
} else {
resolve(response);
}
});
});
}
export async function requestPullRequestReview(repositoryName: string, prId: number): Promise<Response<PullRequestsCreateReviewRequestResponse>> {
const octokit = getAuthenticatedClient();
const params: PullRequestsCreateReviewRequestParams = {
owner: _repositoryOwner,
repo: repositoryName,
number: prId,
reviewers: [ "daschult", "amarzavery", "sergey-shandar" ]
};
return new Promise<Response<PullRequestsCreateReviewRequestResponse>>((resolve, reject) => {
octokit.pullRequests.createReviewRequest(params, (error, response) => {
if (error) {
reject(error);
} else {
resolve(response);
}
});
});
}
export async function commitAndCreatePullRequest(
repository: Repository,
packageName: string,
commitMessage: string,
repositoryName: string,
pullRequestTitle: string,
pullRequestDescription:string,
validate?: ValidateFunction,
validateEach?: ValidateEachFunction): Promise<string> {
await createNewUniqueBranch(repository, `generated/${packageName}`, true);
await commitSpecificationChanges(repository, commitMessage, validate, validateEach);
const newBranch = await repository.getCurrentBranch();
_logger.logInfo(`Committed changes successfully on ${newBranch.name()} branch`);
await pushToNewBranch(repository, newBranch.name());
_logger.logInfo(`Pushed changes successfully to ${newBranch.name()} branch`);
const pullRequestResponse = await createPullRequest(repositoryName, pullRequestTitle, pullRequestDescription, newBranch.name());
_logger.logInfo(`Created pull request successfully - ${pullRequestResponse.data.html_url}`);
const reviewResponse = await requestPullRequestReview(repositoryName, pullRequestResponse.data.number);
_logger.logInfo(`Requested preview on pull request successfully - ${reviewResponse.data.html_url}`);
return reviewResponse.data.html_url;
}

180
.scripts/gulp.ts Normal file
Просмотреть файл

@ -0,0 +1,180 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import { SdkType, getCommandLineOptions } from "./commandLine";
import { findAzureRestApiSpecsRepositoryPath, findSdkDirectory, saveContentToFile, findMissingSdks } from "./generateSdks";
import { copyExistingNodeJsReadme, updateTypeScriptReadmeFile, findReadmeTypeScriptMdFilePaths, getPackageNamesFromReadmeTypeScriptMdFileContents, getAbsolutePackageFolderPathFromReadmeFileContents, updateMainReadmeFile, getSinglePackageName } from "./readme";
import * as fs from "fs";
import * as path from "path";
import { contains, npmInstall } from "./common";
import { execSync } from "child_process";
import { getLogger } from "./logger";
import { refreshRepository, getValidatedRepository, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction } from "./git";
import { commitAndCreatePullRequest } from "./github";
const _logger = getLogger();
const _args = getCommandLineOptions();
function containsPackageName(packageNames: string[], packageName: string): boolean {
const result = contains(packageNames, packageName) ||
contains(packageNames, `@azure/${packageName}`) ||
contains(packageNames, `"${packageName}"`) ||
contains(packageNames, `"@azure/${packageName}"`) ||
contains(packageNames, `'${packageName}'`) ||
contains(packageNames, `'@azure/${packageName}'`);
_logger.logTrace(`Comparing package name "${packageName}" to ${JSON.stringify(packageNames)} - Result: ${result}`);
return result;
}
export async function generateSdk(azureRestAPISpecsRoot: string, azureSDKForJSRepoRoot: string, packageName: string, use?: boolean, useDebugger?: boolean) {
const typeScriptReadmeFilePaths: string[] = findReadmeTypeScriptMdFilePaths(azureRestAPISpecsRoot);
for (let i = 0; i < typeScriptReadmeFilePaths.length; ++i) {
const typeScriptReadmeFilePath: string = typeScriptReadmeFilePaths[i];
const typeScriptReadmeFileContents: string = await fs.promises.readFile(typeScriptReadmeFilePath, { encoding: 'utf8' });
const packageNames: string[] = getPackageNamesFromReadmeTypeScriptMdFileContents(typeScriptReadmeFileContents);
const packageNamesString: string = JSON.stringify(packageNames);
if (!packageName || containsPackageName(packageNames, packageName)) {
_logger.log(`>>>>>>>>>>>>>>>>>>> Start: "${packageNamesString}" >>>>>>>>>>>>>>>>>>>>>>>>>`);
const readmeFilePath: string = path.resolve(path.dirname(typeScriptReadmeFilePath), 'readme.md');
let cmd = `autorest --typescript --typescript-sdks-folder=${azureSDKForJSRepoRoot} --license-header=MICROSOFT_MIT_NO_VERSION ${readmeFilePath}`;
if (use) {
cmd += ` --use=${use}`;
}
else {
const localAutorestTypeScriptFolderPath = path.resolve(azureSDKForJSRepoRoot, '..', 'autorest.typescript');
if (fs.existsSync(localAutorestTypeScriptFolderPath) && fs.lstatSync(localAutorestTypeScriptFolderPath).isDirectory()) {
cmd += ` --use=${localAutorestTypeScriptFolderPath}`;
}
}
if (useDebugger) {
cmd += ` --typescript.debugger`;
}
try {
_logger.log('Executing command:');
_logger.log('------------------------------------------------------------');
_logger.log(cmd);
_logger.log('------------------------------------------------------------');
const commandOutput = execSync(cmd, { encoding: "utf8" });
_logger.log(commandOutput);
_logger.log('Installing dependencies...');
const packageFolderPath: string = getAbsolutePackageFolderPathFromReadmeFileContents(azureSDKForJSRepoRoot, typeScriptReadmeFileContents);
npmInstall(packageFolderPath);
} catch (err) {
_logger.log('Error:');
_logger.log(`An error occurred while generating client for packages: "${packageNamesString}":\nErr: ${err}\nStderr: "${err.stderr}"`);
}
_logger.log(`>>>>>>>>>>>>>>>>>>> End: "${packageNamesString}" >>>>>>>>>>>>>>>>>>>>>>>>>`);
_logger.log();
}
}
}
export async function generateTsReadme(packageName: string, sdkType: SdkType): Promise<{ pullRequestUrl?: string, typescriptReadmePath?: string }> {
if (_args["skip-spec"]) {
_logger.log(`Skipping spec generation`);
return { };
}
const azureRestApiSpecsRepositoryPath: string = await findAzureRestApiSpecsRepositoryPath();
const azureRestApiSpecRepository = await getValidatedRepository(azureRestApiSpecsRepositoryPath);
_logger.log(`Found azure-rest-api-specs repository in ${azureRestApiSpecsRepositoryPath}`);
await refreshRepository(azureRestApiSpecRepository);
_logger.log(`Refreshed ${azureRestApiSpecsRepositoryPath} repository successfully`);
const sdkPath: string = await findSdkDirectory(azureRestApiSpecsRepositoryPath, packageName, sdkType);
_logger.log(`Found specification in ${sdkPath}`);
await waitAndLockGitRepository(azureRestApiSpecRepository);
const typescriptReadmePath: string = await copyExistingNodeJsReadme(sdkPath);
_logger.log(`Copied readme file successfully`);
const newContent: string = await updateTypeScriptReadmeFile(typescriptReadmePath, _args.getSdkType());
_logger.log(`Generated content of the new TypeScript readme file successfully`);
await saveContentToFile(typescriptReadmePath, newContent);
_logger.log(`Content saved successfully to ${typescriptReadmePath}`);
const readmeFilePath = path.resolve(sdkPath, "readme.md");
const updatedReadmeContent: string = await updateMainReadmeFile(readmeFilePath);
_logger.log(`Updated content of the readme file successfully`);
await saveContentToFile(readmeFilePath, updatedReadmeContent);
_logger.log(`Content saved successfully to ${readmeFilePath}`);
const pullRequestTitle = `Add ${packageName}/${sdkType}/readme.typescript.md`;
const pullRequestDescription = "Autogenerated";
const validate: ValidateFunction = statuses => statuses.length == 2;
const validateEach: ValidateEachFunction = el => el.path().startsWith(`specification/${packageName}`);
const pullRequestUrl = await commitAndCreatePullRequest(azureRestApiSpecRepository, packageName, pullRequestTitle, "azure-rest-api-specs", pullRequestTitle, pullRequestDescription, validate, validateEach);
await unlockGitRepository(azureRestApiSpecRepository);
return { pullRequestUrl: pullRequestUrl, typescriptReadmePath: typescriptReadmePath };
}
export async function generateMissingSdk(azureSdkForJsRepoPath: string, packageName: string, sdkType: SdkType): Promise<string> {
const readmeGenerationResult = await generateTsReadme(packageName, sdkType);
if (_args["skip-sdk"]) {
_logger.log(`Skipping sdk generation`);
return "";
}
if (readmeGenerationResult.typescriptReadmePath) {
const generatedPackageName = await getSinglePackageName(readmeGenerationResult.typescriptReadmePath);
packageName = generatedPackageName;
}
const azureRestApiSpecsRepositoryPath: string = await findAzureRestApiSpecsRepositoryPath();
_logger.log(`Found azure-rest-api-specs repository in ${azureRestApiSpecsRepositoryPath}`);
const azureSdkForJsRepository = await getValidatedRepository(azureSdkForJsRepoPath);
await refreshRepository(azureSdkForJsRepository);
_logger.log(`Refreshed ${azureRestApiSpecsRepositoryPath} repository successfully`);
await waitAndLockGitRepository(azureSdkForJsRepository);
await generateSdk(azureRestApiSpecsRepositoryPath, azureSdkForJsRepoPath, packageName);
_logger.log(`Generated ${packageName} SDK successfully`);
const pullRequestTitle = `Generate ${packageName} package`;
const pullRequestDescription =
`Autogenerated. Matching specification pull request - ${readmeGenerationResult.pullRequestUrl}\n\n\n
\`\`\`
${_logger.getCapturedText()}
\`\`\``
const validate: ValidateFunction = changes => changes.length > 0;
const validateEach: ValidateEachFunction = el => el.path().startsWith(`packages/${packageName}`);
const pullRequestUrl = await commitAndCreatePullRequest(azureSdkForJsRepository, packageName, pullRequestTitle, "azure-sdk-for-js", pullRequestTitle, pullRequestDescription, validate, validateEach);
await unlockGitRepository(azureSdkForJsRepository);
return pullRequestUrl;
}
export async function generateAllMissingSdks(azureSdkForJsRepoPath: string, azureRestApiSpecsRepository: string) {
const missingSdks = await findMissingSdks(azureRestApiSpecsRepository);
_logger.log(`Found ${missingSdks.length} missing specifications`);
for (const missingSdk of missingSdks) {
try {
await generateMissingSdk(azureSdkForJsRepoPath, missingSdk.sdkName, missingSdk.sdkType);
} catch (error) {
_logger.logError(error);
continue;
}
}
}

Просмотреть файл

@ -5,48 +5,106 @@
*/
import * as colors from "colors";
import { CommandLineOptions } from "./commandLineOptions";
import { CommandLineOptions, getCommandLineOptions } from "./commandLine";
export enum Color {
Red,
Green
export enum LoggingLevel {
All = 0,
Trace = 0,
Debug = 1,
Info = 2,
Warn = 3,
Error = 4
}
export class Logger {
private _colorsMap = {
[Color.Red]: colors.red,
[Color.Green]: colors.green
colors.setTheme({
positive: "green",
negative: "red",
debug: "bgCyan",
info: "bgGreen"
});
declare global {
interface String {
positive: string;
negative: string;
debug: string;
info: string;
}
}
export class Logger {
private _cache: string[];
_loggingLevel: LoggingLevel;
constructor(options: CommandLineOptions) {
const lowerCaseLevel = options["logging-level"].toLowerCase();
const capitalizedLevel = lowerCaseLevel.charAt(0).toUpperCase() + lowerCaseLevel.slice(1);
this._loggingLevel = LoggingLevel[capitalizedLevel];
this._cache = [];
}
constructor(private _options: CommandLineOptions) {
log(text?: string): void {
console.log(text);
this._capture(text);
}
log(text: string, color?: Color): void {
if (color !== undefined) {
const coloredText = this._colorsMap[color](text);
console.log(coloredText);
} else {
console.log(text);
clearCapturedText(): void {
this._cache = [];
}
getCapturedText(): string {
return this._cache.join("\n");
}
private _capture(text?: string): void {
this._cache.push(text);
}
logInfo(text?: string) {
this.log(text.info);
}
logRed(text?: string): void {
this.log(text.red);
}
logGreen(text?: string): void {
this.log(text.green);
}
logError(text?: string): void {
this.log(text.bgRed);
}
logWarn(text?: string): void {
if (this._loggingLevel <= LoggingLevel.Warn) {
this.log(text.bgYellow);
}
}
logRed(text: string): void {
this.log(text, Color.Red)
}
logGreen(text: string): void {
this.log(text, Color.Green)
}
logVerbose(text: string, color?: Color): void {
if (this._options.verbose) {
this.log(text, color);
}
}
logDebug(text?: string): void {
if (this._loggingLevel <= LoggingLevel.Debug) {
this.log(text);
}
}
logDebug(text: string, color?: Color): void {
if (this._options.debug) {
this.log(text, color);
}
}
}
logWithDebugDetails(text?: string, details?: string): void {
const greyDetails = `(${details})`.grey;
const textToLog = (this._loggingLevel <= LoggingLevel.Debug) ? `${text} ${greyDetails}` : (text);
this.log(textToLog);
}
logTrace(text?: string) {
if (this._loggingLevel <= LoggingLevel.Trace) {
this.log(text.gray);
}
}
logWithPath(path: string, message: string): void {
console.log(`[${path}]> ${message}`);
}
}
export function getLogger() {
return new Logger(getCommandLineOptions());
}

205
.scripts/readme.ts Normal file
Просмотреть файл

@ -0,0 +1,205 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for
* license information.
*/
import { getLogger } from "./logger";
import { pathExists, startsWith } from "./common";
import { promises as fs } from "fs";
import * as glob from "glob";
import * as path from "path";
import * as yaml from "js-yaml";
import { SdkType } from "./commandLine";
const _logger = getLogger();
interface ReadmeSettings {
"nodejs": {
"azure-arm": boolean;
"license-header": string;
"payload-flattening-threshold": number;
"package-name": string;
"output-folder": string;
"generate-license-txt": boolean | undefined;
"generate-package-json": boolean | undefined;
"generate-readme-md": boolean | undefined;
"generate-metadata": boolean | undefined;
} | undefined;
}
export async function getYamlSection(buffer: Buffer, sectionBeginning: string, sectionEnd: string): Promise<Buffer> {
const beginningIndex = buffer.indexOf(sectionBeginning);
const trimmedBuffer = buffer.slice(beginningIndex + (sectionBeginning.length));
const endIndex = trimmedBuffer.indexOf(sectionEnd, 3);
const sectionBuffer = trimmedBuffer.slice(0, endIndex);
return sectionBuffer;
}
export async function doesReadmeMdFileSpecifiesTypescriptSdk(readmeMdPath: string): Promise<boolean> {
const readmeMdBuffer = await fs.readFile(readmeMdPath);
const sectionBuffer = await getYamlSection(readmeMdBuffer, "``` yaml $(swagger-to-sdk)", "```");
if (sectionBuffer.includes("azure-sdk-for-js")) {
return true;
}
return false;
}
export async function copyExistingNodeJsReadme(sdkPath: string): Promise<string> {
const nodeJsReadmePath = path.resolve(sdkPath, "readme.nodejs.md");
if (!(await pathExists(nodeJsReadmePath))) {
return Promise.reject(`${nodeJsReadmePath} doesn't exists`)
}
const typescriptReadmePath = path.resolve(sdkPath, "readme.typescript.md");
_logger.logDebug(`Copying ${nodeJsReadmePath} to ${typescriptReadmePath}`)
if (await pathExists(typescriptReadmePath)) {
return Promise.reject(`${typescriptReadmePath} file already exists`)
}
await fs.copyFile(nodeJsReadmePath, typescriptReadmePath);
return typescriptReadmePath;
}
export async function getSinglePackageName(typescriptReadmePath: string): Promise<string> {
const readmeBuffer: Buffer = await fs.readFile(typescriptReadmePath);
const yamlSectionBuffer = await getYamlSection(readmeBuffer, "``` yaml $(typescript)", "```");
const yamlSectionText = yamlSectionBuffer.toString();
const yamlSection = yaml.safeLoad(yamlSectionText);
return yamlSection["typescript"]["package-name"];
}
async function updatePackageName(settings: ReadmeSettings, sdkType: SdkType): Promise<ReadmeSettings> {
let packageName = settings.nodejs["package-name"]
if (packageName.startsWith("azure-")) {
packageName = packageName.replace("azure-", "");
}
if (sdkType == SdkType.ResourceManager && !packageName.startsWith("arm-")) {
packageName = `arm-${packageName}`
}
settings.nodejs["package-name"] = `"@azure/${packageName}"`
return settings;
}
async function updateMetadataFields(settings: ReadmeSettings): Promise<ReadmeSettings> {
settings.nodejs["generate-metadata"] = true;
delete settings.nodejs["generate-license-txt"]
delete settings.nodejs["generate-package-json"]
delete settings.nodejs["generate-readme-md"];
return settings;
}
function stripExtraQuotes(text: string): string {
return text.replace(/'/g, "");
}
async function updateOutputFolder(settings: ReadmeSettings): Promise<ReadmeSettings> {
const outputName = settings.nodejs["package-name"].replace(/"/g, "");
settings.nodejs["output-folder"] = `"$(typescript-sdks-folder)/packages/${outputName}"`;
return settings;
}
async function updateYamlSection(sectionText: string, sdkType: SdkType): Promise<string> {
const section = yaml.safeLoad(sectionText);
await updatePackageName(section, sdkType);
await updateMetadataFields(section);
await updateOutputFolder(section);
section["typescript"] = section.nodejs;
delete section.nodejs;
return yaml.safeDump(section).trim();
}
export async function updateTypeScriptReadmeFile(typescriptReadmePath: string, sdkType: SdkType): Promise<string> {
const readmeBuffer: Buffer = await fs.readFile(typescriptReadmePath);
let outputReadme: string = readmeBuffer.toString();
const yamlSection = await getYamlSection(readmeBuffer, "``` yaml $(nodejs)", "```");
const sectionText = yamlSection.toString().trim();
let updatedYamlSection = await updateYamlSection(sectionText, sdkType);
updatedYamlSection = stripExtraQuotes(updatedYamlSection);
outputReadme = outputReadme.replace(sectionText, updatedYamlSection);
outputReadme = outputReadme.replace("azure-sdk-for-node", "azure-sdk-for-js");
outputReadme = outputReadme.replace("Node.js", "TypeScript");
outputReadme = outputReadme.replace("$(nodejs)", "$(typescript)");
outputReadme = outputReadme.replace("nodejs", "typescript");
outputReadme = outputReadme.replace("Node", "TypeScript");
outputReadme = outputReadme.replace("node", "typescript");
return outputReadme;
}
export async function updateMainReadmeFile(readmeFilePath: string) {
const readmeBuffer: Buffer = await fs.readFile(readmeFilePath);
let outputReadme: string = readmeBuffer.toString();
const yamlSection = await getYamlSection(readmeBuffer, "``` yaml $(swagger-to-sdk)", "```");
const sectionText = yamlSection.toString().trim();
let lines = sectionText.split("\r\n");
let nodeLineIndex = lines.findIndex(el => el.includes("- repo: azure-sdk-for-node"));
if (nodeLineIndex == -1) {
lines.push(" - repo: azure-sdk-for-node");
nodeLineIndex = lines.length - 1;
}
const nodeLine = lines[nodeLineIndex];
lines.splice(nodeLineIndex, 0, nodeLine.replace("node", "js"));
const updatedYamlSection = lines.join("\r\n");
outputReadme = outputReadme.replace(sectionText, updatedYamlSection);
return outputReadme;
}
export function getPackageNamesFromReadmeTypeScriptMdFileContents(readmeTypeScriptMdFileContents: string): string[] {
const packageNamePattern: RegExp = /package-name: (\S*)/g;
const matches: string[] = readmeTypeScriptMdFileContents.match(packageNamePattern) || [];
_logger.logDebug(`"package-name" matches: ${JSON.stringify(matches)}`.debug);
for (let i = 0; i < matches.length; ++i) {
matches[i] = matches[i].substring("package-name: ".length);
}
_logger.logDebug(`"package-name" matches trimmed: ${JSON.stringify(matches)}`.debug);
return matches;
}
export function findReadmeTypeScriptMdFilePaths(azureRestAPISpecsRoot: string): string[] {
_logger.logDebug(`Looking for "readme.typescript.md" files in "${azureRestAPISpecsRoot}"...`.debug);
const specificationFolderPath: string = path.resolve(azureRestAPISpecsRoot, 'specification');
const readmeTypeScriptMdFilePaths: string[] = glob.sync('**/readme.typescript.md', { absolute: true, cwd: specificationFolderPath });
if (readmeTypeScriptMdFilePaths) {
for (let i = 0; i < readmeTypeScriptMdFilePaths.length; ++i) {
const readmeTypeScriptMdFilePath: string = readmeTypeScriptMdFilePaths[i];
_logger.logDebug(` Found "${readmeTypeScriptMdFilePath}".`.debug);
if (readmeTypeScriptMdFilePath && !startsWith(readmeTypeScriptMdFilePath, specificationFolderPath)) {
const resolvedReadmeTypeScriptMdFilePath: string = path.resolve(specificationFolderPath, readmeTypeScriptMdFilePath);
_logger.logDebug(` Updating to "${resolvedReadmeTypeScriptMdFilePath}".`.debug);
readmeTypeScriptMdFilePaths[i] = resolvedReadmeTypeScriptMdFilePath;
}
}
}
return readmeTypeScriptMdFilePaths;
}
export function getOutputFolderFromReadmeTypeScriptMdFileContents(readmeTypeScriptMdFileContents: string): string {
return readmeTypeScriptMdFileContents.match(/output-folder: (\S*)/)[1];
}
export function getAbsolutePackageFolderPathFromReadmeFileContents(azureSDKForJSRepoRoot: string, typeScriptReadmeFileContents: string): string {
const outputFolderPath: string = getOutputFolderFromReadmeTypeScriptMdFileContents(typeScriptReadmeFileContents);
const outputFolderPathRelativeToAzureSDKForJSRepoRoot: string = outputFolderPath.substring('$(typescript-sdks-folder)/'.length + 1, outputFolderPath.length - 1);
return path.resolve(azureSDKForJSRepoRoot, outputFolderPathRelativeToAzureSDKForJSRepoRoot);
}

Просмотреть файл

@ -4,93 +4,27 @@
* license information.
*/
import { execSync } from "child_process";
import { contains, endsWith, npmInstall, npmRunBuild } from "./.scripts/common";
import { getCommandLineOptions } from "./.scripts/commandLine";
import { findAzureRestApiSpecsRepositoryPath, findMissingSdks } from "./.scripts/generateSdks";
import { generateTsReadme, generateSdk, generateMissingSdk, generateAllMissingSdks } from "./.scripts/gulp";
import { getPackageNamesFromReadmeTypeScriptMdFileContents, findReadmeTypeScriptMdFilePaths, getAbsolutePackageFolderPathFromReadmeFileContents } from "./.scripts/readme";
import { getLogger } from "./.scripts/logger";
import * as fs from "fs";
import * as glob from "glob";
import * as gulp from "gulp";
import * as path from "path";
import { argv } from "yargs";
import { findAzureRestApiSpecsRepository, findSdkDirectory, findMissingSdks, copyExistingNodeJsReadme, updateTypeScriptReadmeFile, saveContentToFile } from "./.scripts/generate-sdks";
import { execSync } from "child_process";
const azureSDKForJSRepoRoot: string = __dirname;
const azureRestAPISpecsRoot: string = argv['azure-rest-api-specs-root'] || path.resolve(azureSDKForJSRepoRoot, '..', 'azure-rest-api-specs');
const packageArg: string = argv['package'];
const use: string = argv['use'];
const whatif: boolean = argv['whatif'];
const useDebugger: boolean = argv["debugger"];
const _logger = getLogger();
const args = getCommandLineOptions();
const azureSDKForJSRepoRoot: string = args["azure-sdk-for-js-repo-root"] || __dirname;
const azureRestAPISpecsRoot: string = args["azure-rest-api-specs-root"] || path.resolve(azureSDKForJSRepoRoot, '..', 'azure-rest-api-specs');
function findReadmeTypeScriptMdFilePaths(azureRestAPISpecsRoot: string): string[] {
// console.log(`Looking for "readme.typescript.md" files in "${azureRestAPISpecsRoot}"...`);
const specificationFolderPath: string = path.resolve(azureRestAPISpecsRoot, 'specification');
const readmeTypeScriptMdFilePaths: string[] = glob.sync('**/readme.typescript.md', { absolute: true, cwd: specificationFolderPath });
if (readmeTypeScriptMdFilePaths) {
for (let i = 0; i < readmeTypeScriptMdFilePaths.length; ++i) {
const readmeTypeScriptMdFilePath: string = readmeTypeScriptMdFilePaths[i];
// console.log(` Found "${readmeTypeScriptMdFilePath}".`);
if (readmeTypeScriptMdFilePath && !startsWith(readmeTypeScriptMdFilePath, specificationFolderPath)) {
const resolvedReadmeTypeScriptMdFilePath: string = path.resolve(specificationFolderPath, readmeTypeScriptMdFilePath);
// console.log(` Updating to "${resolvedReadmeTypeScriptMdFilePath}".`);
readmeTypeScriptMdFilePaths[i] = resolvedReadmeTypeScriptMdFilePath;
}
}
}
return readmeTypeScriptMdFilePaths;
}
function getPackageNamesFromReadmeTypeScriptMdFileContents(readmeTypeScriptMdFileContents: string): string[] {
const packageNamePattern: RegExp = /package-name: (\S*)/g;
const matches: string[] = readmeTypeScriptMdFileContents.match(packageNamePattern) || [];
// console.log(`"package-name" matches: ${JSON.stringify(matches)}`);
for (let i = 0; i < matches.length; ++i) {
matches[i] = matches[i].substring("package-name: ".length);
}
// console.log(`"package-name" matches trimmed: ${JSON.stringify(matches)}`);
return matches;
}
function getOutputFolderFromReadmeTypeScriptMdFileContents(readmeTypeScriptMdFileContents: string): string {
return readmeTypeScriptMdFileContents.match(/output-folder: (\S*)/)[1];
}
function execute(command: string, packageFolderPath: string): void {
if (!fs.existsSync(packageFolderPath)) {
log(packageFolderPath, "Folder not found.");
} else {
execSync(command, { cwd: packageFolderPath, stdio: "inherit" });
}
}
function npmRunBuild(packageFolderPath: string): void {
execute("npm run build", packageFolderPath);
}
function npmInstall(packageFolderPath: string): void {
execute("npm install", packageFolderPath);
}
function getAbsolutePackageFolderPathFromReadmeFileContents(typeScriptReadmeFileContents: string): string {
const outputFolderPath: string = getOutputFolderFromReadmeTypeScriptMdFileContents(typeScriptReadmeFileContents);
const outputFolderPathRelativeToAzureSDKForJSRepoRoot: string = outputFolderPath.substring('$(typescript-sdks-folder)/'.length);
return path.resolve(azureSDKForJSRepoRoot, outputFolderPathRelativeToAzureSDKForJSRepoRoot);
}
function startsWith(value: string, prefix: string): boolean {
return value && prefix && value.indexOf(prefix) === 0;
}
function endsWith(value: string, suffix: string): boolean {
return value && suffix && value.length >= suffix.length && value.lastIndexOf(suffix) === value.length - suffix.length;
}
function contains(values: string[], searchString: string): boolean {
return values.indexOf(searchString) !== -1;
}
function getPackgeFolderPathFromPackageArgument(packageArgument: string | undefined): string | undefined {
function getPackageFolderPathFromPackageArgument(): string | undefined {
let packageFolderPath: string | undefined;
if (!packageArg) {
console.log(`No --package specified.`);
if (!args.package) {
_logger.log(`No --package specified.`);
} else {
const typeScriptReadmeFilePaths: string[] = findReadmeTypeScriptMdFilePaths(azureRestAPISpecsRoot);
@ -102,126 +36,65 @@ function getPackgeFolderPathFromPackageArgument(packageArgument: string | undefi
const typeScriptReadmeFileContents: string = fs.readFileSync(typeScriptReadmeFilePath, 'utf8');
const packageNames: string[] = getPackageNamesFromReadmeTypeScriptMdFileContents(typeScriptReadmeFileContents);
if (contains(packageNames, packageArg)) {
if (contains(packageNames, args.package)) {
foundPackage = true;
packageFolderPath = getAbsolutePackageFolderPathFromReadmeFileContents(typeScriptReadmeFileContents);
packageFolderPath = getAbsolutePackageFolderPathFromReadmeFileContents(azureSDKForJSRepoRoot, typeScriptReadmeFileContents);
}
}
if (!foundPackage) {
console.log(`No package found with the name "${packageArg}".`);
_logger.log(`No package found with the name "${args.package}".`);
}
}
return packageFolderPath;
}
function log(path: string, message: string): void {
console.log(`[${path}]> ${message}`);
}
gulp.task('default', () => {
console.log('gulp build --package <package-name>');
console.log(' --package');
console.log(' NPM package to run "npm run build" on.');
console.log();
console.log('gulp install --package <package name>');
console.log(' --package');
console.log(' NPM package to run "npm install" on.');
console.log();
console.log('gulp codegen [--azure-rest-api-specs-root <azure-rest-api-specs root>] [--use <autorest.typescript root>] [--package <package name>]');
console.log(' --azure-rest-api-specs-root');
console.log(' Root location of the local clone of the azure-rest-api-specs-root repository.');
console.log(' --use');
console.log(' Root location of autorest.typescript repository. If this is not specified, then the latest installed generator for TypeScript will be used.');
console.log(' --package');
console.log(' NPM package to regenerate. If no package is specified, then all packages will be regenerated.');
console.log();
console.log('gulp publish [--package <package name>] [--whatif]');
console.log(' --package');
console.log(' The name of the package to publish. If no package is specified, then all packages will be published.');
console.log(' --whatif');
console.log(' Don\'t actually publish packages, but just indicate which packages would be published.');
_logger.log('gulp build --package <package-name>');
_logger.log(' --package');
_logger.log(' NPM package to run "npm run build" on.');
_logger.log();
_logger.log('gulp install --package <package name>');
_logger.log(' --package');
_logger.log(' NPM package to run "npm install" on.');
_logger.log();
_logger.log('gulp codegen [--azure-rest-api-specs-root <azure-rest-api-specs root>] [--use <autorest.typescript root>] [--package <package name>]');
_logger.log(' --azure-rest-api-specs-root');
_logger.log(' Root location of the local clone of the azure-rest-api-specs-root repository.');
_logger.log(' --use');
_logger.log(' Root location of autorest.typescript repository. If this is not specified, then the latest installed generator for TypeScript will be used.');
_logger.log(' --package');
_logger.log(' NPM package to regenerate. If no package is specified, then all packages will be regenerated.');
_logger.log();
_logger.log('gulp publish [--package <package name>] [--whatif]');
_logger.log(' --package');
_logger.log(' The name of the package to publish. If no package is specified, then all packages will be published.');
_logger.log(' --whatif');
_logger.log(' Don\'t actually publish packages, but just indicate which packages would be published.');
});
gulp.task("install", () => {
const packageFolderPath: string | undefined = getPackgeFolderPathFromPackageArgument(packageArg);
const packageFolderPath: string | undefined = getPackageFolderPathFromPackageArgument();
if (packageFolderPath) {
log(packageFolderPath, "npm install");
_logger.logWithPath(packageFolderPath, "npm install");
npmInstall(packageFolderPath);
}
});
gulp.task("build", () => {
const packageFolderPath: string | undefined = getPackgeFolderPathFromPackageArgument(packageArg);
const packageFolderPath: string | undefined = getPackageFolderPathFromPackageArgument();
if (packageFolderPath) {
log(packageFolderPath, "npm run build");
_logger.logWithPath(packageFolderPath, "npm run build");
npmRunBuild(packageFolderPath);
}
});
function containsPackageName(packageNames: string[], packageName: string): boolean {
return contains(packageNames, packageName) ||
contains(packageNames, `@azure/${packageName}`) ||
contains(packageNames, `"${packageName}"`) ||
contains(packageNames, `"@azure/${packageName}"`) ||
contains(packageNames, `'${packageName}'`) ||
contains(packageNames, `'@azure/${packageName}'`);
}
// This task is used to generate libraries based on the mappings specified above.
gulp.task('codegen', () => {
const typeScriptReadmeFilePaths: string[] = findReadmeTypeScriptMdFilePaths(azureRestAPISpecsRoot);
for (let i = 0; i < typeScriptReadmeFilePaths.length; ++i) {
const typeScriptReadmeFilePath: string = typeScriptReadmeFilePaths[i];
const typeScriptReadmeFileContents: string = fs.readFileSync(typeScriptReadmeFilePath, 'utf8');
const packageNames: string[] = getPackageNamesFromReadmeTypeScriptMdFileContents(typeScriptReadmeFileContents);
const packageNamesString: string = JSON.stringify(packageNames);
// console.log(`In "${typeScriptReadmeFilePath}", found package names "${packageNamesString}".`);
if (!packageArg || containsPackageName(packageNames, packageArg)) {
console.log(`>>>>>>>>>>>>>>>>>>> Start: "${packageNamesString}" >>>>>>>>>>>>>>>>>>>>>>>>>`);
const readmeFilePath: string = path.resolve(path.dirname(typeScriptReadmeFilePath), 'readme.md');
let cmd = `autorest --typescript --typescript-sdks-folder=${azureSDKForJSRepoRoot} --license-header=MICROSOFT_MIT_NO_VERSION ${readmeFilePath}`;
if (use) {
cmd += ` --use=${use}`;
}
else {
const localAutorestTypeScriptFolderPath = path.resolve(azureSDKForJSRepoRoot, '..', 'autorest.typescript');
if (fs.existsSync(localAutorestTypeScriptFolderPath) && fs.lstatSync(localAutorestTypeScriptFolderPath).isDirectory()) {
cmd += ` --use=${localAutorestTypeScriptFolderPath}`;
}
}
if (useDebugger) {
cmd += ` --typescript.debugger`;
}
try {
console.log('Executing command:');
console.log('------------------------------------------------------------');
console.log(cmd);
console.log('------------------------------------------------------------');
execSync(cmd, { encoding: "utf8", stdio: "inherit" });
console.log('Installing dependencies...');
const packageFolderPath: string = getAbsolutePackageFolderPathFromReadmeFileContents(typeScriptReadmeFileContents);
npmInstall(packageFolderPath);
} catch (err) {
console.log('Error:');
console.log(`An error occurred while generating client for packages: "${packageNamesString}":\n Stderr: "${err.stderr}"`);
}
console.log(`>>>>>>>>>>>>>>>>>>> End: "${packageNamesString}" >>>>>>>>>>>>>>>>>>>>>>>>>`);
console.log();
}
}
_logger.log(`Passed arguments: ${process.argv}`);
generateSdk(azureRestAPISpecsRoot, azureSDKForJSRepoRoot, args.package);
});
gulp.task('publish', () => {
@ -234,28 +107,28 @@ gulp.task('publish', () => {
for (let i = 0; i < typeScriptReadmeFilePaths.length; ++i) {
const typeScriptReadmeFilePath: string = typeScriptReadmeFilePaths[i];
// console.log(`INFO: Processing ${typeScriptReadmeFilePath}`);
_logger.logTrace(`INFO: Processing ${typeScriptReadmeFilePath}`);
const typeScriptReadmeFileContents: string = fs.readFileSync(typeScriptReadmeFilePath, 'utf8');
const packageFolderPath: string = getAbsolutePackageFolderPathFromReadmeFileContents(typeScriptReadmeFileContents);
const packageFolderPath: string = getAbsolutePackageFolderPathFromReadmeFileContents(azureSDKForJSRepoRoot, typeScriptReadmeFileContents);
if (!fs.existsSync(packageFolderPath)) {
console.log(`ERROR: Package folder ${packageFolderPath} has not been generated.`);
_logger.log(`ERROR: Package folder ${packageFolderPath} has not been generated.`);
errorPackages++;
}
else {
const packageJsonFilePath: string = `${packageFolderPath}/package.json`;
if (!fs.existsSync(packageJsonFilePath)) {
console.log(`ERROR: Package folder ${packageFolderPath} is missing its package.json file.`);
_logger.log(`ERROR: Package folder ${packageFolderPath} is missing its package.json file.`);
errorPackages++;
}
else {
const packageJson: { [propertyName: string]: any } = require(packageJsonFilePath);
const packageName: string = packageJson.name;
if (!packageArg || packageArg === packageName || endsWith(packageName, `-${packageArg}`)) {
if (!args.package || args.package === packageName || endsWith(packageName, `-${args.package}`)) {
const localPackageVersion: string = packageJson.version;
if (!localPackageVersion) {
console.log(`ERROR: "${packageJsonFilePath}" doesn't have a version specified.`);
_logger.log(`ERROR: "${packageJsonFilePath}" doesn't have a version specified.`);
errorPackages++;
}
else {
@ -272,8 +145,8 @@ gulp.task('publish', () => {
upToDatePackages++;
}
else {
console.log(`Publishing package "${packageName}" with version "${localPackageVersion}"...${whatif ? " (SKIPPED)" : ""}`);
if (!whatif) {
_logger.log(`Publishing package "${packageName}" with version "${localPackageVersion}"...${args.whatif ? " (SKIPPED)" : ""}`);
if (!args.whatif) {
try {
npmInstall(packageFolderPath);
execSync(`npm publish`, { cwd: packageFolderPath });
@ -292,46 +165,52 @@ gulp.task('publish', () => {
}
}
console.log();
console.log(`Error packages: ${errorPackages}`);
console.log(`Up to date packages: ${upToDatePackages}`);
console.log(`Published packages: ${publishedPackages}`);
console.log(`Published packages skipped: ${publishedPackagesSkipped}`);
_logger.log();
_logger.log(`Error packages: ${errorPackages}`);
_logger.log(`Up to date packages: ${upToDatePackages}`);
_logger.log(`Published packages: ${publishedPackages}`);
_logger.log(`Published packages skipped: ${publishedPackagesSkipped}`);
});
gulp.task("find-missing-sdks", async () => {
try {
console.log(`Passed arguments: ${process.argv}`);
_logger.log(`Passed arguments: ${process.argv}`);
const azureRestApiSpecsRepository = await findAzureRestApiSpecsRepository();
console.log(`Found azure-rest-api-specs repository in ${azureRestApiSpecsRepository}`);
const azureRestApiSpecsRepositoryPath = await findAzureRestApiSpecsRepositoryPath();
_logger.log(`Found azure-rest-api-specs repository in ${azureRestApiSpecsRepositoryPath}`);
await findMissingSdks(azureRestApiSpecsRepository);
await findMissingSdks(azureRestApiSpecsRepositoryPath);
} catch (error) {
console.error(error);
_logger.logError(error);
}
});
gulp.task("generate-ts-readme", async () => {
try {
console.log(`Passed arguments: ${process.argv}`);
const azureRestApiSpecsRepository: string = await findAzureRestApiSpecsRepository();
console.log(`Found azure-rest-api-specs repository in ${azureRestApiSpecsRepository}`);
const sdkPath: string = await findSdkDirectory(azureRestApiSpecsRepository);
console.log(`Found specification in ${sdkPath}`);
const typescriptReadmePath: string = await copyExistingNodeJsReadme(sdkPath);
console.log(`Copied readme file successfully`);
const newContent: string = await updateTypeScriptReadmeFile(typescriptReadmePath);
console.log(`Generated content of the new readme file successfully`);
await saveContentToFile(typescriptReadmePath, newContent);
console.log(`Content saved successfully to ${typescriptReadmePath}`);
_logger.log(`Passed arguments: ${process.argv}`);
await generateTsReadme(args.package, args.getSdkType());
}
catch (error) {
console.error(error);
_logger.logError(error);
}
});
gulp.task("generate-missing-sdk", async () => {
try {
_logger.log(`Passed arguments: ${process.argv}`);
await generateMissingSdk(azureSDKForJSRepoRoot, args.package, args.getSdkType());
}
catch (error) {
_logger.logError(error);
}
});
gulp.task("generate-all-missing-sdks", async () => {
try {
_logger.log(`Passed arguments: ${process.argv}`);
const azureRestApiSpecsRepositoryPath = await findAzureRestApiSpecsRepositoryPath();
await generateAllMissingSdks(azureSDKForJSRepoRoot, azureRestApiSpecsRepositoryPath);
} catch (error) {
_logger.logError(error);
}
});

Просмотреть файл

@ -28,15 +28,18 @@
"url": "http://github.com/Azure/azure-sdk-for-js/issues"
},
"devDependencies": {
"@octokit/rest": "^15.13.0",
"@types/colors": "^1.2.1",
"@types/js-yaml": "^3.11.2",
"@types/minimist": "^1.2.0",
"@types/node": "^10.11.4",
"@types/nodegit": "^0.22.3",
"colors": "^1.3.2",
"fs": "0.0.1-security",
"gulp": "^3.9.1",
"js-yaml": "^3.12.0",
"minimist": "^1.2.0",
"nodegit": "^0.23.0-alpha.1",
"path": "^0.12.7",
"ts-node": "^7.0.1",
"typescript": "^3.0.3",