read manual autorest config for dpg (#3442)
This commit is contained in:
Wei Dong
GitHub
Родитель
0915c46ea2
Коммит
2324fdc09a
|
|
@ -53,7 +53,7 @@ RUN npm install -g typescript
|
|||
RUN npm install -g @microsoft/rush
|
||||
RUN npm install -g autorest
|
||||
|
||||
RUN mkdir "/mock-host" && cd /mock-host && npm install @azure-tools/mock-service-host
|
||||
RUN mkdir "/mock-host" && cd /mock-host && npm install @azure-tools/mock-service-host@0.1.16
|
||||
|
||||
# install chrome
|
||||
RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
|
||||
|
|
|
|||
|
|
@ -21,17 +21,18 @@ The docker image will be used in different scenarios:
|
|||
Command
|
||||
|
||||
```shell
|
||||
docker run -it --privileged -v {local_spec_repo_path}:/spec-repo -v {local_work_folder}:/work-dir docker.image:latest --readme={relative_readme} --sdk={sdk_to_generate}
|
||||
docker run -it --privileged -v {local_spec_repo_path}:/spec-repo -v {local_work_folder}:/work-dir -v {local_autorest_config}:/autorest.md docker.image:latest --readme={relative_readme} --sdk={sdk_to_generate}
|
||||
```
|
||||
|
||||
Parameter description:
|
||||
|
||||
| Parameter | Description | Example |
|
||||
|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------|
|
||||
| { local_spec_repo_path } | Required. It's used to point to the swagger folder. | /home/test/azure-rest-api-specs |
|
||||
| { local_work_folder} | Required. It's used to point to the work folder, which will store all sdk repositories. If there is no sdk repository in the folder, the docker image will clone it | /home/test/sdk-repos |
|
||||
| { relative_readme } | Required. It's used to specify the readme.md file and docker image uses it to generate SDKs. it's the relative path from {path_to_local_spec_repo} | specification/agrifood/resource-manager/readme.md |
|
||||
| { sdk_to_generate } | Required. It's used to specify which language of sdk you want to generate. Supported value for management sdk: js, java, python, .net, and go. Supported value for dataplane sdk: js, java, python, and .net. If you want to generate multi-packages, use comma to separate them. (__Not recommend to generate multi packages in one docker container because the docker will failed when encoutering error in generating one sdk, and the remaining sdk will be generated__) | js,java |
|
||||
| Parameter | Description | Example |
|
||||
|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------|
|
||||
| { local_spec_repo_path } | Required. It's used to point to the swagger folder. | /home/test/azure-rest-api-specs |
|
||||
| { local_work_folder} | Required. It's used to point to the work folder, which will store all sdk repositories. If there is no sdk repository in the folder, the docker image will clone it | /home/test/sdk-repos |
|
||||
| { local_autorest_config } | Optional. When you generate data-plane sdk, and there is no autorest configuration in sdk repository or you want to change the autorest configuration, you can set new autorest config in a file and mount it to the docker container. About the content of file, please refer to [document](https://github.com/Azure/azure-rest-api-specs/blob/dpg-doc/documentation/onboard-dpg-in-sdkautomation/add-autorest-configuration-in-spec-comment.md) | /home/test/autorest.md ([Example file](./autorest-config-file-sample.md)) |
|
||||
| { relative_readme } | Required. It's used to specify the readme.md file and docker image uses it to generate SDKs. it's the relative path from {path_to_local_spec_repo} | specification/agrifood/resource-manager/readme.md |
|
||||
| { sdk_to_generate } | Required. It's used to specify which language of sdk you want to generate. Supported value for management sdk: js, java, python, .net, and go. Supported value for dataplane sdk: js, java, python, and .net. If you want to generate multi-packages, use comma to separate them. (__Not recommend to generate multi packages in one docker container because the docker will failed when encoutering error in generating one sdk, and the remaining sdk will be generated__) | js,java |
|
||||
|
||||
Example Command:
|
||||
```shell
|
||||
|
|
@ -67,15 +68,16 @@ Parameter description:
|
|||
|
||||
Then run docker commands to do grow up development:
|
||||
```shell
|
||||
docker run -it --privileged -v {local_spec_repo_path}:/spec-repo -v {local_work_folder}:/work-dir docker.image:latest --readme={relative_readme}
|
||||
docker run -it --privileged -v {local_spec_repo_path}:/spec-repo -v {local_work_folder}:/work-dir -v {local_autorest_config}:/autorest.md docker.image:latest --readme={relative_readme}
|
||||
```
|
||||
Parameter description:
|
||||
|
||||
| Parameter | Description | Example |
|
||||
|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------|
|
||||
| { local_spec_repo_path } | Optional. If you want to change the swagger and re-generate codes, you need to mount the swagger repo. If you only want to do grow up development, no need to mount it. | /home/test/azure-rest-api-specs |
|
||||
| { local_work_folder } | Required. It's used to point to the work folder, which stores all sdk repositories. | /home/test/work-dir |
|
||||
| { relative_readme } | Optional. It's used to specify the readme.md file and docker image uses it to start mock server. it's the relative path from {path_to_local_spec_repo}. If not specified, mock server will not start. | specification/agrifood/resource-manager/readme.md |
|
||||
| Parameter | Description | Example |
|
||||
|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------|
|
||||
| { local_spec_repo_path } | Optional. If you want to change the swagger and re-generate codes, you need to mount the swagger repo. If you only want to do grow up development, no need to mount it. | /home/test/azure-rest-api-specs |
|
||||
| { local_work_folder } | Required. It's used to point to the work folder, which stores all sdk repositories. | /home/test/work-dir |
|
||||
| { local_autorest_config } | Optional. When you generate data-plane sdk, and there is no autorest configuration in sdk repository or you want to change the autorest configuration, you can set new autorest config in a file and mount it to the docker container. About the content of file, please refer to [document](https://github.com/Azure/azure-rest-api-specs/blob/dpg-doc/documentation/onboard-dpg-in-sdkautomation/add-autorest-configuration-in-spec-comment.md) | /home/test/autorest.md ([Example file](./autorest-config-file-sample.md)) |
|
||||
| { relative_readme } | Optional. It's used to specify the readme.md file and docker image uses it to start mock server. it's the relative path from {path_to_local_spec_repo}. If not specified, mock server will not start. | specification/agrifood/resource-manager/readme.md |
|
||||
|
||||
Example Command:
|
||||
```shell
|
||||
|
|
@ -99,20 +101,21 @@ Before running docker command, pipeline must prepare the spec repo and sdk repo.
|
|||
Command:
|
||||
|
||||
```shell
|
||||
docker run --privileged -v {spec_repo_path}:/spec-repo -v {sdk_repo_path}:/sdk-repo -v {output_folder_path}:/tmp/output docker.image:latest --readme={relative_readme}
|
||||
docker run --privileged -v {spec_repo_path}:/spec-repo -v {sdk_repo_path}:/sdk-repo -v {local_autorest_config}:/autorest.md -v {output_folder_path}:/tmp/output docker.image:latest --readme={relative_readme}
|
||||
```
|
||||
|
||||
Parameter description:
|
||||
|
||||
| Parameter | Description | Example |
|
||||
|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------|
|
||||
| { spec_repo_path } | Required. It's used to point to the swagger folder. | /home/test/azure-rest-api-specs |
|
||||
| { sdk_repo_path } | Required. It's used to point to the sdk repository. | /home/test/sdk-repos |
|
||||
| { relative_readme } | Required. It's used to specify the readme.md file and docker image uses it to generate SDKs. it's the relative path from {path_to_local_spec_repo} | specification/agrifood/resource-manager/readme.md |
|
||||
| Parameter | Description | Example |
|
||||
|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------|
|
||||
| { spec_repo_path } | Required. It's used to point to the swagger folder. | /home/test/azure-rest-api-specs |
|
||||
| { sdk_repo_path } | Required. It's used to point to the sdk repository. | /home/test/sdk-repos |
|
||||
| { local_autorest_config } | Optional. When you generate data-plane sdk, and there is no autorest configuration in sdk repository or you want to change the autorest configuration, you can set new autorest config in a file and mount it to the docker container. About the content of file, please refer to [document](https://github.com/Azure/azure-rest-api-specs/blob/dpg-doc/documentation/onboard-dpg-in-sdkautomation/add-autorest-configuration-in-spec-comment.md) | /home/test/autorest.md ([Example file](./autorest-config-file-sample.md)) |
|
||||
| { relative_readme } | Required. It's used to specify the readme.md file and docker image uses it to generate SDKs. it's the relative path from {path_to_local_spec_repo} | specification/agrifood/resource-manager/readme.md |
|
||||
|
||||
Example Command:
|
||||
```shell
|
||||
docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/vsts/work/azure-rest-api-specs:/spec-repo -v /home/vsts/work/azure-sdk-for-js:/sdk-repo -v /home/vsts/work/output:/tmp/output docker.image:latest --readme=specification/agrifood/resource-manager/readme.md
|
||||
docker run --privileged -v /home/vsts/work/azure-rest-api-specs:/spec-repo -v /home/vsts/work/azure-sdk-for-js:/sdk-repo -v /home/vsts/work/output:/tmp/output docker.image:latest --readme=specification/agrifood/resource-manager/readme.md
|
||||
```
|
||||
|
||||
After running the command in pipeline, docker will execute tasks automatically. Also, there will be output files generated, which will be used by pipeline's other job, such as upload codes, parsing logs.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
# azure-sdk-for-js
|
||||
``` yaml
|
||||
output-folder: sdk/deviceupdate/iot-device-update-rest
|
||||
require:
|
||||
- specification/deviceupdate/data-plane/readme.md
|
||||
```
|
||||
|
|
@ -16,6 +16,7 @@ export class DockerContext {
|
|||
workDir?: string;
|
||||
sdkRepo?: string;
|
||||
resultOutputFolder?: string;
|
||||
autorestConfigFilePath?: string;
|
||||
logger: Logger;
|
||||
|
||||
/*
|
||||
|
|
@ -32,7 +33,7 @@ export class DockerContext {
|
|||
this.workDir = inputParams.workDir;
|
||||
this.sdkRepo = inputParams.sdkRepo;
|
||||
this.resultOutputFolder = inputParams.resultOutputFolder;
|
||||
|
||||
this.autorestConfigFilePath = inputParams.autorestConfigFilePath;
|
||||
this.logger = initializeLogger(path.join(inputParams.resultOutputFolder, inputParams.dockerLogger), 'docker');
|
||||
|
||||
if (this.sdkList?.length === 0 && fs.existsSync(this.workDir)) {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { writeFileSync } from 'fs';
|
|||
import * as path from 'path';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
import { extractAutorestConfigs } from '../../../utils/autorestConfigExtractorUtils';
|
||||
import { GitOperationWrapper } from '../../../utils/GitOperationWrapper';
|
||||
import { dockerTaskEngineInput } from '../schema/dockerTaskEngineInput';
|
||||
import { DockerContext } from './DockerContext';
|
||||
|
|
@ -43,6 +44,7 @@ export class DockerTaskEngineContext {
|
|||
taskResultJsonPath: string;
|
||||
changeOwner: boolean;
|
||||
mode: DockerRunningModel;
|
||||
autorestConfig: string | undefined;
|
||||
|
||||
public async initialize(dockerContext: DockerContext) {
|
||||
// before execute task engine, safe spec repos and sdk repos because they may be owned by others
|
||||
|
|
@ -75,6 +77,7 @@ export class DockerTaskEngineContext {
|
|||
this.taskResultJsonPath = path.join(dockerContext.resultOutputFolder, dockerTaskEngineConfigProperties.taskResultJson);
|
||||
this.changeOwner = dockerTaskEngineConfigProperties.changeOwner;
|
||||
this.mode = dockerContext.mode;
|
||||
this.autorestConfig = extractAutorestConfigs(dockerContext.autorestConfigFilePath, dockerContext.sdkRepo, dockerContext.logger);
|
||||
}
|
||||
|
||||
public async beforeRunTaskEngine() {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,8 @@ export class GenerateAndBuildTask implements SDKGenerationTaskBase {
|
|||
headRef: this.context.specRepo.headRef,
|
||||
repoHttpsUrl: this.context.specRepo.repoHttpsUrl,
|
||||
relatedReadmeMdFile: relatedReadmeMdFileRelativePath,
|
||||
serviceType: this.context.serviceType
|
||||
serviceType: this.context.serviceType,
|
||||
autorestConfig: this.context.autorestConfig
|
||||
};
|
||||
const inputJson = JSON.stringify(inputContent, undefined, 2);
|
||||
this.context.logger.info(`Get ${path.basename(this.context.generateAndBuildInputJsonFile)}:`);
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export class DockerCliInput {
|
|||
sdkRepo: string;
|
||||
resultOutputFolder: string;
|
||||
dockerLogger: string;
|
||||
autorestConfigFilePath: string;
|
||||
}
|
||||
|
||||
export const dockerCliInput = convict<DockerCliInput>({
|
||||
|
|
@ -70,5 +71,11 @@ export const dockerCliInput = convict<DockerCliInput>({
|
|||
arg: 'docker-logger',
|
||||
format: String,
|
||||
doc: 'the path of docker.log. it will concat with resultOutputFolder'
|
||||
},
|
||||
autorestConfigFilePath: {
|
||||
default: '/autorest.md',
|
||||
env: 'AUTOREST_CONFIG_FILE_PATH',
|
||||
format: String,
|
||||
doc: `The absolute path to autorest configuration file. It's required when you want to input your own autorest config in generating data-plane sdk.`
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import winston from 'winston';
|
||||
|
||||
export function extractAutorestConfigs(autorestConfigFilePath: string, sdkRepo: string, logger: winston.Logger): string | undefined {
|
||||
if (!fs.existsSync(autorestConfigFilePath)) {
|
||||
return undefined;
|
||||
}
|
||||
const autorestConfigFileContent = fs.readFileSync(autorestConfigFilePath, 'utf-8');
|
||||
try {
|
||||
let autorestConfigs: string[] = autorestConfigFileContent.split(/#+ *azure-sdk-for-/);
|
||||
if (autorestConfigs.length < 2) {
|
||||
throw new Error(`Parse autorest config file failed.`);
|
||||
}
|
||||
autorestConfigs = autorestConfigs.slice(1);
|
||||
if (!path.basename(sdkRepo).startsWith('azure-sdk-for-')) {
|
||||
if (autorestConfigs.length > 1) {
|
||||
logger.warn(`Docker is running in pipeline, but get autorest config for more than 1 language of sdk. So only get the first autorest config`);
|
||||
}
|
||||
return `azure-sdk-for-${autorestConfigs[0]}`;
|
||||
}
|
||||
for (const autorestConfig of autorestConfigs) {
|
||||
let autorestFullConfig = `# azure-sdk-for-${autorestConfig}`;
|
||||
if (autorestFullConfig.startsWith(`# ${path.basename(sdkRepo)}`)) {
|
||||
logger.info(`Find autorest config for ${path.basename(sdkRepo)} in ${autorestConfigFilePath}: \n${autorestFullConfig}`);
|
||||
if (os.EOL == '\n') {
|
||||
autorestFullConfig = autorestFullConfig.replace(/\n/g, '\r\n');
|
||||
}
|
||||
return autorestFullConfig;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
logger.error(`Parse ${autorestConfigFilePath} failed: ${e.message}`);
|
||||
throw e;
|
||||
}
|
||||
logger.warn(`Cannot find autorest config for ${path.basename(sdkRepo)} in ${autorestConfigFilePath}.`);
|
||||
return undefined;
|
||||
}
|
||||
|
|
@ -18,7 +18,8 @@ describe('task engine', async () => {
|
|||
workDir: '/work-dir',
|
||||
sdkRepo: path.join(tmpFolder, 'sdk-repo'),
|
||||
resultOutputFolder: path.join(tmpFolder, 'output'),
|
||||
dockerLogger: 'docker.log'
|
||||
dockerLogger: 'docker.log',
|
||||
autorestConfigFilePath: path.join(path.resolve('.'), 'test', 'unit', 'utils', 'autorest-single-config.md')
|
||||
});
|
||||
const dockerTaskEngineContext = new DockerTaskEngineContext();
|
||||
await dockerTaskEngineContext.initialize(dockerContext);
|
||||
|
|
@ -32,6 +33,7 @@ describe('task engine', async () => {
|
|||
expect(dockerTaskEngineContext.generateAndBuildTaskLog).toBe(path.join(tmpFolder, 'output', 'generateAndBuild-task.log'));
|
||||
expect(dockerTaskEngineContext.mockTestTaskLog).toBe(path.join(tmpFolder, 'output', 'mockTest-task.log'));
|
||||
expect(dockerTaskEngineContext.readmeMdPath).toBe('specification/agrifood/resource-manager/readme.md');
|
||||
expect(dockerTaskEngineContext.autorestConfig?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should get task list', async () => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
import * as path from 'path';
|
||||
|
||||
const logger = {
|
||||
error: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
info: jest.fn()
|
||||
};
|
||||
|
||||
jest.mock('winston', () => ({
|
||||
format: {
|
||||
colorize: jest.fn(),
|
||||
combine: jest.fn(),
|
||||
label: jest.fn(),
|
||||
timestamp: jest.fn(),
|
||||
printf: jest.fn()
|
||||
},
|
||||
createLogger: jest.fn().mockReturnValue(logger),
|
||||
transports: {
|
||||
Console: jest.fn()
|
||||
}
|
||||
}));
|
||||
|
||||
import * as winston from 'winston';
|
||||
|
||||
import { extractAutorestConfigs } from '../../dist/utils/autorestConfigExtractorUtils';
|
||||
|
||||
const autorestSingleConfigFilePath = path.join(path.resolve('.'), 'test', 'unit', 'utils', 'autorest-single-config.md');
|
||||
const autorestMultiConfigFilePath = path.join(path.resolve('.'), 'test', 'unit', 'utils', 'autorest-multi-config.md');
|
||||
const loggerMock: winston.Logger = winston.createLogger();
|
||||
|
||||
describe('autorest config extractor util test', () => {
|
||||
it('should extract autorest config from autorest file', async () => {
|
||||
const sdkRepo = './azure-sdk-for-js';
|
||||
const result = extractAutorestConfigs(autorestSingleConfigFilePath, sdkRepo, loggerMock);
|
||||
expect(result?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should extract the first autorest config from autorest file', async () => {
|
||||
const sdkRepo = './sdk-repo';
|
||||
const result = extractAutorestConfigs(autorestMultiConfigFilePath, sdkRepo, loggerMock);
|
||||
expect(result?.length).toBeGreaterThan(0);
|
||||
expect(result.includes('azure-sdk-for-js')).toBe(true);
|
||||
});
|
||||
|
||||
it('cannot extract autorest config from autorest file', async () => {
|
||||
const sdkRepo = './azure-sdk-for-go';
|
||||
const result = extractAutorestConfigs(autorestSingleConfigFilePath, sdkRepo, loggerMock);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
# azure-sdk-for-js
|
||||
``` yaml
|
||||
output-folder: sdk/deviceupdate/iot-device-update-rest
|
||||
require:
|
||||
- specification/deviceupdate/data-plane/readme.md
|
||||
```
|
||||
|
||||
# azure-sdk-for-python
|
||||
``` yaml
|
||||
output-folder: sdk/deviceupdate/iot-device-update-rest
|
||||
require:
|
||||
- specification/deviceupdate/data-plane/readme.md
|
||||
```
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# azure-sdk-for-js
|
||||
``` yaml
|
||||
output-folder: sdk/deviceupdate/iot-device-update-rest
|
||||
require:
|
||||
- specification/deviceupdate/data-plane/readme.md
|
||||
```
|
||||
|
|
@ -10,6 +10,7 @@ export type GenerateAndBuildInput = {
|
|||
repoHttpsUrl: string;
|
||||
relatedReadmeMdFile: string;
|
||||
serviceType: string;
|
||||
autorestConfig: string;
|
||||
};
|
||||
|
||||
export const getGenerateAndBuildInput = getTypeTransformer<GenerateAndBuildInput>(
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@
|
|||
// The type of related swagger: resource-manager or data-plane
|
||||
"type": "string",
|
||||
"enum": ["resource-manager", "data-plane"]
|
||||
},
|
||||
"autorestConfig": {
|
||||
// The autorest config got from /autorest.md, which is mounted by user
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["specFolder", "headSha", "headRef", "repoHttpsUrl", "relatedReadmeMdFile"]
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче