Azure
/
oav
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

make runner run api sync (#648) 

* runner run test scenario sequentially

* add runner architecture

* bump version and update changelog

* remove install gif update runApiTest.gif
This commit is contained in:
Ruoxuan Wang 2021-08-10 14:49:25 +08:00 коммит произвёл GitHub
Родитель 7320e528a0
Коммит 6c9aab3879
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 305 добавлений и 82 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
ChangeLog.md
Просмотреть файл

@ -1,5 +1,10 @@
# Change Log - oav
## 08/09/2021 2.7.2
- Add runner architecture drawio
- Oav runner run test scenarios sequentially
## 08/03/2021 2.7.1
- Disable checkUnderFileRoot in `Validate Traffic` command

2
README.md
Просмотреть файл

@ -19,8 +19,6 @@ You can install the latest stable release of node.js from [here](https://nodejs.
npm install -g oav@latest
```
![](./documentation/installOav.gif)
### Run API test
OAV support run API test against Azure and validate request and response. You could define test scenario file which compose with severval swagger example file and then use oav to run it. For more details about API test, please refer to this [API testing doc](https://github.com/Azure/azure-rest-api-specs/tree/test-scenario-main/documentation/test-scenario).

Двоичные данные
documentation/installOav.gif
Просмотреть файл

Двоичный файл не отображается.
Режим "рядом"

До

Ширина:  |  Высота:  |  Размер: 413 KiB

210
documentation/postmanRunner.drawio Normal file
Просмотреть файл

@ -0,0 +1,210 @@
<mxfile host="65bd71144e">
<diagram id="wCy_WC5Ljue5KlhTAN_c" name="Page-1">
<mxGraphModel dx="1128" dy="514" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0"/>
<mxCell id="1" parent="0"/>
<mxCell id="2" value="" style="aspect=fixed;html=1;points=[];align=center;image;fontSize=12;image=img/lib/azure2/identity/Azure_Active_Directory.svg;" vertex="1" parent="1">
<mxGeometry x="770" y="258" width="70" height="64" as="geometry"/>
</mxCell>
<mxCell id="44" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="5" target="43">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="5" value="AAD Auth" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="460" y="266" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="13" value="Test Scenario File" style="swimlane;fontStyle=0;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" vertex="1" parent="1">
<mxGeometry x="30" y="33" width="160" height="78" as="geometry"/>
</mxCell>
<mxCell id="14" value="Step: CreateSignalService" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="13">
<mxGeometry y="26" width="160" height="26" as="geometry"/>
</mxCell>
<mxCell id="15" value="Step: DeleteSignalService" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="13">
<mxGeometry y="52" width="160" height="26" as="geometry"/>
</mxCell>
<mxCell id="23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="17" target="21">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="17" value="Swagger readme" style="swimlane;fontStyle=0;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" vertex="1" parent="1">
<mxGeometry x="30" y="148" width="160" height="78" as="geometry"/>
</mxCell>
<mxCell id="18" value="SignalR swagger" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="17">
<mxGeometry y="26" width="160" height="26" as="geometry"/>
</mxCell>
<mxCell id="58" value="SignalR swagger examples" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="17">
<mxGeometry y="52" width="160" height="26" as="geometry"/>
</mxCell>
<mxCell id="26" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="21" target="25">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="21" value="OAV: Test Secneario Loader" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="460" y="80" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="22" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="14" target="21">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="27" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="25" target="5">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="25" value="OAV: Postman Runner" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="460" y="170" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="29" value="Envoirment variables" style="swimlane;fontStyle=0;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" vertex="1" parent="1">
<mxGeometry x="30" y="266" width="160" height="106" as="geometry"/>
</mxCell>
<mxCell id="30" value="SubscriptionId" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="29">
<mxGeometry y="26" width="160" height="26" as="geometry"/>
</mxCell>
<mxCell id="31" value="AAD ApplicationId" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="29">
<mxGeometry y="52" width="160" height="28" as="geometry"/>
</mxCell>
<mxCell id="32" value="AAD ApplicationSecret" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="29">
<mxGeometry y="80" width="160" height="26" as="geometry"/>
</mxCell>
<mxCell id="35" value="" style="endArrow=classic;html=1;" edge="1" parent="1" target="2">
<mxGeometry relative="1" as="geometry">
<mxPoint x="580" y="289.5" as="sourcePoint"/>
<mxPoint x="740" y="289.5" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="36" value="Get Bearer Token" style="edgeLabel;resizable=0;html=1;align=center;verticalAlign=middle;" connectable="0" vertex="1" parent="35">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="39" value="Runtime Env" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="250" y="266" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="40" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="31" target="39">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="42" value="" style="shape=flexArrow;endArrow=classic;startArrow=classic;html=1;" edge="1" parent="1">
<mxGeometry width="100" height="100" relative="1" as="geometry">
<mxPoint x="370" y="300" as="sourcePoint"/>
<mxPoint x="460" y="300" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="46" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="43" target="45">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="43" value="Create Resource Group" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="460" y="350" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="48" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="45" target="47">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="45" value="Create SignalR" style="whiteSpace=wrap;html=1;rounded=1;" vertex="1" parent="1">
<mxGeometry x="460" y="440" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="50" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="47" target="49">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="53" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="47" target="52">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="47" value="LRO Poller" style="whiteSpace=wrap;html=1;rounded=1;" vertex="1" parent="1">
<mxGeometry x="460" y="520" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="49" value="ARM" style="whiteSpace=wrap;html=1;rounded=1;" vertex="1" parent="1">
<mxGeometry x="710" y="520" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="51" value="Write token to runtime env" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="370" y="230" width="100" height="42" as="geometry"/>
</mxCell>
<mxCell id="55" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="52" target="54">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="52" value="Delete SignalR" style="whiteSpace=wrap;html=1;rounded=1;" vertex="1" parent="1">
<mxGeometry x="460" y="610" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="57" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="54" target="56">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="54" value="Delete Resource Group" style="whiteSpace=wrap;html=1;rounded=1;" vertex="1" parent="1">
<mxGeometry x="460" y="700" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="61" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="56" target="62">
<mxGeometry relative="1" as="geometry">
<mxPoint x="370" y="460" as="targetPoint"/>
<Array as="points">
<mxPoint x="400" y="820"/>
<mxPoint x="400" y="470"/>
</Array>
</mxGeometry>
</mxCell>
<mxCell id="56" value="Response Validator" style="whiteSpace=wrap;html=1;rounded=1;" vertex="1" parent="1">
<mxGeometry x="460" y="790" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="72" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.46;entryY=0.021;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="62" target="63">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="90" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="62" target="75">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="180" y="520"/>
<mxPoint x="290" y="520"/>
</Array>
</mxGeometry>
</mxCell>
<mxCell id="62" value="Traffic validator" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="120" y="440" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="73" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="63" target="70">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="63" value="validate response" style="swimlane;fontStyle=0;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" vertex="1" parent="1">
<mxGeometry x="10" y="576" width="190" height="90" as="geometry"/>
</mxCell>
<object label="Compare with example resposne" a="" id="65">
<mxCell style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="63">
<mxGeometry y="26" width="190" height="26" as="geometry"/>
</mxCell>
</object>
<object label="Ignore random generate value&#10;(readonly, secret, etc)" a="" id="95">
<mxCell style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="63">
<mxGeometry y="52" width="190" height="38" as="geometry"/>
</mxCell>
</object>
<mxCell id="82" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="70" target="81">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="85" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="70" target="84">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="70" value="report generator" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="120" y="700" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="80" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="75" target="70">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="330" y="680"/>
<mxPoint x="180" y="680"/>
</Array>
</mxGeometry>
</mxCell>
<mxCell id="75" value="validate request" style="swimlane;fontStyle=0;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" vertex="1" parent="1">
<mxGeometry x="230" y="580" width="160" height="52" as="geometry"/>
</mxCell>
<mxCell id="76" value="Status code should be 2xx" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" vertex="1" parent="75">
<mxGeometry y="26" width="160" height="26" as="geometry"/>
</mxCell>
<mxCell id="88" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="81" target="87">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="81" value="JSON report" style="whiteSpace=wrap;html=1;rounded=1;" vertex="1" parent="1">
<mxGeometry x="30" y="790" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="89" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="84" target="87">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="84" value="Markdown report" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="190" y="790" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="87" value="Done" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="110" y="910" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="96" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="95" target="76">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>

Двоичные данные
documentation/runApiTest.gif
Просмотреть файл

Двоичный файл не отображается.
Режим "рядом" Свайп Оверлей

До

Ширина:  |  Высота:  |  Размер: 2.4 MiB

После

Ширина:  |  Высота:  |  Размер: 1.3 MiB

10
lib/testScenario/postmanCollectionRunnerClient.ts
Просмотреть файл

@ -221,6 +221,9 @@ export class PostmanCollectionRunnerClient implements TestScenarioRunnerClient {
url: "",
body: { mode: "raw" } as RequestBodyDefinition,
});
if (step.step === "Deployments_CreateOrUpdate") {
console.log(JSON.stringify(stepEnv.env, null, 2));
}
item.description = step.operation.operationId || "";
const queryParams: QueryParamDefinition[] = [];
const urlVariables: VariableDefinition[] = [];
@ -539,6 +542,9 @@ export class PostmanCollectionRunnerClient implements TestScenarioRunnerClient {
if (this.opts.from || this.opts.to) {
runtimeEnvManager.repopulateCollectionItems(this.opts.from, this.opts.to);
}
const newmanRun = async () => {
return new Promise((resolve) => {
newman
.run(
{
@ -618,7 +624,11 @@ export class PostmanCollectionRunnerClient implements TestScenarioRunnerClient {
)}' was not cleaned up.`
);
}
resolve(_summary);
});
});
};
await newmanRun();
}
private generatedGetOperationItem(

2
package-lock.json сгенерированный
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "oav",
"version": "2.7.1",
"version": "2.7.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
package.json
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "oav",
"version": "2.7.1",
"version": "2.7.2",
"author": {
"name": "Microsoft Corporation",
"email": "azsdkteam@microsoft.com",