Move TestAllCombinations into a mode (#221)
Co-authored-by: stas <statis@microsoft.com>
This commit is contained in:
Stas
GitHub
Родитель
558c924153
Коммит
585a5dc6ba
|
|
@ -274,10 +274,6 @@
|
|||
"type": "boolean",
|
||||
"description": "Ignore feedback from response"
|
||||
},
|
||||
"testAllCombinations":{
|
||||
"type": "boolean",
|
||||
"description": "By default, test mode will try to execute each request successfully once. This means that, if there are 5 possible values for a parameter, and the request\r\nis successfully executed when passing the first value, the remaining 4 will not be tested.\r\nIn some cases, such as for differential regression testing, it is desired to test all of the specified parameter values in Test mode.\r\nSetting TestAllCombinations to true in test mode in order to try all parameter values\r\n(up to max_combinations).\r\nResults for all parameter combinations will be reported in the spec coverage file."
|
||||
},
|
||||
"includeUserAgent":{
|
||||
"type": "boolean",
|
||||
"description": "Include RESTler user agent"
|
||||
|
|
@ -339,7 +335,7 @@
|
|||
"type": "string",
|
||||
"description": "RESTLer task types",
|
||||
"nullable": false,
|
||||
"enum" : ["compile", "test", "fuzz", "replay", "testFuzzLean", "fuzzRandomWalk", "fuzzBfsCheap"]
|
||||
"enum" : ["compile", "test", "fuzz", "replay", "testFuzzLean", "testAllCombinations", "fuzzRandomWalk", "fuzzBfsCheap"]
|
||||
},
|
||||
"compileConfiguration": {
|
||||
"$ref": "#/components/schemas/CompileConfiguration"
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
"tasks": [
|
||||
{
|
||||
"toolName": "RESTler",
|
||||
"outputFolder": "RESTLer-test-fuzz-lean-1",
|
||||
"outputFolder": "RESTler-test-fuzz-lean-1",
|
||||
"keyVaultSecrets": [ "RaftServicePrincipal" ],
|
||||
"authenticationMethod": {
|
||||
"MSAL": "RaftServicePrincipal"
|
||||
|
|
@ -28,7 +28,23 @@
|
|||
},
|
||||
{
|
||||
"toolName": "RESTler",
|
||||
"outputFolder": "RESTler-test-2",
|
||||
"outputFolder": "RESTler-test-all-combinations-2",
|
||||
"keyVaultSecrets": [ "RaftServicePrincipal" ],
|
||||
"authenticationMethod": {
|
||||
"MSAL": "RaftServicePrincipal"
|
||||
},
|
||||
"toolConfiguration": {
|
||||
"task": "TestAllCombinations",
|
||||
"runConfiguration": {
|
||||
"inputFolderPath": "/job-compile/RESTler-compile",
|
||||
"authenticationTokenRefreshIntervalSeconds": 300,
|
||||
"maxRequestExecutionTime" : 300
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"toolName": "RESTler",
|
||||
"outputFolder": "RESTler-test-3",
|
||||
"keyVaultSecrets": [ "RaftServicePrincipal" ],
|
||||
"authenticationMethod": {
|
||||
"MSAL": "RaftServicePrincipal"
|
||||
|
|
@ -37,8 +53,7 @@
|
|||
"task": "Test",
|
||||
"runConfiguration": {
|
||||
"inputFolderPath": "/job-compile/RESTler-compile",
|
||||
"maxRequestExecutionTime" : 300,
|
||||
"testAllCombinations" : true
|
||||
"maxRequestExecutionTime" : 300
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -895,7 +895,7 @@ see: https://docs.microsoft.com/en-us/azure/container-instances/container-instan
|
|||
</div>
|
||||
<div class="model">
|
||||
<h3><a name="RunConfiguration"><code>RunConfiguration</code></a> <a class="up" href="#__Models">Up</a></h3>
|
||||
<div class='model-description'>RESTler job Test, Fuzz or Replay configuration</div>
|
||||
<div class='model-description'>RESTler job Test, TestFuzzLean, TestAllCombinations, Fuzz or Replay configuration</div>
|
||||
<div class="field-items">
|
||||
<div class="param">grammarPy (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> Path to grammar py relative to compile folder path. If not set then default "grammar.py" grammar is assumed </div>
|
||||
<div class="param">inputFolderPath (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> For Test or Fuzz tasks: Grammar is produced by compile step. The compile step
|
||||
|
|
@ -911,7 +911,6 @@ For Replay task: path to RESTler Fuzz or Test run that contains bug buckets to r
|
|||
<div class="param">maxRequestExecutionTime (optional)</div><div class="param-desc"><span class="param-type"><a href="#integer">Integer</a></span> Maximum request execution time </div>
|
||||
<div class="param">ignoreDependencies (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Ignore resource dependencies </div>
|
||||
<div class="param">ignoreFeedback (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Ignore feedback from responses </div>
|
||||
<div class="param">testAllCombinations (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> By default, test mode will try to execute each request successfully once. This means that, if there are 5 possible values for a parameter, and the request is successfully executed when passing the first value, the remaining 4 will not be tested. In some cases, such as for differential regression testing, it is desired to test all of the specified parameter values in Test mode. Setting TestAllCombinations to true in test mode in order to try all parameter values (up to max_combinations). Results for all parameter combinations will be reported in the spec coverage file </div>
|
||||
</div> <!-- field-items -->
|
||||
</div>
|
||||
<div class="model">
|
||||
|
|
|
|||
|
|
@ -880,31 +880,35 @@ let main argv =
|
|||
match jobConfiguration.InputFolderPath with
|
||||
| Some p -> copyDir p workDirectory (set [taskConfigurationPath])
|
||||
| None -> ()
|
||||
|
||||
let testMode =
|
||||
match jobConfiguration.TestAllCombinations with
|
||||
| None | Some false -> "directed-smoke-test"
|
||||
| Some true -> "test-all-combinations"
|
||||
|
||||
let testMode = "directed-smoke-test"
|
||||
printfn "Running %s" testMode
|
||||
do! report Raft.JobEvents.Running (None, None)
|
||||
do! test testMode [] jobConfiguration
|
||||
let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
|
||||
return Raft.JobEvents.TaskCompleted, None, 0, summary
|
||||
|
||||
| TaskType.TestAllCombinations ->
|
||||
let jobConfiguration =
|
||||
match restlerPayload.RunConfiguration with
|
||||
| None -> RunConfiguration.Empty
|
||||
| Some jobConfiguration -> jobConfiguration
|
||||
match jobConfiguration.InputFolderPath with
|
||||
| Some p -> copyDir p workDirectory (set [taskConfigurationPath])
|
||||
| None -> ()
|
||||
let testMode = "test-all-combinations"
|
||||
printfn "Running %s" testMode
|
||||
do! report Raft.JobEvents.Running (None, None)
|
||||
do! test testMode [] jobConfiguration
|
||||
let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
|
||||
return Raft.JobEvents.TaskCompleted, None, 0, summary
|
||||
|
||||
| TaskType.TestFuzzLean ->
|
||||
|
||||
| TaskType.TestFuzzLean ->
|
||||
printfn "Running test fuzz lean"
|
||||
let jobConfiguration =
|
||||
match restlerPayload.RunConfiguration with
|
||||
| None -> RunConfiguration.Empty
|
||||
| Some jobConfiguration -> jobConfiguration
|
||||
|
||||
match jobConfiguration.TestAllCombinations with
|
||||
| None | Some false ->
|
||||
printfn "WARNING: Test All Combination settings is not supported in test fuzz lean mode"
|
||||
| Some true -> ()
|
||||
|
||||
match jobConfiguration.InputFolderPath with
|
||||
| Some p -> copyDir p workDirectory (set [taskConfigurationPath])
|
||||
| None -> ()
|
||||
|
|
|
|||
|
|
@ -139,9 +139,6 @@ type RunConfiguration =
|
|||
/// Specifies whether to show contents of auth token in RESTler logs
|
||||
ShowAuthToken : bool option
|
||||
|
||||
/// Specifies whether to test all combinations in 'test' mode
|
||||
TestAllCombinations : bool option
|
||||
|
||||
/// Token Refresh Interval
|
||||
AuthenticationTokenRefreshIntervalSeconds: int option
|
||||
|
||||
|
|
@ -179,7 +176,6 @@ type RunConfiguration =
|
|||
GrammarPy = None
|
||||
InputFolderPath = None
|
||||
ProducerTimingDelay = None
|
||||
TestAllCombinations = None
|
||||
UseSsl = None
|
||||
AuthenticationTokenRefreshIntervalSeconds = None
|
||||
PathRegex = None
|
||||
|
|
@ -209,6 +205,7 @@ type TaskType =
|
|||
| Compile
|
||||
| Test
|
||||
| TestFuzzLean
|
||||
| TestAllCombinations
|
||||
| Fuzz
|
||||
| FuzzRandomWalk
|
||||
| FuzzBfsCheap
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче