microsoft
/
rest-api-fuzz-testing
зеркало из https://github.com/microsoft/rest-api-fuzz-testing.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Switch BVT runs to petstore container (rather than permanent live web service) (#78) 

Co-authored-by: stas <statis@microsoft.com>
This commit is contained in:
Stas 2020-12-07 14:12:20 -08:00 коммит произвёл GitHub
Родитель 7d7adc939f
Коммит 2b2cb591f2
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
14 изменённых файлов: 191 добавлений и 119 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

39
Scripts/Tests/bvt-petstore.py → Scripts/Tests/bvt-petstore3.py
Просмотреть файл

@ -33,30 +33,30 @@ def webhook_triggers_results(job_id, test_url):
def time_span(t_start, t_end):
return time.strftime("%H:%M:%S", time.gmtime(t_end - t_start))
def bvt(cli, definitions):
def bvt(cli, definitions, subs):
print('Getting available wehook events')
webhook_events = cli.list_available_webhooks_events()
try:
test_url = webhooks_test_url(definitions.subscription, definitions.resource_group, definitions.test_infra)
for event in webhook_events:
print(f'Setting webhook for {event}')
compile_webhook = cli.set_webhooks_subscription('petstore-compile', event, test_url)
fuzz_webhook = cli.set_webhooks_subscription('petstore-fuzz', event, test_url)
compile_webhook = cli.set_webhooks_subscription('petstore3-compile', event, test_url)
fuzz_webhook = cli.set_webhooks_subscription('petstore3-fuzz', event, test_url)
added_compile = cli.list_webhooks('petstore-compile', event)
added_compile = cli.list_webhooks('petstore3-compile', event)
if len(added_compile) == 0:
raise Exception('Expected petstore-compile webhooks not to be empty after creation')
raise Exception('Expected petstore3-compile webhooks not to be empty after creation')
added_fuzz = cli.list_webhooks('petstore-fuzz', event)
added_fuzz = cli.list_webhooks('petstore3-fuzz', event)
if len(added_fuzz) == 0:
raise Exception('Expected petstore-fuzz webhooks not to be empty after creation')
raise Exception('Expected petstore3-fuzz webhooks not to be empty after creation')
t_pre_compile = time.time()
print('Compile')
compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.compile.json'))
compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'compile.json'))
compile_config = raft.RaftJobConfig(file_path=compile_config_path)
compile_config = raft.RaftJobConfig(file_path=compile_config_path, substitutions=subs)
compile_job = cli.new_job(compile_config)
cli.poll(compile_job['jobId'], 10)
@ -81,11 +81,9 @@ def bvt(cli, definitions):
f' {after_compile_pre_fuzz}')
print('Fuzz')
fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.fuzz.json'))
subs = {}
fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'fuzz.json'))
subs['{compile.jobId}'] = compile_job['jobId']
fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs)
fuzz_config.config['duration'] = '00:20:00'
fuzz_job = cli.new_job(fuzz_config)
cli.poll(fuzz_job['jobId'], 10)
@ -104,8 +102,8 @@ def bvt(cli, definitions):
raise Exception('Expected job to be in completed state when retrieved job list.'
f'{after_fuzz}')
if m != 2:
raise Exception('Expected 2 after compile job step'
if m != 3:
raise Exception('Expected 3 after compile job step'
f' for job {fuzz_job["jobId"]}'
f' got {m}'
f' {after_fuzz}')
@ -136,7 +134,7 @@ def bvt(cli, definitions):
print('Validating that bugs posted events matches total bugs found in job status')
total_bugs_found = 0
for r in job_status_events:
if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId']:
if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId'] and r['Data']['Tool'] == 'RESTler':
total_bugs_found += r['Data']['Metrics']['TotalBugBucketsCount']
print(f'Total bugs found: {total_bugs_found}')
@ -161,12 +159,11 @@ def bvt(cli, definitions):
if len(deleted_fuzz) > 0:
raise Exception('Expected petstore-fuzz webhooks to be empty after deletion, instead got %s', deleted_compile)
if __name__ == "__main__":
formatter = argparse.ArgumentDefaultsHelpFormatter
parser = argparse.ArgumentParser(description='bvt', formatter_class=formatter)
raft.add_defaults_and_secret_args(parser)
parser.add_argument('--build', required=True)
args = parser.parse_args()
if args.defaults_context_json:
@ -179,4 +176,10 @@ if __name__ == "__main__":
definitions = RaftDefinitions(defaults)
defaults['secret'] = args.secret
cli = RaftCLI(defaults)
bvt(cli, definitions)
subs = {
"{build-url}" : os.environ.get('SYSTEM_COLLECTIONURI'),
"{build-id}" : os.environ.get('BUILD_BUILDID'),
"{ci-run}" : args.build.replace('.', '-')
}
print(f"SUBS: {subs}")
bvt(cli, definitions, subs)

2
ado/stages/test/steps/bvt.yml
Просмотреть файл

@ -13,4 +13,4 @@ steps:
azureSubscription: $(raft-subscription)
scriptType: 'pscore'
scriptLocation: 'inlineScript'
inlineScript: "python Scripts/Tests/bvt.py --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret) --host $(bvt-host)"
inlineScript: "python Scripts/Tests/bvt-petstore3.py --build $(Build.BuildNumber) --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret)"

5
cli/samples/restler/self-contained/swagger-petstore/restler.compile.json
Просмотреть файл

@ -3,10 +3,7 @@
"URL" : "http://localhost:8080/api/swagger.json"
},
"webhook": {
"name": "petstore-compile",
"metadata": {
"action" : "compile"
}
"name": "petstore-compile"
},
"testTargets" : {
"targets" : [

7
cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json
Просмотреть файл

@ -1,12 +1,7 @@
{
"host": "localhost",
"webhook": {
"name": "petstore-fuzz",
"metadata": {
"app" : "petstore",
"swagger_version" : "v2",
"action" : "fuzz"
}
"name": "petstore-fuzz"
},
"readonlyFileShareMounts": [

7
cli/samples/restler/self-contained/swagger-petstore/restler.test.json
Просмотреть файл

@ -2,12 +2,7 @@
"host": "localhost",
"webhook": {
"name": "petstore",
"metadata": {
"app" : "petstore",
"swagger_version" : "v2",
"action" : "test"
}
"name": "petstore-test"
},
"readonlyFileShareMounts": [
{

4
cli/samples/restler/self-contained/swagger-petstore/run.py
Просмотреть файл

@ -14,8 +14,6 @@ def run(compile, test, fuzz, replay):
cli = RaftCLI()
# Create compilation job configuration
compile_job_config = RaftJobConfig(file_path=compile)
# add webhook metadata that will be included in every triggered webhook by Compile job
compile_job_config.add_metadata({"branch":"wizbangFeature"})
print('Compile')
# submit a new job with the Compile config and get new job ID
compile_job = cli.new_job(compile_job_config)
@ -37,8 +35,6 @@ def run(compile, test, fuzz, replay):
# create a new job config with Fuzz configuration JSON
fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
print('Fuzz')
# add webhook metadata that will included in every triggered webhook by Fuzz job
fuzz_job_config.add_metadata({"branch":"wizbangFeature"})
# create new fuzz job configuration
fuzz_job = cli.new_job(fuzz_job_config)

11
cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json → cli/samples/restler/self-contained/swagger-petstore3/compile.json
Просмотреть файл

@ -1,18 +1,11 @@
{
"rootFileShare" : "{ci-run}",
"namePrefix" : "petstore3-compile-",
"swaggerLocation": {
"URL" : "http://localhost:8082/api/v3/openapi.json"
},
"resources": {
"Cores": 4,
"MemoryGBs": 8
},
"testTargets" : {
"resources" : {
"Cores" : 2,
"MemoryGBs" : 4
},
"targets" : [
{
"Container" : "swaggerapi/petstore3",

40
cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json → cli/samples/restler/self-contained/swagger-petstore3/fuzz.json
Просмотреть файл

@ -1,4 +1,6 @@
{
"rootFileShare" : "{ci-run}",
"namePrefix" : "petstore3-test-fuzz-lean-",
"host": "localhost",
"resources": {
@ -6,10 +8,17 @@
"MemoryGBs": 8
},
"webhook":{
"name" : "petstore3-fuzz",
"metadata" : {
"buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results"
}
},
"readonlyFileShareMounts": [
{
"FileShareName": "{compile.jobId}",
"MountPath": "/job-compile"
"FileShareName": "{ci-run}",
"MountPath": "/ci-run"
}
],
@ -26,21 +35,16 @@
"Run" : {
"Command" : "/bin/sh",
"Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"]
},
"Shell" : "/bin/sh",
"OutputFolder" : "restler_petstore3"
}
},
{
"Container" : "swaggerapi/petstore3",
"Port" : 8080,
"ExpectedDurationUntilReady" : "00:00:30",
"PostRun" : {
"Port" : 8082,
"ExpectedDurationUntilReady" : "00:02:00",
"Run" : {
"Command" : "/bin/sh",
"Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"],
"ExpectedRunDuration" : "00:00:10"
},
"Shell" : "/bin/sh",
"OutputFolder" : "zap_petstore3"
"Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8082 /swagger-petstore/server.war"]
}
}
]
},
@ -48,7 +52,7 @@
{
"toolName": "RESTler",
"outputFolder": "fuzz",
"duration": "00:30:00",
"duration": "00:10:00",
"toolConfiguration": {
"task": "Fuzz",
"runConfiguration": {
@ -56,15 +60,15 @@
"Port" : 8081
},
"useSsl" : false,
"inputFolderPath": "/job-compile/compile"
"inputFolderPath": "/ci-run/{compile.jobId}/compile"
}
}
},
{
"toolName" : "ZAP",
"outputFolder" : "zap",
"toolName": "ZAP",
"outputFolder": "zap",
"swaggerLocation": {
"URL" : "http://localhost:8080/api/v3/openapi.json"
"URL" : "http://localhost:8082/api/v3/openapi.json"
}
}
]

77
cli/samples/restler/self-contained/swagger-petstore3/run.py
Просмотреть файл

@ -4,47 +4,60 @@
import pathlib
import sys
import os
import json
cur_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..'))
from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError
sys.path.append(os.path.join(cur_dir, '..'))
from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError, RaftDefinitions
def run(compile, test, fuzz):
# instantiate RAFT CLI
cli = RaftCLI()
def run(cli, config, subs):
# Create compilation job configuration
compile_job_config = RaftJobConfig(file_path=compile)
print('Compile')
job_config = RaftJobConfig(file_path=config, substitutions=subs)
print(f'Running {config}')
# submit a new job with the Compile config and get new job ID
compile_job = cli.new_job(compile_job_config)
job = cli.new_job(job_config)
# wait for a job with ID from compile_job to finish the run
cli.poll(compile_job['jobId'])
# use compile job as input for fuzz job
subs = {}
subs['{compile.jobId}'] = compile_job['jobId']
test_job_config = RaftJobConfig(file_path=test, substitutions=subs)
print('Test')
# create new fuzz job configuration
test_job = cli.new_job(test_job_config)
# wait for job ID from fuzz_job to finish the run
cli.poll(test_job['jobId'])
# create a new job config with Fuzz configuration JSON
fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
print('Fuzz')
# create new fuzz job configuration
fuzz_job = cli.new_job(fuzz_job_config)
# wait for job ID from fuzz_job to finish the run
cli.poll(fuzz_job['jobId'])
cli.poll(job['jobId'])
return job['jobId']
if __name__ == "__main__":
try:
run(os.path.join(cur_dir, "restler.compile.json"),
os.path.join(cur_dir, "restler.test.json"),
os.path.join(cur_dir, "restler.zap.fuzz.json"))
defaults = None
if sys.argv[1] == '--build':
build_id = sys.argv[2].replace(".", "-")
print(f"BUILD ID : {build_id}")
with open(os.path.join(cur_dir, '..', 'defaults.json'), 'r') as defaults_json:
defaults = json.load(defaults_json)
if sys.argv[3] == '--secret':
defaults['secret'] = sys.argv[4]
# instantiate RAFT CLI
cli = RaftCLI(defaults)
defs = RaftDefinitions(defaults)
compile_job_id = None
subs = {
"{ci-run}" : f"{build_id}",
"{build-url}" : os.environ['SYSTEM_COLLECTIONURI'],
"{build-id}" : os.environ['BUILD_BUILDID'],
"{raft-subscription}": defs.subscription,
"{raft-resource-group}" : defs.resource_group,
"{raft-storage-account}" : defs.storage_account
}
for arg in sys.argv[1:]:
if arg == 'compile':
compile_job_id = run(cli, os.path.join(cur_dir, 'compile.json'), subs)
subs['{compile.jobId}'] = compile_job_id
if arg == 'test':
run(cli, os.path.join(cur_dir, "test.json"), subs),
if arg == 'test-fuzz-lean':
run(cli, os.path.join(cur_dir, "test-fuzz-lean.json"), subs),
except RaftJobError as ex:
print(f'ERROR: {ex.message}')
sys.exit(1)

59
cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json Normal file
Просмотреть файл

@ -0,0 +1,59 @@
{
"rootFileShare" : "{ci-run}",
"namePrefix" : "petstore3-test-fuzz-lean-",
"host": "localhost",
"resources": {
"Cores": 4,
"MemoryGBs": 8
},
"webhook":{
"name" : "petstore3-test-fuzz-lean",
"metadata" : {
"buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results"
}
},
"readonlyFileShareMounts": [
{
"FileShareName": "{ci-run}",
"MountPath": "/ci-run"
}
],
"testTargets" : {
"resources" : {
"Cores" : 2,
"MemoryGBs" : 4
},
"targets" : [
{
"Container" : "swaggerapi/petstore3",
"Port" : 8081,
"ExpectedDurationUntilReady" : "00:02:00",
"Run" : {
"Command" : "/bin/sh",
"Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"]
}
}
]
},
"tasks": [
{
"toolName": "RESTler",
"outputFolder": "test-fuzz-lean",
"duration": "01:00:00",
"toolConfiguration": {
"task": "TestFuzzLean",
"runConfiguration": {
"targetEndpointConfiguration" : {
"Port" : 8081
},
"useSsl" : false,
"inputFolderPath": "/ci-run/{compile.jobId}/compile"
}
}
}
]
}

17
cli/samples/restler/self-contained/swagger-petstore3/restler.test.json → cli/samples/restler/self-contained/swagger-petstore3/test.json
Просмотреть файл

@ -1,4 +1,6 @@
{
"rootFileShare" : "{ci-run}",
"namePrefix" : "petstore3-test-",
"host": "localhost",
"resources": {
@ -6,10 +8,17 @@
"MemoryGBs": 8
},
"webhook":{
"name" : "petstore3-test",
"metadata" : {
"buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results"
}
},
"readonlyFileShareMounts": [
{
"FileShareName": "{compile.jobId}",
"MountPath": "/job-compile"
"FileShareName": "{ci-run}",
"MountPath": "/ci-run"
}
],
@ -29,7 +38,7 @@
"ExpectedRunDuration" : "00:00:10"
},
"Shell" : "/bin/sh",
"OutputFolder" : "zap_petstore3"
"OutputFolder" : "petstore3"
}
]
},
@ -44,7 +53,7 @@
"Port" : 8080
},
"useSsl" : false,
"inputFolderPath": "/job-compile/compile"
"inputFolderPath": "/ci-run/{compile.jobId}/compile"
}
}
}

5
src/Agent/RESTlerAgent/RESTlerDriver.fs
Просмотреть файл

@ -151,6 +151,7 @@ module private RESTlerInternal =
if Seq.isEmpty experiments then
None
else
try
let startedExperiments =
experiments
|> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime)
@ -160,6 +161,10 @@ module private RESTlerInternal =
printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run."
(Seq.length startedExperiments) startedExperiments runStartTime
startedExperiments |> Seq.tryHead
with
| :? System.IO.IOException as ioex ->
printfn "Getting experiment folder interrupted due to : %s" ioex.Message
None
else
None

2
src/Contracts/Job.fs
Просмотреть файл

@ -87,7 +87,7 @@ type FileShareMount =
type Webhook =
{
Name : string
Metadata : Map<string, string>
Metadata : Map<string, string> option
}

9
src/Orchestrator/OrchestratorLogic/Orchestrator.fs
Просмотреть файл

@ -1316,10 +1316,13 @@ module ContainerInstances =
let webhookDefinition = Microsoft.FSharpLu.Json.Compact.deserialize<Raft.Job.Webhook option>(jobEntity.Webhook)
match webhookDefinition with
| Some webhook ->
if webhook.Metadata.IsEmpty then
match webhook.Metadata with
| None ->
return None
else
return Some webhook.Metadata
| Some m when m.IsEmpty ->
return None
| Some m ->
return Some m
| None ->
return None
else