Switch BVT runs to petstore container (rather than permanent live web service) (#78)

Co-authored-by: stas <statis@microsoft.com>

Scripts/Tests/bvt-petstore3.py

@@ -33,30 +33,30 @@ def webhook_triggers_results(job_id, test_url):
 def time_span(t_start, t_end):
     return time.strftime("%H:%M:%S", time.gmtime(t_end - t_start))
 
-def bvt(cli, definitions):
+def bvt(cli, definitions, subs):
     print('Getting available wehook events')
     webhook_events = cli.list_available_webhooks_events()
     try:
         test_url = webhooks_test_url(definitions.subscription, definitions.resource_group, definitions.test_infra)
         for event in webhook_events:
             print(f'Setting webhook for {event}')
-            compile_webhook = cli.set_webhooks_subscription('petstore-compile', event, test_url)
+            compile_webhook = cli.set_webhooks_subscription('petstore3-compile', event, test_url)
-            fuzz_webhook = cli.set_webhooks_subscription('petstore-fuzz', event, test_url)
+            fuzz_webhook = cli.set_webhooks_subscription('petstore3-fuzz', event, test_url)
 
-        added_compile = cli.list_webhooks('petstore-compile', event)
+        added_compile = cli.list_webhooks('petstore3-compile', event)
         if len(added_compile) == 0:
-            raise Exception('Expected petstore-compile webhooks not to be empty after creation')
+            raise Exception('Expected petstore3-compile webhooks not to be empty after creation')
 
-        added_fuzz = cli.list_webhooks('petstore-fuzz', event)
+        added_fuzz = cli.list_webhooks('petstore3-fuzz', event)
         if len(added_fuzz) == 0:
-            raise Exception('Expected petstore-fuzz webhooks not to be empty after creation')
+            raise Exception('Expected petstore3-fuzz webhooks not to be empty after creation')
 
         t_pre_compile = time.time()
 
         print('Compile')
-        compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.compile.json'))
+        compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'compile.json'))
 
-        compile_config = raft.RaftJobConfig(file_path=compile_config_path)
+        compile_config = raft.RaftJobConfig(file_path=compile_config_path, substitutions=subs)
         compile_job = cli.new_job(compile_config)
         cli.poll(compile_job['jobId'], 10)
 
@@ -81,11 +81,9 @@ def bvt(cli, definitions):
                             f' {after_compile_pre_fuzz}')
 
         print('Fuzz')
-        fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.fuzz.json'))
+        fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'fuzz.json'))
-        subs = {}
         subs['{compile.jobId}'] = compile_job['jobId']
         fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs)
-        fuzz_config.config['duration'] = '00:20:00'
         fuzz_job = cli.new_job(fuzz_config)
         cli.poll(fuzz_job['jobId'], 10)
 
@@ -104,8 +102,8 @@ def bvt(cli, definitions):
                     raise Exception('Expected job to be in completed state when retrieved job list.'
                                     f'{after_fuzz}')
         
-        if m != 2:
+        if m != 3:
-            raise Exception('Expected 2 after compile job step'
+            raise Exception('Expected 3 after compile job step'
                             f' for job {fuzz_job["jobId"]}'
                             f' got {m}'
                             f' {after_fuzz}')
@@ -136,7 +134,7 @@ def bvt(cli, definitions):
         print('Validating that bugs posted events matches total bugs found in job status')
         total_bugs_found = 0
         for r in job_status_events:
-            if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId']:
+            if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId'] and r['Data']['Tool'] == 'RESTler':
                 total_bugs_found += r['Data']['Metrics']['TotalBugBucketsCount']
 
         print(f'Total bugs found: {total_bugs_found}')
@@ -161,12 +159,11 @@ def bvt(cli, definitions):
         if len(deleted_fuzz) > 0:
             raise Exception('Expected petstore-fuzz webhooks to be empty after deletion, instead got %s', deleted_compile)
 
-
-
 if __name__ == "__main__":
     formatter = argparse.ArgumentDefaultsHelpFormatter
     parser = argparse.ArgumentParser(description='bvt', formatter_class=formatter)
     raft.add_defaults_and_secret_args(parser)
+    parser.add_argument('--build', required=True)
     args = parser.parse_args()
 
     if args.defaults_context_json:
@@ -179,4 +176,10 @@ if __name__ == "__main__":
     definitions = RaftDefinitions(defaults)
     defaults['secret'] = args.secret
     cli = RaftCLI(defaults)
-    bvt(cli, definitions)
+    subs = {
+        "{build-url}" : os.environ.get('SYSTEM_COLLECTIONURI'),
+        "{build-id}" : os.environ.get('BUILD_BUILDID'),
+        "{ci-run}" : args.build.replace('.', '-')
+    }
+    print(f"SUBS: {subs}")
+    bvt(cli, definitions, subs)

ado/stages/test/steps/bvt.yml

@@ -13,4 +13,4 @@ steps:
       azureSubscription: $(raft-subscription)
       scriptType: 'pscore'
       scriptLocation: 'inlineScript'
-      inlineScript: "python Scripts/Tests/bvt.py --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret) --host $(bvt-host)"
+      inlineScript: "python Scripts/Tests/bvt-petstore3.py --build $(Build.BuildNumber) --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret)"

cli/samples/restler/self-contained/swagger-petstore/restler.compile.json

@@ -3,10 +3,7 @@
     "URL" : "http://localhost:8080/api/swagger.json"
   },
   "webhook": {
-    "name": "petstore-compile",
+    "name": "petstore-compile"
-    "metadata": {
-	    "action" : "compile"
-    }
   },
   "testTargets" : {
     "targets" : [

cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json

@@ -1,12 +1,7 @@
 {
   "host": "localhost",
   "webhook": {
-    "name": "petstore-fuzz",
+    "name": "petstore-fuzz"
-    "metadata": {
-      "app" : "petstore",
-      "swagger_version" : "v2",
-      "action" : "fuzz"	
-    }
   },
 
   "readonlyFileShareMounts": [

cli/samples/restler/self-contained/swagger-petstore/restler.test.json

@@ -2,12 +2,7 @@
   "host": "localhost",
 
   "webhook": {
-    "name": "petstore",
+    "name": "petstore-test"
-    "metadata": {
-      "app" : "petstore",
-      "swagger_version" : "v2",
-      "action" : "test"	
-    }
   },
   "readonlyFileShareMounts": [
     {

cli/samples/restler/self-contained/swagger-petstore/run.py

@@ -14,8 +14,6 @@ def run(compile, test, fuzz, replay):
     cli = RaftCLI()
     # Create compilation job configuration
     compile_job_config = RaftJobConfig(file_path=compile)
-    # add webhook metadata that will be included in every triggered webhook by Compile job
-    compile_job_config.add_metadata({"branch":"wizbangFeature"})
     print('Compile')
     # submit a new job with the Compile config and get new job ID
     compile_job = cli.new_job(compile_job_config)
@@ -37,8 +35,6 @@ def run(compile, test, fuzz, replay):
     # create a new job config with Fuzz configuration JSON
     fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
     print('Fuzz')
-    # add webhook metadata that will included in every triggered webhook by Fuzz job
-    fuzz_job_config.add_metadata({"branch":"wizbangFeature"})
     # create new fuzz job configuration
     fuzz_job = cli.new_job(fuzz_job_config)
 

cli/samples/restler/self-contained/swagger-petstore3/compile.json

@@ -1,18 +1,11 @@
-{
+{ 
+  "rootFileShare" : "{ci-run}",
+  "namePrefix" : "petstore3-compile-",
   "swaggerLocation": {
     "URL" : "http://localhost:8082/api/v3/openapi.json"
   },
 
-  "resources": {
-    "Cores": 4,
-    "MemoryGBs": 8
-  },
-
   "testTargets" : {
-    "resources" : {
-      "Cores" : 2,
-      "MemoryGBs" : 4
-    },
     "targets" : [
       {
         "Container" : "swaggerapi/petstore3",

cli/samples/restler/self-contained/swagger-petstore3/fuzz.json

@@ -1,4 +1,6 @@
 {
+  "rootFileShare" : "{ci-run}",
+  "namePrefix" : "petstore3-test-fuzz-lean-",
   "host": "localhost",
 
   "resources": {
@@ -6,10 +8,17 @@
     "MemoryGBs": 8
   },
 
+  "webhook":{
+    "name" : "petstore3-fuzz",
+    "metadata" : {
+      "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results"
+    }
+  },
+
   "readonlyFileShareMounts": [
     {
-      "FileShareName": "{compile.jobId}",
+      "FileShareName": "{ci-run}",
-      "MountPath": "/job-compile"
+      "MountPath": "/ci-run"
     }
   ],
 
@@ -26,21 +35,16 @@
         "Run" : {
           "Command" : "/bin/sh",
           "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"]
-        },
+        }
-        "Shell" : "/bin/sh",
-        "OutputFolder" : "restler_petstore3"
       },
       {
         "Container" : "swaggerapi/petstore3",
-        "Port" : 8080,
+        "Port" : 8082,
-        "ExpectedDurationUntilReady" : "00:00:30",
+        "ExpectedDurationUntilReady" : "00:02:00",
-        "PostRun" : {
+        "Run" : {
-          "Command" : "/bin/sh", 
+          "Command" : "/bin/sh",
-          "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"],
+          "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8082 /swagger-petstore/server.war"]
-          "ExpectedRunDuration" : "00:00:10"
+        }
-        },
-        "Shell" : "/bin/sh",
-        "OutputFolder" : "zap_petstore3"
       }
     ]
   },
@@ -48,7 +52,7 @@
     {
       "toolName": "RESTler",
       "outputFolder": "fuzz",
-      "duration": "00:30:00",
+      "duration": "00:10:00",
       "toolConfiguration": {
         "task": "Fuzz",
         "runConfiguration": {
@@ -56,16 +60,16 @@
             "Port" : 8081 
           },
           "useSsl" : false,
-          "inputFolderPath": "/job-compile/compile"
+          "inputFolderPath": "/ci-run/{compile.jobId}/compile"
         }
       }
-    },
+    },  
     {
-      "toolName" : "ZAP",
+      "toolName": "ZAP",
-      "outputFolder" : "zap",
+      "outputFolder": "zap",
       "swaggerLocation": {
-        "URL" : "http://localhost:8080/api/v3/openapi.json"
+        "URL" : "http://localhost:8082/api/v3/openapi.json"
       }
     }
   ]
-}
+}

cli/samples/restler/self-contained/swagger-petstore3/run.py

@@ -4,47 +4,60 @@
 import pathlib
 import sys
 import os
+import json
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..'))
+sys.path.append(os.path.join(cur_dir, '..'))
-from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError
+from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError, RaftDefinitions
 
-def run(compile, test, fuzz):
-    # instantiate RAFT CLI
-    cli = RaftCLI()
 
+def run(cli, config, subs):
     # Create compilation job configuration
-    compile_job_config = RaftJobConfig(file_path=compile)
+    job_config = RaftJobConfig(file_path=config, substitutions=subs)
-    print('Compile')
+    print(f'Running {config}')
     # submit a new job with the Compile config and get new job ID
-    compile_job = cli.new_job(compile_job_config)
+    job = cli.new_job(job_config)
     # wait for a job with ID from compile_job to finish the run
-    cli.poll(compile_job['jobId'])
+    cli.poll(job['jobId'])
-
+    return job['jobId']
-    # use compile job as input for fuzz job
-    subs = {}
-    subs['{compile.jobId}'] = compile_job['jobId']
-
-    test_job_config = RaftJobConfig(file_path=test, substitutions=subs)
-    print('Test')
-    # create new fuzz job configuration
-    test_job = cli.new_job(test_job_config)
-    # wait for job ID from fuzz_job to finish the run
-    cli.poll(test_job['jobId'])
-
-    # create a new job config with Fuzz configuration JSON
-    fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
-    print('Fuzz')
-    # create new fuzz job configuration
-    fuzz_job = cli.new_job(fuzz_job_config)
-
-    # wait for job ID from fuzz_job to finish the run
-    cli.poll(fuzz_job['jobId'])
 
 if __name__ == "__main__":
     try:
-        run(os.path.join(cur_dir, "restler.compile.json"),
+        defaults = None
-            os.path.join(cur_dir, "restler.test.json"), 
+
-            os.path.join(cur_dir, "restler.zap.fuzz.json"))
+        if sys.argv[1] == '--build':
+            build_id = sys.argv[2].replace(".", "-")
+        print(f"BUILD ID : {build_id}")
+
+        with open(os.path.join(cur_dir, '..', 'defaults.json'), 'r') as defaults_json:
+            defaults = json.load(defaults_json)
+            if sys.argv[3] == '--secret':
+                defaults['secret'] = sys.argv[4]
+
+        # instantiate RAFT CLI
+        cli = RaftCLI(defaults)
+        defs = RaftDefinitions(defaults)
+
+        compile_job_id = None
+        subs = {
+            "{ci-run}" : f"{build_id}",
+            "{build-url}" : os.environ['SYSTEM_COLLECTIONURI'],
+            "{build-id}" : os.environ['BUILD_BUILDID'],
+            "{raft-subscription}": defs.subscription,
+            "{raft-resource-group}" : defs.resource_group,
+            "{raft-storage-account}" : defs.storage_account
+        }
+        for arg in sys.argv[1:]:
+            if arg == 'compile':
+                compile_job_id = run(cli, os.path.join(cur_dir, 'compile.json'), subs)
+                subs['{compile.jobId}'] = compile_job_id
+
+            if arg == 'test':
+                run(cli, os.path.join(cur_dir, "test.json"), subs), 
+
+            if arg == 'test-fuzz-lean':
+                run(cli, os.path.join(cur_dir, "test-fuzz-lean.json"), subs), 
+
     except RaftJobError as ex:
-        print(f'ERROR: {ex.message}')
+        print(f'ERROR: {ex.message}')
+        sys.exit(1)

cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json

@@ -0,0 +1,59 @@
+{
+  "rootFileShare" : "{ci-run}",
+  "namePrefix" : "petstore3-test-fuzz-lean-",
+  "host": "localhost",
+
+  "resources": {
+    "Cores": 4,
+    "MemoryGBs": 8
+  },
+
+  "webhook":{
+    "name" : "petstore3-test-fuzz-lean",
+    "metadata" : {
+      "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results"
+    }
+  },
+
+  "readonlyFileShareMounts": [
+    {
+      "FileShareName": "{ci-run}",
+      "MountPath": "/ci-run"
+    }
+  ],
+
+ "testTargets" : { 
+    "resources" : {
+      "Cores" : 2,
+      "MemoryGBs" : 4
+    },
+    "targets" : [
+      {
+        "Container" : "swaggerapi/petstore3",
+        "Port" : 8081,
+        "ExpectedDurationUntilReady" : "00:02:00",
+        "Run" : {
+          "Command" : "/bin/sh",
+          "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"]
+        }
+      }
+    ]
+  },
+  "tasks": [
+    {
+      "toolName": "RESTler",
+      "outputFolder": "test-fuzz-lean",
+      "duration": "01:00:00",
+      "toolConfiguration": {
+        "task": "TestFuzzLean",
+        "runConfiguration": {
+          "targetEndpointConfiguration" : {
+            "Port" : 8081 
+          },
+          "useSsl" : false,
+          "inputFolderPath": "/ci-run/{compile.jobId}/compile"
+        }
+      }
+    }  
+  ]
+}

cli/samples/restler/self-contained/swagger-petstore3/test.json

@@ -1,4 +1,6 @@
 {
+  "rootFileShare" : "{ci-run}",
+  "namePrefix" : "petstore3-test-",
   "host": "localhost",
 
   "resources": {
@@ -6,10 +8,17 @@
     "MemoryGBs": 8
   },
 
+  "webhook":{
+    "name" : "petstore3-test",
+    "metadata" : {
+      "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results"
+    }
+  },
+
   "readonlyFileShareMounts": [
     {
-      "FileShareName": "{compile.jobId}",
+      "FileShareName": "{ci-run}",
-      "MountPath": "/job-compile"
+      "MountPath": "/ci-run"
     }
   ],
 
@@ -29,7 +38,7 @@
           "ExpectedRunDuration" : "00:00:10"
         },
         "Shell" : "/bin/sh",
-        "OutputFolder" : "zap_petstore3"
+        "OutputFolder" : "petstore3"
       }
     ]
   },
@@ -44,7 +53,7 @@
             "Port" : 8080 
           },
           "useSsl" : false,
-          "inputFolderPath": "/job-compile/compile"
+          "inputFolderPath": "/ci-run/{compile.jobId}/compile"
         }
       }
     }

src/Agent/RESTlerAgent/RESTlerDriver.fs

@@ -151,15 +151,20 @@ module private RESTlerInternal =
             if Seq.isEmpty experiments then
                 None
             else
-                let startedExperiments =
+                try
-                    experiments 
+                    let startedExperiments =
-                    |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime)
+                        experiments 
-                    |> Seq.sortBy ( fun e -> e.CreationTimeUtc )
+                        |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime)
+                        |> Seq.sortBy ( fun e -> e.CreationTimeUtc )
 
-                if (Seq.length startedExperiments > 1) then
+                    if (Seq.length startedExperiments > 1) then
-                    printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." 
+                        printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." 
-                                (Seq.length startedExperiments) startedExperiments runStartTime
+                                    (Seq.length startedExperiments) startedExperiments runStartTime
-                startedExperiments |> Seq.tryHead
+                    startedExperiments |> Seq.tryHead
+                with
+                | :? System.IO.IOException as ioex ->
+                    printfn "Getting experiment folder interrupted due to : %s" ioex.Message
+                    None
         else
             None
  

src/Contracts/Job.fs

@@ -87,7 +87,7 @@ type FileShareMount =
 type Webhook =
     {
         Name : string
-        Metadata : Map<string, string>
+        Metadata : Map<string, string> option
     }
 
 

src/Orchestrator/OrchestratorLogic/Orchestrator.fs

@@ -1316,10 +1316,13 @@ module ContainerInstances =
                         let webhookDefinition = Microsoft.FSharpLu.Json.Compact.deserialize<Raft.Job.Webhook option>(jobEntity.Webhook)
                         match webhookDefinition with
                         | Some webhook ->
-                            if webhook.Metadata.IsEmpty then
+                            match webhook.Metadata with
+                            | None ->
+                                return None
+                            | Some m when m.IsEmpty ->
                                  return None
-                            else
+                            | Some m ->
-                                return Some webhook.Metadata
+                                return Some m
                         | None ->
                             return None
                     else