Add new settings to include and exclude specific requests. (#614)

This is required for maintaining specific endpoints that should not be exercised, and is a cleaner way
to do so than the existing path_regex switch.

Also:
- fixed a bug in path_regex.

Testing:
- manual testing on demo_server.
- updated unit tests

Partially implements #282.  A new property will be added separately to completely exclude DELETEs from running.

docs/user-guide/SettingsFile.md

@@ -178,6 +178,8 @@ __max_examples__
 For request types where one or more examples are provided, this option limits
 the number of examples that will be tested.
 
+### max_sequence_length: int (default 100)
+The maximum length of fuzzed request sequences.
 
 ### add_fuzzable_dates: bool (default False)
 Set to True to generate additional dates
@@ -199,6 +201,39 @@ Example: `(\w*)/virtualNetworks/(\w*)`
 
 Example: `disk|virtualNetwork`
 
+### include_requests: list (default empty list=No filtering)
+Filters the grammar to include only the specified endpoints and methods.  If no ```methods``` key is specified, all methods are included.
+Note: if the included request depends on pre-requisite resources that are created by other
+requests, all requests required to create the dependency will be exercised. For example, the endpoint
+below requires a ```postId``` that is obtained by executing ```POST /api/blog/posts``` - this request will
+also be executed, even though it is not included in the list below.  A future improvement will filter out
+such requests from fuzzing, but currently they will be fuzzed as well.
+
+```json
+  "include_requests": [
+    {
+      "endpoint": "/api/blog/posts/{postId}",
+    }
+  ]
+```
+
+### exclude_requests: list (default empty list=No filtering)
+Filters the grammar to exclude the specified endpoints and methods.
+
+Note: although the ```DELETE``` is excluded from fuzzing below, it will still be executed by the RESTler
+garbage collector to clean up the blog posts that were created in order to test the other requests with
+endpoint ```/api/blog/posts/{postId}```.  To completely exclude ```DELETE```s from running, you must filter them
+manually from grammar.py.
+
+```json
+  "exclude_requests": [
+    {
+      "endpoint": "/api/blog/posts/{postId}",
+      "methods": ["GET", "DELETE"]
+    }
+  ]
+```
+
 ### save_results_in_fixed_dirname: bool (default False)
 Save the results in a directory with a fixed name (skip the 'experiment\<pid\>' subdir).
 
@@ -425,7 +460,7 @@ value_generators = {
     "global_producer_timing_delay": 2,
     "dyn_objects_cache_size":20,
     "fuzzing_mode": "directed-smoke-test",
-    "path_regex": "(\\w*)/ddosProtectionPlans(\\w*)",
+    "path_regex": "(\\w*)/blog/posts(\\w*)",
     "per_resource_settings": {
         "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/dnsZones/{zoneName}/{recordType}/{relativeRecordSetName}": {
             "producer_timing_delay": 1,

restler/engine/core/preprocessing.py

@@ -51,16 +51,26 @@ def create_fuzzing_req_collection(path_regex):
 
     """
     fuzz_reqs = fuzzing_requests.FuzzingRequestCollection()
-    if path_regex:
-        included_requests = []
-        for request in GrammarRequestCollection():
-            if re.findall(path_regex, request.endpoint):
-                reqs = driver.compute_request_goal_seq(
-                    request, GrammarRequestCollection())
-                for req in reqs:
-                    included_requests.append(req)
-    else:
+    included_requests= set()
+    included_all_reqs = True
+    for request in GrammarRequestCollection():
+        include_req = Settings().include_request(request.endpoint_no_dynamic_objects, request.method)
+        if include_req and path_regex:
+            include_req = re.findall(path_regex, request.endpoint_no_dynamic_objects)
+        if include_req:
+            reqs = driver.compute_request_goal_seq(
+                request, GrammarRequestCollection())
+            for req in reqs:
+                if req not in included_requests:
+                    included_requests.add(req)
+        else:
+            included_all_reqs = False
+    if included_all_reqs:
+        # TODO: For ordering backwards compatibility - once test baselines are updated,
+        # this should be removed.
         included_requests = list (GrammarRequestCollection()._requests)
+    else:
+        included_requests = list (included_requests)
 
     # Sort the request list by hex definition so the requests are
     # always traversed in the same order.

restler/restler.py

@@ -59,7 +59,7 @@ def import_grammar(path):
 
     return req_collection
 
-def get_checker_list(req_collection, fuzzing_requests, enable_list, disable_list, set_enable_first, custom_checkers, enable_default_checkers=True):
+def get_checker_list(req_collection, fuzzing_requests, enable_list, disable_list, set_enable_first, custom_checkers):
     """ Initializes all of the checkers, sets the appropriate checkers
     as enabled/disabled, and returns a list of checker objects
 
@@ -98,10 +98,6 @@ def get_checker_list(req_collection, fuzzing_requests, enable_list, disable_list
     @type  set_enable_first: Bool
     @param custom_checkers: List of paths to custom checker python files
     @type  custom_checkers: List[str]
-    @param enable_default_checkers: If set to False, each checker will be disabled by default,
-                                    otherwise, checkers will be enabled/disabled based on their
-                                    default settings.
-    @type  enable_default_checkers: Bool
 
     @return: List of Checker objects to apply
     @rtype : List[Checker]
@@ -151,8 +147,6 @@ def get_checker_list(req_collection, fuzzing_requests, enable_list, disable_list
     # Iterate through each checker and search for its friendly name
     # in each list of enabled/disabled
     for checker in available_checkers:
-        if not enable_default_checkers:
-            checker.enabled = False
         if checker.friendly_name in first_list:
             checker.enabled = first_enable
         if checker.friendly_name in second_list:
@@ -487,7 +481,7 @@ if __name__ == '__main__':
         set_enable_first = args.enable_checkers is not None
 
     checkers = get_checker_list(req_collection, fuzzing_requests, args.enable_checkers or [], args.disable_checkers or [],\
-        set_enable_first, settings.custom_checkers, enable_default_checkers=args.fuzzing_mode != 'directed-smoke-test')
+        set_enable_first, settings.custom_checkers)
 
     # Initialize request count for each checker
     for checker in checkers:

restler/restler_settings.py

@@ -442,6 +442,11 @@ class RestlerSettings(object):
         self._no_tokens_in_logs = SettingsArg('no_tokens_in_logs', bool, True, user_args)
         ## Save the results in a dir with a fixed name (skip 'experiment<pid>' subdir)
         self._save_results_in_fixed_dirname = SettingsArg('save_results_in_fixed_dirname', bool, False, user_args)
+        ## Include only the specified requests
+        self._include_requests = SettingsArg('include_requests', list, [], user_args)
+        ## Exclude the specified requests
+        self._exclude_requests = SettingsArg('exclude_requests', list, [], user_args)
+
         ## Limit restler grammars only to endpoints whose paths contain a given substring
         self._path_regex = SettingsArg('path_regex', str, None, user_args)
         ## Custom value generator module file path
@@ -701,6 +706,35 @@ class RestlerSettings(object):
     def wait_for_async_resource_creation(self):
         return self._wait_for_async_resource_creation.val
 
+    def include_request(self, endpoint, method):
+        """"Returns whether the specified endpoint and method should be tested according to
+        the include/exclude settings
+        """
+        def contains_request(req_list):
+            for req in req_list:
+                if endpoint == req["endpoint"]:
+                    return "methods" not in req or method in req["methods"]
+            return False
+
+        def exclude_req():
+            return contains_request(self._exclude_requests.val)
+
+        def include_req():
+            return contains_request(self._include_requests.val)
+        # A request is included if
+        # - the include/exclude lists are not specified
+        # - only the include list is specified, and includes this request
+        # - only the exclude list is specified, and does not include this request
+        # - both lists are specified, the requst is in the include list and not in the exclude list
+        if not (self._include_requests.val or self._exclude_requests.val):
+            return True
+        elif not self._exclude_requests.val:
+            return include_req()
+        elif not self._include_requests.val:
+            return not exclude_req()
+        else:
+            return include_req() and not exclude_req()
+
     def get_cached_prefix_request_settings(self, endpoint, method):
         def get_settings():
             if 'create_prefix_once' in self._seq_rendering_settings.val:

restler/unit_tests/log_baseline_test_files/test_fuzz_settings.json

@@ -0,0 +1,6 @@
+{
+  "test_combinations_settings": {
+    "max_schema_combinations":  1
+  },
+  "max_sequence_length": 3
+}

restler/unit_tests/restler_user_settings.json

@@ -33,7 +33,7 @@
   "disable_cert_validation": true,
   "disable_logging": true,
   "no_tokens_in_logs": true,
-  "path_regex": "(\\w*)/ddosProtectionPlans(\\w*)",
+  "path_regex": "(\\w*)/blog/posts(\\w*)",
   "ignore_decoding_failures": true,
   "request_throttle_ms": 500,
   "custom_retry_settings": {
@@ -73,6 +73,24 @@
       }
     ]
   },
+  "include_requests": [
+    {
+      "endpoint": "/blog/included/1"
+    },
+    {
+      "endpoint": "/blog/included/2",
+      "methods": ["PUT"]
+    }
+  ],
+  "exclude_requests": [
+    {
+      "endpoint": "/blog/excluded/1"
+    },
+    {
+      "endpoint": "/blog/included/1",
+      "methods": ["GET"]
+    }
+  ],
   "checkers": {
     "namespacerule": {
       "mode": "exhaustive"

restler/unit_tests/test_basic_functionality_end_to_end.py

@@ -545,7 +545,7 @@ class FunctionalityTests(unittest.TestCase):
         """
         Fuzz_Time = 0.1 # 6 minutes
         Num_Sequences = 300
-        settings_file_path = os.path.join(Test_File_Directory, "test_one_schema_settings.json")
+        settings_file_path = os.path.join(Test_File_Directory, "test_fuzz_settings.json")
 
         args = Common_Settings + [
             '--fuzzing_mode', 'bfs-cheap',

restler/unit_tests/test_restler_settings.py

@@ -459,6 +459,16 @@ class RestlerSettingsTest(unittest.TestCase):
         self.assertEqual(5, settings.get_producer_timing_delay(hex_def(request2)))
         self.assertEqual(2, settings.get_producer_timing_delay(hex_def("test_unknown_request_id")))
 
+        # Test that inclusion/exclusion is correctly determined based on a request endpoint and method
+        self.assertTrue(settings.include_request("/blog/included/1", "POST"))
+        self.assertTrue(settings.include_request("/blog/included/1", "PUT"))
+        self.assertTrue(settings.include_request("/blog/included/2", "PUT"))
+        self.assertFalse(settings.include_request("/blog/included/2", "POST"))
+
+        self.assertFalse(settings.include_request("/blog/excluded/1", "GET"))
+        self.assertFalse(settings.include_request("/blog/excluded/2", "GET"))
+        self.assertFalse(settings.include_request("/blog/included/1", "GET"))
+
         custom_dicts = settings.get_endpoint_custom_mutations_paths()
         self.assertEqual("c:\\restler\\custom_dict1.json", custom_dicts[hex_def(request1)])
         self.assertEqual("c:\\restler\\custom_dict2.json", custom_dicts[hex_def(request2)])
@@ -491,7 +501,7 @@ class RestlerSettingsTest(unittest.TestCase):
         self.assertFalse(settings.connection_settings.use_ssl)
         self.assertTrue(settings.connection_settings.disable_cert_validation)
         self.assertTrue(settings.no_tokens_in_logs)
-        self.assertEqual('(\w*)/ddosProtectionPlans(\w*)', settings.path_regex)
+        self.assertEqual('(\w*)/blog/posts(\w*)', settings.path_regex)
         self.assertEqual(500, settings.request_throttle_ms)
         self.assertEqual('100.100.100.100', settings.connection_settings.target_ip)
         self.assertEqual(500, settings.connection_settings.target_port)