Add settings to limit the number of schema combinations and examples. (#544)

These settings can be used to more conveniently limit the total number of different schemas tested
per request.

Testing:
- manual testing.
- added settings unit test.

docs/user-guide/SettingsFile.md

@@ -157,8 +157,8 @@ header parameter combinations (see above).
 
 __example_payloads__
 
-For request types where one or more examples are provided, this option enables testing all of
-the examples instead of just the first one.
+For request types where one or more examples are provided, this option enables
+testing all of the examples instead of just the first one.
 
 ```json
 "test_combinations_settings": {
@@ -169,6 +169,15 @@ the examples instead of just the first one.
 ```
 The supported ```payload_kind``` value is 'all'.
 
+__max_schema_combinations__
+When RESTler explores more than one schema (for example, because parameter
+combinations are being tested, as specified in ```test_combinations_settings```),
+this option limits the number of schemas that will be tested.
+
+__max_examples__
+For request types where one or more examples are provided, this option limits
+the number of examples that will be tested.
+
 
 ### add_fuzzable_dates: bool (default False)
 Set to True to generate additional dates

restler/engine/core/requests.py

@@ -702,7 +702,8 @@ class Request(object):
         example_payloads = Settings().example_payloads
         if example_payloads is not None and self.examples is not None:
             tested_example_payloads = True
-            for ex in self.get_example_payloads(example_payloads):
+            example_schemas = self.get_example_payloads()
+            for ex in itertools.islice(example_schemas, Settings().max_examples):
                 yield ex
 
         tested_param_combinations = False
@@ -931,7 +932,8 @@ class Request(object):
         # constructed, since not all of them may be needed, e.g. during smoke test mode.
         next_combination = 0
         schema_idx = -1
-        for req in self.get_schema_combinations():
+        schema_combinations = itertools.islice(self.get_schema_combinations(), Settings().max_schema_combinations)
+        for req in schema_combinations:
             schema_idx += 1
             parser = None
             fuzzable_request_blocks = []

restler/restler_settings.py

@@ -301,6 +301,9 @@ MAX_ASYNC_RESOURCE_CREATION_TIME_DEFAULT = 20
 # first-time use, such as in Test mode.  Users are expected to increase this value
 # as needed for more extensive fuzzing.
 MAX_COMBINATIONS_DEFAULT = 20
+MAX_SCHEMA_COMBINATIONS_DEFAULT = 20
+MAX_EXAMPLES_DEFAULT = 20
+
 MAX_SEQUENCE_LENGTH_DEFAULT = 100
 TARGET_PORT_MAX = (1<<16)-1
 TIME_BUDGET_DEFAULT = 24.0*30 # ~1 month
@@ -572,6 +575,18 @@ class RestlerSettings(object):
     def max_combinations(self):
         return self._max_combinations.val
 
+    @property
+    def max_schema_combinations(self):
+        if 'max_schema_combinations' in self._combinations_args.val:
+            return self._combinations_args.val['max_schema_combinations']
+        return MAX_SCHEMA_COMBINATIONS_DEFAULT
+
+    @property
+    def max_examples(self):
+        if 'max_examples' in self._combinations_args.val:
+            return self._combinations_args.val['max_examples']
+        return MAX_EXAMPLES_DEFAULT
+
     @property
     def header_param_combinations(self):
         if 'header_param_combinations' in self._combinations_args.val:

restler/unit_tests/restler_user_settings.json

@@ -12,7 +12,9 @@
     },
     "example_payloads": {
       "payload_kind": "all"
-    }
+    },
+    "max_examples": 100,
+    "max_schema_combinations": 30
   },
   "max_request_execution_time": 90,
   "save_results_in_fixed_dirname": false,

restler/unit_tests/test_restler_settings.py

@@ -464,6 +464,8 @@ class RestlerSettingsTest(unittest.TestCase):
         self.assertEqual("c:\\restler\\custom_dict2.json", custom_dicts[hex_def(request2)])
 
         self.assertEqual(20, settings.max_combinations)
+        self.assertEqual(100, settings.max_examples)
+        self.assertEqual(30, settings.max_schema_combinations)
         self.assertEqual("optional", settings.header_param_combinations['param_kind'])
         self.assertEqual(10, settings.header_param_combinations['max_combinations'])
         self.assertEqual("required", settings.query_param_combinations['param_kind'])