Allow specifying max_combinations settings for specific test combinations settings. (#320)

**BREAKING CHANGE** the settings for test combinations now have an object instead of a single value.

This additional setting allows configuring different RESTler runs with different maximum values for
combinations.  For example, shorter CI/CD invocations may be configured with fewer combinations than
longer or one-off regression tests.

docs/user-guide/SettingsFile.md

@@ -110,10 +110,13 @@ Testing
 different combinations of headers is supported via the following property:
 ```json
 "test_combinations_settings": {
-      "header_param_combinations" : "optional"
+     "header_param_combinations": {
+      "max_combinations": 50,
+      "param_kind": "optional"
+    }
 }
 ```
-The supported values are 'optional', 'required', and 'all'.
+The supported ```param_kind``` values are 'optional', 'required', and 'all'.
 
 - optional: test all combinations of optional parameters, always sending the required parameters.
 - required: test combinations of required parameters, and omit all optional parameters.
@@ -124,10 +127,13 @@ Testing
 different combinations of queries is supported via the following property:
 ```json
 "test_combinations_settings": {
-      "query_param_combinations" : "required"
+     "query_param_combinations": {
+      "max_combinations": 50,
+      "param_kind": "required"
+    }
 }
 ```
-The supported values are 'optional', 'required', and 'all'.  These have the same meaning as for
+The supported ```param_kind``` values are 'optional', 'required', and 'all'.  These have the same meaning as for
 header parameter combinations (see above).
 
 __example_payloads__
@@ -137,10 +143,12 @@ the examples instead of just the first one.
 
 ```json
 "test_combinations_settings": {
-      "example_payloadds" : "all"
+      "example_payloadds" : {
+          "payload_kind": "all"
+      }
 }
 ```
-The supported value is 'all'.
+The supported ```payload_kind``` value is 'all'.
 
 
 ### max_request_execution_time: float (default 120, max 600)

restler/engine/fuzzing_parameters/param_combinations.py

@@ -9,16 +9,19 @@ import json
 import engine.primitives as primitives
 import utils.logger as logger
 
-def get_param_list_combinations(param_list):
+def get_param_list_combinations(param_list, max_combinations):
     """
     Generator that takes the specified list and returns all combinations of the elements.
     """
-    for i in range(1, len(param_list) + 2):
-        num_items = i - 1
-        combinations_num_i = itertools.combinations(param_list, num_items)
-        for new_param_list in combinations_num_i:
-            new_param_list = list(new_param_list)
-            yield new_param_list
+    def generate_combinations():
+        for i in range(1, len(param_list) + 2):
+            num_items = i - 1
+            combinations_num_i = itertools.combinations(param_list, num_items)
+            for new_param_list in combinations_num_i:
+                new_param_list = list(new_param_list)
+                yield new_param_list
+
+    return itertools.islice(generate_combinations(), 0, max_combinations)
 
 def get_param_combinations(req, param_combinations_setting, param_list, param_type):
     """
@@ -30,22 +33,33 @@ def get_param_combinations(req, param_combinations_setting, param_list, param_ty
             if p.is_required == required_val:
                 rp.append(p)
         return rp
-    if param_combinations_setting == "all":
+
+
+    if 'param_kind' in param_combinations_setting:
+        param_kind = param_combinations_setting['param_kind']
+    else:
+        param_kind = "all"
+    if 'max_combinations' in param_combinations_setting:
+        max_combinations = param_combinations_setting['max_combinations']
+    else:
+        max_combinations = Settings().max_combinations
+
+    if param_kind == "all":
         # Send combinations of all available parameters.
-        for x in get_param_list_combinations(param_list):
+        for x in get_param_list_combinations(param_list, max_combinations):
             yield x
-    elif param_combinations_setting == "required":
+    elif param_kind == "required":
         # Only send required parameter combinations, and omit optional parameters.
         required_params_list = filter_required(param_list)
-        for x in get_param_list_combinations(required_params_list):
+        for x in get_param_list_combinations(required_params_list, max_combinations):
             yield x
-    elif param_combinations_setting == "optional":
+    elif param_kind == "optional":
         # Send required parameters, and additionally send combinations
         # of optional parameters.
         required_params_list = filter_required(param_list)
         optional_params_list = filter_required(param_list, required_val=False)
 
-        optional_param_combinations = get_param_list_combinations(optional_params_list)
+        optional_param_combinations = get_param_list_combinations(optional_params_list, max_combinations)
 
         for opc in optional_param_combinations:
             yield required_params_list + opc

restler/unit_tests/restler_user_settings.json

@@ -1,11 +1,19 @@
 {
     "client_certificate_path": "\\path\\to\\file.pem",
     "max_combinations": 20,
-    "test_combinations_settings": {
-      "header_param_combinations" : "optional",
-      "query_param_combinations": "required",
-      "example_payloads":  "all"
+  "test_combinations_settings": {
+    "header_param_combinations": {
+      "max_combinations": 10,
+      "param_kind": "optional"
     },
+    "query_param_combinations": {
+      "max_combinations": 15,
+      "param_kind": "required"
+    },
+    "example_payloads": {
+      "payload_kind": "all"
+    }
+  },
     "max_request_execution_time": 90,
     "save_results_in_fixed_dirname": false,
     "global_producer_timing_delay": 2,

restler/unit_tests/test_restler_settings.py

@@ -464,9 +464,11 @@ class RestlerSettingsTest(unittest.TestCase):
         self.assertEqual("c:\\restler\\custom_dict2.json", custom_dicts[hex_def(request2)])
 
         self.assertEqual(20, settings.max_combinations)
-        self.assertEqual("optional", settings.header_param_combinations)
-        self.assertEqual("required", settings.query_param_combinations)
-        self.assertEqual("all", settings.example_payloads)
+        self.assertEqual("optional", settings.header_param_combinations['param_kind'])
+        self.assertEqual(10, settings.header_param_combinations['max_combinations'])
+        self.assertEqual("required", settings.query_param_combinations['param_kind'])
+        self.assertEqual(15, settings.query_param_combinations['max_combinations'])
+        self.assertEqual("all", settings.example_payloads['payload_kind'])
         self.assertEqual(90, settings.max_request_execution_time)
 
         self.assertEqual(False, settings.save_results_in_fixed_dirname)