Support testing all examples as part of the main RESTler algorithm. (#274)

To use this in 'Test' mode:

--test_all_combinations must be specified.

in the engine settings, also specify this property:

"test_combinations_settings": {
      "example_payloads":  "all"
    },

docs/user-guide/SettingsFile.md

@@ -103,9 +103,13 @@ The maximum amount of time, in seconds, to wait for a resource to be created bef
 The maximum number of parameter value combinations for parameters within a given request payload.
 
 ### test_combinations_settings: dict (default empty)
-The settings for advanced testing of parameter combinations.  Currently, testing
+The settings for advanced testing of parameter combinations.
+
+__header_param_combinations__
+Currently, testing
 different combinations of headers is supported via the following property:
-```"test_combinations_settings": {
+```json
+"test_combinations_settings": {
       "header_param_combinations" : "optional"
 }
 ```
@@ -115,6 +119,18 @@ The supported values are 'optional', 'required', and 'all'.
 - required: test combinations of required parameters, and omit all optional parameters.
 - all: test all combinations of headers, regardless of whether they are required or optional.
 
+__example_payloads__
+
+For request types where one or more examples are provided, this option enables testing all of
+the examples instead of just the first one.
+
+```json
+"test_combinations_settings": {
+      "example_payloadds" : "all"
+}
+```
+The supported value is 'all'.
+
 
 ### max_request_execution_time: float (default 120, max 600)
 The maximum amount of time, in seconds, to wait for a response after sending a request.

restler/engine/core/requests.py

@@ -605,6 +605,13 @@ class Request(object):
         @rtype : (Request)
 
         """
+        tested_example_payloads = False
+        example_payloads = Settings().example_payloads
+        if example_payloads is not None and self.examples is not None:
+            tested_example_payloads = True
+            for ex in self.get_example_payloads(example_payloads):
+                yield ex
+
         tested_param_combinations = False
         header_param_combinations = Settings().header_param_combinations
         if header_param_combinations is not None:
@@ -612,7 +619,7 @@ class Request(object):
             for hpc in self.get_header_param_combinations(header_param_combinations):
                 yield hpc
 
-        if not tested_param_combinations:
+        if not (tested_param_combinations or tested_example_payloads):
             yield self
 
     def init_fuzzable_values(self, req_definition, candidate_values_pool, preprocessing=False):
@@ -984,6 +991,42 @@ class Request(object):
                 # In such cases, skip the combination.
                 logger.write_to_main(f"Warning: could not substitute header parameters.")
 
+    def get_example_payloads(self, example_payloads_setting):
+        """
+        Replaces the body and query of this request by the available examples.
+        Does not currently modify the headers, because header examples are not yet supported.
+        """
+        # The length of all the example lists is currently expected to be the same.
+        num_payloads = len(self.examples.query_examples)
+        for payload_idx in range(num_payloads):
+            logger.write_to_main(f"Found example {payload_idx}.")
+            body_example = self.examples.body_examples[payload_idx]
+            query_example = self.examples.query_examples[payload_idx]
+            # TODO: header examples are not supported yet.
+            #  header_example = examples.header_examples[payload_idx]
+
+            # Copy the request definition and reset it here.
+            body_blocks = body_example.get_blocks()
+            query_blocks = []
+            for idx, query in enumerate(query_example.param_list):
+                query_blocks += query.get_blocks()
+                if idx < len(query_example.queries) - 1:
+                    # Add the query separator
+                    query_blocks.append(primitives.restler_static_string('&'))
+
+            new_request = self.substitute_query(query_blocks)
+
+            # Only substitute the body if there is a body.
+            if body_blocks:
+                new_request = new_request.substitute_body(body_blocks)
+
+            if new_request:
+                yield new_request
+            else:
+                # For malformed requests, it is possible that the place to insert query parameters is not found,
+                # so the query parameters cannot be inserted. In such cases, skip the example.
+                logger.write_to_main(f"Warning: could not substitute example parameters for example {payload_idx}.")
+
     def get_body_start(self):
         """ Get the starting index of the request body
 
@@ -1105,8 +1148,8 @@ class Request(object):
             if new_query_blocks:
                 new_query_blocks.insert(0, primitives.restler_static_string('?'))
 
-        # If the new query is empty, remove the '?'
+        # If the new query is empty, remove the '?' if it exists
-        if not new_query_blocks:
+        if not new_query_blocks and (start_idx != end_idx):
             start_idx = start_idx - 1
         new_definition = old_request.definition[:start_idx] + new_query_blocks + old_request.definition[end_idx:]
         new_definition += [old_request.metadata.copy()]

restler/engine/fuzzing_parameters/request_examples.py

@@ -25,8 +25,8 @@ class RequestExamples():
 
         """
         # initialization
-        self._query_examples: set = set()   # {QueryList}
+        self._query_examples: list = []   # {QueryList}
-        self._body_examples: set = set()    # {BodySchema}
+        self._body_examples: list = []    # {BodySchema}
 
         # process the request schema
         try:
@@ -83,7 +83,7 @@ class RequestExamples():
                 # Set each query parameter of the query
                 for query in des_query_param(query_parameter[1]):
                     query_list.append(query)
-                self._query_examples.add(query_list)
+                self._query_examples.append(query_list)
 
     def _set_body_params(self, body_parameters):
         """ Deserializes and populates the body parameters
@@ -102,4 +102,4 @@ class RequestExamples():
                 if payload:
                     body_example = des_param_payload(payload)
                     if body_example:
-                        self._body_examples.add(BodySchema(param=body_example))
+                        self._body_examples.append(BodySchema(param=body_example))

restler/restler_settings.py

@@ -538,6 +538,12 @@ class RestlerSettings(object):
             return self._combinations_args.val['header_param_combinations']
         return None
 
+    @property
+    def example_payloads(self):
+        if 'example_payloads' in self._combinations_args.val:
+            return self._combinations_args.val['example_payloads']
+        return None
+
     @property
     def max_request_execution_time(self):
         return self._max_request_execution_time.val

restler/unit_tests/restler_user_settings.json

@@ -3,7 +3,8 @@
     "max_combinations": 20,
     "test_combinations_settings": {
       "header_param_combinations" : "optional",
-        "query_param_combinations": "required"
+      "query_param_combinations": "required",
+      "example_payloads":  "all"
     },
     "max_request_execution_time": 90,
     "save_results_in_fixed_dirname": false,

restler/unit_tests/test_restler_settings.py

@@ -464,6 +464,8 @@ class RestlerSettingsTest(unittest.TestCase):
         self.assertEqual("c:\\restler\\custom_dict2.json", custom_dicts[hex_def(request2)])
 
         self.assertEqual(20, settings.max_combinations)
+        self.assertEqual("optional", settings.header_param_combinations)
+        self.assertEqual("all", settings.example_payloads)
         self.assertEqual(90, settings.max_request_execution_time)
 
         self.assertEqual(False, settings.save_results_in_fixed_dirname)