From e6b5e4ee48c457512748958569c858c1bee27559 Mon Sep 17 00:00:00 2001 From: marina-p Date: Wed, 28 Jul 2021 14:07:55 -0700 Subject: [PATCH] Support testing all examples as part of the main RESTler algorithm. (#274) To use this in 'Test' mode: --test_all_combinations must be specified. in the engine settings, also specify this property: "test_combinations_settings": { "example_payloads": "all" }, --- docs/user-guide/SettingsFile.md | 20 +++++++- restler/engine/core/requests.py | 49 +++++++++++++++++-- .../fuzzing_parameters/request_examples.py | 8 +-- restler/restler_settings.py | 6 +++ restler/unit_tests/restler_user_settings.json | 5 +- restler/unit_tests/test_restler_settings.py | 2 + 6 files changed, 79 insertions(+), 11 deletions(-) diff --git a/docs/user-guide/SettingsFile.md b/docs/user-guide/SettingsFile.md index 0e651ca..a7fd55d 100644 --- a/docs/user-guide/SettingsFile.md +++ b/docs/user-guide/SettingsFile.md @@ -103,9 +103,13 @@ The maximum amount of time, in seconds, to wait for a resource to be created bef The maximum number of parameter value combinations for parameters within a given request payload. ### test_combinations_settings: dict (default empty) -The settings for advanced testing of parameter combinations. Currently, testing +The settings for advanced testing of parameter combinations. + +__header_param_combinations__ +Currently, testing different combinations of headers is supported via the following property: -```"test_combinations_settings": { +```json +"test_combinations_settings": { "header_param_combinations" : "optional" } ``` @@ -115,6 +119,18 @@ The supported values are 'optional', 'required', and 'all'. - required: test combinations of required parameters, and omit all optional parameters. - all: test all combinations of headers, regardless of whether they are required or optional. +__example_payloads__ + +For request types where one or more examples are provided, this option enables testing all of +the examples instead of just the first one. + +```json +"test_combinations_settings": { + "example_payloadds" : "all" +} +``` +The supported value is 'all'. + ### max_request_execution_time: float (default 120, max 600) The maximum amount of time, in seconds, to wait for a response after sending a request. diff --git a/restler/engine/core/requests.py b/restler/engine/core/requests.py index b10fe72..bfeefd5 100644 --- a/restler/engine/core/requests.py +++ b/restler/engine/core/requests.py @@ -605,6 +605,13 @@ class Request(object): @rtype : (Request) """ + tested_example_payloads = False + example_payloads = Settings().example_payloads + if example_payloads is not None and self.examples is not None: + tested_example_payloads = True + for ex in self.get_example_payloads(example_payloads): + yield ex + tested_param_combinations = False header_param_combinations = Settings().header_param_combinations if header_param_combinations is not None: @@ -612,7 +619,7 @@ class Request(object): for hpc in self.get_header_param_combinations(header_param_combinations): yield hpc - if not tested_param_combinations: + if not (tested_param_combinations or tested_example_payloads): yield self def init_fuzzable_values(self, req_definition, candidate_values_pool, preprocessing=False): @@ -984,6 +991,42 @@ class Request(object): # In such cases, skip the combination. logger.write_to_main(f"Warning: could not substitute header parameters.") + def get_example_payloads(self, example_payloads_setting): + """ + Replaces the body and query of this request by the available examples. + Does not currently modify the headers, because header examples are not yet supported. + """ + # The length of all the example lists is currently expected to be the same. + num_payloads = len(self.examples.query_examples) + for payload_idx in range(num_payloads): + logger.write_to_main(f"Found example {payload_idx}.") + body_example = self.examples.body_examples[payload_idx] + query_example = self.examples.query_examples[payload_idx] + # TODO: header examples are not supported yet. + # header_example = examples.header_examples[payload_idx] + + # Copy the request definition and reset it here. + body_blocks = body_example.get_blocks() + query_blocks = [] + for idx, query in enumerate(query_example.param_list): + query_blocks += query.get_blocks() + if idx < len(query_example.queries) - 1: + # Add the query separator + query_blocks.append(primitives.restler_static_string('&')) + + new_request = self.substitute_query(query_blocks) + + # Only substitute the body if there is a body. + if body_blocks: + new_request = new_request.substitute_body(body_blocks) + + if new_request: + yield new_request + else: + # For malformed requests, it is possible that the place to insert query parameters is not found, + # so the query parameters cannot be inserted. In such cases, skip the example. + logger.write_to_main(f"Warning: could not substitute example parameters for example {payload_idx}.") + def get_body_start(self): """ Get the starting index of the request body @@ -1105,8 +1148,8 @@ class Request(object): if new_query_blocks: new_query_blocks.insert(0, primitives.restler_static_string('?')) - # If the new query is empty, remove the '?' - if not new_query_blocks: + # If the new query is empty, remove the '?' if it exists + if not new_query_blocks and (start_idx != end_idx): start_idx = start_idx - 1 new_definition = old_request.definition[:start_idx] + new_query_blocks + old_request.definition[end_idx:] new_definition += [old_request.metadata.copy()] diff --git a/restler/engine/fuzzing_parameters/request_examples.py b/restler/engine/fuzzing_parameters/request_examples.py index e864b17..3021bf9 100644 --- a/restler/engine/fuzzing_parameters/request_examples.py +++ b/restler/engine/fuzzing_parameters/request_examples.py @@ -25,8 +25,8 @@ class RequestExamples(): """ # initialization - self._query_examples: set = set() # {QueryList} - self._body_examples: set = set() # {BodySchema} + self._query_examples: list = [] # {QueryList} + self._body_examples: list = [] # {BodySchema} # process the request schema try: @@ -83,7 +83,7 @@ class RequestExamples(): # Set each query parameter of the query for query in des_query_param(query_parameter[1]): query_list.append(query) - self._query_examples.add(query_list) + self._query_examples.append(query_list) def _set_body_params(self, body_parameters): """ Deserializes and populates the body parameters @@ -102,4 +102,4 @@ class RequestExamples(): if payload: body_example = des_param_payload(payload) if body_example: - self._body_examples.add(BodySchema(param=body_example)) + self._body_examples.append(BodySchema(param=body_example)) diff --git a/restler/restler_settings.py b/restler/restler_settings.py index fded4b5..92017a1 100644 --- a/restler/restler_settings.py +++ b/restler/restler_settings.py @@ -538,6 +538,12 @@ class RestlerSettings(object): return self._combinations_args.val['header_param_combinations'] return None + @property + def example_payloads(self): + if 'example_payloads' in self._combinations_args.val: + return self._combinations_args.val['example_payloads'] + return None + @property def max_request_execution_time(self): return self._max_request_execution_time.val diff --git a/restler/unit_tests/restler_user_settings.json b/restler/unit_tests/restler_user_settings.json index b1ebbb5..abf5892 100644 --- a/restler/unit_tests/restler_user_settings.json +++ b/restler/unit_tests/restler_user_settings.json @@ -2,8 +2,9 @@ "client_certificate_path": "\\path\\to\\file.pem", "max_combinations": 20, "test_combinations_settings": { - "header_param_combinations" : "optional", - "query_param_combinations": "required" + "header_param_combinations" : "optional", + "query_param_combinations": "required", + "example_payloads": "all" }, "max_request_execution_time": 90, "save_results_in_fixed_dirname": false, diff --git a/restler/unit_tests/test_restler_settings.py b/restler/unit_tests/test_restler_settings.py index bd8e637..5c3c195 100644 --- a/restler/unit_tests/test_restler_settings.py +++ b/restler/unit_tests/test_restler_settings.py @@ -464,6 +464,8 @@ class RestlerSettingsTest(unittest.TestCase): self.assertEqual("c:\\restler\\custom_dict2.json", custom_dicts[hex_def(request2)]) self.assertEqual(20, settings.max_combinations) + self.assertEqual("optional", settings.header_param_combinations) + self.assertEqual("all", settings.example_payloads) self.assertEqual(90, settings.max_request_execution_time) self.assertEqual(False, settings.save_results_in_fixed_dirname)