Fix issue #348 (#364)

Do not add dates by default; add an engine setting that may be specified to keep the current behavior. (Note: a further fix to the current behavior is needed to generate dates in the format corresponding to the ones specified in the spec or dictionary.) - Also fix flaky engine test.
2021-10-06 17:52:26 -07:00 · 2021-10-06 17:52:26 -07:00 · 900cc0f81f
--- a/docs/user-guide/SettingsFile.md
+++ b/docs/user-guide/SettingsFile.md
@ -155,6 +155,13 @@ the examples instead of just the first one.
 The supported ```payload_kind``` value is 'all'.


+### add_fuzzable_dates: bool (default False)
+Set to True to generate additional dates
+near the current date (e.g. one in the future) that will be used for fuzzable date types in addition to
+the values specified in the dictionary.
+Since some API parameters require a current or future date,
+this setting can be used to generate those values, without having to modify the dictionary.
+
 ### max_request_execution_time: float (default 120, max 600)
 The maximum amount of time, in seconds, to wait for a response after sending a request.

@ -162,7 +169,7 @@ The maximum amount of time, in seconds, to wait for a response after sending a r
 Set to True to disable SSL for requests

 ### path_regex: str (default None=No regex filtering)
-Filters the grammar to only use endpoints whose paths contain the given regex string. 
+Filters the grammar to only use endpoints whose paths contain the given regex string.

 Example: `(\w*)/virtualNetworks/(\w*)`

--- a/restler/engine/primitives.py
+++ b/restler/engine/primitives.py
@ -171,7 +171,7 @@ class CandidateValuesPool(object):
        @rtype : None

        """
-        if FUZZABLE_DATETIME in candidate_values:
+        if FUZZABLE_DATETIME in candidate_values and Settings().add_fuzzable_dates:
            candidate_values[FUZZABLE_DATETIME].values.append(self._future_date)
            candidate_values[FUZZABLE_DATETIME].values.append(self._past_date)

--- a/restler/restler_settings.py
+++ b/restler/restler_settings.py
@ -439,6 +439,8 @@ class RestlerSettings(object):
        self._test_server = SettingsArg('test_server', str, DEFAULT_TEST_SERVER_ID, user_args)
        ## Stops fuzzing after given time (hours)
        self._time_budget = SettingsArg('time_budget', (int, float), TIME_BUDGET_DEFAULT, user_args, minval=0)
+        ## Add current dates in addition to the ones specified in the dictionary
+        self._add_fuzzable_dates = SettingsArg('add_fuzzable_dates', bool, False, user_args)
        ## The command to execute in order to refresh the authentication token
        self._token_refresh_cmd = SettingsArg('token_refresh_cmd', str, None, user_args)
        ## Interval to periodically refresh the authentication token (seconds)
@ -594,6 +596,10 @@ class RestlerSettings(object):
    def time_budget(self):
        return self._time_budget.val

+    @property
+    def add_fuzzable_dates(self):
+        return self._add_fuzzable_dates.val
+
    @property
    def token_refresh_cmd(self):
        return self._token_refresh_cmd.val
--- a/restler/test_servers/log_parser.py
+++ b/restler/test_servers/log_parser.py
@ -4,6 +4,8 @@
 """ Module includes classes used to parse logs for testing purposes """
 from test_servers.parsed_requests import *

+import copy
+
 SENDING = ': Sending: '
 GENERATION = 'Generation-'
 RENDERING_SEQUENCE = ': Rendering Sequence-'
@ -116,11 +118,12 @@ class FuzzingLogParser(LogParser):
        """

        def get_diff(left_seq_list, right_seq_list, description_str):
+            right_seq_list_copy = copy.copy(right_seq_list)
            for left in left_seq_list:
                found = False
                found_idx = -1
                found_count = 0
-                for idx, right in enumerate(right_seq_list):
+                for idx, right in enumerate(right_seq_list_copy):
                    if left.requests == right.requests and \
                        left.checker_requests == right.checker_requests:
                        found = True
@ -128,7 +131,7 @@ class FuzzingLogParser(LogParser):
                        found_count += 1
                        break
                if found:
-                    right_seq_list.pop(found_idx)
+                    right_seq_list_copy.pop(found_idx)
                if not found:
                    print(f"+++ Found item not in {description_str} set +++")
                    print("+++ Requests: +++")
@ -140,6 +143,8 @@ class FuzzingLogParser(LogParser):
                            print(f"Checker name: {name}")
                            for req in reqs:
                                print(f"endpoint: {req.endpoint}, method: {req.method}, body: {req.body}")
+                        else:
+                            print(f"Checker {name} is enabled, but did not kick in.")
                    return False
                if found_count > 1:
                    print(f"+++ Found item {found_count} (more than once) times in {description_str} set +++")
@ -194,6 +199,7 @@ class FuzzingLogParser(LogParser):
        with open(self._path, 'r') as file:
            try:
                line = file.readline()
+
                while line:
                    # Find the start of a new sequence
                    if GENERATION in line and RENDERING_SEQUENCE in line:
@ -201,7 +207,8 @@ class FuzzingLogParser(LogParser):
                        num_reqs = int(line.split(GENERATION)[1].split(': ')[0])
                        seq_num = int(line.split(RENDERING_SEQUENCE)[1])

-                        if max_seq > 0 and seq_num > max_seq:
+                        if max_seq > 0 and len(self._seq_list) >= max_seq:
+                            print(f"Testing max={max_seq} sequences.")
                            return

                        seq = ParsedSequence([])
@ -223,7 +230,7 @@ class FuzzingLogParser(LogParser):
                                break

                        # Extend the list of sequences in this log
-                        self._seq_list += [seq]
+                        self._seq_list.append(seq)
                    elif CHECKER_START in line:
                        self._handle_checker(seq, line, file)
                        line = file.readline()
--- a/restler/unit_tests/log_baseline_test_files/test_dict.json
+++ b/restler/unit_tests/log_baseline_test_files/test_dict.json
@ -3,6 +3,7 @@
    "fuzzstring",
    "enable"
  ],
+  "restler_fuzzable_datetime": [ "2021-01-01", "2021-10-01" ],
  "restler_fuzzable_int": [
    "0",
    "1"
--- a/restler/unit_tests/restler_user_settings.json
+++ b/restler/unit_tests/restler_user_settings.json
@ -1,6 +1,6 @@
 {
-    "client_certificate_path": "\\path\\to\\file.pem",
-    "max_combinations": 20,
+  "client_certificate_path": "\\path\\to\\file.pem",
+  "max_combinations": 20,
  "test_combinations_settings": {
    "header_param_combinations": {
      "max_combinations": 10,
@ -14,70 +14,71 @@
      "payload_kind": "all"
    }
  },
-    "max_request_execution_time": 90,
-    "save_results_in_fixed_dirname": false,
-    "global_producer_timing_delay": 2,
-    "dyn_objects_cache_size": 200,
-    "fuzzing_jobs": 2,
-    "fuzzing_mode": "directed-smoke-test",
-    "garbage_collection_interval": 30,
-    "ignore_dependencies": true,
-    "ignore_feedback": true,
-    "include_user_agent": true,
-    "max_async_resource_creation_time": 45,
-    "max_sequence_length": 11,
-    "no_ssl": true,
-    "disable_cert_validation": true,
-    "no_tokens_in_logs": true,
-    "path_regex": "(\\w*)/ddosProtectionPlans(\\w*)",
-    "ignore_decoding_failures": true,
-    "request_throttle_ms": 500,
-    "target_ip": "100.100.100.100",
-    "target_port": 500,
-    "time_budget": 12,
-    "token_refresh_cmd": "some refresh command",
-    "token_refresh_interval": 60,
-    "wait_for_async_resource_creation": false,
-    "per_resource_settings": {
-        "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/dnsZones/{zoneName}/{recordType}/{relativeRecordSetName}": {
-            "producer_timing_delay": 1,
-            "create_once": 1,
-            "custom_dictionary": "c:\\restler\\custom_dict1.json"
-        },
-        "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/dnsZones/{zoneName}": {
-            "producer_timing_delay": 5,
-            "custom_dictionary": "c:\\restler\\custom_dict2.json"
-        }
+  "max_request_execution_time": 90,
+  "save_results_in_fixed_dirname": false,
+  "global_producer_timing_delay": 2,
+  "dyn_objects_cache_size": 200,
+  "fuzzing_jobs": 2,
+  "fuzzing_mode": "directed-smoke-test",
+  "garbage_collection_interval": 30,
+  "ignore_dependencies": true,
+  "ignore_feedback": true,
+  "include_user_agent": true,
+  "max_async_resource_creation_time": 45,
+  "max_sequence_length": 11,
+  "no_ssl": true,
+  "disable_cert_validation": true,
+  "no_tokens_in_logs": true,
+  "path_regex": "(\\w*)/ddosProtectionPlans(\\w*)",
+  "ignore_decoding_failures": true,
+  "request_throttle_ms": 500,
+  "target_ip": "100.100.100.100",
+  "target_port": 500,
+  "time_budget": 12,
+  "token_refresh_cmd": "some refresh command",
+  "add_fuzzable_dates": true,
+  "token_refresh_interval": 60,
+  "wait_for_async_resource_creation": false,
+  "per_resource_settings": {
+    "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/dnsZones/{zoneName}/{recordType}/{relativeRecordSetName}": {
+      "producer_timing_delay": 1,
+      "create_once": 1,
+      "custom_dictionary": "c:\\restler\\custom_dict1.json"
    },
-    "checkers": {
-        "namespacerule" : {
-            "mode" : "exhaustive"
-        },
-        "useafterfree" : {
-            "mode" : "exhaustive"
-        },
-        "leakagerule" : {
-            "mode" : "exhaustive"
-        },
-        "resourcehierarchy" : {
-            "mode" : "exhaustive"
-        },
-        "payloadbody" : {
-            "mode" : "normal",
-            "start_with_valid" : true,
-            "start_with_examples" : true,
-            "size_dep_budget" : false,
-            "use_feedback" : true,
-            "recipe_file" : "C:\\restler\\restlerpayloadbody\\recipe_custom.json"
-        }
+    "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/dnsZones/{zoneName}": {
+      "producer_timing_delay": 5,
+      "custom_dictionary": "c:\\restler\\custom_dict2.json"
+    }
+  },
+  "checkers": {
+    "namespacerule": {
+      "mode": "exhaustive"
    },
-    "custom_bug_codes": [
-        "400",
-        "2?4",
-        "3*"
-    ],
-    "custom_non_bug_codes": [
-        "404",
-        "500"
-    ]
+    "useafterfree": {
+      "mode": "exhaustive"
+    },
+    "leakagerule": {
+      "mode": "exhaustive"
+    },
+    "resourcehierarchy": {
+      "mode": "exhaustive"
+    },
+    "payloadbody": {
+      "mode": "normal",
+      "start_with_valid": true,
+      "start_with_examples": true,
+      "size_dep_budget": false,
+      "use_feedback": true,
+      "recipe_file": "C:\\restler\\restlerpayloadbody\\recipe_custom.json"
+    }
+  },
+  "custom_bug_codes": [
+    "400",
+    "2?4",
+    "3*"
+  ],
+  "custom_non_bug_codes": [
+    "404",
+    "500"
+  ]
 }
--- a/restler/unit_tests/test_basic_functionality_end_to_end.py
+++ b/restler/unit_tests/test_basic_functionality_end_to_end.py
@ -393,23 +393,25 @@ class FunctionalityTests(unittest.TestCase):
        args = Common_Settings + [
            '--fuzzing_mode', 'bfs-cheap',
            '--restler_grammar',f'{os.path.join(Test_File_Directory, "test_grammar.py")}',
-            '--time_budget', f'{Fuzz_Time}', '--enable_checkers', '*',
+            '--time_budget', f'{Fuzz_Time}',
+            '--enable_checkers', '*',
            '--disable_checkers', 'namespacerule'
        ]

        result = subprocess.run(args, capture_output=True)
        if result.stderr:
-            self.fail(result.stderr)
+            self.fail(f"{result.stderr}")
        try:
            result.check_returncode()
        except subprocess.CalledProcessError:
-            self.fail(f"Restler returned non-zero exit code: {result.returncode}")
+            self.fail(f"Restler returned non-zero exit code: {result.returncode}, Stdout: {result.stdout}")

        experiments_dir = self.get_experiments_dir()
-
+        #experiments_dir = "D:\git\restler-fuzzer\restler\RestlerResults\experiment31916"
        try:
            default_parser = FuzzingLogParser(os.path.join(Test_File_Directory, "fuzz_testing_log.txt"), max_seq=Num_Sequences)
            test_parser = FuzzingLogParser(self.get_network_log_path(experiments_dir, logger.LOG_TYPE_TESTING), max_seq=Num_Sequences)
+            #test_parser = FuzzingLogParser("D:\\git\\restler-fuzzer\\restler\\RestlerResults\\experiment31916\\logs\\network.testing.23496.1.txt", max_seq=Num_Sequences)
            self.assertTrue(default_parser.diff_log(test_parser))
        except TestFailedException:
            self.fail("Fuzz failed: Fuzzing")
--- a/restler/unit_tests/test_restler_settings.py
+++ b/restler/unit_tests/test_restler_settings.py
@ -494,6 +494,7 @@ class RestlerSettingsTest(unittest.TestCase):
        self.assertEqual('100.100.100.100', settings.connection_settings.target_ip)
        self.assertEqual(500, settings.connection_settings.target_port)
        self.assertEqual(12, settings.time_budget)
+        self.assertEqual(True, settings.add_fuzzable_dates)
        self.assertEqual('some refresh command', settings.token_refresh_cmd)
        self.assertEqual(60, settings.token_refresh_interval)
        self.assertEqual(False, settings.wait_for_async_resource_creation)