Fixes to get_body_start (#612)

1) The body could be declared as an object, and contain a constant in a restler_fuzzable_object.

2) The authentication token element should not be deleted, because currently it is used as a body delimiter.

3) Handle the case when the body is not json (e.g. a string).
This case is not currently supported in RESTler, but it should not crash.

restler/engine/core/request_utilities.py

@@ -256,14 +256,19 @@ def resolve_dynamic_primitives(values, candidate_values_pool):
             )
             if not isinstance(token_dict, dict):
                 raise Exception("Refreshable token was not specified as a setting, but a request was expecting it.")
-            token_refresh_interval = token_dict['token_refresh_interval']
-            token_refresh_cmd = token_dict['token_refresh_cmd']
-            if int(time.time()) - last_refresh > token_refresh_interval:
-                execute_token_refresh_cmd(token_refresh_cmd)
-                last_refresh = int(time.time())
-                #print("-{}-\n-{}-".format(repr(latest_token_value),
-                #                          repr(latest_shadow_token_value)))
-            values[i] = latest_token_value
+            if token_dict:
+                token_refresh_interval = token_dict['token_refresh_interval']
+                token_refresh_cmd = token_dict['token_refresh_cmd']
+                if int(time.time()) - last_refresh > token_refresh_interval:
+                    execute_token_refresh_cmd(token_refresh_cmd)
+                    last_refresh = int(time.time())
+                    #print("-{}-\n-{}-".format(repr(latest_token_value),
+                    #                          repr(latest_shadow_token_value)))
+                values[i] = latest_token_value
+            else:
+                # If the dictionary is empty, there is no authentication specified.
+                # Simply return the empty string.
+                values[i] = ""
 
     return values
 

restler/engine/core/requests.py

@@ -1477,13 +1477,18 @@ class Request(object):
                     auth_token_index = auth_tokens[0]
                     for idx in range(auth_token_index + 1, len(request.definition)-1):
                         if request.definition[idx] not in body_delim_patterns and\
-                                request.definition[idx][0] == primitives.STATIC_STRING:
+                                request.definition[idx][0] in [primitives.STATIC_STRING, primitives.FUZZABLE_OBJECT]:
                             if request.definition[idx][1].startswith("{"):
                                 dict_index = idx
                                 break
                             if request.definition[idx][1].startswith("["):
                                 array_index = idx
                                 break
+                    # If the body was not found using the above method, simply assume that the body starts
+                    # after the authentication token delimiter.  This is a best-effort workaround for the currently
+                    # unsupported case of non-json bodies.
+                    if dict_index == -1 and array_index == -1 and len(request.definition) > auth_token_index + 1:
+                        return auth_token_index + 2
             except Exception:
                 pass
 
@@ -1761,19 +1766,6 @@ class RequestCollection(object):
         if value_generators_file_path:
             self.candidate_values_pool.set_value_generators(value_generators_file_path)
 
-    def remove_authentication_tokens(self):
-        """ Removes the authentication token line from each request in the collection
-
-        @return: None
-        @rtype : None
-
-        """
-        for req in self._requests:
-            for line in req.definition:
-                if line[0] == primitives.REFRESHABLE_AUTHENTICATION_TOKEN:
-                    req._definition.remove(line)
-                    break
-
     @property
     def request_id_collection(self):
         """ Returns the request id collection, which is a dictionary of request IDs

restler/restler.py

@@ -430,7 +430,13 @@ if __name__ == '__main__':
             }
         )
     else:
-        req_collection.remove_authentication_tokens()
+        req_collection.candidate_values_pool.set_candidate_values(
+            {
+                'restler_refreshable_authentication_token':
+                    {
+                    }
+            }
+        )
 
     # Initialize the fuzzing monitor
     monitor = fuzzing_monitor.FuzzingMonitor()

restler/unit_tests/test_grammar_schema_parser.py

@@ -31,10 +31,6 @@ def get_python_grammar(grammar_name):
     grammar = importlib.import_module(grammar_name)
     req_collection = getattr(grammar, "req_collection")
 
-    # The line below is required to avoid key errors on the auth token
-    # TODO: remove this constraint from the code, so the token refresh grammar element
-    # can also be tested here.
-    req_collection.remove_authentication_tokens()
     return req_collection
 
 def set_grammar_schema(grammar_file_name, request_collection):