Fix crash due to incorrect body substitution. (#652)

The functions that generate the python grammar for different schema combinations use
ad-hoc parsing to determine the boundaries of body, header, and query parameters.
This is error prone and will be refactored to use grammar delimiters in the future.

As a quick fix, make `substitute_headers` keep the original location of the authentication token,
which `substitute_body` is using as a delimiter for currently unsupported body content types.

Closes #650.

Testing:
- added regression test

restler/engine/core/requests.py

@@ -1272,10 +1272,11 @@ class Request(object):
         # These special headers are not fuzzed, and should not be replaced
         skipped_headers_str = ["Accept", "Host"]
         required_header_blocks = []
+        auth_token = []
         append_header = False
         for line in old_request.definition[header_start_index : header_end_index]:
             if line[0] == "restler_refreshable_authentication_token":
-                required_header_blocks.append(line)
+                auth_token.append(line)
                 continue
             if append_header:
                 required_header_blocks.append(line)
@@ -1288,9 +1289,15 @@ class Request(object):
                         # Continue to append
                         append_header = True
 
+        # Note: currently, the required header blocks must be placed at the end, since the auth primitive is being
+        # used as a delimiter.
         # Make sure there is still a delimiter between headers and the remainder of the payload
-        new_header_blocks.append(primitives.restler_static_string("\r\n"))
-        new_definition = old_request.definition[:header_start_index] + required_header_blocks + new_header_blocks + old_request.definition[header_end_index:]
+        new_definition = old_request.definition[:header_start_index] +\
+                         required_header_blocks +\
+                         new_header_blocks +\
+                         auth_token +\
+                         [ primitives.restler_static_string("\r\n") ] +\
+                         old_request.definition[header_end_index:]
         new_definition += [old_request.metadata.copy()]
         new_request = Request(new_definition)
         # Update the new Request object with the create once requests data of the old Request,

restler/engine/transport_layer/messaging.py

@@ -184,8 +184,12 @@ class HttpSock(object):
             return header + message[_get_start_of_body(message):]
 
         if "Content-Length: " not in message:
-            contentlen = len(message[_get_start_of_body(message):])
-            message = _append_to_header(message, f"Content-Length: {contentlen}")
+            try:
+                contentlen = len(message[_get_start_of_body(message):])
+                message = _append_to_header(message, f"Content-Length: {contentlen}")
+            except Exception as error:
+                RAW_LOGGING(f'Failed to append Content-Length header to message: {message!r}\n')
+                raise error
         if self.connection_settings.include_user_agent:
             message = _append_to_header(message, f"User-Agent: restler/{Settings().version}")
 

restler/unit_tests/grammar_schema_test_files/substitute_body_regression_test_grammar.json

@@ -0,0 +1,119 @@
+{
+  "Requests": [
+    {
+      "id": {
+        "endpoint": "/Ping",
+        "method": "Post"
+      },
+      "method": "Post",
+      "basePath": "",
+      "path": [
+        {
+          "Constant": [
+            "String",
+            "Ping"
+          ]
+        }
+      ],
+      "queryParameters": [
+        [
+          "Schema",
+          {
+            "ParameterList": []
+          }
+        ]
+      ],
+      "bodyParameters": [
+        [
+          "Schema",
+          {
+            "ParameterList": [
+              {
+                "name": "__body__",
+                "payload": {
+                  "LeafNode": {
+                    "name": "",
+                    "payload": {
+                      "Fuzzable": {
+                        "primitiveType": "String",
+                        "defaultValue": "fuzzstring",
+                        "exampleValue": "BBBBCCCCC"
+                      }
+                    },
+                    "isRequired": true,
+                    "isReadOnly": false
+                  }
+                }
+              }
+            ]
+          }
+        ]
+      ],
+      "headerParameters": [
+        [
+          "Schema",
+          {
+            "ParameterList": [
+              {
+                "name": "Content-Type",
+                "payload": {
+                  "LeafNode": {
+                    "name": "",
+                    "payload": {
+                      "Fuzzable": {
+                        "primitiveType": "String",
+                        "defaultValue": "fuzzstring",
+                        "exampleValue": "application/json"
+                      }
+                    },
+                    "isRequired": true,
+                    "isReadOnly": false
+                  }
+                }
+              }
+            ]
+          }
+        ],
+        [
+          "DictionaryCustomPayload",
+          {
+            "ParameterList": [
+              {
+                "name": "Content-Type",
+                "payload": {
+                  "LeafNode": {
+                    "name": "",
+                    "payload": {
+                      "Constant": [
+                        "String",
+                        "application/json"
+                      ]
+                    },
+                    "isRequired": true,
+                    "isReadOnly": false
+                  }
+                }
+              }
+            ]
+          }
+        ]
+      ],
+      "token": "Refreshable",
+      "headers": [
+        [
+          "Accept",
+          "application/json"
+        ],
+        [
+          "Host",
+          "{{fums-reporting}}"
+        ]
+      ],
+      "httpVersion": "1.1",
+      "requestMetadata": {
+        "isLongRunningOperation": false
+      }
+    }
+
+  ]
+}

restler/unit_tests/grammar_schema_test_files/substitute_body_regression_test_grammar.py

@@ -0,0 +1,52 @@
+""" THIS IS AN AUTOMATICALLY GENERATED FILE!"""
+from __future__ import print_function
+import json
+from engine import primitives
+from engine.core import requests
+from engine.errors import ResponseParsingException
+from engine import dependencies
+req_collection = requests.RequestCollection([])
+# Endpoint: /Ping, method: Post
+request = requests.Request([
+    primitives.restler_static_string("POST "),
+    primitives.restler_basepath(""),
+    primitives.restler_static_string("/"),
+    primitives.restler_static_string("Ping"),
+    primitives.restler_static_string(" HTTP/1.1\r\n"),
+    primitives.restler_static_string("Accept: application/json\r\n"),
+    primitives.restler_static_string("Host: {{fums-reporting}}\r\n"),
+    primitives.restler_static_string("Content-Type: "),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["application/json"]),
+    primitives.restler_static_string("\r\n"),
+    primitives.restler_static_string("Content-Type: "),
+    primitives.restler_static_string("application/json"),
+    primitives.restler_static_string("\r\n"),
+    primitives.restler_refreshable_authentication_token("authentication_token_tag"),
+    primitives.restler_static_string("\r\n"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=True, examples=["BBBBCCCCC"]),
+    primitives.restler_static_string("\r\n"),
+
+],
+requestId="/Ping"
+)
+req_collection.add_request(request)
+
+# Endpoint: /Ping, method: Get
+request = requests.Request([
+    primitives.restler_static_string("GET "),
+    primitives.restler_basepath(""),
+    primitives.restler_static_string("/"),
+    primitives.restler_static_string("Ping"),
+    primitives.restler_static_string(" HTTP/1.1\r\n"),
+    primitives.restler_static_string("Accept: application/json\r\n"),
+    primitives.restler_static_string("Host: {{fums-reporting}}\r\n"),
+    primitives.restler_static_string("Content-Type: "),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["application/json"]),
+    primitives.restler_static_string("\r\n"),
+    primitives.restler_refreshable_authentication_token("authentication_token_tag"),
+    primitives.restler_static_string("\r\n"),
+
+],
+requestId="/Ping"
+)
+#req_collection.add_request(request)

restler/unit_tests/test_grammar_schema_parser.py

@@ -344,6 +344,7 @@ class SchemaParserTest(unittest.TestCase):
             "simple_swagger_all_param_data_types_local_examples_grammar",
             "simple_swagger_with_annotations_grammar",
             "demo_server_grammar",
+            "substitute_body_regression_test_grammar"
              # TODO: enable this after abstracting uuid suffix in the equivalence check
              # "simple_swagger_annotations_uuid4_suffix" # todo rename 'grammar'
         ]