Fix quoting issue for dependencies. (#263)

Closes #260
2021-07-14 15:52:00 -07:00 · 2021-07-14 15:52:00 -07:00 · 4c9ff663e9
--- a/restler/end_to_end_tests/test_quick_start.py
+++ b/restler/end_to_end_tests/test_quick_start.py
@ -51,7 +51,7 @@ if __name__ == '__main__':
            raise QuickStartFailedException(f"Failing because restler-quick-start exited with a non-zero return code: {output.returncode!s}")

        stdout = str(output.stdout)
-        if 'Request coverage (successful / total): 5 / 6' not in stdout or\
+        if 'Request coverage (successful / total): 6 / 6' not in stdout or\
        'No bugs were found.' not in stdout or\
        'Task Test succeeded.' not in stdout:
            stdout = stdout.replace('\\r\\n', '\r\n')
--- a/src/compiler/Restler.Compiler.Test/CodeGeneratorTests.fs
+++ b/src/compiler/Restler.Compiler.Test/CodeGeneratorTests.fs
@ -51,7 +51,7 @@ module CodeGenerator =
            let pathPayload =
                [ (Constant (PrimitiveType.String, "api"))
                  (Constant (PrimitiveType.String, "accounts"))
-                  (DynamicObject "accountId") ]
+                  (DynamicObject (PrimitiveType.Int, "accountId")) ]
            let requestElements =
                [
                    Method OperationMethod.Get
--- a/src/compiler/Restler.Compiler.Test/DependencyTests.fs
+++ b/src/compiler/Restler.Compiler.Test/DependencyTests.fs
@ -113,9 +113,10 @@ module Dependencies =
        /// Test that a full path annotation only to a specified body parameter works.
        [<Fact>]
        let ``path annotation to body parameter`` () =
+            let grammarOutputDirectoryPath = ctx.testRootDirPath
            let config = { Restler.Config.SampleConfig with
                             IncludeOptionalParameters = true
-                             GrammarOutputDirectoryPath = Some ctx.testRootDirPath
+                             GrammarOutputDirectoryPath = Some grammarOutputDirectoryPath
                             ResolveBodyDependencies = true
                             UseBodyExamples = Some true
                             SwaggerSpecFilePath = Some [(Path.Combine(Environment.CurrentDirectory, @"swagger\annotationTests\pathAnnotation.json"))]
@ -128,7 +129,7 @@ module Dependencies =
            //
            let expectedGrammarFilePath = Path.Combine(Environment.CurrentDirectory,
                                                       @"baselines\dependencyTests\path_annotation_grammar.py")
-            let actualGrammarFilePath = Path.Combine(ctx.testRootDirPath,
+            let actualGrammarFilePath = Path.Combine(grammarOutputDirectoryPath,
                                                     Restler.Workflow.Constants.DefaultRestlerGrammarFileName)
            let grammarDiff = getLineDifferences expectedGrammarFilePath actualGrammarFilePath
            let message = sprintf "Grammar Does not match baseline.  First difference: %A" grammarDiff
--- a/src/compiler/Restler.Compiler.Test/baselines/dependencyTests/path_annotation_grammar.py
+++ b/src/compiler/Restler.Compiler.Test/baselines/dependencyTests/path_annotation_grammar.py
@ -73,7 +73,7 @@ request = requests.Request([
    primitives.restler_static_string("Host: localhost:8888\r\n"),
    primitives.restler_refreshable_authentication_token("authentication_token_tag"),
    primitives.restler_static_string("\r\n"),
-
+    
    {
        'post_send':
        {
@ -100,7 +100,7 @@ request = requests.Request([
    primitives.restler_static_string("/"),
    primitives.restler_static_string("stores"),
    primitives.restler_static_string("/"),
-    primitives.restler_static_string(_stores_post_id.reader()),
+    primitives.restler_static_string(_stores_post_id.reader(), quoted=False),
    primitives.restler_static_string("/"),
    primitives.restler_static_string("order"),
    primitives.restler_static_string(" HTTP/1.1\r\n"),
@ -120,9 +120,8 @@ request = requests.Request([
    "storeProperties":
        {
            "tags":"""),
-    primitives.restler_static_string('"'),
-    primitives.restler_static_string(_stores_post_metadata.reader()),
-    primitives.restler_static_string(""""
+    primitives.restler_static_string(_stores_post_metadata.reader(), quoted=False),
+    primitives.restler_static_string("""
        }
    ,
    "deliveryProperties":
@ -145,9 +144,8 @@ request = requests.Request([
    primitives.restler_fuzzable_string("fuzzstring", quoted=True),
    primitives.restler_static_string(""",
            "deliveryTags":"""),
-    primitives.restler_static_string('"'),
-    primitives.restler_static_string(_stores_post_delivery_metadata.reader()),
-    primitives.restler_static_string("""",
+    primitives.restler_static_string(_stores_post_delivery_metadata.reader(), quoted=False),
+    primitives.restler_static_string(""",
            "code":"""),
    primitives.restler_fuzzable_int("1"),
    primitives.restler_static_string(""",
--- a/src/compiler/Restler.Compiler.Test/baselines/dependencyTests/path_in_dictionary_payload_grammar.py
+++ b/src/compiler/Restler.Compiler.Test/baselines/dependencyTests/path_in_dictionary_payload_grammar.py
@ -49,7 +49,7 @@ request = requests.Request([
    primitives.restler_static_string("Host: localhost:8888\r\n"),
    primitives.restler_refreshable_authentication_token("authentication_token_tag"),
    primitives.restler_static_string("\r\n"),
-
+    
    {
        'post_send':
        {
@ -74,7 +74,7 @@ request = requests.Request([
    primitives.restler_static_string("/"),
    primitives.restler_static_string("stores"),
    primitives.restler_static_string("/"),
-    primitives.restler_static_string(_stores_post_id.reader()),
+    primitives.restler_static_string(_stores_post_id.reader(), quoted=False),
    primitives.restler_static_string("/"),
    primitives.restler_static_string("order"),
    primitives.restler_static_string(" HTTP/1.1\r\n"),
--- a/src/compiler/Restler.Compiler.Test/baselines/dictionaryTests/quoted_primitives_grammar.py
+++ b/src/compiler/Restler.Compiler.Test/baselines/dictionaryTests/quoted_primitives_grammar.py
@ -74,7 +74,7 @@ request = requests.Request([
    primitives.restler_static_string("/"),
    primitives.restler_static_string("stores"),
    primitives.restler_static_string("/"),
-    primitives.restler_static_string(_stores_post_id.reader()),
+    primitives.restler_static_string(_stores_post_id.reader(), quoted=False),
    primitives.restler_static_string(" HTTP/1.1\r\n"),
    primitives.restler_static_string("Accept: application/json\r\n"),
    primitives.restler_static_string("Host: localhost:8888\r\n"),
@ -114,7 +114,7 @@ request = requests.Request([
    primitives.restler_static_string("/"),
    primitives.restler_static_string("stores"),
    primitives.restler_static_string("/"),
-    primitives.restler_static_string(_stores_post_id.reader()),
+    primitives.restler_static_string(_stores_post_id.reader(), quoted=False),
    primitives.restler_static_string("/"),
    primitives.restler_static_string("order"),
    primitives.restler_static_string("?"),
--- a/src/compiler/Restler.Compiler.Test/baselines/exampleTests/array_example_grammar.py
+++ b/src/compiler/Restler.Compiler.Test/baselines/exampleTests/array_example_grammar.py
@ -110,7 +110,7 @@ request = requests.Request([
        "awesome"
    ]}"""),
    primitives.restler_static_string("\r\n"),
-    
+
    {
        'post_send':
        {
@ -139,7 +139,7 @@ request = requests.Request([
    primitives.restler_static_string("/"),
    primitives.restler_static_string("order"),
    primitives.restler_static_string("/"),
-    primitives.restler_static_string(_stores__storeId__order_post_id.reader()),
+    primitives.restler_static_string(_stores__storeId__order_post_id.reader(), quoted=False),
    primitives.restler_static_string(" HTTP/1.1\r\n"),
    primitives.restler_static_string("Accept: application/json\r\n"),
    primitives.restler_static_string("Host: localhost:8888\r\n"),
--- a/src/compiler/Restler.Compiler/CodeGenerator.fs
+++ b/src/compiler/Restler.Compiler/CodeGenerator.fs
@ -28,7 +28,7 @@ module Types =
    /// IMPORTANT ! All primitives must be supported in restler/engine/primitives.py
    type RequestPrimitiveType =
        | Restler_static_string_constant of string
-        | Restler_static_string_variable of string
+        | Restler_static_string_variable of string * bool
        | Restler_static_string_jtoken_delim of string
        | Restler_fuzzable_string of RequestPrimitiveTypeData
        | Restler_fuzzable_datetime of RequestPrimitiveTypeData
@ -96,8 +96,8 @@ let rec getRestlerPythonPayload (payload:FuzzingPayload) (isQuoted:bool) : Reque
                Restler_custom_payload_uuid4_suffix c.payloadValue
            | CustomPayloadType.Header ->
                Restler_custom_payload_header c.payloadValue  // TODO: need test
-        | DynamicObject s ->
-            Restler_static_string_variable (sprintf "%s.reader()" s)
+        | DynamicObject (primitiveType, s) ->
+            Restler_static_string_variable (sprintf "%s.reader()" s, isQuoted)
        | PayloadParts p ->
            raise (invalidArg "p" "expected primitive payload")

@ -317,7 +317,7 @@ let generatePythonParameter includeOptionalParameters parameterKind (requestPara
                else
                    [ formatPropertyName p.name ]

-            let needQuotes, isFuzzable =
+            let needQuotes, isFuzzable, isDynamicObject =
                match p.payload with
                | FuzzingPayload.Custom c ->
                    let isFuzzable = (c.payloadType = CustomPayloadType.String)
@ -325,27 +325,27 @@ let generatePythonParameter includeOptionalParameters parameterKind (requestPara
                    let needQuotes =
                        (not c.isObject) &&
                        (isPrimitiveTypeQuoted c.primitiveType false)
-                    needQuotes, isFuzzable
+                    needQuotes, isFuzzable, false
                | FuzzingPayload.Constant (PrimitiveType.String, s) ->
                    // TODO: improve the metadata of FuzzingPayload.Constant to capture whether
                    // the constant represents an object,
                    // rather than relying on the formatting behavior of JToken.ToString.
                    not (isNull s) &&
                    not (s.Contains("\n")),
-                    false
+                    false, false
                | FuzzingPayload.Constant (primitiveType, v) ->
                    isPrimitiveTypeQuoted primitiveType (isNull v),
-                    false
+                    false, false
                | FuzzingPayload.Fuzzable (primitiveType, _, _,_) ->
                    // Note: this is a current RESTler limitation -
                    // fuzzable values may not be set to null without changing the grammar.
                    isPrimitiveTypeQuoted primitiveType false,
-                    true
-                | FuzzingPayload.DynamicObject (_)
-                | FuzzingPayload.PayloadParts (_) ->
-                    // Note: dynamic objects as the query parameter (e.g. api-version) do not need quotes.
-                    // That case is handled elsewhere.
                    true, false
+                | FuzzingPayload.DynamicObject (primitiveType, _) ->
+                    isPrimitiveTypeQuoted primitiveType false,
+                    false, true
+                | FuzzingPayload.PayloadParts (_) ->
+                    false, false, false

            let tabSeq =
                if parameterKind = ParameterKind.Body then
@ -362,11 +362,12 @@ let generatePythonParameter includeOptionalParameters parameterKind (requestPara
                    // the user may choose to fuzz with quoted or unquoted values.
                    let needStaticStringQuotes =
                        parameterKind = ParameterKind.Body &&
-                        needQuotes && not isFuzzable
+                        needQuotes && not isFuzzable && not isDynamicObject
                    if needStaticStringQuotes then
                        yield Restler_static_string_constant "\""

-                    let isQuoted = parameterKind = ParameterKind.Body && needQuotes && isFuzzable
+                    let isQuoted = parameterKind = ParameterKind.Body && needQuotes &&
+                                    (isFuzzable || isDynamicObject)
                    for k in getRestlerPythonPayload p.payload isQuoted do
                        yield k

@ -854,8 +855,10 @@ let getRequests(requests:Request list) includeOptionalParameters =
                            else s
                        quoteStringForPythonGrammar rawValue
                    sprintf "primitives.restler_static_string(%s%s%s)" delim s delim
-            | Restler_static_string_variable s ->
-                sprintf "primitives.restler_static_string(%s)" s
+            | Restler_static_string_variable (s, isQuoted) ->
+                sprintf "primitives.restler_static_string(%s, quoted=%s)"
+                        s
+                        (if isQuoted then "True" else "False")
            | Restler_fuzzable_string s ->
                if String.IsNullOrEmpty s.defaultValue then
                    printfn "ERROR: fuzzable strings should not be empty.  Skipping."
--- a/src/compiler/Restler.Compiler/Dependencies.fs
+++ b/src/compiler/Restler.Compiler/Dependencies.fs
@ -1136,7 +1136,14 @@ module DependencyLookup =
        | None -> defaultPayload
        | Some (ResponseObject responseProducer) ->
            let variableName = generateDynamicObjectVariableName responseProducer.id.RequestId (Some responseProducer.id.AccessPathParts) "_"
-            DynamicObject variableName
+            // Mark the type of the dynamic object to be the type of the input parameter if available
+            let primitiveType =
+                match defaultPayload with
+                | FuzzingPayload.Fuzzable (primitiveType, _, _, _) -> primitiveType
+                | _ ->
+                    printfn "Warning: primitive type not available for %A [resource: %s]" requestId consumerResourceName
+                    responseProducer.id.PrimitiveType
+            DynamicObject (primitiveType, variableName)
        | Some (DictionaryPayload (customPayload, primitiveType, resourceName, isObject)) ->
            Custom { payloadType = customPayload
                     primitiveType = primitiveType
--- a/src/compiler/Restler.Compiler/Grammar.fs
+++ b/src/compiler/Restler.Compiler/Grammar.fs
@ -142,7 +142,7 @@ type FuzzingPayload =
    /// The custom payload, as specified in the fuzzing dictionary
    | Custom of CustomPayload

-    | DynamicObject of string
+    | DynamicObject of PrimitiveType * string

    /// In some cases, a payload may need to be split into multiple payload parts
    | PayloadParts of FuzzingPayload list