Generate fuzzables for examples (#590)

This change is needed for the invalid values checker to work correctly.

Instead of generating constant examples in the grammar, generate fuzzable elements with example values.  The engine has been updated to use only the constant example value in the main algorithm.

The checker can then find the fuzzable elements that need to be fuzzed.

Testing:
- manual testing

restler/engine/core/requests.py

@@ -716,7 +716,7 @@ class Request(object):
         parameter combinations, there will be a total of 8 combinations.
 
         @return: (One or more copies of the request, each with a unique schema)
-        @rtype : (Request)
+        @rtype : List[(Request, is_example)]
 
         """
         def get_all_example_schemas():
@@ -729,20 +729,20 @@ class Request(object):
         if Settings().example_payloads is not None:
             tested_example_payloads = True
             for ex in get_all_example_schemas():
-                yield ex
+                yield ex, True
 
         tested_param_combinations = False
         header_param_combinations = Settings().header_param_combinations
         if header_param_combinations is not None:
             tested_param_combinations = True
             for hpc in self.get_header_param_combinations(header_param_combinations):
-                yield hpc
+                yield hpc, False
 
         query_param_combinations = Settings().query_param_combinations
         if query_param_combinations is not None:
             tested_param_combinations = True
             for hpc in self.get_query_param_combinations(query_param_combinations):
-                yield hpc
+                yield hpc, False
 
         if not (tested_param_combinations or tested_example_payloads):
             # When no test combination settings are specified, RESTler will try
@@ -762,7 +762,7 @@ class Request(object):
                     tested_all_params = True
                 else:
                     tested_first_example = True
-                yield self
+                yield self, tested_first_example
 
             # If examples are available, test all the examples (up to the maximum in the settings)
             example_schemas = get_all_example_schemas()
@@ -771,7 +771,7 @@ class Request(object):
             if tested_first_example:
                 next(example_schemas)
             for ex in example_schemas:
-                yield ex
+                yield ex, True
 
             if not tested_all_params:
                 param_schema_combinations = {
@@ -779,14 +779,14 @@ class Request(object):
                     "param_kind": "all",
                     "choose_n": "max"
                 }
-                yield self.get_parameters_from_schema(param_schema_combinations)
+                yield self.get_parameters_from_schema(param_schema_combinations), False
 
             # Test all required parameters (obtained from the schema, without examples)
             param_schema_combinations = {
                 "max_combinations": 1,
                 "param_kind": "optional"
             }
-            yield self.get_parameters_from_schema(param_schema_combinations)
+            yield self.get_parameters_from_schema(param_schema_combinations), False
 
     def init_fuzzable_values(self, req_definition, candidate_values_pool, preprocessing=False, log_dict_err_to_main=True):
         def _raise_dict_err(type, tag):
@@ -841,14 +841,23 @@ class Request(object):
                 values = [(primitives.restler_fuzzable_uuid4, quoted, writer_variable)]
             # Handle enums that have a list of values instead of one default val
             elif primitive_type == primitives.FUZZABLE_GROUP:
+                values = []
+                # Handle example values
+                for ex_value in examples:
+                    if ex_value is None:
+                        ex_value = "null"
+                    elif quoted:
+                        ex_value = f'"{ex_value}"'
+                    values.append(ex_value)
                 if quoted:
-                    values = [f'"{val}"' for val in default_val]
+                    enum_values = [f'"{val}"' for val in default_val]
                 else:
-                    values = list(default_val)
+                    enum_values = list(default_val)
+                values.extend(enum_values)
             # Handle static whose value is the field name
             elif primitive_type == primitives.STATIC_STRING:
                 val = default_val
-                if val == None:
+                if val is None:
                     # the examplesChecker may inject None/null, so replace these with the string 'null'
                     logger.raw_network_logging(f"Warning: there is a None value in a STATIC_STRING.")
                     val = 'null'
@@ -1010,7 +1019,7 @@ class Request(object):
         schema_combinations = itertools.islice(self.get_schema_combinations(), Settings().max_schema_combinations)
         remaining_combinations_count = Settings().max_combinations - skip
 
-        for req in schema_combinations:
+        for (req, is_example) in schema_combinations:
             schema_idx += 1
             parser = None
             fuzzable_request_blocks = []
@@ -1048,7 +1057,10 @@ class Request(object):
                 # Keep plugging in values from the static combinations pool while dynamic
                 # values are available.
                 combinations_pool = itertools.cycle(combinations_pool)
-            combinations_pool = itertools.islice(combinations_pool, Settings().max_combinations)
+            # If this is an example payload, only use the first combination.  This contains the original example
+            # values.
+            max_combinations = 1 if is_example else Settings().max_combinations
+            combinations_pool = itertools.islice(combinations_pool, max_combinations)
 
             # skip combinations, if asked to
             while next_combination < skip:
@@ -1360,9 +1372,6 @@ class Request(object):
         check_example_schema_is_valid(num_query_payloads, max_example_payloads, "query")
         check_example_schema_is_valid(num_body_payloads, max_example_payloads, "body")
 
-        fuzzing_config = FuzzingConfig()
-        fuzzing_config.use_constant_enum_value = True
-
         for payload_idx in range(max_example_payloads):
             body_example = None
             query_example = None
@@ -1386,7 +1395,6 @@ class Request(object):
             query_blocks = None
             header_blocks = None
             if body_example:
-                body_example.set_config(fuzzing_config)
                 body_blocks = body_example.get_blocks()
                 # Only substitute the body if there is a body.
                 if body_blocks:

restler/engine/fuzzing_parameters/fuzzing_config.py

@@ -26,7 +26,6 @@ class FuzzingConfig(object):
         self.merge_fuzzable_values = False
         self.max_depth = sys.maxsize
         self.filter_fn = None
-        self.use_constant_enum_value = False
         # Traversal depth state
         self.depth = 0
 
@@ -89,7 +88,6 @@ class FuzzingConfig(object):
         new_config.merge_fuzzable_values = self.merge_fuzzable_values
         new_config.max_depth = self.max_depth
         new_config.filter_fn = self.filter_fn
-        new_config.use_constant_enum_value = self.use_constant_enum_value
 
         return new_config
 

restler/engine/fuzzing_parameters/request_params.py

@@ -1362,7 +1362,8 @@ class ParamEnum(ParamValue):
         @rtype : List[str]
 
         """
-        return [primitives.restler_fuzzable_group(self._enum_name, self._get_fuzzable_group_values())]
+        return [primitives.restler_fuzzable_group(self._enum_name, self._get_fuzzable_group_values(),
+                                                  quoted=self.is_quoted, examples=self.example_values)]
 
     def get_blocks(self, config=None):
         """ Gets request blocks for the Enum Parameters
@@ -1371,16 +1372,8 @@ class ParamEnum(ParamValue):
         @rtype : List[str]
 
         """
-        contents_str = []
-
-        if config is not None and config.use_constant_enum_value:
-            if self._is_quoted and (self.content_type in ['String', 'Uuid', 'DateTime', 'Date']):
-                content_str = f'"{self._content}"'
-            else:
-                content_str = self._content
-            return [primitives.restler_static_string(content_str)]
-        else:
-            return [primitives.restler_fuzzable_group(FUZZABLE_GROUP_TAG, self._get_fuzzable_group_values())]
+        return [primitives.restler_fuzzable_group(FUZZABLE_GROUP_TAG, self._get_fuzzable_group_values(),
+                                                  quoted=self.is_quoted, examples=self.example_values)]
 
     def get_fuzzing_pool(self, fuzzer, config):
         """ Returns the fuzzing pool

restler/unit_tests/test_grammar_schema_parser.py

@@ -258,7 +258,7 @@ class SchemaParserTest(unittest.TestCase):
         print(f"req{req_with_body.endpoint} {req_with_body.method}")
 
         # Just go through and get all schema combinations.  This makes sure there are no crashes.
-        for x in req_with_body.get_schema_combinations(use_grammar_py_schema=False):
+        for x, is_example in req_with_body.get_schema_combinations(use_grammar_py_schema=False):
             self.assertTrue(len(x.definition) > 0)
 
 
@@ -275,5 +275,5 @@ class SchemaParserTest(unittest.TestCase):
         print(f"req{req_with_body.endpoint} {req_with_body.method}")
 
         # Just go through and get all schema combinations.  This makes sure there are no crashes.
-        for x in req_with_body.get_schema_combinations(use_grammar_py_schema=False):
+        for x, is_example in req_with_body.get_schema_combinations(use_grammar_py_schema=False):
             self.assertTrue(len(x.definition) > 0)

src/compiler/Restler.Compiler.Test/ExampleTests.fs

@@ -118,7 +118,8 @@ module Examples =
                                                Restler.Workflow.Constants.DefaultRestlerGrammarFileName)
             let grammar = File.ReadAllText(grammarFilePath)
             // Check that the grammar contains the object example
-            Assert.True(grammar.Contains("\"tag1\":\"value1\"") && grammar.Contains("\"tag2\":\"value2\""))
+            //examples=["{\"tag1\":\"value1\",\"tag2\":\"value2\"}"]
+            Assert.True(grammar.Contains("\\\"tag1\\\":\\\"value1\\\"") && grammar.Contains("\\\"tag2\\\":\\\"value2\\\""))
 
         [<Fact>]
         let ``allof property omitted in example`` () =
@@ -188,7 +189,8 @@ module Examples =
             // computerName is missing from the example
             Assert.False(grammar.Contains("primitives.restler_static_string(\"computerName: \")"))
             Assert.True(grammar.Contains("primitives.restler_static_string(\"computerDimensions: \")"))
-            Assert.True(grammar.Contains("primitives.restler_static_string(\"\\\"quotedString\\\"\")"))
+            // primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["\"quotedString\""]),
+            Assert.True(grammar.Contains("primitives.restler_fuzzable_string(\"fuzzstring\", quoted=False, examples=[\"\\\"quotedString\\\"\"])"))
 
             // The grammar should contain the array items from the example
             Assert.True(grammar.Contains("1.11"))
@@ -496,12 +498,18 @@ module Examples =
                 exactCopy = true
             }
 
+            let exactCopyTestConfig = { config with ExampleConfigFiles = Some [ exampleConfigFile ] }
+            Restler.Workflow.generateRestlerGrammar None exactCopyTestConfig
+            let grammarFilePath = Path.Combine(grammarOutputDirectoryPath, Restler.Workflow.Constants.DefaultRestlerGrammarFileName)
+            let grammar = File.ReadAllText(grammarFilePath)
+            Assert.True(grammar.Contains("primitives.restler_static_string(\"2020-02-02\")"))
+
             let testConfig = { config with ExampleConfigFiles = Some [ {exampleConfigFile with exactCopy = false }] }
             Restler.Workflow.generateRestlerGrammar None testConfig
 
             let grammarFilePath = Path.Combine(grammarOutputDirectoryPath, Restler.Workflow.Constants.DefaultRestlerGrammarFileName)
             let grammar = File.ReadAllText(grammarFilePath)
-            Assert.True(grammar.Contains("primitives.restler_static_string(\"2020-02-02\"),"))
+            Assert.True(grammar.Contains("primitives.restler_fuzzable_string(\"fuzzstring\", quoted=False, examples=[\"2020-02-02\"])"))
             /// Also test to make sure that 'exactCopy : true' filters out the parameters that are not declared in the spec
             Assert.False(grammar.Contains("ddd"))
 

src/compiler/Restler.Compiler.Test/baselines/exampleTests/array_example_grammar.py

@@ -75,28 +75,28 @@ request = requests.Request([
     primitives.restler_static_string("order"),
     primitives.restler_static_string("?"),
     primitives.restler_static_string("apiVersion="),
-    primitives.restler_static_string("2020-02-02"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["2020-02-02"]),
     primitives.restler_static_string("&"),
     primitives.restler_static_string("expiration="),
-    primitives.restler_static_string("10"),
+    primitives.restler_fuzzable_int("1", examples=["10"]),
     primitives.restler_static_string("&"),
     primitives.restler_static_string("arrayQueryParameter="),
     primitives.restler_static_string("&"),
     primitives.restler_static_string("arrayQueryParameter2="),
-    primitives.restler_static_string("a"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["a"]),
     primitives.restler_static_string("&"),
     primitives.restler_static_string("arrayQueryParameter2="),
-    primitives.restler_static_string("b"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["b"]),
     primitives.restler_static_string("&"),
     primitives.restler_static_string("arrayQueryParameter2="),
-    primitives.restler_static_string("c"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["c"]),
     primitives.restler_static_string("&"),
     primitives.restler_static_string("arrayQueryParameter3="),
-    primitives.restler_static_string("ddd"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["ddd"]),
     primitives.restler_static_string(","),
-    primitives.restler_static_string("eee"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["eee"]),
     primitives.restler_static_string(","),
-    primitives.restler_static_string("fff"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["fff"]),
     primitives.restler_static_string(" HTTP/1.1\r\n"),
     primitives.restler_static_string("Accept: application/json\r\n"),
     primitives.restler_static_string("Host: localhost:8888\r\n"),
@@ -107,16 +107,26 @@ request = requests.Request([
     primitives.restler_static_string("\r\n"),
     primitives.restler_static_string("{"),
     primitives.restler_static_string("""
-    "storeId":"23456",
-    "rush":"True",
-    "bagType":"paperfestive",
+    "storeId":"""),
+    primitives.restler_fuzzable_int("1", examples=['"23456"']),
+    primitives.restler_static_string(""",
+    "rush":"""),
+    primitives.restler_fuzzable_bool("true", examples=['"True"']),
+    primitives.restler_static_string(""",
+    "bagType":"""),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=True, examples=["paperfestive"]),
+    primitives.restler_static_string(""",
     "item_descriptions":
     [
     ],
     "item_feedback":
     [
-        "great",
-        "awesome"
+        """),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=True, examples=["great"]),
+    primitives.restler_static_string(""",
+        """),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=True, examples=["awesome"]),
+    primitives.restler_static_string("""
     ]}"""),
     primitives.restler_static_string("\r\n"),
     

src/compiler/Restler.Compiler.Test/baselines/schemaTests/xMsPaths_grammar.json

@@ -174,10 +174,11 @@
                   "LeafNode": {
                     "name": "",
                     "payload": {
-                      "Constant": [
-                        "String",
-                        "2020-03-02"
-                      ]
+                      "Fuzzable": {
+                        "primitiveType": "String",
+                        "defaultValue": "fuzzstring",
+                        "exampleValue": "2020-03-02"
+                      }
                     },
                     "isRequired": true,
                     "isReadOnly": false
@@ -190,10 +191,11 @@
                   "LeafNode": {
                     "name": "",
                     "payload": {
-                      "Constant": [
-                        "String",
-                        "1.2.3.4"
-                      ]
+                      "Fuzzable": {
+                        "primitiveType": "String",
+                        "defaultValue": "fuzzstring",
+                        "exampleValue": "1.2.3.4"
+                      }
                     },
                     "isRequired": true,
                     "isReadOnly": false

src/compiler/Restler.Compiler.Test/baselines/schemaTests/xMsPaths_grammar.py

@@ -43,10 +43,10 @@ request = requests.Request([
     primitives.restler_static_string("Accept: application/json\r\n"),
     primitives.restler_static_string("Host: \r\n"),
     primitives.restler_static_string("api-version: "),
-    primitives.restler_static_string("2020-03-02"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["2020-03-02"]),
     primitives.restler_static_string("\r\n"),
     primitives.restler_static_string("resource-version: "),
-    primitives.restler_static_string("1.2.3.4"),
+    primitives.restler_fuzzable_string("fuzzstring", quoted=False, examples=["1.2.3.4"]),
     primitives.restler_static_string("\r\n"),
     primitives.restler_refreshable_authentication_token("authentication_token_tag"),
     primitives.restler_static_string("\r\n"),

src/compiler/Restler.Compiler/Compiler.fs

@@ -321,7 +321,7 @@ module private Parameters =
                                 else
                                     let parameterGrammarElement =
                                         generateGrammarElementForSchema declaredParameter.ActualSchema
-                                                                        (Some payloadValue, false)
+                                                                        (Some payloadValue, true)
                                                                         (trackParameters, None)
                                                                         (declaredParameter.IsRequired, (parameterIsReadOnly declaredParameter))
                                                                         []

src/compiler/Restler.Compiler/SwaggerVisitors.fs

@@ -704,7 +704,7 @@ module SwaggerVisitors =
                                 |> Seq.map (fun example ->
                                                 generateGrammarElementForSchema
                                                     schema.Item.ActualSchema
-                                                    (Some example, false)
+                                                    (Some example, true)
                                                     (trackParameters, jsonPropertyMaxDepth)
                                                     (isRequired, isReadOnly)
                                                     (schema::parents)
@@ -716,7 +716,7 @@ module SwaggerVisitors =
 
             let allOfParameterSchemas =
                 schema.AllOf
-                |> Seq.map (fun ao -> ao, generateGrammarElementForSchema ao.ActualSchema (exampleValue, false) (trackParameters, jsonPropertyMaxDepth) (isRequired, isReadOnly) (schema::parents) schemaCache id)
+                |> Seq.map (fun ao -> ao, generateGrammarElementForSchema ao.ActualSchema (exampleValue, true) (trackParameters, jsonPropertyMaxDepth) (isRequired, isReadOnly) (schema::parents) schemaCache id)
                 |> Seq.cache
 
             // For AnyOf, take the first schema.
@@ -724,7 +724,7 @@ module SwaggerVisitors =
             let anyOfParameterSchema =
                 schema.AnyOf
                 |> Seq.truncate 1
-                |> Seq.map (fun ao -> ao, generateGrammarElementForSchema ao.ActualSchema (exampleValue, false) (trackParameters, jsonPropertyMaxDepth) (isRequired, isReadOnly) (schema::parents) schemaCache id)
+                |> Seq.map (fun ao -> ao, generateGrammarElementForSchema ao.ActualSchema (exampleValue, true) (trackParameters, jsonPropertyMaxDepth) (isRequired, isReadOnly) (schema::parents) schemaCache id)
                 |> Seq.cache
 
             let getSchemaAndProperties schemas =