Track parameter combinations in the spec coverage file. (#244)

* Track parameter combinations in the spec coverage file.

This change adds a new option to track parameter combinations to the engine.

When the fuzzing mode is 'test-all-combinations',
a new property 'tracked_parameters' will appear in speccov.json.
This property contains key-value pairs of all of the parameters
for which more than one combination is being tested.

In this commit, the parameters for which tracking is supported are:

- enums
- custom payloads

For example, an enum 'per_page' with several values will appear as:

        "tracked_parameters": {
            "per_page_14": "2"
        }

The suffix '14' is used to disambiguate between several primitives
of the same name appearing in the payload, and is the position of the argument
in the request definition.

Full support for tracking fuzzable primitives will be enabled in a future
update.  For fuzzable primitives, the name of the parameter or property in the payload
is not yet included, and a default 'tracked_param' name is used instead.

For example:

        "tracked_parameters": {
            "tracked_param_14": "1"
        }

* Fix failing unit test - update needed to payload body checker.

* update doc
This commit is contained in:
marina-p 2021-06-16 16:21:15 -07:00 коммит произвёл GitHub
Родитель 0f45b0e0d2
Коммит 52144c601a
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
17 изменённых файлов: 154 добавлений и 51 удалений

Просмотреть файл

@ -85,6 +85,7 @@ During each Test run a `speccov.json` file will be created in the logs directory
"status_code": "400",
"status_text": "BAD REQUEST",
"error_message": "{\n \"errors\": {\n \"id\": \"'5882' is not of type 'integer'\"\n },\n \"message\": \"Input payload validation failed\"\n}\n",
"request_order": 4,
"sample_request": {
"request_sent_timestamp": null,
"response_received_timestamp": "2021-03-31 18:20:14",
@ -103,7 +104,9 @@ During each Test run a `speccov.json` file will be created in the logs directory
],
"response_body": "{\n \"errors\": {\n \"id\": \"'5882' is not of type 'integer'\"\n },\n \"message\": \"Input payload validation failed\"\n}\n"
},
"request_order": 4
"tracked_parameters": {
"per_page_9": "2"
}
},
```
@ -129,6 +132,10 @@ the coverage data is being reported.
* The __"error_message"__ value will be set to the response body if the request was not "valid".
* The __"request_order"__ value is the 0 indexed order that the request was sent.
* Requests sent during "preprocessing" or "postprocessing" will explicitely say so.
* The __"tracked_parameters"__ property is optional and generated only when using
`Test` mode with ```test-all-combinations```.
This property contains key-value pairs of all of the parameters
for which more than one value is being tested.
#### Postprocessing Scripts:
The `utilities` directory contains a sub-directory called `speccovparsing` that contains scripts for postprocessing speccov files.

Просмотреть файл

@ -110,7 +110,7 @@ class CheckerBase:
@rtype : Tuple(HttpResponse, HttpResponse)
"""
rendered_data, parser = request.render_current(self._req_collection.candidate_values_pool)
rendered_data, parser, tracked_parameters = request.render_current(self._req_collection.candidate_values_pool)
rendered_data = seq.resolve_dependencies(rendered_data)
response = self._send_request(parser, rendered_data)
response_to_parse = response

Просмотреть файл

@ -58,7 +58,7 @@ class InvalidDynamicObjectChecker(CheckerBase):
InvalidDynamicObjectChecker.generation_executed_requests[generation].add(last_request.hex_definition)
# Get the current rendering of the sequence, which will be the valid rendering of the last request
last_rendering, last_request_parser = last_request.render_current(self._req_collection.candidate_values_pool)
last_rendering, last_request_parser, tracked_parameters = last_request.render_current(self._req_collection.candidate_values_pool)
# Execute the sequence up until the last request
new_seq = self._execute_start_of_sequence()

Просмотреть файл

@ -74,7 +74,7 @@ class NameSpaceRuleChecker(CheckerBase):
self._checker_log.checker_print("\nRe-rendering start of original sequence")
for request in seq.requests[:-1]:
rendered_data, parser = request.render_current(
rendered_data, parser, tracked_parameters = request.render_current(
self._req_collection.candidate_values_pool
)
rendered_data = seq.resolve_dependencies(rendered_data)
@ -102,8 +102,8 @@ class NameSpaceRuleChecker(CheckerBase):
# Check if last request contains any trigger_object
last_request = self._sequence.last_request
last_rendering, last_parser = last_request.render_current(self._req_collection.candidate_values_pool)
last_rendering, last_parser, _ = last_request.render_current(self._req_collection.candidate_values_pool)
last_request_contains_a_trigger_object = False
for obj in self._trigger_objects:
if last_rendering.find(obj) != -1:
@ -118,7 +118,7 @@ class NameSpaceRuleChecker(CheckerBase):
if not self._trigger_on_dynamic_objects:
return
# Here, trigger_on_dynamic_objects is True.
# Exit the checker if there are no consumed_types
# Exit the checker if there are no consumed_types
# # in the entire sequence.
if not consumed_types:
return
@ -178,7 +178,7 @@ class NameSpaceRuleChecker(CheckerBase):
for i in range(stopping_length):
request = self._sequence.requests[i]
rendered_data, parser = request.render_current(
rendered_data, parser, tracked_parameters = request.render_current(
self._req_collection.candidate_values_pool
)
rendered_data = self._sequence.resolve_dependencies(rendered_data)
@ -202,7 +202,7 @@ class NameSpaceRuleChecker(CheckerBase):
"""
self._checker_log.checker_print("Hijack request rendering")
rendered_data, parser = req.render_current(
rendered_data, parser, tracked_parameters = req.render_current(
self._req_collection.candidate_values_pool
)
rendered_data = self._sequence.resolve_dependencies(rendered_data)

Просмотреть файл

@ -1109,7 +1109,7 @@ class PayloadBodyChecker(CheckerBase):
cnt = 0
# iterate through different value combinations
for rendered_data, parser in new_request.render_iter(
for rendered_data, parser,_ in new_request.render_iter(
self._req_collection.candidate_values_pool
):
# check time budget

Просмотреть файл

@ -109,7 +109,7 @@ class UseAfterFreeChecker(CheckerBase):
"""
request = seq.last_request
for rendered_data, parser in\
for rendered_data, parser,_ in\
request.render_iter(self._req_collection.candidate_values_pool,
skip=request._current_combination_id):
# Hold the lock (because other workers may be rendering the same

Просмотреть файл

@ -102,6 +102,7 @@ class SmokeTestStats(object):
self.status_text = None
self.sample_request = RenderedRequestStats()
self.tracked_parameters = {}
def set_matching_prefix(self, sequence_prefix):
# Set the prefix of the request, if it exists.
@ -136,6 +137,12 @@ class SmokeTestStats(object):
self.error_msg = response_body
self.set_matching_prefix(renderings.sequence.prefix)
# Set tracked parameters
last_req = renderings.sequence.last_request
# extract the custom payloads and enums
for property_name, property_value in last_req._tracked_parameters.items():
self.tracked_parameters[property_name] = property_value
class Request(object):
""" Request Class. """
@ -166,6 +173,7 @@ class Request(object):
self._produces = set()
self._set_constraints()
self._create_once_requests = []
self._tracked_parameters = {}
# Check for empty request before assigning ids
if self._definition:
@ -535,7 +543,7 @@ class Request(object):
@type preprocessing: Bool
@return: (rendered request's payload, response's parser function)
@rtype : (Str, Function Pointer)
@rtype : (Str, Function Pointer, List[Str])
"""
def _raise_dict_err(type, tag):
@ -574,11 +582,30 @@ class Request(object):
parser = self.metadata['post_send']['parser']
fuzzable = []
# The following list will contain name-value pairs of properties whose combinations
# are tracked for coverage reporting purposes.
# First, in the loop below, the index of the property in the values list will be added.
# Then, at the time of returning the specific combination of values, a new list with
# the values will be created
tracked_parameters = {}
for request_block in definition:
primitive_type = request_block[0]
default_val = request_block[1]
quoted = request_block[2]
examples = request_block[3]
if primitive_type == primitives.FUZZABLE_GROUP:
field_name = request_block[1]
default_val = request_block[2]
quoted = request_block[3]
examples = request_block[4]
elif primitive_type in [ primitives.CUSTOM_PAYLOAD,
primitives.CUSTOM_PAYLOAD_HEADER,
primitives.CUSTOM_PAYLOAD_UUID4_SUFFIX ]:
field_name = request_block[1]
quoted = request_block[2]
examples = request_block[3]
else:
field_name = None
default_val = request_block[1]
quoted = request_block[2]
examples = request_block[3]
values = []
# Handling dynamic primitives that need fresh rendering every time
@ -617,40 +644,40 @@ class Request(object):
elif primitive_type == primitives.CUSTOM_PAYLOAD:
try:
current_fuzzable_values = candidate_values_pool.\
get_candidate_values(primitive_type, request_id=self._request_id, tag=default_val, quoted=quoted)
get_candidate_values(primitive_type, request_id=self._request_id, tag=field_name, quoted=quoted)
# handle case where custom payload have more than one values
if isinstance(current_fuzzable_values, list):
values = current_fuzzable_values
else:
values = [current_fuzzable_values]
except primitives.CandidateValueException:
_raise_dict_err(primitive_type, default_val)
_raise_dict_err(primitive_type, field_name)
except Exception as err:
_handle_exception(primitive_type, default_val, err)
_handle_exception(primitive_type, field_name, err)
# Handle custom (user defined) static payload on header (Adds \r\n)
elif primitive_type == primitives.CUSTOM_PAYLOAD_HEADER:
try:
current_fuzzable_values = candidate_values_pool.\
get_candidate_values(primitive_type, request_id=self._request_id, tag=default_val, quoted=quoted)
get_candidate_values(primitive_type, request_id=self._request_id, tag=field_name, quoted=quoted)
# handle case where custom payload have more than one values
if isinstance(current_fuzzable_values, list):
values = current_fuzzable_values
else:
values = [current_fuzzable_values]
except primitives.CandidateValueException:
_raise_dict_err(primitive_type, default_val)
_raise_dict_err(primitive_type, field_name)
except Exception as err:
_handle_exception(primitive_type, default_val, err)
_handle_exception(primitive_type, field_name, err)
# Handle custom (user defined) static payload with uuid4 suffix
elif primitive_type == primitives.CUSTOM_PAYLOAD_UUID4_SUFFIX:
try:
current_fuzzable_value = candidate_values_pool.\
get_candidate_values(primitive_type, request_id=self._request_id, tag=default_val, quoted=quoted)
get_candidate_values(primitive_type, request_id=self._request_id, tag=field_name, quoted=quoted)
values = [primitives.restler_custom_payload_uuid4_suffix(current_fuzzable_value)]
except primitives.CandidateValueException:
_raise_dict_err(primitive_type, default_val)
_raise_dict_err(primitive_type, field_name)
except Exception as err:
_handle_exception(primitive_type, default_val, err)
_handle_exception(primitive_type, field_name, err)
elif primitive_type == primitives.REFRESHABLE_AUTHENTICATION_TOKEN:
values = [primitives.restler_refreshable_authentication_token]
# Handle all the rest
@ -662,6 +689,17 @@ class Request(object):
if len(values) == 0:
_raise_dict_err(primitive_type, current_fuzzable_tag)
# When testing all combinations, update tracked parameters.
if Settings().fuzzing_mode == 'test-all-combinations':
param_idx = len(fuzzable)
# Only track the parameter if there are multiple values being combined
if len(values) > 1:
if not field_name:
field_name = "tracked_param"
field_name = f"{field_name}_{param_idx}"
tracked_parameters[field_name] = param_idx
fuzzable.append(values)
# lazy generation of pool for candidate values
@ -678,8 +716,14 @@ class Request(object):
for ind, values in enumerate(combinations_pool):
values = list(values)
values = request_utilities.resolve_dynamic_primitives(values, candidate_values_pool)
tracked_parameter_values = {}
for (k, idx) in tracked_parameters.items():
tracked_parameter_values[k] = values[idx]
rendered_data = "".join(values)
yield rendered_data, parser
yield rendered_data, parser, tracked_parameter_values
def render_current(self, candidate_values_pool, preprocessing=False):
""" Renders the next combination for the current request.
@ -690,7 +734,7 @@ class Request(object):
@type preprocessing: Bool
@return: (rendered request's payload, response's parser function)
@rtype : (Str, Function Pointer)
@rtype : (Str, Function Pointer, List[Str])
"""
return next(self.render_iter(candidate_values_pool,

Просмотреть файл

@ -323,7 +323,8 @@ class Sequence(object):
CUSTOM_LOGGING(self, candidate_values_pool)
self._sent_request_data_list = []
for rendered_data, parser in\
for rendered_data, parser, tracked_parameters in\
request.render_iter(candidate_values_pool,
skip=request._current_combination_id,
preprocessing=preprocessing):
@ -349,15 +350,18 @@ class Sequence(object):
dependencies.reset_tlb()
sequence_failed = False
request._tracked_parameters = tracked_parameters
# Step A: Static template rendering
# Render last known valid combination of primitive type values
# for every request until the last
for i in range(len(self.requests) - 1):
prev_request = self.requests[i]
prev_rendered_data, prev_parser =\
prev_rendered_data, prev_parser, tracked_parameters =\
prev_request.render_current(candidate_values_pool,
preprocessing=preprocessing)
request._tracked_parameters.update(tracked_parameters)
# substitute reference placeholders with resolved values
if not Settings().ignore_dependencies:
prev_rendered_data =\

Просмотреть файл

@ -392,7 +392,7 @@ class GarbageCollectorThread(threading.Thread):
# Iterate in reverse to give priority to newest resources
for value in reversed(self.overflowing[type]):
rendered_data, _ = destructor.\
rendered_data, _ , _ = destructor.\
render_current(self.req_collection.candidate_values_pool)
# replace dynamic parameters

Просмотреть файл

@ -186,11 +186,25 @@ class BodySchema():
acc = ''
sets = []
for block in blocks:
primitive_type = block[0]
value = block[1]
if len(block) > 2:
quoted = block[2]
for request_block in blocks:
primitive_type = request_block[0]
if primitive_type == primitives.FUZZABLE_GROUP:
field_name = request_block[1]
value = request_block[2]
quoted = request_block[3]
examples = request_block[4]
elif primitive_type in [ primitives.CUSTOM_PAYLOAD,
primitives.CUSTOM_PAYLOAD_HEADER,
primitives.CUSTOM_PAYLOAD_UUID4_SUFFIX ]:
field_name = request_block[1]
quoted = request_block[2]
examples = request_block[3]
value = None
else:
field_name = None
value = request_block[1]
quoted = request_block[2]
examples = request_block[3]
# accumulate
if primitive_type == primitives.STATIC_STRING:

Просмотреть файл

@ -523,6 +523,7 @@ def restler_fuzzable_group(*args, **kwargs):
@rtype : Tuple
"""
field_name = args[0]
try:
enum_vals = args[1]
except IndexError:
@ -535,7 +536,7 @@ def restler_fuzzable_group(*args, **kwargs):
examples=None
if EXAMPLES_ARG in kwargs:
examples = kwargs[EXAMPLES_ARG]
return sys._getframe().f_code.co_name, enum_vals, quoted, examples
return sys._getframe().f_code.co_name, field_name, enum_vals, quoted, examples
def restler_fuzzable_uuid4(*args, **kwargs):

Просмотреть файл

@ -348,7 +348,7 @@ class FunctionalityTests(unittest.TestCase):
try:
result.check_returncode()
except subprocess.CalledProcessError:
self.fail(f"Restler returned non-zero exit code: {result.returncode}")
self.fail(f"Restler returned non-zero exit code: {result.returncode} {result.stdout}")
experiments_dir = self.get_experiments_dir()

Просмотреть файл

@ -135,7 +135,7 @@ class SpecCoverageLog(object):
SpecCoverageLog.__instance = self
def _get_request_coverage_summary_stats(self, rendered_request, req_hash):
def _get_request_coverage_summary_stats(self, rendered_request, req_hash, log_tracked_parameters=False):
""" Constructs a json object with the coverage information for a request
from the rendered request. This info will be reported in a spec coverage file.
@ -181,6 +181,11 @@ class SpecCoverageLog(object):
req_spec['request_order'] = req.stats.request_order
req_spec['sample_request'] = vars(req.stats.sample_request)
if log_tracked_parameters:
req_spec['tracked_parameters'] = {}
for k, v in req.stats.tracked_parameters.items():
req_spec['tracked_parameters'][k] = v
return coverage_data
def log_request_coverage_incremental(self, request=None, rendered_sequence=None, log_rendered_hash=True):
@ -222,7 +227,7 @@ class SpecCoverageLog(object):
write_to_main("ERROR: spec coverage is being logged twice for the same rendering.", True)
return
req_coverage = self._get_request_coverage_summary_stats(req, req_hash)
req_coverage = self._get_request_coverage_summary_stats(req, req_hash, log_tracked_parameters=log_rendered_hash)
self._renderings_logged[req_hash] = req_coverage[req_hash]['valid']
coverage_as_json = json.dumps(req_coverage, indent=4)
@ -463,9 +468,23 @@ def custom_network_logging(sequence, candidate_values_pool, **kwargs):
f"{request._current_combination_id} / {request.num_combinations(candidate_values_pool)})")
for request_block in definition:
primitive = request_block[0]
default_val = request_block[1]
quoted = request_block[2]
examples = request_block[3]
if primitive == primitives.FUZZABLE_GROUP:
field_name = request_block[1]
default_val = request_block[2]
quoted = request_block[3]
examples = request_block[4]
elif primitive in [ primitives.CUSTOM_PAYLOAD,
primitives.CUSTOM_PAYLOAD_HEADER,
primitives.CUSTOM_PAYLOAD_UUID4_SUFFIX ]:
field_name = request_block[1]
quoted = request_block[2]
examples = request_block[3]
else:
field_name = None
default_val = request_block[1]
quoted = request_block[2]
examples = request_block[3]
# Handling dynamic primitives that need fresh rendering every time
if primitive == "restler_fuzzable_uuid4":
values = [primitives.restler_fuzzable_uuid4]
@ -478,7 +497,7 @@ def custom_network_logging(sequence, candidate_values_pool, **kwargs):
default_val = '_OMITTED_BINARY_DATA_'
# Handle custom payload
elif primitive == "restler_custom_payload_header":
current_fuzzable_tag = default_val
current_fuzzable_tag = field_name
values = candidate_values_pool.get_candidate_values(primitive, request_id=request.request_id, tag=current_fuzzable_tag, quoted=quoted)
if not isinstance(values, list):
values = [values]
@ -486,7 +505,7 @@ def custom_network_logging(sequence, candidate_values_pool, **kwargs):
default_val = values[0]
# Handle custom payload
elif primitive == "restler_custom_payload":
current_fuzzable_tag = default_val
current_fuzzable_tag = field_name
values = candidate_values_pool.get_candidate_values(primitive, request_id=request.request_id, tag=current_fuzzable_tag, quoted=quoted)
if not isinstance(values, list):
values = [values]
@ -494,7 +513,7 @@ def custom_network_logging(sequence, candidate_values_pool, **kwargs):
default_val = values[0]
# Handle custom payload with uuid4 suffix
elif primitive == "restler_custom_payload_uuid4_suffix":
current_fuzzable_tag = default_val
current_fuzzable_tag = field_name
values = candidate_values_pool.get_candidate_values(primitive, request_id=request.request_id, tag=current_fuzzable_tag, quoted=quoted)
default_val = values[0]
# Handle all the rest

Просмотреть файл

@ -74,14 +74,14 @@ let rec getRestlerPythonPayload (payload:FuzzingPayload) (isQuoted:bool) : Reque
| Number -> Restler_fuzzable_number { defaultValue = v ; isQuoted = false ; exampleValue = exv }
| Uuid ->
Restler_fuzzable_uuid4 { defaultValue = v ; isQuoted = isQuoted ; exampleValue = exv }
| PrimitiveType.Enum (_, enumeration, defaultValue) ->
// TODO: should this be generating unique fuzzable group tags? Why is one needed?
| PrimitiveType.Enum (enumPropertyName, _, enumeration, defaultValue) ->
let defaultStr =
match defaultValue with
| Some v -> sprintf ", default_enum=\"%s\"" v
| None -> ""
let groupValue =
(sprintf "\"fuzzable_group_tag\", [%s] %s "
(sprintf "\"%s\", [%s] %s "
enumPropertyName
(enumeration |> List.map (fun s -> sprintf "'%s'" s) |> String.concat ",")
defaultStr
)
@ -296,7 +296,7 @@ let generatePythonParameter includeOptionalParameters parameterKind (requestPara
| PrimitiveType.DateTime
| PrimitiveType.Uuid ->
true
| PrimitiveType.Enum (enumType, _, _) ->
| PrimitiveType.Enum (_, enumType, _, _) ->
isPrimitiveTypeQuoted enumType isNullValue
| PrimitiveType.Object
| PrimitiveType.Int

Просмотреть файл

@ -290,6 +290,19 @@ module private Parameters =
let parameterPayload = generateGrammarElementForSchema
p.ActualSchema
(specExampleValue, true) [] id
// Add the name to the parameter payload
let parameterPayload =
match parameterPayload with
| LeafNode leafProperty ->
let leafNodePayload =
match leafProperty.payload with
| Fuzzable (Enum(propertyName, propertyType, values, defaultValue), x, y) ->
Fuzzable (Enum(p.Name, propertyType, values, defaultValue), x, y)
| _ -> leafProperty.payload
LeafNode { leafProperty with payload = leafNodePayload }
| InternalNode (internalNode, children) ->
// TODO: need enum test to see if body enum is fine.
parameterPayload
{
name = p.Name
payload = parameterPayload

Просмотреть файл

@ -102,7 +102,8 @@ type PrimitiveType =
| DateTime
/// The enum type specifies the list of possible enum values
/// and the default value, if specified.
| Enum of PrimitiveType * string list * string option
/// (tag, data type, possible values, default value if present)
| Enum of string * PrimitiveType * string list * string option
type NestedType =
| Array

Просмотреть файл

@ -145,7 +145,7 @@ module SwaggerVisitors =
match enumValues with
| [] -> "null"
| h::rest -> h
Fuzzable (PrimitiveType.Enum (grammarPrimitiveType, enumValues, defaultValue), defaultFuzzableEnumValue, exv)
Fuzzable (PrimitiveType.Enum (propertyName, grammarPrimitiveType, enumValues, defaultValue), defaultFuzzableEnumValue, exv)
| NJsonSchema.JsonObjectType.Object
| NJsonSchema.JsonObjectType.None ->
// Example of JsonObjectType.None: "content": {} without a type specified in Swagger.
@ -214,7 +214,7 @@ module SwaggerVisitors =
| _ when v.Type = JTokenType.Null -> null
| PrimitiveType.String
| PrimitiveType.DateTime
| PrimitiveType.Enum (PrimitiveType.String, _, _)
| PrimitiveType.Enum (_, PrimitiveType.String, _, _)
| PrimitiveType.Uuid ->
// Remove the start and end quotes, which are preserved with 'Formatting.None'.
if rawValue.Length > 1 then