microsoft
/
restler-fuzzer
зеркало из https://github.com/microsoft/restler-fuzzer.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Initial implementation for header dependencies (#342) 

1) Compiler changes:
- add kwargs and parse out headers from the parser arguments
- add ability to infer header dependencies using the header names, similar to
query parameters
- update test baselines
- add new tests

2) Engine changes:
- parse the headers and invoke the parser with the additional named argument
This commit is contained in:
marina-p 2021-09-30 12:33:15 -07:00 коммит произвёл GitHub
Родитель a7f342477c
Коммит d546780111
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
19 изменённых файлов: 848 добавлений и 179 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

9
restler/engine/core/request_utilities.py
Просмотреть файл

@ -262,7 +262,14 @@ def call_response_parser(parser, response, request=None):
# parse response and set dependent variables (for garbage collector)
try:
if parser:
parser(response.json_body)
# For backwards compatibility, check if the parser accepts named arguments.
# If not, this is an older grammar that only supports a json body as the argument
import inspect
args, varargs, varkw, defaults = inspect.getargspec(parser)
if varkw=='kwargs':
parser(response.json_body, headers=response.headers_dict)
else:
parser(response.json_body)
# Check request's producers to verify dynamic objects were set
if request:
for producer in request.produces:

21
restler/engine/transport_layer/response.py
Просмотреть файл

@ -81,6 +81,27 @@ class HttpResponse(object):
except:
return None
@property
def headers_dict(self):
""" The parsed name-value pairs of the headers of the response
Headers which are not in the expected format are ignored.
@return: The headers
@rtype : Dict[Str, Str]
"""
headers_dict = {}
for header in self.headers:
payload_start_idx = header.index(":")
try:
header_name = header[0:payload_start_idx]
header_val = header[payload_start_idx+1:]
headers_dict[header_name] = header_val
except Exception as error:
print(f"Error parsing header: {x}", x)
pass
return headers_dict
@property
def json_body(self):
""" The json portion of the body if exists.

39
src/compiler/Restler.Compiler.Test/DependencyTests.fs
Просмотреть файл

@ -373,5 +373,44 @@ module Dependencies =
Assert.True(grammar.Contains("""restler_custom_payload("/subnets/{subnetName}/get/__body__", quoted=False)"""))
[<Fact>]
let ``response headers can be used as producers`` () =
let grammarOutputDirPath = ctx.testRootDirPath
let config = { Restler.Config.SampleConfig with
IncludeOptionalParameters = true
GrammarOutputDirectoryPath = Some grammarOutputDirPath
ResolveBodyDependencies = true
UseBodyExamples = Some true
SwaggerSpecFilePath = Some [(Path.Combine(Environment.CurrentDirectory, @"swagger\dependencyTests\response_headers.json"))]
CustomDictionaryFilePath = None
AnnotationFilePath = None
AllowGetProducers = true
}
Restler.Workflow.generateRestlerGrammar None config
let grammarFilePath = Path.Combine(grammarOutputDirPath,
Restler.Workflow.Constants.DefaultRestlerGrammarFileName)
let grammar = File.ReadAllText(grammarFilePath)
let expectedGrammarFilePath = Path.Combine(Environment.CurrentDirectory,
@"baselines\dependencyTests\header_response_writer_grammar.py")
let actualGrammarFilePath = Path.Combine(grammarOutputDirPath,
Restler.Workflow.Constants.DefaultRestlerGrammarFileName)
let grammarDiff = getLineDifferences expectedGrammarFilePath actualGrammarFilePath
let message = sprintf "Grammar (test without annotations) does not match baseline. First difference: %A" grammarDiff
Assert.True(grammarDiff.IsNone, message)
// Confirm the same works with annotations
let configWithAnnotations = { config with
AnnotationFilePath = Some (Path.Combine(Environment.CurrentDirectory, @"swagger\dependencyTests\response_headers_annotations.json"))}
Restler.Workflow.generateRestlerGrammar None configWithAnnotations
let expectedGrammarFilePath = Path.Combine(Environment.CurrentDirectory,
@"baselines\dependencyTests\header_response_writer_annotation_grammar.py")
let grammarDiff = getLineDifferences expectedGrammarFilePath actualGrammarFilePath
let message = sprintf "Grammar (test with annotations) does not match baseline. First difference: %A" grammarDiff
Assert.True(grammarDiff.IsNone, message)
interface IClassFixture<Fixtures.TestSetupAndCleanup>

12
src/compiler/Restler.Compiler.Test/Restler.Compiler.Test.fsproj
Просмотреть файл

@ -38,6 +38,12 @@
<Content Include="baselines\dependencyTests\path_in_dictionary_payload_grammar.py">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="baselines\dependencyTests\header_response_writer_grammar.py">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="baselines\dependencyTests\header_response_writer_annotation_grammar.py">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="swagger\dependencyTests\post_patch_dependency.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
@ -125,6 +131,12 @@
<Content Include="swagger\DependencyTests\lowercase_paths.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="swagger\DependencyTests\response_headers.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="swagger\DependencyTests\response_headers_annotations.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="swagger\DependencyTests\inconsistent_casing_paths.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

145
src/compiler/Restler.Compiler.Test/baselines/dependencyTests/header_response_writer_annotation_grammar.py Normal file
Просмотреть файл

@ -0,0 +1,145 @@
""" THIS IS AN AUTOMATICALLY GENERATED FILE!"""
from __future__ import print_function
import json
from engine import primitives
from engine.core import requests
from engine.errors import ResponseParsingException
from engine import dependencies
_service_user_post_Location_header = dependencies.DynamicVariable("_service_user_post_Location_header")
def parse_serviceuserpost(data, **kwargs):
""" Automatically generated response parser """
# Declare response variables
temp_7262 = None
if 'headers' in kwargs:
headers = kwargs['headers']
# Parse body if needed
if data:
pass
# Try to extract each dynamic object
if headers:
# Try to extract dynamic objects from headers
try:
temp_7262 = str(headers["Location"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
pass
# If no dynamic objects were extracted, throw.
if not (temp_7262):
raise ResponseParsingException("Error: all of the expected dynamic objects were not present in the response.")
# Set dynamic variables
if temp_7262:
dependencies.set_variable("_service_user_post_Location_header", temp_7262)
req_collection = requests.RequestCollection([])
# Endpoint: /service/user, method: Post
request = requests.Request([
primitives.restler_static_string("POST "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
{
'post_send':
{
'parser': parse_serviceuserpost,
'dependencies':
[
_service_user_post_Location_header.writer()
]
}
},
],
requestId="/service/user"
)
req_collection.add_request(request)
# Endpoint: /service/user/{userId}, method: Get
request = requests.Request([
primitives.restler_static_string("GET "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string("/"),
primitives.restler_static_string(_service_user_post_Location_header.reader(), quoted=False),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
],
requestId="/service/user/{userId}"
)
req_collection.add_request(request)
# Endpoint: /service/user/{userId}, method: Put
request = requests.Request([
primitives.restler_static_string("PUT "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string("/"),
primitives.restler_static_string(_service_user_post_Location_header.reader(), quoted=False),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
],
requestId="/service/user/{userId}"
)
req_collection.add_request(request)
# Endpoint: /service/user/{userId}, method: Delete
request = requests.Request([
primitives.restler_static_string("DELETE "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string("/"),
primitives.restler_static_string(_service_user_post_Location_header.reader(), quoted=False),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
],
requestId="/service/user/{userId}"
)
req_collection.add_request(request)

145
src/compiler/Restler.Compiler.Test/baselines/dependencyTests/header_response_writer_grammar.py Normal file
Просмотреть файл

@ -0,0 +1,145 @@
""" THIS IS AN AUTOMATICALLY GENERATED FILE!"""
from __future__ import print_function
import json
from engine import primitives
from engine.core import requests
from engine.errors import ResponseParsingException
from engine import dependencies
_service_user_post_userId_header = dependencies.DynamicVariable("_service_user_post_userId_header")
def parse_serviceuserpost(data, **kwargs):
""" Automatically generated response parser """
# Declare response variables
temp_7262 = None
if 'headers' in kwargs:
headers = kwargs['headers']
# Parse body if needed
if data:
pass
# Try to extract each dynamic object
if headers:
# Try to extract dynamic objects from headers
try:
temp_7262 = str(headers["userId"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
pass
# If no dynamic objects were extracted, throw.
if not (temp_7262):
raise ResponseParsingException("Error: all of the expected dynamic objects were not present in the response.")
# Set dynamic variables
if temp_7262:
dependencies.set_variable("_service_user_post_userId_header", temp_7262)
req_collection = requests.RequestCollection([])
# Endpoint: /service/user, method: Post
request = requests.Request([
primitives.restler_static_string("POST "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
{
'post_send':
{
'parser': parse_serviceuserpost,
'dependencies':
[
_service_user_post_userId_header.writer()
]
}
},
],
requestId="/service/user"
)
req_collection.add_request(request)
# Endpoint: /service/user/{userId}, method: Get
request = requests.Request([
primitives.restler_static_string("GET "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string("/"),
primitives.restler_static_string(_service_user_post_userId_header.reader(), quoted=False),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
],
requestId="/service/user/{userId}"
)
req_collection.add_request(request)
# Endpoint: /service/user/{userId}, method: Put
request = requests.Request([
primitives.restler_static_string("PUT "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string("/"),
primitives.restler_static_string(_service_user_post_userId_header.reader(), quoted=False),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
],
requestId="/service/user/{userId}"
)
req_collection.add_request(request)
# Endpoint: /service/user/{userId}, method: Delete
request = requests.Request([
primitives.restler_static_string("DELETE "),
primitives.restler_static_string("/"),
primitives.restler_static_string("api"),
primitives.restler_static_string("/"),
primitives.restler_static_string("service"),
primitives.restler_static_string("/"),
primitives.restler_static_string("user"),
primitives.restler_static_string("/"),
primitives.restler_static_string(_service_user_post_userId_header.reader(), quoted=False),
primitives.restler_static_string(" HTTP/1.1\r\n"),
primitives.restler_static_string("Accept: application/json\r\n"),
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
],
requestId="/service/user/{userId}"
)
req_collection.add_request(request)

57
src/compiler/Restler.Compiler.Test/baselines/dependencyTests/path_annotation_grammar.py
Просмотреть файл

@ -12,40 +12,51 @@ _stores_post_id = dependencies.DynamicVariable("_stores_post_id")
_stores_post_metadata = dependencies.DynamicVariable("_stores_post_metadata")
def parse_storespost(data):
def parse_storespost(data, **kwargs):
""" Automatically generated response parser """
# Declare response variables
temp_7262 = None
temp_8173 = None
temp_7680 = None
# Parse the response into json
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
if 'headers' in kwargs:
headers = kwargs['headers']
# Parse body if needed
if data:
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
pass
# Try to extract each dynamic object
try:
temp_7262 = str(data["delivery"]["metadata"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_7262 = str(data["delivery"]["metadata"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_8173 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_8173 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_7680 = str(data["metadata"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_7680 = str(data["metadata"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
# If no dynamic objects were extracted, throw.
@ -73,7 +84,7 @@ request = requests.Request([
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
{
'post_send':
{

33
src/compiler/Restler.Compiler.Test/baselines/dependencyTests/path_in_dictionary_payload_grammar.py
Просмотреть файл

@ -8,24 +8,33 @@ from engine import dependencies
_stores_post_id = dependencies.DynamicVariable("_stores_post_id")
def parse_storespost(data):
def parse_storespost(data, **kwargs):
""" Automatically generated response parser """
# Declare response variables
temp_7262 = None
# Parse the response into json
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
if 'headers' in kwargs:
headers = kwargs['headers']
# Parse body if needed
if data:
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
pass
# Try to extract each dynamic object
try:
temp_7262 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_7262 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
# If no dynamic objects were extracted, throw.
@ -49,7 +58,7 @@ request = requests.Request([
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
{
'post_send':
{

33
src/compiler/Restler.Compiler.Test/baselines/dictionaryTests/quoted_primitives_grammar.py
Просмотреть файл

@ -8,24 +8,33 @@ from engine import dependencies
_stores_post_id = dependencies.DynamicVariable("_stores_post_id")
def parse_storespost(data):
def parse_storespost(data, **kwargs):
""" Automatically generated response parser """
# Declare response variables
temp_7262 = None
# Parse the response into json
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
if 'headers' in kwargs:
headers = kwargs['headers']
# Parse body if needed
if data:
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
pass
# Try to extract each dynamic object
try:
temp_7262 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_7262 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
# If no dynamic objects were extracted, throw.
@ -49,7 +58,7 @@ request = requests.Request([
primitives.restler_static_string("Host: localhost:8888\r\n"),
primitives.restler_refreshable_authentication_token("authentication_token_tag"),
primitives.restler_static_string("\r\n"),
{
'post_send':
{

33
src/compiler/Restler.Compiler.Test/baselines/exampleTests/array_example_grammar.py
Просмотреть файл

@ -8,24 +8,33 @@ from engine import dependencies
_stores__storeId__order_post_id = dependencies.DynamicVariable("_stores__storeId__order_post_id")
def parse_storesstoreIdorderpost(data):
def parse_storesstoreIdorderpost(data, **kwargs):
""" Automatically generated response parser """
# Declare response variables
temp_7262 = None
# Parse the response into json
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
if 'headers' in kwargs:
headers = kwargs['headers']
# Parse body if needed
if data:
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))
pass
# Try to extract each dynamic object
try:
temp_7262 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
try:
temp_7262 = str(data["id"])
except Exception as error:
# This is not an error, since some properties are not always returned
pass
# If no dynamic objects were extracted, throw.
@ -112,7 +121,7 @@ request = requests.Request([
"awesome"
]}"""),
primitives.restler_static_string("\r\n"),
{
'post_send':
{

14
src/compiler/Restler.Compiler.Test/baselines/schemaTests/xMsPaths_grammar.json
Просмотреть файл

@ -121,6 +121,7 @@
"httpVersion": "1.1",
"responseParser": {
"writerVariables": [],
"headerWriterVariables": [],
"inputWriterVariables": [
{
"requestId": {
@ -137,7 +138,8 @@
"path"
]
},
"primitiveType": "String"
"primitiveType": "String",
"kind": "InputParameter"
}
]
},
@ -157,7 +159,8 @@
"path"
]
},
"primitiveType": "String"
"primitiveType": "String",
"kind": "InputParameter"
}
],
"requestMetadata": {
@ -285,6 +288,7 @@
"httpVersion": "1.1",
"responseParser": {
"writerVariables": [],
"headerWriterVariables": [],
"inputWriterVariables": [
{
"requestId": {
@ -301,7 +305,8 @@
"path"
]
},
"primitiveType": "String"
"primitiveType": "String",
"kind": "InputParameter"
}
]
},
@ -321,7 +326,8 @@
"path"
]
},
"primitiveType": "String"
"primitiveType": "String",
"kind": "InputParameter"
}
],
"requestMetadata": {

85
src/compiler/Restler.Compiler.Test/swagger/dependencyTests/response_headers.json Normal file
Просмотреть файл

@ -0,0 +1,85 @@
{
"basePath": "/api",
"consumes": [
"application/json"
],
"host": "localhost:8888",
"info": {
"description": "Small example for header dependencies.",
"title": "The title.",
"version": "1.0.0"
},
"paths": {
"/service/user": {
"post": {
"responses": {
"201": {
"headers": {
"Location": {
"description": "The location (URI) of the new resource",
"schema": {
"type": "string",
"format": "uri"
}
},
"userId": {
"description": "The ID of the new user.",
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/service/user/{userId}": {
"get": {
"parameters": [
{
"name": "userId",
"in": "path",
"type": "string",
"required": true
}
],
"responses": {
"200": {
"description": "Success"
}
}
},
"put": {
"parameters": [
{
"name": "userId",
"in": "path",
"type": "string",
"required": true
}
],
"responses": {
"201": {
"description": "Success"
}
}
},
"delete": {
"responses": {
"200": {
"description": "Success"
}
},
"parameters": [
{
"name": "userId",
"in": "path",
"type": "string",
"required": true
}
]
}
}
},
"swagger": "2.0"
}

10
src/compiler/Restler.Compiler.Test/swagger/dependencyTests/response_headers_annotations.json Normal file
Просмотреть файл

@ -0,0 +1,10 @@
{
"x-restler-global-annotations": [
{
"producer_endpoint": "/service/user",
"producer_method": "POST",
"producer_resource_name": "Location",
"consumer_param": "userId"
}
]
}

26
src/compiler/Restler.Compiler/ApiResourceTypes.fs
Просмотреть файл

@ -50,6 +50,8 @@ type ResourceReference =
| QueryResource of string
/// A body parameter
| BodyResource of JsonParameterReference
/// A header parameter
| HeaderResource of string
let private pluralizer = Pluralize.NET.Core.Pluralizer()
@ -92,10 +94,12 @@ type ApiResource(requestId:RequestId,
match resourceReference with
| PathResource pr ->
getContainerPartFromBody pr.responsePath
| QueryResource qr ->
| QueryResource _ ->
None
| BodyResource br ->
getContainerPartFromBody br.fullPath
| HeaderResource _ ->
None
let resourceName =
match resourceReference with
@ -105,6 +109,8 @@ type ApiResource(requestId:RequestId,
qr
| BodyResource br ->
br.name
| HeaderResource hr ->
hr
let isNestedBodyResource =
match resourceReference with
@ -114,13 +120,16 @@ type ApiResource(requestId:RequestId,
false
| BodyResource br ->
br.fullPath.getPathPropertyNameParts().Length > 1
| HeaderResource hr ->
false
let getContainerName() =
let containerNamePart =
match resourceReference with
| PathResource pr ->
getContainerPartFromPath pr.pathToParameter
| QueryResource qr ->
| QueryResource _
| HeaderResource _ ->
getContainerPartFromPath endpointParts
| BodyResource br ->
// If the path to property contains at least 2 identifiers, then it has a body container.
@ -244,25 +253,32 @@ type ApiResource(requestId:RequestId,
| BodyResource b -> b.fullPath.getJsonPointer()
| PathResource p ->
p.responsePath.getJsonPointer()
| QueryResource q -> None
| QueryResource _
| HeaderResource _ ->
None
member x.AccessPathParts =
match resourceReference with
| BodyResource b -> b.fullPath
| PathResource p -> p.responsePath
| QueryResource q -> { AccessPath.path = Array.empty }
| QueryResource _
| HeaderResource _ ->
{ AccessPath.path = Array.empty }
member x.getParentAccessPath() =
match resourceReference with
| BodyResource b -> b.fullPath.getParentPath()
| PathResource p -> p.responsePath.getParentPath()
| QueryResource q -> { path = Array.empty }
| QueryResource _
| HeaderResource _ ->
{ path = Array.empty }
member x.ResourceName =
match resourceReference with
| BodyResource b -> b.name
| PathResource p -> p.name
| QueryResource q -> q
| HeaderResource h -> h
// Gets the variable name that should be present in the response
// Example: /api/accounts/{accountId}

124
src/compiler/Restler.Compiler/CodeGenerator.fs
Просмотреть файл

@ -535,9 +535,9 @@ let generatePythonFromRequestElement includeOptionalParameters (requestId:Reques
| Some responseParser ->
let generateWriterStatement var =
sprintf "%s.writer()" var
let variablesReferencedInParser = responseParser.writerVariables @ responseParser.headerWriterVariables
let parserStatement =
match responseParser.writerVariables with
match variablesReferencedInParser with
| [] -> ""
| writerVariable::rest ->
sprintf @"'parser': %s,"
@ -545,7 +545,7 @@ let generatePythonFromRequestElement includeOptionalParameters (requestId:Reques
let postSend =
let writerVariablesList =
responseParser.writerVariables @ responseParser.inputWriterVariables
variablesReferencedInParser @ responseParser.inputWriterVariables
// TODO: generate this ID only once if possible.
|> List.map (fun producerWriter ->
let stmt = generateWriterStatement (generateDynamicObjectVariableName producerWriter.requestId (Some producerWriter.accessPathParts) "_")
@ -742,6 +742,10 @@ let getDynamicObjectDefinitions (writerVariables:seq<DynamicObjectWriterVariable
}
|> Seq.toList
type ResponseVariableKind =
| Body
| Header
let getResponseParsers (requests: Request list) =
let random = System.Random(0)
@ -749,20 +753,26 @@ let getResponseParsers (requests: Request list) =
let responseParsers = requests |> Seq.choose (fun r -> r.responseParser)
// First, define the dynamic variables initialized by the response parser
let dynamicObjectDefinitionsFromResponses =
let dynamicObjectDefinitionsFromBodyResponses =
getDynamicObjectDefinitions (responseParsers |> Seq.map (fun r -> r.writerVariables |> seq) |> Seq.concat)
let dynamicObjectDefinitionsFromHeaderResponses =
getDynamicObjectDefinitions (responseParsers |> Seq.map (fun r -> r.headerWriterVariables |> seq) |> Seq.concat)
let dynamicObjectDefinitionsFromInputParameters =
getDynamicObjectDefinitions (responseParsers |> Seq.map (fun r -> r.inputWriterVariables |> seq) |> Seq.concat)
let formatParserFunction (parser:ResponseParser) =
let functionName = NameGenerators.generateProducerEndpointResponseParserFunctionName
parser.writerVariables.[0].requestId
let functionName =
let writerVariables = parser.writerVariables @ parser.headerWriterVariables
NameGenerators.generateProducerEndpointResponseParserFunctionName writerVariables.[0].requestId
// Go through the producer fields and parse them all out of the response
let responseParsingStatements =
// STOPPED HERE:
// also do 'if true' for header parsing and body parsing where 'true' is if there are actually variables to parse out of there.
let getResponseParsingStatements writerVariables (variableKind:ResponseVariableKind) =
[
for w in parser.writerVariables do
for w in writerVariables do
let dynamicObjectVariableName = generateDynamicObjectVariableName w.requestId (Some w.accessPathParts) "_"
let tempVariableName = sprintf "temp_%d" (random.Next(10000))
let emptyInitStatement = sprintf "%s = None" tempVariableName
@ -774,10 +784,18 @@ let getResponseParsers (requests: Request list) =
else
sprintf "[\"%s\"]" part
let extractData =
w.accessPathParts.path |> Array.map getPath
|> String.concat ""
let parsingStatement = sprintf "%s = str(data%s)" tempVariableName extractData
let parsingStatement =
let dataSource, accessPath =
match variableKind with
| ResponseVariableKind.Body -> "data", w.accessPathParts.path
| ResponseVariableKind.Header -> "headers", w.accessPathParts.path |> Array.truncate 1
let extractData =
accessPath
|> Array.map getPath
|> String.concat ""
sprintf "%s = str(%s%s)" tempVariableName dataSource extractData
let initCheck = sprintf "if %s:" tempVariableName
let initStatement = sprintf "dependencies.set_variable(\"%s\", %s)"
dynamicObjectVariableName
@ -790,34 +808,62 @@ let getResponseParsers (requests: Request list) =
yield (emptyInitStatement, parsingStatement, initCheck, initStatement, tempVariableName, booleanConversionStatement)
]
let responseBodyParsingStatements = getResponseParsingStatements parser.writerVariables ResponseVariableKind.Body
let responseHeaderParsingStatements = getResponseParsingStatements parser.headerWriterVariables ResponseVariableKind.Header
let parsingStatementWithTryExcept parsingStatement (booleanConversionStatement:string option) =
sprintf "
try:
%s
%s
except Exception as error:
# This is not an error, since some properties are not always returned
pass
"
parsingStatement
try:
%s
%s
except Exception as error:
# This is not an error, since some properties are not always returned
pass
" parsingStatement
(if booleanConversionStatement.IsSome then booleanConversionStatement.Value else "")
let getParseBodyStatement() =
"""
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException("Exception parsing response, data was not valid json: {}".format(error))"""
let getHeaderParsingStatements responseHeaderParsingStatements =
let parsingStatements =
responseHeaderParsingStatements
|> List.map(fun (_,parsingStatement,_,_,_,booleanConversionStatement) ->
parsingStatementWithTryExcept parsingStatement booleanConversionStatement)
|> String.concat "\n"
sprintf """
if headers:
# Try to extract dynamic objects from headers
%s
pass
"""
parsingStatements
let functionDefinition = sprintf "
def %s(data):
def %s(data, **kwargs):
\"\"\" Automatically generated response parser \"\"\"
# Declare response variables
%s
# Parse the response into json
try:
data = json.loads(data)
except Exception as error:
raise ResponseParsingException(\"Exception parsing response, data was not valid json: {}\".format(error))
%s
if 'headers' in kwargs:
headers = kwargs['headers']
# Parse body if needed
if data:
%s
pass
# Try to extract each dynamic object
%s
%s
# If no dynamic objects were extracted, throw.
if not (%s):
raise ResponseParsingException(\"Error: all of the expected dynamic objects were not present in the response.\")
@ -826,30 +872,38 @@ def %s(data):
%s
"
functionName
(responseParsingStatements
// Response variable declarations (body and header)
(responseBodyParsingStatements
|> List.map(fun (emptyInitStatement,_,_,_,_,_) -> (TAB + emptyInitStatement)) |> String.concat "\n")
(responseHeaderParsingStatements
|> List.map(fun (emptyInitStatement,_,_,_,_,_) -> (TAB + emptyInitStatement)) |> String.concat "\n")
(responseParsingStatements
// Statement to parse the body
(if parser.writerVariables.Length > 0 then getParseBodyStatement() else "")
(responseBodyParsingStatements
|> List.map(fun (_,parsingStatement,_,_,_,booleanConversionStatement) ->
parsingStatementWithTryExcept parsingStatement booleanConversionStatement)
|> String.concat "\n")
(responseParsingStatements
|> List.map(fun (_,_,_,_,tempVariableName,_) ->
tempVariableName)
(if parser.headerWriterVariables.Length > 0 then getHeaderParsingStatements responseHeaderParsingStatements else "")
(responseBodyParsingStatements @ responseHeaderParsingStatements
|> List.map(fun (_,_,_,_,tempVariableName,_) ->
tempVariableName)
|> String.concat " or ")
(responseParsingStatements
(responseBodyParsingStatements @ responseHeaderParsingStatements
|> List.map(fun (_,_,initCheck,initStatement,_,_) ->
(TAB + initCheck + "\n" + TAB + TAB + initStatement)) |> String.concat "\n")
PythonGrammarElement.ResponseParserDefinition functionDefinition
let responseParsersWithParserFunction =
responseParsers |> Seq.filter (fun rp -> rp.writerVariables.Length > 0)
responseParsers |> Seq.filter (fun rp -> rp.writerVariables.Length + rp.headerWriterVariables.Length > 0)
[
yield! dynamicObjectDefinitionsFromResponses
yield! dynamicObjectDefinitionsFromBodyResponses
yield! dynamicObjectDefinitionsFromHeaderResponses
yield! dynamicObjectDefinitionsFromInputParameters
yield! (responseParsersWithParserFunction |> Seq.map (fun r -> formatParserFunction r) |> Seq.toList)
]

117
src/compiler/Restler.Compiler/Compiler.fs
Просмотреть файл

@ -50,7 +50,7 @@ type UserSpecifiedRequestConfig =
annotations: ProducerConsumerAnnotation option
}
let getWriterVariable (producer:Producer) =
let getWriterVariable (producer:Producer) (kind:DynamicObjectVariableKind) =
match producer with
| InputParameter (iop, _) ->
@ -58,12 +58,24 @@ let getWriterVariable (producer:Producer) =
requestId = iop.id.RequestId
accessPathParts = iop.getInputParameterAccessPath()
primitiveType = iop.id.PrimitiveType
kind = kind
}
| ResponseObject rp ->
let accessPathParts =
match rp.id.ResourceReference with
| HeaderResource hr ->
{ Restler.AccessPaths.AccessPath.path =
[|
hr
"header" // handle ambiguity with body
|] }
| _ ->
rp.id.AccessPathParts
{
requestId = rp.id.RequestId
accessPathParts = rp.id.AccessPathParts
accessPathParts = accessPathParts
primitiveType = rp.id.PrimitiveType
kind = kind
}
| _ ->
raise (invalidArg "producer" "only input parameter and response producers have an associated dynamic object")
@ -75,36 +87,48 @@ let getResponseParsers (dependencies:seq<ProducerConsumerDependency>) =
// Generate the parser for all the consumer variables (Note this means we need both producer
// and consumer pairs. A response parser is only generated if there is a consumer for one or more of the
// response properties.)
dependencies
|> Seq.choose (fun dep ->
match dep.producer with
| Some (ResponseObject _) ->
let writerVariable = getWriterVariable dep.producer.Value
Some (writerVariable, true)
| Some (InputParameter (_, _)) ->
let writerVariable = getWriterVariable dep.producer.Value
Some (writerVariable, false)
| _ -> None)
dependencies
|> Seq.filter (fun dep -> dep.producer.IsSome)
|> Seq.choose (fun dep ->
let writerVariableKind =
match dep.producer.Value with
| ResponseObject ro ->
match ro.id.ResourceReference with
| HeaderResource _ -> Some DynamicObjectVariableKind.Header
| _ -> Some DynamicObjectVariableKind.BodyResponseProperty
| InputParameter (_, _) ->
Some DynamicObjectVariableKind.InputParameter
| _ -> None
match writerVariableKind with
| Some v ->
getWriterVariable dep.producer.Value v
|> Some
| None -> None)
// Remove duplicates
// Producer may be linked to multiple consumers in separate dependency pairs
|> Seq.distinct
|> Seq.groupBy (fun (writerVariable, _) -> writerVariable.requestId)
|> Seq.groupBy (fun writerVariable -> writerVariable.requestId)
|> Map.ofSeq
|> Map.iter (fun requestId writerVariables ->
let writerVariables, inputWriterVariables =
writerVariables
|> Seq.fold
(fun (writerVariables, inputWriterVariables)
(writerVariable, isResponseObject) ->
if isResponseObject then
(writerVariable::writerVariables, inputWriterVariables)
else
(writerVariables, writerVariable::inputWriterVariables)) ([], [])
|> Map.iter (fun requestId allWriterVariables ->
let groupedWriterVariables = allWriterVariables |> Seq.groupBy (fun x -> x.kind)
|> Map.ofSeq
let parser =
{
writerVariables = writerVariables
inputWriterVariables = inputWriterVariables
writerVariables =
match groupedWriterVariables |> Map.tryFind DynamicObjectVariableKind.BodyResponseProperty with
| None -> []
| Some x -> x |> Seq.toList
inputWriterVariables =
match groupedWriterVariables |> Map.tryFind DynamicObjectVariableKind.InputParameter with
| None -> []
| Some x -> x |> Seq.toList
headerWriterVariables =
match groupedWriterVariables |> Map.tryFind DynamicObjectVariableKind.Header with
| None -> []
| Some x -> x |> Seq.toList
}
parsers.Add(requestId, parser))
@ -986,19 +1010,39 @@ let generateRequestGrammar (swaggerDocs:Types.ApiSpecFuzzingConfig list)
config.TrackFuzzedParameterNames
}
let allResponseProperties = seq {
for r in m.Value.Responses do
if validResponseCodes |> List.contains r.Key && not (isNull r.Value.ActualResponse.Schema) then
yield generateGrammarElementForSchema r.Value.ActualResponse.Schema (None, false) false
(true (*isRequired*), false (*isReadOnly*)) []
id
let allResponses = seq {
let responses = m.Value.Responses
|> Seq.filter (fun r -> validResponseCodes |> List.contains r.Key)
|> Seq.sortBy (fun r ->
let hasResponseBody = if isNull r.Value.ActualResponse.Schema then 1 else 0
let hasResponseHeaders = if r.Value.Headers |> Seq.isEmpty then 1 else 0
// Prefer the responses that have a response schema defined.
hasResponseBody, hasResponseHeaders, r.Key)
for r in responses do
let headerResponseSchema =
r.Value.Headers
|> Seq.map (fun h -> let headerSchema =
generateGrammarElementForSchema h.Value (None, false) false
(true (*isRequired*), false (*isReadOnly*)) []
id
h.Key, headerSchema)
|> Seq.toList
let bodyResponseSchema =
if isNull r.Value.ActualResponse.Schema then None
else
generateGrammarElementForSchema r.Value.ActualResponse.Schema (None, false) false
(true (*isRequired*), false (*isReadOnly*)) []
id
|> Some
{| bodyResponse = bodyResponseSchema
headerResponse = headerResponseSchema |}
}
// 'allResponseProperties' contains the schemas of all possible responses
// Pick just the first one for now
// TODO: capture all of them and generate cases for each one in the response parser
let responseProperties = allResponseProperties |> Seq.tryHead
let response = allResponses |> Seq.tryHead
let localAnnotations = Restler.Annotations.getAnnotationsFromExtensionData m.Value.ExtensionData "x-restler-annotations"
@ -1012,7 +1056,14 @@ let generateRequestGrammar (swaggerDocs:Types.ApiSpecFuzzingConfig list)
yield (requestId, { RequestData.requestParameters = requestParameters
localAnnotations = localAnnotations
responseProperties = responseProperties
responseProperties =
match response with
| None -> None
| Some r -> r.bodyResponse
responseHeaders =
match response with
| None -> []
| Some r -> r.headerResponse
requestMetadata = requestMetadata
exampleConfig = exampleConfig })
}

96
src/compiler/Restler.Compiler/Dependencies.fs
Просмотреть файл

@ -625,22 +625,36 @@ let findProducer (producers:Producers)
[ consumer.id.ProducerParameterName ; consumer.id.ResourceName ]
let matchingProducers =
possibleProducerParameterNames
|> Seq.distinct
|> Seq.choose (
fun producerParameterName ->
let mutationsDictionary, producer =
findProducerWithResourceName
producers
consumer
dictionary
allowGetProducers
perRequestDictionary
producerParameterName
if producer.IsSome then
Some (mutationsDictionary, producer)
else None
)
let possibleProducers =
possibleProducerParameterNames
|> Seq.distinct
|> Seq.choose (
fun producerParameterName ->
let mutationsDictionary, producer =
findProducerWithResourceName
producers
consumer
dictionary
allowGetProducers
perRequestDictionary
producerParameterName
if producer.IsSome then
Some (mutationsDictionary, producer)
else None
)
// Workaround: prefer a response over a dictionary payload.
// over a dictionary payload.
// TODO: this workaround should be removed when the
// producer-consumer dependency algorithm is improved to process
// dependencies grouped by paths, rather than independently.
possibleProducers
|> Seq.sortBy (fun (dictionary, producer) ->
match producer.Value with
| ResponseObject _ -> 1
| DictionaryPayload _ -> 2
| _ -> 3)
match matchingProducers |> Seq.tryHead with
| Some result -> result
| None -> dictionary, None
@ -745,20 +759,20 @@ let findAnnotation globalAnnotations
| (Some l, _) -> Some l
| (None, g) -> g
let getPayloadPrimitiveType (payload:FuzzingPayload) =
match payload with
| Constant (t,_) -> t
| Fuzzable (t,_,_,_) -> t
| Custom cp -> cp.primitiveType
| DynamicObject d -> d.primitiveType
| PayloadParts _ ->
PrimitiveType.String
let getProducer (request:RequestId) (response:ResponseProperties) =
// All possible properties in this response
let accessPaths = List<PropertyAccessPath>()
let getPayloadPrimitiveType (payload:FuzzingPayload) =
match payload with
| Constant (t,_) -> t
| Fuzzable (t,_,_,_) -> t
| Custom cp -> cp.primitiveType
| DynamicObject d -> d.primitiveType
| PayloadParts _ ->
PrimitiveType.String
let visitLeaf2 (parentAccessPath:string list) (p:LeafProperty) =
let resourceAccessPath = PropertyAccessPaths.getLeafAccessPathParts parentAccessPath p
let name =
@ -908,6 +922,15 @@ let createPathProducer (requestId:RequestId) (accessPath:PropertyAccessPath)
namingConvention, primitiveType)
}
let createHeaderResponseProducer (requestId:RequestId) (headerParameterName:string)
(namingConvention:NamingConvention option)
(primitiveType:PrimitiveType) =
{
ResponseProducer.id = ApiResource(requestId,
HeaderResource headerParameterName,
namingConvention, primitiveType)
}
/// Given an annotation, create an input-only producer for the specified producer.
let createInputOnlyProducerFromAnnotation (a:ProducerConsumerAnnotation)
@ -1123,6 +1146,20 @@ let extractDependencies (requestData:(RequestId*RequestData)[])
let resourceName = ap.Name
producers.AddResponseProducer(resourceName, producer)
for header in rd.responseHeaders do
// Add the header name as a producer only
let primitiveType =
let headerPayload = (snd header)
match headerPayload with
| Tree.LeafNode lp ->
getPayloadPrimitiveType lp.payload
| Tree.InternalNode (ip, _) ->
PrimitiveType.Object
let resourceName = fst header
let producer = createHeaderResponseProducer r resourceName namingConvention primitiveType
producers.AddResponseProducer(resourceName, producer)
// Also check for input-only producers that should be added for this request.
// At this time, only producers specified in annotations are supported.
// Find the corresponding parameter and add it as a producer.
@ -1371,7 +1408,12 @@ module DependencyLookup =
match producer with
| None -> defaultPayload
| Some (ResponseObject responseProducer) ->
let variableName = generateDynamicObjectVariableName responseProducer.id.RequestId (Some responseProducer.id.AccessPathParts) "_"
let variableName =
match responseProducer.id.ResourceReference with
| HeaderResource hr ->
(generateDynamicObjectVariableName responseProducer.id.RequestId (Some { AccessPath.path = [| hr ; "header"|]} ) "_")
| _ ->
generateDynamicObjectVariableName responseProducer.id.RequestId (Some responseProducer.id.AccessPathParts) "_"
// Mark the type of the dynamic object to be the type of the input parameter if available
let primitiveType =
match defaultPayload with
@ -1600,6 +1642,8 @@ let writeDependencies dependenciesFilePath dependencies (unresolvedOnly:bool) =
p.name
| QueryResource q ->
q
| HeaderResource h ->
h
| BodyResource b ->
b.fullPath.getJsonPointer().Value

1
src/compiler/Restler.Compiler/DependencyAnalysisTypes.fs
Просмотреть файл

@ -16,6 +16,7 @@ type RequestData =
requestParameters : RequestParameters
localAnnotations : seq<ProducerConsumerAnnotation>
responseProperties : ResponseProperties option
responseHeaders : (string * ResponseProperties) list
requestMetadata : RequestMetadata
exampleConfig : ExampleRequestPayload list option
}

27
src/compiler/Restler.Compiler/Grammar.fs
Просмотреть файл

@ -222,21 +222,6 @@ type ProducerConsumerAnnotation =
consumerParameter : AnnotationResourceReference
exceptConsumerId: RequestId list option
}
//with
// /// Given the 'consumerParameter', returns the access path or None if the consumer parameter
// /// is a name.
// member x.tryGetConsumerAccessPath =
// match x.consumerParameter with
// | ResourceName _ -> None
// | ResourcePath parts ->
// Some (parts |> String.concat ";")
// member x.tryGetProducerAccessPath =
// match x.producerParameter with
// | ResourceName _ -> None
// | ResourcePath parts ->
// Some (parts |> String.concat ";")
/// A property that does not have any nested properties
type LeafProperty =
@ -327,6 +312,11 @@ type TokenKind =
| Static of string
| Refreshable
/// The type of dynamic object variable
type DynamicObjectVariableKind =
| BodyResponseProperty
| Header
| InputParameter
type DynamicObjectWriterVariable =
{
@ -338,6 +328,9 @@ type DynamicObjectWriterVariable =
/// The type of the variable
primitiveType : PrimitiveType
/// The kind of the variable (e.g. header or response property)
kind : DynamicObjectVariableKind
}
/// Information needed to generate a response parser
@ -346,6 +339,9 @@ type ResponseParser =
/// The writer variables returned in the response
writerVariables : DynamicObjectWriterVariable list
/// The writer variables returned in the response headers
headerWriterVariables : DynamicObjectWriterVariable list
/// The writer variables that are written when the request is sent, and which
/// are not returned in the response
inputWriterVariables : DynamicObjectWriterVariable list
@ -417,7 +413,6 @@ type GrammarDefinition =
Requests : Request list
}
let generateDynamicObjectVariableName (requestId:RequestId) (accessPath:AccessPath option) delimiter =
// split endpoint, add "id" at the end. TBD: jobs_0 vs jobs_1 - where is the increment?
// See restler_parser.py line 800