Always try to parse responses from GET after async polling (#382)
Previously, if async polling from a PUT response in a certain format was received, that was used as the final response from which to parse dynamic objects, otherwise the GET request was used. This did not always work - in some cases, there were properties missing from the response or an empty body was returned. After this change, the first response on which the response parser will be invoked will be the one from the GET request. If the GET failed or parsing the response of the GET, the response from the PUT will still be tried as before. Testing: ad-hoc testing using original repro (in progress).
This commit is contained in:
marina-p
GitHub
Родитель
af3b6ea272
Коммит
9e91e8810b
|
|
@ -117,9 +117,9 @@ class CheckerBase:
|
|||
async_wait = Settings().get_max_async_resource_creation_time(request.request_id)
|
||||
|
||||
if check_async:
|
||||
response_to_parse, _, _ = async_request_utilities.try_async_poll(
|
||||
responses_to_parse, _, _ = async_request_utilities.try_async_poll(
|
||||
rendered_data, response, async_wait)
|
||||
request_utilities.call_response_parser(parser, response_to_parse)
|
||||
request_utilities.call_response_parser(parser, None, responses=responses_to_parse)
|
||||
seq.append_data_to_sent_list(rendered_data, parser, response, producer_timing_delay=0, max_async_wait_time=async_wait)
|
||||
return response, response_to_parse
|
||||
|
||||
|
|
|
|||
|
|
@ -1166,9 +1166,9 @@ class PayloadBodyChecker(CheckerBase):
|
|||
# send out the request and parse the response
|
||||
response = self._send_request(parser, rendered_data)
|
||||
async_wait = Settings().get_max_async_resource_creation_time(request.request_id)
|
||||
response_to_parse, _, _ = async_request_utilities.try_async_poll(
|
||||
responses_to_parse, _, _ = async_request_utilities.try_async_poll(
|
||||
rendered_data, response, async_wait)
|
||||
request_utilities.call_response_parser(parser, response_to_parse)
|
||||
request_utilities.call_response_parser(parser, None, responses=responses_to_parse)
|
||||
|
||||
self._set_refresh_req(request, response)
|
||||
|
||||
|
|
|
|||
|
|
@ -141,9 +141,13 @@ def try_async_poll(request_data, response, max_async_wait_time):
|
|||
@type max_async_wait_time: Int
|
||||
|
||||
@return: A tuple containing:
|
||||
- The response for parsing, which will either be the @response
|
||||
passed to this function or the response from the GET request
|
||||
if it was an async request.
|
||||
- The list of responses for parsing, which will either contain the @response
|
||||
passed to this function, or in the case of an async request, the
|
||||
response from the end of the async operation (if finished),
|
||||
and the response from the GET request.
|
||||
It is possible that the response of the async request when finished does not
|
||||
contain all of the expected fields in the schema of the async response, so as a
|
||||
workaround the GET response is also returned to try to obtain this additional information.
|
||||
- A boolean that is True if there was an error when creating the
|
||||
resource
|
||||
- A Boolean that tells the caller whether or not async polling was
|
||||
|
|
@ -154,7 +158,7 @@ def try_async_poll(request_data, response, max_async_wait_time):
|
|||
from utils.logger import raw_network_logging as RAW_LOGGING
|
||||
from utils.logger import print_async_results as LOG_RESULTS
|
||||
|
||||
response_to_parse = response
|
||||
responses_to_parse = []
|
||||
resource_error = False
|
||||
async_waited = False
|
||||
if Settings().wait_for_async_resource_creation and max_async_wait_time > 0\
|
||||
|
|
@ -178,11 +182,22 @@ def try_async_poll(request_data, response, max_async_wait_time):
|
|||
if poll_response.status_code in ['200' ,'201']:
|
||||
LOG_RESULTS(request_data,
|
||||
f"Resource creation succeeded after {time_str} seconds.")
|
||||
# Break and return the polling response to be parsed
|
||||
response_to_parse = poll_response
|
||||
|
||||
responses_to_parse.append(poll_response)
|
||||
# Also, attempt to execute a GET request corresponding to this resource and
|
||||
# return the response. This is used in case all of the expected properties are
|
||||
# not present in the async response.
|
||||
RAW_LOGGING("Attempting to get resources from GET request...")
|
||||
get_response = try_parse_GET_request(request_data)
|
||||
if get_response:
|
||||
# Use response from the GET request for parsing. If the GET request failed,
|
||||
# the caller will know to try and use the original PUT response for parsing
|
||||
responses_to_parse.insert(0, get_response)
|
||||
# Break and return the responses to be parsed
|
||||
break
|
||||
else:
|
||||
# The data is not in the polling response and must be fetched separately.
|
||||
# Try to execute a corresponding GET request and obtain
|
||||
# information from the response
|
||||
# This comes from Azure-Async responses that do not contain a Location: field
|
||||
# Check for the status of the resource
|
||||
response_body = json.loads(poll_response.json_body)
|
||||
|
|
@ -192,13 +207,14 @@ def try_async_poll(request_data, response, max_async_wait_time):
|
|||
f"Resource creation succeeded after {time_str} seconds.")
|
||||
get_response = try_parse_GET_request(request_data)
|
||||
if get_response:
|
||||
response_to_parse = get_response
|
||||
break
|
||||
responses_to_parse.append(get_response)
|
||||
elif done == "failed":
|
||||
LOG_RESULTS(request_data,
|
||||
f"The server reported that the resource creation Failed after {time_str} seconds.")
|
||||
resource_error = True
|
||||
break
|
||||
|
||||
# Break and return the responses to be parsed
|
||||
break
|
||||
except json.JSONDecodeError as err:
|
||||
LOG_RESULTS(request_data, "Failed to parse body of async response, retrying.")
|
||||
# This may have been due to a connection failure. Retry until max_async_wait_time.
|
||||
|
|
@ -228,6 +244,8 @@ def try_async_poll(request_data, response, max_async_wait_time):
|
|||
if get_response:
|
||||
# Use response from the GET request for parsing. If the GET request failed,
|
||||
# the caller will know to try and use the original PUT response for parsing
|
||||
response_to_parse = get_response
|
||||
responses_to_parse.append(get_response)
|
||||
|
||||
return response_to_parse, resource_error, async_waited
|
||||
if len(responses_to_parse) == 0:
|
||||
responses_to_parse.append(response)
|
||||
return responses_to_parse, resource_error, async_waited
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ def send_request_data(rendered_data):
|
|||
|
||||
return response
|
||||
|
||||
def call_response_parser(parser, response, request=None):
|
||||
def call_response_parser(parser, response, request=None, responses=None):
|
||||
""" Calls a specified parser on a response
|
||||
|
||||
@param parser: The parser function to calls
|
||||
|
|
@ -253,6 +253,9 @@ def call_response_parser(parser, response, request=None):
|
|||
@type response: HttpResponse
|
||||
@param request: The request whose parser is being called
|
||||
@type request: Request (None ok)
|
||||
@param responses: A list of responses
|
||||
This parameter is used if the response is not specified
|
||||
@type responses: List[HttpResponse]
|
||||
|
||||
@return False if there was a parser exception
|
||||
@rtype Boolean
|
||||
|
|
@ -260,27 +263,36 @@ def call_response_parser(parser, response, request=None):
|
|||
"""
|
||||
from utils.logger import write_to_main
|
||||
# parse response and set dependent variables (for garbage collector)
|
||||
try:
|
||||
if parser:
|
||||
# For backwards compatibility, check if the parser accepts named arguments.
|
||||
# If not, this is an older grammar that only supports a json body as the argument
|
||||
import inspect
|
||||
args, varargs, varkw, defaults = inspect.getargspec(parser)
|
||||
if varkw=='kwargs':
|
||||
parser(response.json_body, headers=response.headers_dict)
|
||||
else:
|
||||
parser(response.json_body)
|
||||
# Check request's producers to verify dynamic objects were set
|
||||
if request:
|
||||
for producer in request.produces:
|
||||
if dependencies.get_variable(producer) == 'None':
|
||||
err_str = f'Failed to parse {producer}; it is now set to None.'
|
||||
write_to_main(err_str)
|
||||
_RAW_LOGGING(err_str)
|
||||
except (ResponseParsingException, AttributeError) as error:
|
||||
_RAW_LOGGING(str(error))
|
||||
return False
|
||||
return True
|
||||
|
||||
if responses is None:
|
||||
responses = []
|
||||
responses.append(response)
|
||||
|
||||
for response in responses:
|
||||
try:
|
||||
if parser:
|
||||
# For backwards compatibility, check if the parser accepts named arguments.
|
||||
# If not, this is an older grammar that only supports a json body as the argument
|
||||
import inspect
|
||||
args, varargs, varkw, defaults = inspect.getargspec(parser)
|
||||
|
||||
if varkw=='kwargs':
|
||||
parser(response.json_body, headers=response.headers_dict)
|
||||
else:
|
||||
parser(response.json_body)
|
||||
# Print a diagnostic message if some dynamic objects were not set.
|
||||
# The parser only fails if all of the objects were not set.
|
||||
if request:
|
||||
for producer in request.produces:
|
||||
if dependencies.get_variable(producer) == 'None':
|
||||
err_str = f'Failed to parse {producer}; it is now set to None.'
|
||||
write_to_main(err_str)
|
||||
_RAW_LOGGING(err_str)
|
||||
return True
|
||||
except (ResponseParsingException, AttributeError) as error:
|
||||
_RAW_LOGGING(str(error))
|
||||
|
||||
return False
|
||||
|
||||
def get_hostname_from_line(line):
|
||||
""" Gets the hostname from a request definition's Host: line
|
||||
|
|
|
|||
|
|
@ -380,12 +380,12 @@ class Sequence(object):
|
|||
prev_producer_timing_delay = Settings().get_producer_timing_delay(prev_request.request_id)
|
||||
|
||||
prev_response = request_utilities.send_request_data(prev_rendered_data)
|
||||
prev_response_to_parse, resource_error, async_waited = async_request_utilities.try_async_poll(
|
||||
prev_responses_to_parse, resource_error, async_waited = async_request_utilities.try_async_poll(
|
||||
prev_rendered_data, prev_response, prev_req_async_wait)
|
||||
prev_parser_threw_exception = False
|
||||
# Response may not exist if there was an error sending the request or a timeout
|
||||
if prev_parser and prev_response_to_parse:
|
||||
prev_parser_threw_exception = not request_utilities.call_response_parser(prev_parser, prev_response_to_parse, prev_request)
|
||||
if prev_parser and prev_responses_to_parse:
|
||||
prev_parser_threw_exception = not request_utilities.call_response_parser(prev_parser, None, request=prev_request, responses=prev_responses_to_parse)
|
||||
prev_status_code = prev_response.status_code
|
||||
|
||||
# If the async logic waited for the resource, this wait already included the required
|
||||
|
|
@ -487,12 +487,12 @@ class Sequence(object):
|
|||
req_async_wait = Settings().get_max_async_resource_creation_time(request.request_id)
|
||||
|
||||
response = request_utilities.send_request_data(rendered_data)
|
||||
response_to_parse, resource_error, _ = async_request_utilities.try_async_poll(
|
||||
responses_to_parse, resource_error, _ = async_request_utilities.try_async_poll(
|
||||
rendered_data, response, req_async_wait)
|
||||
parser_exception_occurred = False
|
||||
# Response may not exist if there was an error sending the request or a timeout
|
||||
if parser and response_to_parse:
|
||||
parser_exception_occurred = not request_utilities.call_response_parser(parser, response_to_parse, request)
|
||||
if parser and responses_to_parse:
|
||||
parser_exception_occurred = not request_utilities.call_response_parser(parser, None, request=request, responses=responses_to_parse)
|
||||
status_code = response.status_code
|
||||
if not status_code:
|
||||
return RenderedSequence(failure_info=FailureInformation.MISSING_STATUS_CODE)
|
||||
|
|
@ -652,10 +652,10 @@ class Sequence(object):
|
|||
""" Gets the token, sends the requst, performs async wait, parses response, returns status code """
|
||||
rendered_data = self.get_request_data_with_token(request_data.rendered_data)
|
||||
response = request_utilities.send_request_data(rendered_data)
|
||||
response_to_parse, _, _ = async_request_utilities.try_async_poll(
|
||||
responses_to_parse, _, _ = async_request_utilities.try_async_poll(
|
||||
rendered_data, response, request_data.max_async_wait_time)
|
||||
if request_data.parser:
|
||||
request_utilities.call_response_parser(request_data.parser, response_to_parse)
|
||||
request_utilities.call_response_parser(request_data.parser, None, responses=responses_to_parse)
|
||||
return response.status_code
|
||||
|
||||
# Send all but the last request in the sequence
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче