Add demo checker. (#556)

Closes #506.

Testing:
- manual testing by adding the checker to the engine settings and invoking it in 'Test' mode by adding ```--enable_checkers demo``` on the command line.

```
  "custom_checkers": ["C:\\git\\restler-fuzzer\\restler\\checkers\\demo_checker.py"]
```

restler/checkers/README.md

@@ -3,7 +3,7 @@ The checkers are created as subclasses to the CheckerBase abstract base class.
 the subclasses' logs and maintaining data members that are shared across all checkers.
 The CheckerBase class also declares and/or defines functions that are required for each checker subclass.
 
-See the checkers in this directory for examples.
+See the checkers in this directory for examples.  A simple checker demo is also included in this directory, which can be used as a template for creating a custom checker.   To run the demo checker, add it to the engine settings file `custom_checkers` setting.
 
 ## Data Members:
 * _checker_log: The CheckerLog log file for the checker
@@ -19,7 +19,7 @@ See the checkers in this directory for examples.
 checker. It can be thought of as a checker's "main" entry point.
 * _send_request: This function sends a request to the service under test and then returns the response. The function request_utilities.call_response_parser() should be called after this function
 in order to update resource dependencies (due to the new request that was just executed) and make these visible to the garbage collector (otherwise resources created by the newly sent request will not be garbage collected).
-* _render_and_send_data: This function renders data for a request, sends the request to the service under test, and then adds that rendered data and its response to a sequence's sent-request-data list. 
+* _render_and_send_data: This function renders data for a request, sends the request to the service under test, and then adds that rendered data and its response to a sequence's sent-request-data list.
   * This sent-request-data list is used exclusively for replaying sequences to test for bug reproducibility. Because checkers tend not
   to use the sequences.render function to render and send requests, the sent-request-data is never added to the list. This means that,
   in order to replay the sequence when a bug is found, this function must be called.
@@ -39,10 +39,10 @@ rule violation is detected.
 # Creating a Checker
 ## To create a checker, the following rules must be adhered to:
 
-* The checker must be defined in its own file and this file must be specified in the settings.json file like this 
+* The checker must be defined in its own file and this file must be specified in the settings.json file like this
+
+  ```"custom_checkers": ["C:\\<path>\\my_new_checker.py"]```
 
-  ```"custom_checkers": ["C:\\<path>\\my_new_checker.py"]``` 
-  
   or be added to checkers/\_\_init\_\_.py (the order of this list defines the order in which the checkers will be called).
 * The checker must inherit from CheckerBase.
 * The checker's class name must end with Checker.

restler/checkers/checker_base.py

@@ -5,11 +5,12 @@ import threading
 
 from abc import ABCMeta, abstractmethod
 from checkers.checker_log import CheckerLog
-
 import engine.core.async_request_utilities as async_request_utilities
 import engine.core.request_utilities as request_utilities
+import engine.core.sequences as sequences
 import engine.transport_layer.messaging as messaging
 from engine.transport_layer.response import *
+from engine.bug_bucketing import BugBuckets
 
 from restler_settings import Settings
 from engine.errors import ResponseParsingException
@@ -209,3 +210,24 @@ class CheckerBase:
             self._checker_log.checker_print(f"\nSuspect sequence: {status_code}")
         for req in seq:
             self._checker_log.checker_print(f"{req.method} {req.endpoint}")
+
+    def _execute_start_of_sequence(self):
+        """ Send all requests in the sequence up until the last request
+
+        @return: Sequence of n predecessor requests send to server
+        @rtype : Sequence
+
+        """
+        if len(self._sequence.requests) > 1:
+            RAW_LOGGING("Re-rendering and sending start of sequence")
+        new_seq = sequences.Sequence([])
+        for request in self._sequence.requests[:-1]:
+            new_seq = new_seq + sequences.Sequence(request)
+            response, _ = self._render_and_send_data(new_seq, request)
+            # Check to make sure a bug wasn't uncovered while executing the sequence
+            if response and response.has_bug_code():
+                self._print_suspect_sequence(new_seq, response)
+                BugBuckets.Instance().update_bug_buckets(new_seq, response.status_code, origin=self.__class__.__name__)
+
+        return new_seq
+

restler/checkers/demo_checker.py

@@ -0,0 +1,126 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+""" A simple demo checker that reports a bug. """
+from __future__ import print_function
+
+from checkers.checker_base import *
+
+from engine.bug_bucketing import BugBuckets
+import engine.core.sequences as sequences
+from engine.errors import TimeOutException
+
+
+class DemoChecker(CheckerBase):
+    """ A simple checker that runs after every request, and reports 2 bugs. """
+    # Dictionary used for determining whether a request has already
+    # been sent for the current generation.
+    # { generation : set(request.hex_definitions) }
+    generation_executed_requests = dict()
+
+    # Keep track of how many bugs were reported
+    # For demo purposes only
+    bugs_reported = 0
+
+    def __init__(self, req_collection, fuzzing_requests):
+        CheckerBase.__init__(self, req_collection, fuzzing_requests)
+
+    def apply(self, rendered_sequence, lock):
+        """ Fuzzes each value in the parameters of this request as specified by
+        the custom dictionary and settings for this checker.
+
+        @param rendered_sequence: Object containing the rendered sequence information
+        @type  rendered_sequence: RenderedSequence
+        @param lock: Lock object used to sync more than one fuzzing job
+        @type  lock: thread.Lock
+
+        @return: None
+        @rtype : None
+
+        """
+        if not rendered_sequence.sequence:
+            return
+
+        # This needs to be set for the base implementation that executes the sequence.
+        self._sequence = rendered_sequence.sequence
+
+        last_request = self._sequence.last_request
+        generation = self._sequence.length
+
+        self._checker_log.checker_print(f"Testing request: {last_request.endpoint} {last_request.method}")
+
+        # Just run this checker once for the endpoint and method
+        # If all schema combinations are desired, use 'last_request.hex_definition' instead below
+        request_hash = last_request.method_endpoint_hex_definition
+        if DemoChecker.generation_executed_requests.get(generation) is None:
+            # This is the first time this checker has seen this generation, create empty set of requests
+            DemoChecker.generation_executed_requests[generation] = set()
+        elif request_hash in DemoChecker.generation_executed_requests[generation]:
+            # This request type has already been tested for this generation
+            return
+        # Add the last request to the generation_executed_requests dictionary for this generation
+        DemoChecker.generation_executed_requests[generation].add(request_hash)
+
+        # Set up pre-requisites required to run the request
+        # The code below sets up the state and re-executes the requests on which this request depends on.
+        req_async_wait = Settings().get_max_async_resource_creation_time(last_request.request_id)
+        new_seq = self._execute_start_of_sequence()
+
+        # Add the last request of the sequence to the new sequence
+        checked_seq = new_seq + sequences.Sequence(last_request)
+        # Add the sent prefix requests for replay
+        checked_seq.set_sent_requests_for_replay(new_seq.sent_request_data_list)
+        # Create a placeholder sent data, so it can be replaced below when bugs are detected for replays
+        checked_seq.append_data_to_sent_list("GET /", None,  HttpResponse(), max_async_wait_time=req_async_wait)
+
+        # Render the current request combination
+        rendered_data, parser, tracked_parameters = \
+            next(last_request.render_iter(self._req_collection.candidate_values_pool,
+                                          skip=last_request._current_combination_id - 1,
+                                          preprocessing=False))
+
+        # Resolve dependencies
+        if not Settings().ignore_dependencies:
+            rendered_data = checked_seq.resolve_dependencies(rendered_data)
+
+        # Exit if time budget exceeded
+        if Monitor().remaining_time_budget <= 0:
+            raise TimeOutException('Exceed Timeout')
+
+        # Send the request and get a response
+        response = request_utilities.send_request_data(rendered_data)
+        responses_to_parse, resource_error, _ = async_request_utilities.try_async_poll(
+            rendered_data, response, req_async_wait)
+
+        # Response may not exist if there was an error sending the request or a timeout
+        if parser and responses_to_parse:
+            # The response parser must be invoked so that dynamic objects created by this
+            # request are initialized, adding them to the list of objects for the GC to clean up.
+            parser_exception_occurred = not request_utilities.call_response_parser(parser, None,
+                                                                                   request=last_request,
+                                                                                   responses=responses_to_parse)
+
+        if response and self._rule_violation(checked_seq, response, valid_response_is_violation=True):
+            checked_seq.replace_last_sent_request_data(rendered_data, parser, response, max_async_wait_time=req_async_wait)
+            self._print_suspect_sequence(checked_seq, response)
+            BugBuckets.Instance().update_bug_buckets(checked_seq, response.status_code, origin=self.__class__.__name__)
+            self.bugs_reported += 1
+
+        self._checker_log.checker_print(f"Tested request"
+                                        f"{last_request.endpoint} {last_request.method}, combination "
+                                        f"{last_request._current_combination_id}.")
+
+    def _false_alarm(self, seq, response):
+        """ For demo purposes, returns 'True' if more than 2 requests were tested by this checker.
+        This causes at most 2 bugs to be reported by this checker.
+
+        @param seq: The sequence that contains the request with the rule violation
+        @type  seq: Sequence
+        @param response: Body of response.
+        @type  response: Str
+
+        @return: True if false alarm detected
+        @rtype : Bool
+
+        """
+        return self.bugs_reported == 2

restler/checkers/invalid_dynamic_object_checker.py

@@ -78,24 +78,6 @@ class InvalidDynamicObjectChecker(CheckerBase):
                 self._print_suspect_sequence(new_seq, response)
 
 
-    def _execute_start_of_sequence(self):
-        """ Send all requests in the sequence up until the last request
-
-        @return: Sequence of n predecessor requests send to server
-        @rtype : Sequence
-
-        """
-        RAW_LOGGING("Re-rendering and sending start of sequence")
-        new_seq = sequences.Sequence([])
-        for request in self._sequence.requests[:-1]:
-            new_seq = new_seq + sequences.Sequence(request)
-            response, _ = self._render_and_send_data(new_seq, request)
-            # Check to make sure a bug wasn't uncovered while executing the sequence
-            if response and response.has_bug_code():
-                self._print_suspect_sequence(new_seq, response)
-                BugBuckets.Instance().update_bug_buckets(new_seq, response.status_code, origin=self.__class__.__name__)
-
-        return new_seq
 
     def _prepare_invalid_requests(self, data):
         """ Prepares requests with invalid dynamic objects.

restler/restler.py

@@ -12,16 +12,15 @@ from subprocess import call
 import os
 import sys
 import signal
-import time
 import json
 import shutil
 import argparse
 import checkers
 import restler_settings
-import atexit
-from threading import Thread
+import traceback
 
 import utils.logger as logger
+
 import engine.bug_bucketing as bug_bucketing
 import engine.dependencies as dependencies
 import engine.core.preprocessing as preprocessing
@@ -111,9 +110,10 @@ def get_checker_list(req_collection, fuzzing_requests, enable_list, disable_list
     # Add any custom checkers
     for custom_checker_file_path in custom_checkers:
         try:
-            utils.import_utilities.load_module('custom_checkers', custom_checker_file_path)
+            import_utilities.load_module('custom_checkers', custom_checker_file_path)
             logger.write_to_main(f"Loaded custom checker from {custom_checker_file_path}", print_to_console=True)
         except Exception as err:
+            traceback.print_exc()
             logger.write_to_main(f"Failed to load custom checker {custom_checker_file_path}: {err!s}", print_to_console=True)
             sys.exit(-1)