Updated to use hostname when making socket connections (#22)

* Updated to use hostname when making socket connections. Restler no longer requires ip address or port

* Code review updates

Co-authored-by: rifiles <v-rifile@microsoft.com>

docs/user-guide/Fuzzing.md

@@ -15,9 +15,9 @@ Inputs: (same as for test)
 
 ## How to invoke RESTler in fuzz-lean or fuzz mode
 
-```C:\RESTler\restler\restler.exe fuzz-lean --grammar_file <RESTLer grammar.py file> --dictionary_file <RESTler fuzzing-dictionary.json file> --target_ip <IP address> --target_port <port number> --token_refresh_interval <time in seconds> --token_refresh_command <command>```
+```C:\RESTler\restler\restler.exe fuzz-lean --grammar_file <RESTLer grammar.py file> --dictionary_file <RESTler fuzzing-dictionary.json file> --token_refresh_interval <time in seconds> --token_refresh_command <command>```
 
-```C:\RESTler\restler\restler.exe fuzz --grammar_file <RESTLer grammar.py file> --dictionary_file <RESTler fuzzing-dictionary.json file> --target_ip <IP address> --target_port <port number> --token_refresh_interval <time in seconds> --token_refresh_command <command> --time_budget <max number of hours (default 1)```
+```C:\RESTler\restler\restler.exe fuzz --grammar_file <RESTLer grammar.py file> --dictionary_file <RESTler fuzzing-dictionary.json file> --token_refresh_interval <time in seconds> --token_refresh_command <command> --time_budget <max number of hours (default 1)```
 
 An optional settings file can also be passed to RESTler by adding the command-line option `--settings <path_to_settings_file.json>`.
 For a list of available settings, see [SettingsFile](SettingsFile.md).

docs/user-guide/Replay.md

@@ -7,10 +7,10 @@ In *Replay* mode, RESTler can replay a sequence from a bug_bucket log that was c
 To reproduce a bug bucket using RESTler,
 send the following command (as an example):
 
-`C:\RESTler\restler\restler.exe replay --replay_log C:\restler-test\Test\RestlerResults\experiment20652\bug_buckets\PayloadBodyChecker_500_1.txt --target_ip 127.0.0.1 --target_port 8888 --token_refresh_command "<command>" --token_refresh_interval 30`
+`C:\RESTler\restler\restler.exe replay --replay_log C:\restler-test\Test\RestlerResults\experiment20652\bug_buckets\PayloadBodyChecker_500_1.txt --token_refresh_command "<command>" --token_refresh_interval 30`
 
-In this example, RESTler will replay the log `PayloadBodyChecker_500_1.txt`
-and send the requests to `127.0.0.1:8888`.
+In this example, RESTler will replay the log `PayloadBodyChecker_500_1.txt`.
+If authentication is required to replay the sequence, the authentication options must be specified during replay.
 
 As you can see above,
 the IP, port, and authorization token refresh command/interval are all used

docs/user-guide/Testing.md

@@ -17,7 +17,7 @@ Authentication options: configuring authentication is described in [Authenticati
 
 How to invoke RESTler in test mode:
 
-`C:\RESTler\restler\Restler.exe test --grammar_file <RESTLer grammar.py file> --dictionary_file <RESTler fuzzing-dictionary.json file> --target_ip <IP address> --target_port <port number> --token_refresh_interval <time in seconds> --token_refresh_command <command>`
+`C:\RESTler\restler\Restler.exe test --grammar_file <RESTLer grammar.py file> --dictionary_file <RESTler fuzzing-dictionary.json file> --token_refresh_interval <time in seconds> --token_refresh_command <command>`
 
 Outputs: see the sub-directory Test
 

docs/user-guide/TutorialDemoServer.md

@@ -26,7 +26,7 @@ For this simple tutorial, we will use the default values automatically generated
 
 Let's run RESTler in test mode to see what specification coverage we get with this default RESTler grammar:
 
-`C:\RESTler\restler\restler.exe test --grammar_file C:\restler-test\Compile\grammar.py --dictionary_file C:\restler-test\Compile\dict.json --settings C:\restler-test\Compile\engine_settings.json --target_ip 127.0.0.1 --target_port 8888 --no_ssl`
+`C:\RESTler\restler\restler.exe test --grammar_file C:\restler-test\Compile\grammar.py --dictionary_file C:\restler-test\Compile\dict.json --settings C:\restler-test\Compile\engine_settings.json --no_ssl`
 
 (For help, run `C:\RESTler\restler\restler.exe --help`)
 
@@ -68,7 +68,7 @@ By looking at `network.testing.<...>.txt`, we can see that RESTler attempts to e
 
 Let's now try to run restler in Fuzz-lean mode.
 
-`C:\RESTler\restler\restler.exe fuzz-lean --grammar_file C:\restler-test\Compile\grammar.py --dictionary_file C:\restler-test\Compile\dict.json --settings C:\restler-test\Compile\engine_settings.json --target_ip 127.0.0.1 --target_port 8888 --no_ssl`
+`C:\RESTler\restler\restler.exe fuzz-lean --grammar_file C:\restler-test\Compile\grammar.py --dictionary_file C:\restler-test\Compile\dict.json --settings C:\restler-test\Compile\engine_settings.json --no_ssl`
 
 The results are in a new `FuzzLean` directory and the experiment results can be found in `C:\restler-test\FuzzLean\RestlerResults\experiment<...>\`. The `logs\` directory should contain the same coverage results as the previous Test run, but you should also now see a new `bug_buckets` directory.
 
@@ -97,7 +97,7 @@ Right above the body in the header you should also see `StructMissing_/id/checks
 
 Now let's try to fuzz:
 
-`C:\RESTler\restler\restler.exe fuzz --grammar_file C:\restler-test\Compile\grammar.py --dictionary_file C:\restler-test\Compile\dict.json --settings C:\restler-test\Compile\engine_settings.json --target_ip 127.0.0.1 --target_port 8888 --no_ssl --time_budget 1`
+`C:\RESTler\restler\restler.exe fuzz --grammar_file C:\restler-test\Compile\grammar.py --dictionary_file C:\restler-test\Compile\dict.json --settings C:\restler-test\Compile\engine_settings.json --no_ssl --time_budget 1`
 
 The new __time_budget__ parameter is the length, in hours, to perform the fuzzing run.
 

restler-quick-start.py

@@ -62,10 +62,14 @@ def test_spec(ip, port, use_ssl, compile_dir, restler_dll_path):
     """
     command = (
         f"dotnet {restler_dll_path} test --grammar_file {compile_dir.joinpath('grammar.py')} --dictionary_file {compile_dir.joinpath('dict.json')}"
-        f" --settings {compile_dir.joinpath('engine_settings.json')} --target_ip {ip} --target_port {port}"
+        f" --settings {compile_dir.joinpath('engine_settings.json')}"
     )
     if not use_ssl:
         command = f"{command} --no_ssl"
+    if ip is not None:
+        command = f"{command} --target_ip {ip}"
+    if port is not None:
+        command = f"{command} --target_port {port}"
 
     with usedir(RESTLER_TEMP_DIR):
         subprocess.run(command, shell=True)
@@ -78,10 +82,10 @@ if __name__ == '__main__':
                         type=str, required=True)
     parser.add_argument('--ip',
                         help='The IP of the service to test',
-                        type=str, required=True)
+                        type=str, required=False, default=None)
     parser.add_argument('--port',
                         help='The port of the service to test',
-                        type=str, required=True)
+                        type=str, required=False, default=None)
     parser.add_argument('--restler_drop_dir',
                         help="The path to the RESTler drop",
                         type=str, required=True)

restler/end_to_end_tests/test_quick_start.py

@@ -36,7 +36,7 @@ if __name__ == '__main__':
     try:
         # Run the quick start script
         output = subprocess.run(
-            f'python ./restler-quick-start.py --api_spec_path {swagger_path} --ip 127.0.0.1 --port 8888 --restler_drop_dir {restler_drop_dir}',
+            f'python ./restler-quick-start.py --api_spec_path {swagger_path} --restler_drop_dir {restler_drop_dir}',
             shell=True, capture_output=True
         )
         # Kill demo server

restler/engine/core/driver.py

@@ -25,6 +25,7 @@ import engine.core.fuzzing_monitor as fuzzing_monitor
 from engine.core.requests import GrammarRequestCollection
 from engine.core.requests import FailureInformation
 from engine.core.request_utilities import execute_token_refresh_cmd
+from engine.core.request_utilities import get_hostname_from_line
 from engine.core.fuzzing_monitor import Monitor
 from engine.errors import TimeOutException
 from engine.errors import ExhaustSeqCollectionException
@@ -720,6 +721,13 @@ def replay_sequence_from_log(replay_log_filename, token_refresh_cmd):
                 line = line.rstrip('\n')
                 line = line.replace('\\r', '\r')
                 line = line.replace('\\n', '\n')
+                if not Settings().host:
+                    # Extract hostname from request
+                    hostname = get_hostname_from_line(line)
+                    if hostname is None:
+                        raise Exception("Host not found in request. The replay log may be corrupted.")
+                    Settings().set_hostname(hostname)
+
                 # Append the request data to the list
                 # None is for the parser, which does not currently run during replays.
                 send_data.append(sequences.SentRequestData(line, None))

restler/engine/core/request_utilities.py

@@ -24,6 +24,8 @@ last_refresh = 0
 latest_token_value = 'NO-TOKEN-SPECIFIED'
 latest_shadow_token_value = 'NO-SHADOW-TOKEN-SPECIFIED'
 
+HOST_PREFIX = 'Host: '
+
 class EmptyTokenException(Exception):
     pass
 
@@ -251,6 +253,20 @@ def call_response_parser(parser, response, request=None):
         return False
     return True
 
+def get_hostname_from_line(line):
+    """ Gets the hostname from a request definition's Host: line
+
+    @param line: The line to extract the hostname
+    @type  line: Str
+    @return: The hostname or None if not found
+    @rtype : Str or None
+
+    """
+    try:
+        return line.split(HOST_PREFIX, 1)[1].split('\r\n', 1)[0]
+    except:
+        return None
+
 def _RAW_LOGGING(log_str):
     """ Wrapper for the raw network logging function.
     Necessary to avoid circular dependency with logging.

restler/engine/core/requests.py

@@ -26,6 +26,9 @@ from enum import Enum
 class EmptyRequestException(Exception):
     pass
 
+class InvalidGrammarException(Exception):
+    pass
+
 class FailureInformation(Enum):
     SEQUENCE = 1
     RESOURCE_CREATION = 2
@@ -374,18 +377,51 @@ class Request(object):
                     self._consumes.remove(var_name)
         self._create_once_requests += rendered_sequence.sequence.sent_request_data_list
 
+    def get_host_index(self):
+        """ Gets the index of the definition line containing the Host parameter
+
+        @return: The index of the Host parameter or -1 if not found
+        @rtype : Int
+
+        """
+        for i, line in enumerate(self._definition):
+            try:
+                if isinstance(line[1], str) and line[1].startswith(request_utilities.HOST_PREFIX):
+                    return i
+            except:
+                # ignore line parsing exceptions - error will be returned if host not found
+                pass
+        return -1
+
     def update_host(self):
         """ Updates the Host field for every request with the one specified in Settings
 
         @return: None
         @rtype : None
 
+        """
+        new_host_line = primitives.restler_static_string(f"{request_utilities.HOST_PREFIX}{Settings().host}\r\n")
+        host_idx = self.get_host_index()
+        if host_idx >= 0:
+            self._definition[host_idx] = new_host_line
+        else:
+            # Host not in grammar, add it
+            header_idx = self.header_start_index()
+            if header_idx < 0:
+                raise InvalidGrammarException
+            self._definition.insert(header_idx, new_host_line)
+
+    def header_start_index(self):
+        """ Gets the index of the first header line in the definition
+
+        @return: The index of the first header line in the definition or -1 if not found
+        @rtype : Int
+
         """
         for i, line in enumerate(self._definition):
-            if isinstance(line[1], str) and line[1].startswith("Host: "):
-                self._definition[i] =\
-                    primitives.restler_static_string(f"Host: {Settings().host}\r\n")
-                break
+            if isinstance(line[1], str) and 'HTTP/1.1' in line[1]:
+                return i + 1
+        return -1
 
     def render_iter(self, candidate_values_pool, skip=0, preprocessing=False):
         """ This is the core method that renders values combinations in a
@@ -681,6 +717,19 @@ class RequestCollection(object):
         for req in self._requests:
             req.update_host()
 
+    def get_host_from_grammar(self):
+        """ Gets the hostname from the grammar
+
+        @return: The hostname or None if not found
+        @rtype : Str or None
+
+        """
+        for req in self._requests:
+            idx = req.get_host_index()
+            if idx >= 0:
+                return request_utilities.get_hostname_from_line(req._definition[idx][1])
+        return None
+
     def add_request(self, request):
         """ Adds a new request in the collection of requests.
 

restler/engine/transport_layer/messaging.py

@@ -37,23 +37,22 @@ class HttpSock(object):
         self._request_throttle_sec = (float)(Settings().request_throttle_ms/1000.0)\
             if Settings().request_throttle_ms else None
 
-        if connection_settings:
-            self.connection_settings = connection_settings
-        else:
-            self.connection_settings = ConnectionSettings('127.0.0.1', 80, use_ssl=True)
+        self.connection_settings = connection_settings
+
         try:
             self._sock = None
+            host = Settings().host
+            target_ip = self.connection_settings.target_ip or host
+            target_port = self.connection_settings.target_port
             if Settings().use_test_socket:
                 self._sock = TestSocket(Settings().test_server)
             elif self.connection_settings.use_ssl:
-                self._sock = ssl.wrap_socket(
-                    socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                )
+                context = ssl.create_default_context()
+                with socket.create_connection((target_ip, target_port or 443)) as sock:
+                    self._sock = context.wrap_socket(sock, server_hostname=host)
             else:
                 self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-
-            server_address = (self.connection_settings.target_ip, self.connection_settings.target_port)
-            self._sock.connect(server_address)
+                self._sock.connect((target_ip, target_port or 80))
         except Exception as error:
             raise TransportLayerException(f"Exception Creating Socket: {error!s}")
 

restler/restler.py

@@ -403,7 +403,27 @@ if __name__ == '__main__':
     monitor.renderings_monitor.set_memoize_invalid_past_renderings_on()
 
     if settings.host:
-        req_collection.update_hosts()
+        try:
+            req_collection.update_hosts()
+        except requests.InvalidGrammarException:
+            sys.exit(-1)
+    else:
+        host = req_collection.get_host_from_grammar()
+        if host is not None:
+            if ':' in host:
+                # If hostname includes port, split it out
+                host_split = host.split(':')
+                host = host_split[0]
+                if settings.connection_settings.target_port is None:
+                    settings.set_port(host_split[1])
+            settings.set_hostname(host)
+        else:
+            logger.write_to_main(
+                "Host not found in grammar. "
+                "Add the host to your spec or launch RESTler with --host parameter.",
+                 print_to_console=True
+            )
+            sys.exit(-1)
 
     # Filter and get the requests to be used for fuzzing
     fuzzing_requests = preprocessing.create_fuzzing_req_collection(args.path_regex)

restler/restler_settings.py

@@ -589,6 +589,29 @@ class RestlerSettings(object):
         self._custom_dictionaries.set_val(custom_dicts)
         self._create_once_endpoints.set_val(create_once_endpoints)
 
+    def set_hostname(self, hostname):
+        """ Sets the hostname
+
+        @param hostname: The hostname to set
+        @type  hostname: Str
+        @return: None
+        @rtype : None
+
+        """
+        self._host.val = hostname
+
+    def set_port(self, port):
+        """ Sets the port
+
+        @param port: The port to set
+        @type  port: Int
+        @return: None
+        @rtype : None
+
+        """
+        self._target_port.val = int(port)
+        self._connection_settings.target_port = int(port)
+
     def in_smoke_test_mode(self) -> bool:
         """ Returns whether or not we are running a smoke test
 
@@ -671,9 +694,5 @@ class RestlerSettings(object):
             raise OptionValidationError("Must specify command to refresh token")
         if self.token_refresh_cmd and not self.token_refresh_interval:
             raise OptionValidationError("Must specify refresh period in seconds")
-        if not self._target_ip.val:
-            raise OptionValidationError("Target IP is required")
-        if not self._target_port.val:
-            raise OptionValidationError("Target Port is required")
         if self.request_throttle_ms and self.fuzzing_jobs != 1:
             raise OptionValidationError("Request throttling not available for multiple fuzzing jobs")

restler/test_servers/README.md

@@ -1,7 +1,7 @@
 ## To run example in test mode:
 
 ```
-python -B C:\...restler\restler.py --fuzzing_mode directed-smoke-test --restler_grammar c:\...restler\test_servers\unit_test_server\test_grammar.py --custom_mutations c:\...restler\test_servers\unit_test_server\test_dict.json --target_ip 1.2.3.4 --target_port 443 --use_test_socket --garbage_collector_interval 30
+python -B C:\...restler\restler.py --fuzzing_mode directed-smoke-test --restler_grammar c:\...restler\test_servers\unit_test_server\test_grammar.py --custom_mutations c:\...restler\test_servers\unit_test_server\test_dict.json --use_test_socket --garbage_collector_interval 30
 ```
 
 test_grammar.py can be updated as any typical RESTler grammar.
@@ -18,7 +18,7 @@ test_grammar.py should pass the smoke test with 33/33 requests rendered,
 but the server is intentionally not free of errors.
 To return a test with planted bugs, run the same command with test_grammar_bugs.py:
 ```
-python -B C:\...restler\restler.py --fuzzing_mode directed-smoke-test --restler_grammar c:\...restler\test_servers\unit_test_server\test_grammar_bugs.py --custom_mutations c:\...restler\test_servers\unit_test_server\test_dict.json --target_ip 1.2.3.4 --target_port 443 --use_test_socket --garbage_collector_interval 30 --enable_checkers leakagerule useafterfree resourcehierarchy
+python -B C:\...restler\restler.py --fuzzing_mode directed-smoke-test --restler_grammar c:\...restler\test_servers\unit_test_server\test_grammar_bugs.py --custom_mutations c:\...restler\test_servers\unit_test_server\test_dict.json --use_test_socket --garbage_collector_interval 30 --enable_checkers leakagerule useafterfree resourcehierarchy
 ```
 
 

restler/unit_tests/test_basic_functionality_end_to_end.py

@@ -28,9 +28,8 @@ Restler_Path = os.path.join(os.path.dirname(__file__), '..', 'restler.py')
 
 Common_Settings = [
     "python", "-B", Restler_Path, "--use_test_socket",
-    '--target_ip', '1', '--target_port', '1',
     '--custom_mutations', f'{os.path.join(Test_File_Directory, "test_dict.json")}',
-    "--garbage_collection_interval", "30"
+    "--garbage_collection_interval", "30", "--host", "unittest"
 ]
 
 class FunctionalityTests(unittest.TestCase):

restler/unit_tests/test_restler_settings.py

@@ -339,18 +339,6 @@ class RestlerSettingsTest(unittest.TestCase):
         with self.assertRaises(InvalidValueError):
             RestlerSettings(user_args)
 
-    def test_missing_ip(self):
-        user_args = {'target_port': 500}
-        settings = RestlerSettings(user_args)
-        with self.assertRaises(OptionValidationError):
-            settings.validate_options()
-
-    def test_missing_port(self):
-        user_args = {'target_ip': '192.168.0.1'}
-        settings = RestlerSettings(user_args)
-        with self.assertRaises(OptionValidationError):
-            settings.validate_options()
-
     def test_random_walk_sequence_length(self):
         user_args = {'target_port': 500,
                      'target_ip': '192.168.0.1',

restler/utils/logger.py

@@ -383,7 +383,7 @@ def update_bug_buckets(bug_buckets, bug_request_data, bug_hash, additional_log_s
             print(f" Hash: {bug_hash}\n", file=bug_file)
             print(" To attempt to reproduce this bug using restler, run restler with the command", file=bug_file)
             print(" line option of --replay_log <path_to_this_log>.", file=bug_file)
-            print(" The other required options are target_ip, target_port, and token_refresh_cmd.", file=bug_file)
+            print(" If an authentication token is required, you must also specify the token_refresh_cmd.", file=bug_file)
             print("\n This log may contain specific values for IDs or names that were generated", file=bug_file)
             print(" during fuzzing, using the fuzzing dictionary. Such names will be re-played", file=bug_file)
             print(" without modification. You must update the replay log manually with any changes", file=bug_file)

src/driver/Program.fs

@@ -45,7 +45,9 @@ let usage() =
         --grammar_file <grammar file>
         --dictionary_file <dictionary file>
         --target_ip <ip>
+            If specified, sets the IP address to this specific value instead of using the hostname.
         --target_port <port>
+            If specified, overrides the default port, which is 443 with SSL, 80 with no SSL.
         --token_refresh_interval <interval with which to refresh the token>
         --token_refresh_command <full command line to refresh token.>
             The command line must be enclosed in double quotes. Paths must be absolute.
@@ -57,7 +59,7 @@ let usage() =
         --no_ssl
             When connecting to the service, do not use SSL.  The default is to connect with SSL.
         --host <Host string>
-            If specified, this string will override the Host in each request.
+            If specified, this string will set or override the Host in each request.
         --settings <engine settings file>
         --enable_checkers <list of checkers>
         --disable_checkers <list of checkers>
@@ -79,8 +81,6 @@ let usage() =
 
     replay options:
         <Required options from 'test' mode as above:
-            --target_ip
-            --target_port
             --token_refresh_cmd. >
         --replay_log <path to the RESTler bug bucket repro file>. "
     exit 1
@@ -168,8 +168,6 @@ module Fuzz =
         [
             sprintf "--restler_grammar \"%s\"" parameters.grammarFilePath
             sprintf "--custom_mutations \"%s\"" parameters.mutationsFilePath
-            sprintf "--target_ip %s" parameters.targetIp
-            sprintf "--target_port %s" parameters.targetPort
             sprintf "--set_version %s" CurrentVersion
 
             (match parameters.refreshableTokenOptions with
@@ -207,7 +205,12 @@ module Fuzz =
              | None -> ""
              | Some t ->
                 sprintf "--time_budget %f" t)
-
+            (match parameters.targetIp with
+             | Some t -> sprintf "--target_ip %s" t
+             | None -> "")
+            (match parameters.targetPort with
+             | Some t -> sprintf "--target_port %s" t
+             | None -> "")
             // internal options
             "--include_user_agent"
             "--no_tokens_in_logs t"
@@ -325,12 +328,6 @@ module Fuzz =
 let rec parseEngineArgs task (args:EngineParameters) = function
     | [] ->
         // Check for unspecified parameters
-        if args.targetIp = DefaultEngineParameters.targetIp then
-            Logging.logError <| sprintf "Target IP must be specified."
-            usage()
-        if args.targetPort = DefaultEngineParameters.targetPort then
-           Logging.logError <| sprintf "Target port must be specified."
-           usage()
         match task with
         | Compile ->
             failwith "Invalid function usage."
@@ -359,9 +356,9 @@ let rec parseEngineArgs task (args:EngineParameters) = function
             usage()
         parseEngineArgs task  { args with mutationsFilePath = Path.GetFullPath(mutationsFilePath) } rest
     | "--target_ip"::targetIp::rest ->
-        parseEngineArgs task { args with targetIp = targetIp } rest
+        parseEngineArgs task { args with targetIp = Some targetIp } rest
     | "--target_port"::targetPort::rest ->
-        parseEngineArgs task { args with targetPort = targetPort } rest
+        parseEngineArgs task { args with targetPort = Some targetPort } rest
     | "--token_refresh_command"::refreshCommand::rest ->
         let parameters = UserSpecifiedCommand refreshCommand
         let options =

src/driver/Types.fs

@@ -29,10 +29,10 @@ type EngineParameters =
         mutationsFilePath : string
 
         /// The IP of the endpoint being fuzzed
-        targetIp : string
+        targetIp : string option
 
         /// The port of the endpoint being fuzzed
-        targetPort : string
+        targetPort : string option
 
         /// The maximum fuzzing time in hours
         maxDurationHours : float
@@ -71,8 +71,8 @@ let DefaultEngineParameters =
     {
         grammarFilePath = ""
         mutationsFilePath = ""
-        targetIp = ""
-        targetPort = ""
+        targetIp = None
+        targetPort = None
         refreshableTokenOptions = None
         maxDurationHours = float 0
         producerTimingDelay = 0