servo: Merge #16131 - Use NetworkConnector directly to account for replaced hosts (from nox:tungstenite); r=jdm

Source-Repo: https://github.com/servo/servo
Source-Revision: 1105100c3beb9cff3a71677937269ecc523f2924

--HG--
extra : subtree_source : https%3A//hg.mozilla.org/projects/converted-servo-linear
extra : subtree_revision : c15f500b44f6ceba2296635274bbc14b2e61753b

servo/components/net/fetch/methods.rs

@@ -192,7 +192,7 @@ pub fn main_fetch(request: Rc<Request>,
             .read()
             .unwrap()
             .is_host_secure(request.current_url().domain().unwrap()) {
-           request.url_list.borrow_mut().last_mut().unwrap().as_mut_url().unwrap().set_scheme("https").unwrap();
+           request.url_list.borrow_mut().last_mut().unwrap().as_mut_url().set_scheme("https").unwrap();
         }
     }
 

servo/components/net/http_loader.rs

@@ -15,6 +15,7 @@ use hsts::HstsList;
 use hyper::Error as HttpError;
 use hyper::LanguageTag;
 use hyper::client::{Pool, Request as HyperRequest, Response as HyperResponse};
+use hyper::client::pool::PooledStream;
 use hyper::header::{AcceptEncoding, AcceptLanguage, AccessControlAllowCredentials};
 use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods};
 use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod};
@@ -24,17 +25,18 @@ use hyper::header::{IfUnmodifiedSince, IfModifiedSince, IfNoneMatch, Location, P
 use hyper::header::{QualityItem, Referer, SetCookie, UserAgent, qitem};
 use hyper::header::Origin as HyperOrigin;
 use hyper::method::Method;
-use hyper::net::Fresh;
+use hyper::net::{Fresh, HttpStream, HttpsStream, NetworkConnector};
 use hyper::status::StatusCode;
 use hyper_serde::Serde;
 use log;
 use msg::constellation_msg::PipelineId;
 use net_traits::{CookieSource, FetchMetadata, NetworkError, ReferrerPolicy};
-use net_traits::hosts::replace_hosts;
+use net_traits::hosts::replace_host;
 use net_traits::request::{CacheMode, CredentialsMode, Destination, Origin};
 use net_traits::request::{RedirectMode, Referrer, Request, RequestMode, ResponseTainting};
 use net_traits::response::{HttpsState, Response, ResponseBody, ResponseType};
 use openssl;
+use openssl::ssl::SslStream;
 use openssl::ssl::error::{OpensslError, SslError};
 use resource_thread::AuthCache;
 use servo_url::{ImmutableOrigin, ServoUrl};
@@ -125,12 +127,18 @@ struct NetworkHttpRequestFactory {
     pub connector: Arc<Pool<Connector>>,
 }
 
+impl NetworkConnector for NetworkHttpRequestFactory {
+    type Stream = PooledStream<HttpsStream<SslStream<HttpStream>>>;
+
+    fn connect(&self, host: &str, port: u16, scheme: &str) -> Result<Self::Stream, HttpError> {
+        self.connector.connect(&replace_host(host), port, scheme)
+    }
+}
+
 impl NetworkHttpRequestFactory {
     fn create(&self, url: ServoUrl, method: Method, headers: Headers)
               -> Result<HyperRequest<Fresh>, NetworkError> {
-        let connection = HyperRequest::with_connector(method,
-                                                      url.clone().into_url().unwrap(),
-                                                      &*self.connector);
+        let connection = HyperRequest::with_connector(method, url.clone().into_url(), self);
 
         if let Err(HttpError::Ssl(ref error)) = connection {
             let error: &(Error + Send + 'static) = &**error;
@@ -222,7 +230,7 @@ fn strict_origin_when_cross_origin(referrer_url: ServoUrl, url: ServoUrl) -> Opt
 fn strip_url(mut referrer_url: ServoUrl, origin_only: bool) -> Option<ServoUrl> {
     if referrer_url.scheme() == "https" || referrer_url.scheme() == "http" {
         {
-            let referrer = referrer_url.as_mut_url().unwrap();
+            let referrer = referrer_url.as_mut_url();
             referrer.set_username("").unwrap();
             referrer.set_password(None).unwrap();
             referrer.set_fragment(None);
@@ -408,7 +416,6 @@ fn obtain_response(request_factory: &NetworkHttpRequestFactory,
                    is_xhr: bool)
                    -> Result<(WrappedHttpResponse, Option<ChromeToDevtoolsControlMsg>), NetworkError> {
     let null_data = None;
-    let connection_url = replace_hosts(&url);
 
     // loop trying connections in connection pool
     // they may have grown stale (disconnected), in which case we'll get
@@ -439,7 +446,7 @@ fn obtain_response(request_factory: &NetworkHttpRequestFactory,
         }
 
         if log_enabled!(log::LogLevel::Info) {
-            info!("{} {}", method, connection_url);
+            info!("{} {}", method, url);
             for header in headers.iter() {
                 info!(" - {}", header);
             }
@@ -448,7 +455,7 @@ fn obtain_response(request_factory: &NetworkHttpRequestFactory,
 
         let connect_start = precise_time_ms();
 
-        let request = try!(request_factory.create(connection_url.clone(), method.clone(),
+        let request = try!(request_factory.create(url.clone(), method.clone(),
                                                   headers.clone()));
 
         let connect_end = precise_time_ms();
@@ -900,7 +907,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
         let headers = &mut *http_request.headers.borrow_mut();
         let host = Host {
             hostname: current_url.host_str().unwrap().to_owned(),
-            port: current_url.port_or_known_default()
+            port: current_url.port()
         };
         headers.set(host);
         // unlike http_loader, we should not set the accept header

servo/components/net/websocket_loader.rs

@@ -9,7 +9,7 @@ use http_loader;
 use hyper::header::{Host, SetCookie};
 use net_traits::{CookieSource, MessageData, WebSocketCommunicate};
 use net_traits::{WebSocketConnectData, WebSocketDomAction, WebSocketNetworkEvent};
-use net_traits::hosts::replace_hosts;
+use net_traits::hosts::replace_host_in_url;
 use servo_url::ServoUrl;
 use std::ascii::AsciiExt;
 use std::sync::{Arc, Mutex, RwLock};
@@ -25,84 +25,6 @@ use websocket::stream::WebSocketStream;
 use websocket::ws::receiver::Receiver as WSReceiver;
 use websocket::ws::sender::Sender as Sender_Object;
 
-// https://fetch.spec.whatwg.org/#concept-websocket-establish
-fn establish_a_websocket_connection(resource_url: &ServoUrl,
-                                    origin: String,
-                                    protocols: Vec<String>,
-                                    cookie_jar: Arc<RwLock<CookieStorage>>)
-                                    -> WebSocketResult<(Option<String>,
-                                                        Sender<WebSocketStream>,
-                                                        Receiver<WebSocketStream>)> {
-    // Steps 1-2 are not really applicable here, given we don't exactly go
-    // through the same infrastructure as the Fetch spec.
-
-    if should_be_blocked_due_to_bad_port(resource_url) {
-        // Subset of steps 11-12, we inline the bad port check here from the
-        // main fetch algorithm for the same reason steps 1-2 are not
-        // applicable.
-        return Err(WebSocketError::RequestError("Request should be blocked due to bad port."));
-    }
-
-    // Steps 3-7.
-    let net_url = replace_hosts(resource_url);
-    let mut request = try!(Client::connect(net_url.as_url()));
-
-    // Client::connect sets the Host header to the host of the URL that is
-    // passed to it, so we need to reset it afterwards to the correct one.
-    request.headers.set(Host {
-        hostname: resource_url.host_str().unwrap().to_owned(),
-        port: resource_url.port(),
-    });
-
-    // Step 8.
-    if !protocols.is_empty() {
-        request.headers.set(WebSocketProtocol(protocols.clone()));
-    }
-
-    // Steps 9-10.
-    // TODO: support for permessage-deflate extension.
-
-    // Subset of step 11.
-    // See step 2 of https://fetch.spec.whatwg.org/#concept-fetch.
-    request.headers.set(Origin(origin));
-
-    // Transitive subset of step 11.
-    // See step 17.1 of https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch.
-    http_loader::set_request_cookies(&resource_url, &mut request.headers, &cookie_jar);
-
-    // Step 11, somewhat.
-    let response = try!(request.send());
-
-    // Step 12, 14.
-    try!(response.validate());
-
-    // Step 13 and transitive subset of step 14.
-    // See step 6 of http://tools.ietf.org/html/rfc6455#section-4.1.
-    let protocol_in_use = response.protocol().and_then(|header| {
-        // https://github.com/whatwg/fetch/issues/515
-        header.first().cloned()
-    });
-    if let Some(ref protocol_name) = protocol_in_use {
-        if !protocols.is_empty() && !protocols.iter().any(|p| (&**p).eq_ignore_ascii_case(protocol_name)) {
-            return Err(WebSocketError::ProtocolError("Protocol in Use not in client-supplied protocol list"));
-        };
-    };
-
-    // Transitive subset of step 11.
-    // See step 15 of https://fetch.spec.whatwg.org/#http-network-fetch.
-    if let Some(cookies) = response.headers.get::<SetCookie>() {
-        let mut jar = cookie_jar.write().unwrap();
-        for cookie in &**cookies {
-            if let Some(cookie) = Cookie::new_wrapped(cookie.clone(), resource_url, CookieSource::HTTP) {
-                jar.push(cookie, resource_url, CookieSource::HTTP);
-            }
-        }
-    }
-
-    let (sender, receiver) = response.begin().split();
-    Ok((protocol_in_use, sender, receiver))
-}
-
 pub fn init(connect: WebSocketCommunicate, connect_data: WebSocketConnectData, cookie_jar: Arc<RwLock<CookieStorage>>) {
     thread::Builder::new().name(format!("WebSocket connection to {}", connect_data.resource_url)).spawn(move || {
         let channel = establish_a_websocket_connection(&connect_data.resource_url,
@@ -182,3 +104,81 @@ pub fn init(connect: WebSocketCommunicate, connect_data: WebSocketConnectData, c
         }
     }).expect("Thread spawning failed");
 }
+
+// https://fetch.spec.whatwg.org/#concept-websocket-establish
+fn establish_a_websocket_connection(resource_url: &ServoUrl,
+                                    origin: String,
+                                    protocols: Vec<String>,
+                                    cookie_jar: Arc<RwLock<CookieStorage>>)
+                                    -> WebSocketResult<(Option<String>,
+                                                        Sender<WebSocketStream>,
+                                                        Receiver<WebSocketStream>)> {
+    // Steps 1-2 are not really applicable here, given we don't exactly go
+    // through the same infrastructure as the Fetch spec.
+
+    if should_be_blocked_due_to_bad_port(resource_url) {
+        // Subset of steps 11-12, we inline the bad port check here from the
+        // main fetch algorithm for the same reason steps 1-2 are not
+        // applicable.
+        return Err(WebSocketError::RequestError("Request should be blocked due to bad port."));
+    }
+
+    // Steps 3-7.
+    let net_url = replace_host_in_url(resource_url.clone());
+    let mut request = try!(Client::connect(net_url.as_url()));
+
+    // Client::connect sets the Host header to the host of the URL that is
+    // passed to it, so we need to reset it afterwards to the correct one.
+    request.headers.set(Host {
+        hostname: resource_url.host_str().unwrap().to_owned(),
+        port: resource_url.port(),
+    });
+
+    // Step 8.
+    if !protocols.is_empty() {
+        request.headers.set(WebSocketProtocol(protocols.clone()));
+    }
+
+    // Steps 9-10.
+    // TODO: support for permessage-deflate extension.
+
+    // Subset of step 11.
+    // See step 2 of https://fetch.spec.whatwg.org/#concept-fetch.
+    request.headers.set(Origin(origin));
+
+    // Transitive subset of step 11.
+    // See step 17.1 of https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch.
+    http_loader::set_request_cookies(&resource_url, &mut request.headers, &cookie_jar);
+
+    // Step 11, somewhat.
+    let response = try!(request.send());
+
+    // Step 12, 14.
+    try!(response.validate());
+
+    // Step 13 and transitive subset of step 14.
+    // See step 6 of http://tools.ietf.org/html/rfc6455#section-4.1.
+    let protocol_in_use = response.protocol().and_then(|header| {
+        // https://github.com/whatwg/fetch/issues/515
+        header.first().cloned()
+    });
+    if let Some(ref protocol_name) = protocol_in_use {
+        if !protocols.is_empty() && !protocols.iter().any(|p| (&**p).eq_ignore_ascii_case(protocol_name)) {
+            return Err(WebSocketError::ProtocolError("Protocol in Use not in client-supplied protocol list"));
+        };
+    };
+
+    // Transitive subset of step 11.
+    // See step 15 of https://fetch.spec.whatwg.org/#http-network-fetch.
+    if let Some(cookies) = response.headers.get::<SetCookie>() {
+        let mut jar = cookie_jar.write().unwrap();
+        for cookie in &**cookies {
+            if let Some(cookie) = Cookie::new_wrapped(cookie.clone(), resource_url, CookieSource::HTTP) {
+                jar.push(cookie, resource_url, CookieSource::HTTP);
+            }
+        }
+    }
+
+    let (sender, receiver) = response.begin().split();
+    Ok((protocol_in_use, sender, receiver))
+}

servo/components/net_traits/hosts.rs

@@ -4,6 +4,7 @@
 
 use parse_hosts::HostsFile;
 use servo_url::ServoUrl;
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::env;
 use std::fs::File;
@@ -56,19 +57,24 @@ pub fn parse_hostsfile(hostsfile_content: &str) -> HashMap<String, IpAddr> {
     host_table
 }
 
-pub fn replace_hosts(url: &ServoUrl) -> ServoUrl {
-    HOST_TABLE.lock().unwrap().as_ref().map_or_else(|| url.clone(),
-                                                    |host_table| host_replacement(host_table, url))
+pub fn replace_host(host: &str) -> Cow<str> {
+    HOST_TABLE.lock().unwrap().as_ref()
+        .and_then(|table| table.get(host))
+        .map_or(host.into(), |replaced_host| replaced_host.to_string().into())
 }
 
-pub fn host_replacement(host_table: &HashMap<String, IpAddr>, url: &ServoUrl) -> ServoUrl {
-    url.domain()
-        .and_then(|domain| {
-            host_table.get(domain).map(|ip| {
-                let mut new_url = url.clone();
-                new_url.set_ip_host(*ip).unwrap();
-                new_url
-            })
-        })
-        .unwrap_or_else(|| url.clone())
+pub fn replace_host_in_url(url: ServoUrl) -> ServoUrl {
+    if let Some(table) = HOST_TABLE.lock().unwrap().as_ref() {
+        host_replacement(table, url)
+    } else {
+        url
+    }
+}
+
+pub fn host_replacement(host_table: &HashMap<String, IpAddr>, mut url: ServoUrl) -> ServoUrl {
+    let replacement = url.domain().and_then(|domain| host_table.get(domain));
+    if let Some(ip) = replacement {
+        url.set_ip_host(*ip).unwrap();
+    }
+    url
 }

servo/components/script/dom/htmlformelement.rs

@@ -384,12 +384,12 @@ impl HTMLFormElement {
     fn mutate_action_url(&self, form_data: &mut Vec<FormDatum>, mut load_data: LoadData, encoding: EncodingRef) {
         let charset = &*encoding.whatwg_name().unwrap();
 
-        if let Some(ref mut url) = load_data.url.as_mut_url() {
-            url.query_pairs_mut().clear()
-               .encoding_override(Some(self.pick_encoding()))
-               .extend_pairs(form_data.into_iter()
-                                      .map(|field| (field.name.clone(), field.replace_value(charset))));
-        }
+        load_data.url
+            .as_mut_url()
+            .query_pairs_mut().clear()
+            .encoding_override(Some(self.pick_encoding()))
+            .extend_pairs(form_data.into_iter()
+                                    .map(|field| (field.name.clone(), field.replace_value(charset))));
 
         self.plan_to_navigate(load_data);
     }
@@ -403,13 +403,12 @@ impl HTMLFormElement {
                 let charset = &*encoding.whatwg_name().unwrap();
                 load_data.headers.set(ContentType::form_url_encoded());
 
-
-                if let Some(ref mut url) = load_data.url.as_mut_url() {
-                    url.query_pairs_mut().clear()
-                       .encoding_override(Some(self.pick_encoding()))
-                       .extend_pairs(form_data.into_iter()
-                       .map(|field| (field.name.clone(), field.replace_value(charset))));
-                }
+                load_data.url
+                    .as_mut_url()
+                    .query_pairs_mut().clear()
+                    .encoding_override(Some(self.pick_encoding()))
+                    .extend_pairs(form_data.into_iter()
+                    .map(|field| (field.name.clone(), field.replace_value(charset))));
 
                 load_data.url.query().unwrap_or("").to_string().into_bytes()
             }

servo/components/script/dom/url.rs

@@ -52,9 +52,8 @@ impl URL {
     }
 
     pub fn set_query_pairs(&self, pairs: &[(String, String)]) {
-        if let Some(ref mut url) = self.url.borrow_mut().as_mut_url() {
-            url.query_pairs_mut().clear().extend_pairs(pairs);
-        }
+        let mut url = self.url.borrow_mut();
+        url.as_mut_url().query_pairs_mut().clear().extend_pairs(pairs);
     }
 }
 

servo/components/script/dom/urlhelper.rs

@@ -45,49 +45,31 @@ impl UrlHelper {
         USVString(quirks::username(url.as_url()).to_owned())
     }
     pub fn SetHash(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            quirks::set_hash(url, &value.0)
-        }
+        quirks::set_hash(url.as_mut_url(), &value.0)
     }
     pub fn SetHost(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            let _ = quirks::set_host(url, &value.0);
-        }
+        let _ = quirks::set_host(url.as_mut_url(), &value.0);
     }
     pub fn SetPort(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            let _ = quirks::set_port(url, &value.0);
-        }
+        let _ = quirks::set_port(url.as_mut_url(), &value.0);
     }
     pub fn SetSearch(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            quirks::set_search(url, &value.0)
-        }
+        quirks::set_search(url.as_mut_url(), &value.0)
     }
     pub fn SetPathname(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            quirks::set_pathname(url, &value.0)
-        }
+        quirks::set_pathname(url.as_mut_url(), &value.0)
     }
     pub fn SetHostname(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            let _ = quirks::set_hostname(url, &value.0);
-        }
+        let _ = quirks::set_hostname(url.as_mut_url(), &value.0);
     }
     pub fn SetPassword(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            let _ = quirks::set_password(url, &value.0);
-        }
+        let _ = quirks::set_password(url.as_mut_url(), &value.0);
     }
     pub fn SetProtocol(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            let _ = quirks::set_protocol(url, &value.0);
-        }
+        let _ = quirks::set_protocol(url.as_mut_url(), &value.0);
     }
     pub fn SetUsername(url: &mut ServoUrl, value: USVString) {
-        if let Some(ref mut url) = url.as_mut_url() {
-            let _ = quirks::set_username(url, &value.0);
-        }
+        let _ = quirks::set_username(url.as_mut_url(), &value.0);
     }
     // https://w3c.github.io/webappsec-secure-contexts/#is-origin-trustworthy
     pub fn is_origin_trustworthy(url: &ServoUrl) -> bool {

servo/components/url/lib.rs

@@ -47,10 +47,8 @@ impl ServoUrl {
         Arc::try_unwrap(self.0).unwrap_or_else(|s| (*s).clone()).into_string()
     }
 
-    // NOTE: These methods return options that are always true temporarily until
-    // we special-case some urls to avoid going through rust-url.
-    pub fn into_url(self) -> Option<Url> {
-        Some(Arc::try_unwrap(self.0).unwrap_or_else(|s| (*s).clone()))
+    pub fn into_url(self) -> Url {
+        Arc::try_unwrap(self.0).unwrap_or_else(|s| (*s).clone())
     }
 
     pub fn as_url(&self) -> &Url {
@@ -94,24 +92,24 @@ impl ServoUrl {
         self.0.as_str()
     }
 
-    pub fn as_mut_url(&mut self) -> Option<&mut Url> {
-        Some(Arc::make_mut(&mut self.0))
+    pub fn as_mut_url(&mut self) -> &mut Url {
+        Arc::make_mut(&mut self.0)
     }
 
     pub fn set_username(&mut self, user: &str) -> Result<(), ()> {
-        Arc::make_mut(&mut self.0).set_username(user)
+        self.as_mut_url().set_username(user)
     }
 
     pub fn set_ip_host(&mut self, addr: IpAddr) -> Result<(), ()> {
-        Arc::make_mut(&mut self.0).set_ip_host(addr)
+        self.as_mut_url().set_ip_host(addr)
     }
 
     pub fn set_password(&mut self, pass: Option<&str>) -> Result<(), ()> {
-        Arc::make_mut(&mut self.0).set_password(pass)
+        self.as_mut_url().set_password(pass)
     }
 
     pub fn set_fragment(&mut self, fragment: Option<&str>) {
-        Arc::make_mut(&mut self.0).set_fragment(fragment)
+        self.as_mut_url().set_fragment(fragment)
     }
 
     pub fn username(&self) -> &str {

servo/tests/unit/net/resource_thread.rs

@@ -151,11 +151,11 @@ fn test_replace_hosts() {
     host_table.insert("servo.test.server".to_owned(), ip("127.0.0.2"));
 
     let url = ServoUrl::parse("http://foo.bar.com:8000/foo").unwrap();
-    assert_eq!(host_replacement(&host_table, &url).host_str().unwrap(), "127.0.0.1");
+    assert_eq!(host_replacement(&host_table, url).host_str().unwrap(), "127.0.0.1");
 
     let url = ServoUrl::parse("http://servo.test.server").unwrap();
-    assert_eq!(host_replacement(&host_table, &url).host_str().unwrap(), "127.0.0.2");
+    assert_eq!(host_replacement(&host_table, url).host_str().unwrap(), "127.0.0.2");
 
     let url = ServoUrl::parse("http://a.foo.bar.com").unwrap();
-    assert_eq!(host_replacement(&host_table, &url).host_str().unwrap(), "a.foo.bar.com");
+    assert_eq!(host_replacement(&host_table, url).host_str().unwrap(), "a.foo.bar.com");
 }