servo: Merge #15354 - Redirect document loads manually (from cynicaldevil:manual-redirect); r=jdm

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes fix #14596 .

r? @jdm

I ran some tests at random from the `navigating-across-documents` folder, and they are passing.

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

Source-Repo: https://github.com/servo/servo
Source-Revision: eac4f407e2bfa0055a698ca438972566f0814e43

--HG--
extra : subtree_source : https%3A//hg.mozilla.org/projects/converted-servo-linear
extra : subtree_revision : 68436f4afccd0313c5d1a0da4e0bbb72106284f5

servo/Cargo.lock

@@ -489,6 +489,7 @@ dependencies = [
  "gaol 0.0.1 (git+https://github.com/servo/gaol)",
  "gfx 0.0.1",
  "gfx_traits 0.0.1",
+ "hyper 0.10.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "layout_traits 0.0.1",

servo/components/constellation/Cargo.toml

@@ -21,6 +21,7 @@ devtools_traits = {path = "../devtools_traits"}
 euclid = "0.11"
 gfx = {path = "../gfx"}
 gfx_traits = {path = "../gfx_traits"}
+hyper = "0.10"
 ipc-channel = "0.7"
 itertools = "0.5"
 layout_traits = {path = "../layout_traits"}

servo/components/constellation/constellation.rs

@@ -91,9 +91,11 @@ use log::{Log, LogLevel, LogLevelFilter, LogMetadata, LogRecord};
 use msg::constellation_msg::{BrowsingContextId, TopLevelBrowsingContextId, FrameType, PipelineId};
 use msg::constellation_msg::{Key, KeyModifiers, KeyState};
 use msg::constellation_msg::{PipelineNamespace, PipelineNamespaceId, TraversalDirection};
-use net_traits::{self, IpcSend, ResourceThreads};
+use net_traits::{self, IpcSend, FetchResponseMsg, ResourceThreads};
 use net_traits::pub_domains::reg_host;
+use net_traits::request::RequestInit;
 use net_traits::storage_thread::{StorageThreadMsg, StorageType};
+use network_listener::NetworkListener;
 use offscreen_gl_context::{GLContextAttributes, GLLimits};
 use pipeline::{InitialPipelineState, Pipeline};
 use profile_traits::mem;
@@ -159,6 +161,12 @@ pub struct Constellation<Message, LTF, STF> {
     /// This is the constellation's view of `layout_sender`.
     layout_receiver: Receiver<Result<FromLayoutMsg, IpcError>>,
 
+    /// A channel for network listener to send messages to the constellation.
+    network_listener_sender: Sender<(PipelineId, FetchResponseMsg)>,
+
+    /// A channel for the constellation to receive messages from network listener.
+    network_listener_receiver: Receiver<(PipelineId, FetchResponseMsg)>,
+
     /// A channel for the constellation to receive messages from the compositor thread.
     compositor_receiver: Receiver<FromCompositorMsg>,
 
@@ -502,6 +510,8 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
             let (ipc_layout_sender, ipc_layout_receiver) = ipc::channel().expect("ipc channel failure");
             let layout_receiver = route_ipc_receiver_to_new_mpsc_receiver_preserving_errors(ipc_layout_receiver);
 
+            let (network_listener_sender, network_listener_receiver) = channel();
+
             let swmanager_receiver = route_ipc_receiver_to_new_mpsc_receiver_preserving_errors(swmanager_receiver);
 
             PipelineNamespace::install(PipelineNamespaceId(0));
@@ -512,6 +522,8 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
                 script_receiver: script_receiver,
                 compositor_receiver: compositor_receiver,
                 layout_receiver: layout_receiver,
+                network_listener_sender: network_listener_sender,
+                network_listener_receiver: network_listener_receiver,
                 compositor_proxy: state.compositor_proxy,
                 debugger_chan: state.debugger_chan,
                 devtools_chan: state.devtools_chan,
@@ -797,6 +809,7 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
             Script(FromScriptMsg),
             Compositor(FromCompositorMsg),
             Layout(FromLayoutMsg),
+            NetworkListener((PipelineId, FetchResponseMsg)),
             FromSWManager(SWManagerMsg),
         }
 
@@ -815,6 +828,7 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
             let receiver_from_script = &self.script_receiver;
             let receiver_from_compositor = &self.compositor_receiver;
             let receiver_from_layout = &self.layout_receiver;
+            let receiver_from_network_listener = &self.network_listener_receiver;
             let receiver_from_swmanager = &self.swmanager_receiver;
             select! {
                 msg = receiver_from_script.recv() =>
@@ -823,6 +837,10 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
                     Ok(Request::Compositor(msg.expect("Unexpected compositor channel panic in constellation"))),
                 msg = receiver_from_layout.recv() =>
                     msg.expect("Unexpected layout channel panic in constellation").map(Request::Layout),
+                msg = receiver_from_network_listener.recv() =>
+                    Ok(Request::NetworkListener(
+                        msg.expect("Unexpected network listener channel panic in constellation")
+                    )),
                 msg = receiver_from_swmanager.recv() =>
                     msg.expect("Unexpected panic channel panic in constellation").map(Request::FromSWManager)
             }
@@ -849,12 +867,31 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
             Request::Layout(message) => {
                 self.handle_request_from_layout(message);
             },
+            Request::NetworkListener(message) => {
+                self.handle_request_from_network_listener(message);
+            },
             Request::FromSWManager(message) => {
                 self.handle_request_from_swmanager(message);
             }
         }
     }
 
+    fn handle_request_from_network_listener(&mut self, message: (PipelineId, FetchResponseMsg)) {
+        let (id, message_) = message;
+        let result = match self.pipelines.get(&id) {
+            Some(pipeline) => {
+                let msg = ConstellationControlMsg::NavigationResponse(id, message_);
+                pipeline.event_loop.send(msg)
+            },
+            None => {
+                return warn!("Pipeline {:?} got fetch data after closure!", id);
+            },
+        };
+        if let Err(e) = result {
+            self.handle_send_error(id, e);
+        }
+    }
+
     fn handle_request_from_swmanager(&mut self, message: SWManagerMsg) {
         match message {
             SWManagerMsg::OwnSender(sw_sender) => {
@@ -956,6 +993,10 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
             FromScriptMsg::PipelineExited(pipeline_id) => {
                 self.handle_pipeline_exited(pipeline_id);
             }
+            FromScriptMsg::InitiateNavigateRequest(req_init, pipeline_id) => {
+                debug!("constellation got initiate navigate request message");
+                self.handle_navigate_request(req_init, pipeline_id);
+            }
             FromScriptMsg::ScriptLoadedURLInIFrame(load_info) => {
                 debug!("constellation got iframe URL load message {:?} {:?} {:?}",
                        load_info.info.parent_pipeline_id,
@@ -1484,6 +1525,18 @@ impl<Message, LTF, STF> Constellation<Message, LTF, STF>
         }
     }
 
+    fn handle_navigate_request(&self,
+                              req_init: RequestInit,
+                              id: PipelineId) {
+        let listener = NetworkListener::new(
+                           req_init,
+                           id,
+                           self.public_resource_threads.clone(),
+                           self.network_listener_sender.clone());
+
+        listener.initiate_fetch();
+    }
+
     // The script thread associated with pipeline_id has loaded a URL in an iframe via script. This
     // will result in a new pipeline being spawned and a child being added to
     // the parent pipeline. This message is never the result of a

servo/components/constellation/lib.rs

@@ -20,6 +20,7 @@ extern crate euclid;
 extern crate gaol;
 extern crate gfx;
 extern crate gfx_traits;
+extern crate hyper;
 extern crate ipc_channel;
 extern crate itertools;
 extern crate layout_traits;
@@ -45,6 +46,7 @@ extern crate webvr_traits;
 mod browsingcontext;
 mod constellation;
 mod event_loop;
+mod network_listener;
 mod pipeline;
 #[cfg(not(target_os = "windows"))]
 mod sandboxing;

servo/components/constellation/network_listener.rs

@@ -0,0 +1,133 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+//! The listener that encapsulates all state for an in-progress document request.
+//! Any redirects that are encountered are followed. Whenever a non-redirect
+//! response is received, it is forwarded to the appropriate script thread.
+
+use hyper::header::Location;
+use ipc_channel::ipc;
+use ipc_channel::router::ROUTER;
+use msg::constellation_msg::PipelineId;
+use net::http_loader::{set_default_accept, set_default_accept_language};
+use net_traits::{CoreResourceMsg, FetchMetadata, FetchResponseMsg};
+use net_traits::{IpcSend, NetworkError, ResourceThreads};
+use net_traits::request::{Destination, RequestInit, Type};
+use net_traits::response::ResponseInit;
+use std::sync::mpsc::Sender;
+
+pub struct NetworkListener {
+    res_init: Option<ResponseInit>,
+    req_init: RequestInit,
+    pipeline_id: PipelineId,
+    resource_threads: ResourceThreads,
+    sender: Sender<(PipelineId, FetchResponseMsg)>,
+    should_send: bool,
+}
+
+impl NetworkListener {
+    pub fn new(req_init: RequestInit,
+               pipeline_id: PipelineId,
+               resource_threads: ResourceThreads,
+               sender: Sender<(PipelineId, FetchResponseMsg)>) -> NetworkListener {
+        NetworkListener {
+            res_init: None,
+            req_init,
+            pipeline_id,
+            resource_threads,
+            sender,
+            should_send: false
+        }
+    }
+
+    pub fn initiate_fetch(&self) {
+        let (ipc_sender, ipc_receiver) = ipc::channel().expect("Failed to create IPC channel!");
+
+        let mut listener = NetworkListener {
+            res_init: self.res_init.clone(),
+            req_init: self.req_init.clone(),
+            resource_threads: self.resource_threads.clone(),
+            sender: self.sender.clone(),
+            pipeline_id: self.pipeline_id.clone(),
+            should_send: false,
+        };
+
+        let msg = match self.res_init {
+            Some(ref res_init_) => CoreResourceMsg::FetchRedirect(
+                                   self.req_init.clone(),
+                                   res_init_.clone(),
+                                   ipc_sender),
+            None => {
+                set_default_accept(Type::None, Destination::Document, &mut listener.req_init.headers);
+                set_default_accept_language(&mut listener.req_init.headers);
+
+                CoreResourceMsg::Fetch(
+                listener.req_init.clone(),
+                ipc_sender)
+            }
+        };
+
+        ROUTER.add_route(ipc_receiver.to_opaque(), box move |message| {
+            let msg = message.to();
+            match msg {
+                Ok(FetchResponseMsg::ProcessResponse(res)) => listener.check_redirect(res),
+                Ok(msg_) => listener.send(msg_),
+                Err(e) => warn!("Error while receiving network listener message: {}", e),
+            };
+        });
+
+        if let Err(e) = self.resource_threads.sender().send(msg) {
+            warn!("Resource thread unavailable ({})", e);
+        }
+    }
+
+    fn check_redirect(&mut self,
+                      message: Result<(FetchMetadata), NetworkError>) {
+        match message {
+            Ok(res_metadata) => {
+                let metadata = match res_metadata {
+                    FetchMetadata::Filtered { ref unsafe_, .. } => unsafe_,
+                    FetchMetadata::Unfiltered(ref m) => m,
+                };
+
+                match metadata.headers {
+                    Some(ref headers) if headers.has::<Location>() => {
+                        if self.req_init.url_list.is_empty() {
+                            self.req_init.url_list.push(self.req_init.url.clone());
+                        }
+                        self.req_init.url_list.push(metadata.final_url.clone());
+
+                        self.req_init.referrer_url = metadata.referrer.clone();
+                        self.req_init.referrer_policy = metadata.referrer_policy;
+
+                        self.res_init = Some(ResponseInit {
+                            url: metadata.final_url.clone(),
+                            headers: headers.clone().into_inner(),
+                            referrer: metadata.referrer.clone(),
+                        });
+
+                        self.initiate_fetch();
+                    },
+                    _ => {
+                        // Response should be processed by script thread.
+                        self.should_send = true;
+                        self.send(FetchResponseMsg::ProcessResponse(Ok(res_metadata.clone())));
+                    }
+                };
+            },
+            Err(e) => {
+                self.should_send = true;
+                self.send(FetchResponseMsg::ProcessResponse(Err(e)))
+            }
+        };
+    }
+
+    fn send(&mut self, msg: FetchResponseMsg) {
+        if self.should_send {
+            if let Err(e) = self.sender.send((self.pipeline_id, msg)) {
+                warn!("Failed to forward network message to pipeline {}: {:?}", self.pipeline_id, e);
+            }
+        }
+    }
+}

servo/components/net/http_loader.rs

@@ -674,14 +674,14 @@ pub fn http_fetch(request: &mut Request,
 }
 
 /// [HTTP redirect fetch](https://fetch.spec.whatwg.org#http-redirect-fetch)
-fn http_redirect_fetch(request: &mut Request,
-                       cache: &mut CorsCache,
-                       response: Response,
-                       cors_flag: bool,
-                       target: Target,
-                       done_chan: &mut DoneChannel,
-                       context: &FetchContext)
-                       -> Response {
+pub fn http_redirect_fetch(request: &mut Request,
+                           cache: &mut CorsCache,
+                           response: Response,
+                           cors_flag: bool,
+                           target: Target,
+                           done_chan: &mut DoneChannel,
+                           context: &FetchContext)
+                           -> Response {
     // Step 1
     assert!(response.return_internal);
 
@@ -749,8 +749,10 @@ fn http_redirect_fetch(request: &mut Request,
     // Step 12
     // TODO implement referrer policy
 
+    let recursive_flag = request.redirect_mode != RedirectMode::Manual;
+
     // Step 13
-    main_fetch(request, cache, cors_flag, true, target, done_chan, context)
+    main_fetch(request, cache, cors_flag, recursive_flag, target, done_chan, context)
 }
 
 fn try_immutable_origin_to_hyper_origin(url_origin: &ImmutableOrigin) -> Option<HyperOrigin> {
@@ -1108,6 +1110,7 @@ fn http_network_fetch(request: &Request,
                                 res.response.status_raw().1.as_bytes().to_vec()));
     response.headers = res.response.headers.clone();
     response.referrer = request.referrer.to_url().cloned();
+    response.referrer_policy = request.referrer_policy.clone();
 
     let res_body = response.body.clone();
 

servo/components/net/lib.rs

@@ -52,7 +52,7 @@ mod data_loader;
 pub mod filemanager_thread;
 mod hosts;
 pub mod hsts;
-mod http_loader;
+pub mod http_loader;
 pub mod image_cache;
 pub mod mime_classifier;
 pub mod resource_thread;

servo/components/net/resource_thread.rs

@@ -8,10 +8,11 @@ use cookie;
 use cookie_rs;
 use cookie_storage::CookieStorage;
 use devtools_traits::DevtoolsControlMsg;
+use fetch::cors_cache::CorsCache;
 use fetch::methods::{FetchContext, fetch};
 use filemanager_thread::{FileManager, TFDProvider};
 use hsts::HstsList;
-use http_loader::HttpState;
+use http_loader::{HttpState, http_redirect_fetch};
 use hyper_serde::Serde;
 use ipc_channel::ipc::{self, IpcReceiver, IpcReceiverSet, IpcSender};
 use net_traits::{CookieSource, CoreResourceThread};
@@ -19,6 +20,7 @@ use net_traits::{CoreResourceMsg, FetchResponseMsg};
 use net_traits::{CustomResponseMediator, ResourceId};
 use net_traits::{ResourceThreads, WebSocketCommunicate, WebSocketConnectData};
 use net_traits::request::{Request, RequestInit};
+use net_traits::response::{Response, ResponseInit};
 use net_traits::storage_thread::StorageThreadMsg;
 use profile_traits::time::ProfilerChan;
 use serde::{Deserialize, Serialize};
@@ -153,8 +155,10 @@ impl ResourceChannelManager {
                    msg: CoreResourceMsg,
                    http_state: &Arc<HttpState>) -> bool {
         match msg {
-            CoreResourceMsg::Fetch(init, sender) =>
-                self.resource_manager.fetch(init, sender, http_state),
+            CoreResourceMsg::Fetch(req_init, sender) =>
+                self.resource_manager.fetch(req_init, None, sender, http_state),
+            CoreResourceMsg::FetchRedirect(req_init, res_init, sender) =>
+                self.resource_manager.fetch(req_init, Some(res_init), sender, http_state),
             CoreResourceMsg::WebsocketConnect(connect, connect_data) =>
                 self.resource_manager.websocket_connect(connect, connect_data, http_state),
             CoreResourceMsg::SetCookieForUrl(request, cookie, source) =>
@@ -318,7 +322,8 @@ impl CoreResourceManager {
     }
 
     fn fetch(&self,
-             init: RequestInit,
+             req_init: RequestInit,
+             res_init_: Option<ResponseInit>,
              mut sender: IpcSender<FetchResponseMsg>,
              http_state: &Arc<HttpState>) {
         let http_state = http_state.clone();
@@ -326,8 +331,8 @@ impl CoreResourceManager {
         let dc = self.devtools_chan.clone();
         let filemanager = self.filemanager.clone();
 
-        thread::Builder::new().name(format!("fetch thread for {}", init.url)).spawn(move || {
-            let mut request = Request::from_init(init);
+        thread::Builder::new().name(format!("fetch thread for {}", req_init.url)).spawn(move || {
+            let mut request = Request::from_init(req_init);
             // XXXManishearth: Check origin against pipeline id (also ensure that the mode is allowed)
             // todo load context / mimesniff in fetch
             // todo referrer policy?
@@ -338,7 +343,20 @@ impl CoreResourceManager {
                 devtools_chan: dc,
                 filemanager: filemanager,
             };
-            fetch(&mut request, &mut sender, &context);
+
+            match res_init_ {
+                Some(res_init) => {
+                    let response = Response::from_init(res_init);
+                    http_redirect_fetch(&mut request,
+                                        &mut CorsCache::new(),
+                                        response,
+                                        true,
+                                        &mut sender,
+                                        &mut None,
+                                        &context);
+                },
+                None => fetch(&mut request, &mut sender, &context),
+            };
         }).expect("Thread spawning failed");
     }
 

servo/components/net_traits/lib.rs

@@ -42,7 +42,7 @@ use ipc_channel::Error as IpcError;
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use ipc_channel::router::ROUTER;
 use request::{Request, RequestInit};
-use response::{HttpsState, Response};
+use response::{HttpsState, Response, ResponseInit};
 use servo_url::ServoUrl;
 use std::error::Error;
 use storage_thread::StorageThreadMsg;
@@ -369,6 +369,8 @@ pub struct WebSocketConnectData {
 #[derive(Deserialize, Serialize)]
 pub enum CoreResourceMsg {
     Fetch(RequestInit, IpcSender<FetchResponseMsg>),
+    /// Initiate a fetch in response to processing a redirection
+    FetchRedirect(RequestInit, ResponseInit, IpcSender<FetchResponseMsg>),
     /// Try to make a websocket connection to a URL.
     WebsocketConnect(WebSocketCommunicate, WebSocketConnectData),
     /// Store a cookie for a given originating URL
@@ -435,6 +437,9 @@ pub struct Metadata {
 
     /// Referrer Url
     pub referrer: Option<ServoUrl>,
+
+    /// Referrer Policy of the Request used to obtain Response
+    pub referrer_policy: Option<ReferrerPolicy>,
 }
 
 impl Metadata {
@@ -449,6 +454,7 @@ impl Metadata {
             status: Some((200, b"OK".to_vec())),
             https_state: HttpsState::None,
             referrer: None,
+            referrer_policy: None,
         }
     }
 

servo/components/net_traits/request.rs

@@ -60,7 +60,7 @@ pub enum Origin {
 }
 
 /// A [referer](https://fetch.spec.whatwg.org/#concept-request-referrer)
-#[derive(Clone, PartialEq, HeapSizeOf)]
+#[derive(Clone, PartialEq, Serialize, Deserialize, HeapSizeOf)]
 pub enum Referrer {
     NoReferrer,
     /// Default referrer if nothing is specified
@@ -158,6 +158,8 @@ pub struct RequestInit {
     pub pipeline_id: Option<PipelineId>,
     pub redirect_mode: RedirectMode,
     pub integrity_metadata: String,
+    // to keep track of redirects
+    pub url_list: Vec<ServoUrl>,
 }
 
 impl Default for RequestInit {
@@ -182,6 +184,7 @@ impl Default for RequestInit {
             pipeline_id: None,
             redirect_mode: RedirectMode::Follow,
             integrity_metadata: "".to_owned(),
+            url_list: vec![],
         }
     }
 }
@@ -290,7 +293,7 @@ impl Request {
     }
 
     pub fn from_init(init: RequestInit) -> Request {
-        let mut req = Request::new(init.url,
+        let mut req = Request::new(init.url.clone(),
                                    Some(Origin::Origin(init.origin.origin())),
                                    false,
                                    init.pipeline_id);
@@ -314,6 +317,12 @@ impl Request {
         req.referrer_policy = init.referrer_policy;
         req.pipeline_id = init.pipeline_id;
         req.redirect_mode = init.redirect_mode;
+        let mut url_list = init.url_list;
+        if url_list.is_empty() {
+            url_list.push(init.url);
+        }
+        req.redirect_count = url_list.len() as u32 - 1;
+        req.url_list = url_list;
         req.integrity_metadata = init.integrity_metadata;
         req
     }

servo/components/net_traits/response.rs

@@ -4,7 +4,7 @@
 
 //! The [Response](https://fetch.spec.whatwg.org/#responses) object
 //! resulting from a [fetch operation](https://fetch.spec.whatwg.org/#concept-fetch)
-use {FetchMetadata, FilteredMetadata, Metadata, NetworkError};
+use {FetchMetadata, FilteredMetadata, Metadata, NetworkError, ReferrerPolicy};
 use hyper::header::{AccessControlExposeHeaders, ContentType, Headers};
 use hyper::status::StatusCode;
 use hyper_serde::Serde;
@@ -74,6 +74,16 @@ pub enum ResponseMsg {
     Errored,
 }
 
+#[derive(Serialize, Deserialize, Clone, HeapSizeOf)]
+pub struct ResponseInit {
+    pub url: ServoUrl,
+    #[serde(deserialize_with = "::hyper_serde::deserialize",
+            serialize_with = "::hyper_serde::serialize")]
+    #[ignore_heap_size_of = "Defined in hyper"]
+    pub headers: Headers,
+    pub referrer: Option<ServoUrl>,
+}
+
 /// A [Response](https://fetch.spec.whatwg.org/#concept-response) as defined by the Fetch spec
 #[derive(Debug, Clone, HeapSizeOf)]
 pub struct Response {
@@ -97,6 +107,7 @@ pub struct Response {
     pub internal_response: Option<Box<Response>>,
     /// whether or not to try to return the internal_response when asked for actual_response
     pub return_internal: bool,
+    pub referrer_policy: Option<ReferrerPolicy>,
 }
 
 impl Response {
@@ -113,11 +124,19 @@ impl Response {
             cache_state: CacheState::None,
             https_state: HttpsState::None,
             referrer: None,
+            referrer_policy: None,
             internal_response: None,
             return_internal: true,
         }
     }
 
+    pub fn from_init(init: ResponseInit) -> Response {
+        let mut res = Response::new(init.url);
+        res.headers = init.headers;
+        res.referrer = init.referrer;
+        res
+    }
+
     pub fn network_error(e: NetworkError) -> Response {
         Response {
             response_type: ResponseType::Error(e),
@@ -131,6 +150,7 @@ impl Response {
             cache_state: CacheState::None,
             https_state: HttpsState::None,
             referrer: None,
+            referrer_policy: None,
             internal_response: None,
             return_internal: true,
         }
@@ -263,6 +283,7 @@ impl Response {
             metadata.status = response.raw_status.clone();
             metadata.https_state = response.https_state;
             metadata.referrer = response.referrer.clone();
+            metadata.referrer_policy = response.referrer_policy.clone();
             metadata
         };
 

servo/components/script/dom/servoparser/mod.rs

@@ -527,6 +527,7 @@ impl Tokenizer {
 
 /// The context required for asynchronously fetching a document
 /// and parsing it progressively.
+#[derive(JSTraceable)]
 pub struct ParserContext {
     /// The parser that initiated the request.
     parser: Option<Trusted<ServoParser>>,

servo/components/script/script_thread.rs

@@ -60,7 +60,7 @@ use dom::worklet::WorkletThreadPool;
 use dom::workletglobalscope::WorkletGlobalScopeInit;
 use euclid::Rect;
 use euclid::point::Point2D;
-use hyper::header::{ContentType, HttpDate, LastModified, Headers};
+use hyper::header::{ContentType, HttpDate, Headers, LastModified};
 use hyper::header::ReferrerPolicy as ReferrerPolicyHeader;
 use hyper::mime::{Mime, SubLevel, TopLevel};
 use hyper_serde::Serde;
@@ -74,12 +74,11 @@ use js::rust::Runtime;
 use mem::heap_size_of_self_and_children;
 use microtask::{MicrotaskQueue, Microtask};
 use msg::constellation_msg::{BrowsingContextId, FrameType, PipelineId, PipelineNamespace, TopLevelBrowsingContextId};
-use net_traits::{CoreResourceMsg, FetchMetadata, FetchResponseListener};
-use net_traits::{IpcSend, Metadata, ReferrerPolicy, ResourceThreads};
+use net_traits::{FetchMetadata, FetchResponseListener, FetchResponseMsg};
+use net_traits::{Metadata, NetworkError, ReferrerPolicy, ResourceThreads};
 use net_traits::image_cache::{ImageCache, PendingImageResponse};
-use net_traits::request::{CredentialsMode, Destination, RequestInit};
+use net_traits::request::{CredentialsMode, Destination, RedirectMode, RequestInit};
 use net_traits::storage_thread::StorageType;
-use network_listener::NetworkListener;
 use profile_traits::mem::{self, OpaqueSender, Report, ReportKind, ReportsChan};
 use profile_traits::time::{self, ProfilerCategory, profile};
 use script_layout_interface::message::{self, NewLayoutThreadInfo, ReflowQueryType};
@@ -105,7 +104,7 @@ use std::option::Option;
 use std::ptr;
 use std::rc::Rc;
 use std::result::Result;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{Receiver, Select, Sender, channel};
 use std::thread;
@@ -411,6 +410,8 @@ pub struct ScriptThread {
     window_proxies: DOMRefCell<HashMap<BrowsingContextId, JS<WindowProxy>>>,
     /// A list of data pertaining to loads that have not yet received a network response
     incomplete_loads: DOMRefCell<Vec<InProgressLoad>>,
+    /// A vector containing parser contexts which have not yet been fully processed
+    incomplete_parser_contexts: DOMRefCell<Vec<(PipelineId, ParserContext)>>,
     /// A map to store service worker registrations for a given origin
     registration_map: DOMRefCell<HashMap<ServoUrl, JS<ServiceWorkerRegistration>>>,
     /// A job queue for Service Workers keyed by their scope url
@@ -574,7 +575,7 @@ impl ScriptThreadFactory for ScriptThread {
             let origin = MutableOrigin::new(load_data.url.origin());
             let new_load = InProgressLoad::new(id, browsing_context_id, top_level_browsing_context_id, parent_info,
                                                layout_chan, window_size, load_data.url.clone(), origin);
-            script_thread.start_page_load(new_load, load_data);
+            script_thread.pre_page_load(new_load, load_data);
 
             let reporter_name = format!("script-reporter-{}", id);
             mem_profiler_chan.run_with_memory_reporting(|| {
@@ -758,6 +759,7 @@ impl ScriptThread {
             documents: DOMRefCell::new(Documents::new()),
             window_proxies: DOMRefCell::new(HashMap::new()),
             incomplete_loads: DOMRefCell::new(vec!()),
+            incomplete_parser_contexts: DOMRefCell::new(vec!()),
             registration_map: DOMRefCell::new(HashMap::new()),
             job_queue_map: Rc::new(JobQueue::new()),
 
@@ -1105,6 +1107,14 @@ impl ScriptThread {
 
     fn handle_msg_from_constellation(&self, msg: ConstellationControlMsg) {
         match msg {
+            ConstellationControlMsg::NavigationResponse(id, fetch_data) => {
+                match fetch_data {
+                    FetchResponseMsg::ProcessResponse(metadata) => self.handle_fetch_metadata(id, metadata),
+                    FetchResponseMsg::ProcessResponseChunk(chunk) => self.handle_fetch_chunk(id, chunk),
+                    FetchResponseMsg::ProcessResponseEOF(eof) => self.handle_fetch_eof(id, eof),
+                    _ => unreachable!(),
+                };
+            },
             ConstellationControlMsg::Navigate(parent_pipeline_id, browsing_context_id, load_data, replace) =>
                 self.handle_navigate(parent_pipeline_id, Some(browsing_context_id), load_data, replace),
             ConstellationControlMsg::SendEvent(id, event) =>
@@ -1396,7 +1406,7 @@ impl ScriptThread {
         if load_data.url.as_str() == "about:blank" {
             self.start_page_load_about_blank(new_load);
         } else {
-            self.start_page_load(new_load, load_data);
+            self.pre_page_load(new_load, load_data);
         }
     }
 
@@ -2225,45 +2235,67 @@ impl ScriptThread {
         window.evaluate_media_queries_and_report_changes();
     }
 
-    /// Initiate a non-blocking fetch for a specified resource. Stores the InProgressLoad
+    /// Instructs the constellation to fetch the document that will be loaded. Stores the InProgressLoad
     /// argument until a notification is received that the fetch is complete.
-    fn start_page_load(&self, incomplete: InProgressLoad, mut load_data: LoadData) {
+    fn pre_page_load(&self, incomplete: InProgressLoad, load_data: LoadData) {
         let id = incomplete.pipeline_id.clone();
-
-        let context = Arc::new(Mutex::new(ParserContext::new(id, load_data.url.clone())));
-        let (action_sender, action_receiver) = ipc::channel().unwrap();
-        let listener = NetworkListener {
-            context: context,
-            task_source: self.networking_task_source.clone(),
-            wrapper: None,
-        };
-        ROUTER.add_route(action_receiver.to_opaque(), box move |message| {
-            listener.notify_fetch(message.to().unwrap());
-        });
-
-        if load_data.url.scheme() == "javascript" {
-            load_data.url = ServoUrl::parse("about:blank").unwrap();
-        }
-
-        let request = RequestInit {
+        let mut req_init = RequestInit {
             url: load_data.url.clone(),
             method: load_data.method,
             destination: Destination::Document,
             credentials_mode: CredentialsMode::Include,
             use_url_credentials: true,
-            origin: load_data.url,
+            origin: load_data.url.clone(),
             pipeline_id: Some(id),
             referrer_url: load_data.referrer_url,
             referrer_policy: load_data.referrer_policy,
             headers: load_data.headers,
             body: load_data.data,
+            redirect_mode: RedirectMode::Manual,
             .. RequestInit::default()
         };
 
-        self.resource_threads.send(CoreResourceMsg::Fetch(request, action_sender)).unwrap();
+        if req_init.url.scheme() == "javascript" {
+            req_init.url = ServoUrl::parse("about:blank").unwrap();
+        }
+
+        let context = ParserContext::new(id, load_data.url);
+        self.incomplete_parser_contexts.borrow_mut().push((id, context));
+
+        self.constellation_chan.send(ConstellationMsg::InitiateNavigateRequest(req_init, id)).unwrap();
         self.incomplete_loads.borrow_mut().push(incomplete);
     }
 
+    fn handle_fetch_metadata(&self, id: PipelineId, fetch_metadata: Result<FetchMetadata, NetworkError>) {
+        match fetch_metadata {
+            Ok(_) => {},
+            Err(ref e) => warn!("Network error: {:?}", e),
+        };
+        let mut incomplete_parser_contexts = self.incomplete_parser_contexts.borrow_mut();
+        let parser = incomplete_parser_contexts.iter_mut().find(|&&mut (pipeline_id, _)| pipeline_id == id);
+        if let Some(&mut (_, ref mut ctxt)) = parser {
+            ctxt.process_response(fetch_metadata);
+        }
+    }
+
+    fn handle_fetch_chunk(&self, id: PipelineId, chunk: Vec<u8>) {
+        let mut incomplete_parser_contexts = self.incomplete_parser_contexts.borrow_mut();
+        let parser = incomplete_parser_contexts.iter_mut().find(|&&mut (pipeline_id, _)| pipeline_id == id);
+        if let Some(&mut (_, ref mut ctxt)) = parser {
+            ctxt.process_response_chunk(chunk);
+        }
+    }
+
+    fn handle_fetch_eof(&self, id: PipelineId, eof: Result<(), NetworkError>) {
+        let idx = self.incomplete_parser_contexts.borrow().iter().position(|&(pipeline_id, _)| {
+            pipeline_id == id
+        });
+        if let Some(idx) = idx {
+            let (_, mut ctxt) = self.incomplete_parser_contexts.borrow_mut().remove(idx);
+            ctxt.process_response_eof(eof);
+        }
+    }
+
     /// Synchronously fetch `about:blank`. Stores the `InProgressLoad`
     /// argument until a notification is received that the fetch is complete.
     fn start_page_load_about_blank(&self, incomplete: InProgressLoad) {

servo/components/script_traits/lib.rs

@@ -55,7 +55,7 @@ use ipc_channel::ipc::{IpcReceiver, IpcSender};
 use libc::c_void;
 use msg::constellation_msg::{BrowsingContextId, TopLevelBrowsingContextId, FrameType, Key, KeyModifiers, KeyState};
 use msg::constellation_msg::{PipelineId, PipelineNamespaceId, TraversalDirection};
-use net_traits::{ReferrerPolicy, ResourceThreads};
+use net_traits::{FetchResponseMsg, ReferrerPolicy, ResourceThreads};
 use net_traits::image::base::Image;
 use net_traits::image_cache::ImageCache;
 use net_traits::response::HttpsState;
@@ -232,6 +232,9 @@ pub enum UpdatePipelineIdReason {
 /// Messages sent from the constellation or layout to the script thread.
 #[derive(Deserialize, Serialize)]
 pub enum ConstellationControlMsg {
+    /// Sends the final response to script thread for fetching after all redirections
+    /// have been resolved
+    NavigationResponse(PipelineId, FetchResponseMsg),
     /// Gives a channel and ID to a layout thread, as well as the ID of that layout's parent
     AttachLayout(NewLayoutInfo),
     /// Window resized.  Sends a DOM event eventually, but first we combine events.
@@ -304,6 +307,7 @@ impl fmt::Debug for ConstellationControlMsg {
     fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
         use self::ConstellationControlMsg::*;
         let variant = match *self {
+            NavigationResponse(..) => "NavigationResponse",
             AttachLayout(..) => "AttachLayout",
             Resize(..) => "Resize",
             ResizeInactive(..) => "ResizeInactive",

servo/components/script_traits/script_msg.rs

@@ -20,6 +20,7 @@ use ipc_channel::ipc::IpcSender;
 use msg::constellation_msg::{BrowsingContextId, TopLevelBrowsingContextId, FrameType, PipelineId, TraversalDirection};
 use msg::constellation_msg::{Key, KeyModifiers, KeyState};
 use net_traits::CoreResourceMsg;
+use net_traits::request::RequestInit;
 use net_traits::storage_thread::StorageType;
 use offscreen_gl_context::{GLContextAttributes, GLLimits};
 use servo_url::ImmutableOrigin;
@@ -67,6 +68,9 @@ pub enum LogEntry {
 /// Messages from the script to the constellation.
 #[derive(Deserialize, Serialize)]
 pub enum ScriptMsg {
+    /// Requests are sent to constellation and fetches are checked manually
+    /// for cross-origin loads
+    InitiateNavigateRequest(RequestInit, PipelineId),
     /// Broadcast a storage event to every same-origin pipeline.
     /// The strings are key, old value and new value.
     BroadcastStorageEvent(PipelineId, StorageType, ServoUrl, Option<String>, Option<String>, Option<String>),