servo: Merge #6416 - Use hosts-replaced URL only when loading resources (from jgraham:hosts_replaced); r=jdm

Source-Repo: https://github.com/servo/servo
Source-Revision: 8602d01af2b2081ea1e9d600abdb1ea609a65038
This commit is contained in:
James Graham 2015-08-05 08:01:03 -06:00
Родитель eec46242dc
Коммит a77654c4c2
15 изменённых файлов: 162 добавлений и 132 удалений

Просмотреть файл

@ -642,10 +642,12 @@ impl<LTF: LayoutTaskFactory, STF: ScriptTaskFactory> Constellation<LTF, STF> {
// FIXME(tkuehn): Need to follow the standardized spec for checking same-origin
// Reuse the script task if the URL is same-origin
if same_script {
debug!("Constellation: loading same-origin iframe at {:?}", url);
debug!("Constellation: loading same-origin iframe, \
parent url {:?}, iframe url {:?}", source_url, url);
Some(source_pipeline.script_chan.clone())
} else {
debug!("Constellation: loading cross-origin iframe at {:?}", url);
debug!("Constellation: loading cross-origin iframe, \
parent url {:?}, iframe url {:?}", source_url, url);
None
}
};

Просмотреть файл

@ -5,8 +5,7 @@
use rustc_serialize::json::{decode};
use time;
use url::Url;
use net_traits::IncludeSubdomains;
use resource_task::{IPV4_REGEX, IPV6_REGEX};
use net_traits::{IncludeSubdomains, IPV4_REGEX, IPV6_REGEX};
use std::str::{from_utf8};

Просмотреть файл

@ -4,6 +4,7 @@
use net_traits::{ControlMsg, CookieSource, LoadData, Metadata, LoadConsumer, IncludeSubdomains};
use net_traits::ProgressMsg::{Payload, Done};
use net_traits::hosts::replace_hosts;
use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, NetworkEvent};
use mime_classifier::MIMEClassifier;
use resource_task::{start_sending_opt, start_sending_sniffed_opt};
@ -95,7 +96,11 @@ fn load(mut load_data: LoadData,
// repository DOES exist, please update this constant to use it.
let max_redirects = 50;
let mut iters = 0;
let mut url = load_data.url.clone();
// URL of the document being loaded, as seen by all the higher-level code.
let mut doc_url = load_data.url.clone();
// URL that we actually fetch from the network, after applying the replacements
// specified in the hosts file.
let mut url = replace_hosts(&load_data.url);
let mut redirected_to = HashSet::new();
// If the URL is a view-source scheme then the scheme data contains the
@ -105,7 +110,8 @@ fn load(mut load_data: LoadData,
let viewing_source = url.scheme == "view-source";
if viewing_source {
let inner_url = load_data.url.non_relative_scheme_data().unwrap();
url = Url::parse(inner_url).unwrap();
doc_url = Url::parse(inner_url).unwrap();
url = replace_hosts(&doc_url);
match &*url.scheme {
"http" | "https" => {}
_ => {
@ -176,8 +182,11 @@ reason: \"certificate verify failed\" }]))";
}
};
// Preserve the `host` header set automatically by Request.
let host = req.headers().get::<Host>().unwrap().clone();
//Ensure that the host header is set from the original url
let host = Host {
hostname: doc_url.serialize_host().unwrap(),
port: doc_url.port_or_default()
};
// Avoid automatically preserving request headers when redirects occur.
// See https://bugzilla.mozilla.org/show_bug.cgi?id=401564 and
@ -204,9 +213,9 @@ reason: \"certificate verify failed\" }]))";
}
let (tx, rx) = ipc::channel().unwrap();
resource_mgr_chan.send(ControlMsg::GetCookiesForUrl(url.clone(),
tx,
CookieSource::HTTP)).unwrap();
resource_mgr_chan.send(ControlMsg::GetCookiesForUrl(doc_url.clone(),
tx,
CookieSource::HTTP)).unwrap();
if let Some(cookie_list) = rx.recv().unwrap() {
let mut v = Vec::new();
v.push(cookie_list.into_bytes());
@ -291,7 +300,7 @@ reason: \"certificate verify failed\" }]))";
if let Some(cookies) = response.headers.get_raw("set-cookie") {
for cookie in cookies.iter() {
if let Ok(cookies) = String::from_utf8(cookie.clone()) {
resource_mgr_chan.send(ControlMsg::SetCookiesForUrl(url.clone(),
resource_mgr_chan.send(ControlMsg::SetCookiesForUrl(doc_url.clone(),
cookies,
CookieSource::HTTP)).unwrap();
}
@ -340,15 +349,16 @@ reason: \"certificate verify failed\" }]))";
}
_ => {}
}
let new_url = match UrlParser::new().base_url(&url).parse(&new_url) {
let new_doc_url = match UrlParser::new().base_url(&doc_url).parse(&new_url) {
Ok(u) => u,
Err(e) => {
send_error(url, e.to_string(), start_chan);
send_error(doc_url, e.to_string(), start_chan);
return;
}
};
info!("redirecting to {}", new_url);
url = new_url;
info!("redirecting to {}", new_doc_url);
url = replace_hosts(&new_doc_url);
doc_url = new_doc_url;
// According to https://tools.ietf.org/html/rfc7231#section-6.4.2,
// historically UAs have rewritten POST->GET on 301 and 302 responses.
@ -358,12 +368,12 @@ reason: \"certificate verify failed\" }]))";
load_data.method = Method::Get;
}
if redirected_to.contains(&url) {
send_error(url, "redirect loop".to_string(), start_chan);
if redirected_to.contains(&doc_url) {
send_error(doc_url, "redirect loop".to_string(), start_chan);
return;
}
redirected_to.insert(url.clone());
redirected_to.insert(doc_url.clone());
continue;
}
None => ()
@ -374,7 +384,7 @@ reason: \"certificate verify failed\" }]))";
if viewing_source {
adjusted_headers.set(ContentType(Mime(TopLevel::Text, SubLevel::Plain, vec![])));
}
let mut metadata: Metadata = Metadata::default(url);
let mut metadata: Metadata = Metadata::default(doc_url);
metadata.set_content_type(match adjusted_headers.get() {
Some(&ContentType(ref mime)) => Some(mime),
None => None

Просмотреть файл

@ -2,7 +2,6 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#![feature(box_raw)]
#![feature(box_syntax)]
#![feature(fnbox)]
#![feature(mpsc_select)]
@ -10,8 +9,6 @@
#![feature(plugin)]
#![feature(vec_push_all)]
#![plugin(regex_macros)]
extern crate net_traits;
extern crate cookie as cookie_rs;
extern crate devtools_traits;
@ -29,8 +26,6 @@ extern crate time;
extern crate url;
extern crate uuid;
extern crate regex;
pub mod about_loader;
pub mod file_loader;
pub mod http_loader;

Просмотреть файл

@ -26,47 +26,12 @@ use hyper::header::{ContentType, Header, SetCookie, UserAgent};
use hyper::mime::{Mime, TopLevel, SubLevel};
use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
use regex::Regex;
use std::borrow::ToOwned;
use std::boxed::FnBox;
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::{BufReader, Read};
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::mpsc::{channel, Sender};
static mut HOST_TABLE: Option<*mut HashMap<String, String>> = None;
pub static IPV4_REGEX: Regex = regex!(
r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"
);
pub static IPV6_REGEX: Regex = regex!(r"^([a-fA-F0-9]{0,4}[:]?){1,8}(/\d{1,3})?$");
pub fn global_init() {
//TODO: handle bad file path
let path = match env::var("HOST_FILE") {
Ok(host_file_path) => host_file_path,
Err(_) => return,
};
let mut file = match File::open(&path) {
Ok(f) => BufReader::new(f),
Err(_) => return,
};
let mut lines = String::new();
match file.read_to_string(&mut lines) {
Ok(_) => (),
Err(_) => return,
};
let host_table = Box::into_raw(parse_hostsfile(&lines));
unsafe {
HOST_TABLE = Some(host_table);
}
}
pub enum ProgressSender {
Channel(IpcSender<ProgressMsg>),
Listener(AsyncResponseTarget),
@ -187,38 +152,6 @@ pub fn new_resource_task(user_agent: Option<String>,
setup_chan
}
pub fn parse_hostsfile(hostsfile_content: &str) -> Box<HashMap<String, String>> {
let mut host_table = HashMap::new();
let lines: Vec<&str> = hostsfile_content.split('\n').collect();
for line in lines.iter() {
let ip_host: Vec<&str> = line.trim().split(|c: char| c == ' ' || c == '\t').collect();
if ip_host.len() > 1 {
if !IPV4_REGEX.is_match(ip_host[0]) && !IPV6_REGEX.is_match(ip_host[0]) { continue; }
let address = ip_host[0].to_owned();
for token in ip_host.iter().skip(1) {
if token.as_bytes()[0] == b'#' {
break;
}
host_table.insert(token.to_owned().to_string(), address.clone());
}
}
}
box host_table
}
pub fn replace_hosts(mut load_data: LoadData, host_table: *mut HashMap<String, String>) -> LoadData {
if let Some(h) = load_data.url.domain_mut() {
unsafe {
if let Some(ip) = (*host_table).get(h) {
*h = ip.clone();
}
}
}
return load_data;
}
struct ResourceChannelManager {
from_client: IpcReceiver<ControlMsg>,
resource_manager: ResourceManager
@ -296,12 +229,6 @@ impl ResourceManager {
}
fn load(&mut self, mut load_data: LoadData, consumer: LoadConsumer) {
unsafe {
if let Some(host_table) = HOST_TABLE {
load_data = replace_hosts(load_data, host_table);
}
}
self.user_agent.as_ref().map(|ua| {
load_data.preserved_headers.set(UserAgent(ua.clone()));
});

Просмотреть файл

@ -34,6 +34,7 @@ features = [ "serde_serialization" ]
[dependencies]
log = "0.3"
euclid = "0.1"
regex = "0.1.33"
regex_macros = "0.1.19"
serde = "0.4"
serde_macros = "0.4"

Просмотреть файл

@ -0,0 +1,79 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::{BufReader, Read};
use url::Url;
use {IPV4_REGEX, IPV6_REGEX};
static mut HOST_TABLE: Option<*mut HashMap<String, String>> = None;
pub fn global_init() {
//TODO: handle bad file path
let path = match env::var("HOST_FILE") {
Ok(host_file_path) => host_file_path,
Err(_) => return,
};
let mut file = match File::open(&path) {
Ok(f) => BufReader::new(f),
Err(_) => return,
};
let mut lines = String::new();
match file.read_to_string(&mut lines) {
Ok(_) => (),
Err(_) => return,
};
unsafe {
let host_table = Box::into_raw(parse_hostsfile(&lines));
HOST_TABLE = Some(host_table);
}
}
pub fn parse_hostsfile(hostsfile_content: &str) -> Box<HashMap<String, String>> {
let mut host_table = HashMap::new();
let lines: Vec<&str> = hostsfile_content.split('\n').collect();
for line in lines.iter() {
let ip_host: Vec<&str> = line.trim().split(|c: char| c == ' ' || c == '\t').collect();
if ip_host.len() > 1 {
if !IPV4_REGEX.is_match(ip_host[0]) && !IPV6_REGEX.is_match(ip_host[0]) { continue; }
let address = ip_host[0].to_owned();
for token in ip_host.iter().skip(1) {
if token.as_bytes()[0] == b'#' {
break;
}
host_table.insert(token.to_owned().to_string(), address.clone());
}
}
}
box host_table
}
pub fn replace_hosts(url: &Url) -> Url {
unsafe {
HOST_TABLE.map(|host_table| {
host_replacement(host_table, url)
}).unwrap_or_else(|| url.clone())
}
}
pub fn host_replacement(host_table: *mut HashMap<String, String>,
url: &Url) -> Url {
unsafe {
url.domain().and_then(|domain|
(*host_table).get(domain).map(|ip| {
let mut net_url = url.clone();
*net_url.domain_mut().unwrap() = ip.clone();
net_url
})).unwrap_or(url.clone())
}
}

Просмотреть файл

@ -4,18 +4,22 @@
#![feature(box_syntax)]
#![feature(custom_derive)]
#![feature(box_raw)]
#![feature(plugin)]
#![feature(slice_patterns)]
#![feature(step_by)]
#![feature(vec_push_all)]
#![plugin(serde_macros)]
#![plugin(regex_macros)]
extern crate euclid;
extern crate hyper;
extern crate ipc_channel;
#[macro_use]
extern crate log;
extern crate png;
extern crate regex;
extern crate serde;
extern crate stb_image;
extern crate url;
@ -28,14 +32,20 @@ use hyper::method::Method;
use hyper::mime::{Mime, Attr};
use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
use msg::constellation_msg::{PipelineId};
use regex::Regex;
use serde::{Deserializer, Serializer};
use url::Url;
use std::thread;
pub mod hosts;
pub mod image_cache_task;
pub mod storage_task;
pub static IPV4_REGEX: Regex = regex!(
r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$");
pub static IPV6_REGEX: Regex = regex!(r"^([a-fA-F0-9]{0,4}[:]?){1,8}(/\d{1,3})?$");
/// Image handling.
///
/// It may be surprising that this goes in the network crate as opposed to the graphics crate.
@ -342,4 +352,3 @@ impl Iterator for ProgressMsgPortIterator {
}
}
}

Просмотреть файл

@ -19,6 +19,7 @@ use dom::bindings::utils::reflect_dom_object;
use dom::closeevent::CloseEvent;
use dom::event::{Event, EventBubbles, EventCancelable, EventHelpers};
use dom::eventtarget::{EventTarget, EventTargetHelpers, EventTargetTypeId};
use net_traits::hosts::replace_hosts;
use script_task::Runnable;
use script_task::ScriptMsg;
use std::cell::{Cell, RefCell};
@ -27,6 +28,7 @@ use util::str::DOMString;
use util::task::spawn_named;
use hyper::header::Host;
use websocket::Message;
use websocket::ws::sender::Sender as Sender_Object;
use websocket::client::sender::Sender;
@ -64,10 +66,20 @@ pub struct WebSocket {
}
/// *Establish a WebSocket Connection* as defined in RFC 6455.
fn establish_a_websocket_connection(url: (Host, String, bool), origin: String)
-> WebSocketResult<(Sender<WebSocketStream>, Receiver<WebSocketStream>)> {
let mut request = try!(Client::connect(url));
fn establish_a_websocket_connection(resource_url: &Url, net_url: (Host, String, bool),
origin: String)
-> WebSocketResult<(Sender<WebSocketStream>, Receiver<WebSocketStream>)> {
// URL that we actually fetch from the network, after applying the replacements
// specified in the hosts file.
let host = Host {
hostname: resource_url.serialize_host().unwrap(),
port: resource_url.port_or_default()
};
let mut request = try!(Client::connect(net_url));
request.headers.set(Origin(origin));
request.headers.set(host);
let response = try!(request.send());
try!(response.validate());
@ -104,8 +116,8 @@ impl WebSocket {
protocols: Option<DOMString>)
-> Fallible<Root<WebSocket>> {
// Step 1.
let parsed_url = try!(Url::parse(&url).map_err(|_| Error::Syntax));
let url = try!(parse_url(&parsed_url).map_err(|_| Error::Syntax));
let resource_url = try!(Url::parse(&url).map_err(|_| Error::Syntax));
let net_url = try!(parse_url(&replace_hosts(&resource_url)).map_err(|_| Error::Syntax));
// Step 2: Disallow https -> ws connections.
// Step 3: Potentially block access to some ports.
@ -123,6 +135,7 @@ impl WebSocket {
if protocols[i+1..].iter().any(|p| p == protocol) {
return Err(Syntax);
}
if protocol.chars().any(|c| c < '\u{0021}' || c > '\u{007E}') {
@ -133,7 +146,7 @@ impl WebSocket {
// Step 6: Origin.
// Step 7.
let ws = WebSocket::new(global, parsed_url);
let ws = WebSocket::new(global, resource_url.clone());
let address = Trusted::new(global.get_cx(), ws.r(), global.script_chan());
let origin = global.get_url().serialize();
@ -142,7 +155,7 @@ impl WebSocket {
// Step 8: Protocols.
// Step 9.
let channel = establish_a_websocket_connection(url, origin);
let channel = establish_a_websocket_connection(&resource_url, net_url, origin);
let (temp_sender, _temp_receiver) = match channel {
Ok(channel) => channel,
Err(e) => {

2
servo/components/servo/Cargo.lock сгенерированный
Просмотреть файл

@ -930,6 +930,8 @@ dependencies = [
"log 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"msg 0.0.1",
"png 0.1.0 (git+https://github.com/servo/rust-png)",
"regex 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"regex_macros 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_macros 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"stb_image 0.1.0 (git+https://github.com/servo/rust-stb-image)",

Просмотреть файл

@ -23,6 +23,7 @@ extern crate servo;
extern crate compositing;
// Servo networking
extern crate net;
extern crate net_traits;
// Servo common utilitiess
extern crate util;
// The window backed by glutin
@ -36,7 +37,7 @@ extern crate android_glue;
use std::rc::Rc;
use util::opts;
use net::resource_task;
use net_traits::hosts;
use servo::Browser;
use compositing::windowing::WindowEvent;
@ -52,7 +53,7 @@ fn main() {
setup_logging();
// Possibly interpret the `HOST_FILE` environment variable
resource_task::global_init();
hosts::global_init();
let window = if opts::get().headless {
None

4
servo/ports/cef/Cargo.lock сгенерированный
Просмотреть файл

@ -894,8 +894,6 @@ dependencies = [
"net_traits 0.0.1",
"openssl 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
"png 0.1.0 (git+https://github.com/servo/rust-png)",
"regex 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"regex_macros 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)",
"url 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
@ -913,6 +911,8 @@ dependencies = [
"log 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"msg 0.0.1",
"png 0.1.0 (git+https://github.com/servo/rust-png)",
"regex 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"regex_macros 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_macros 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"stb_image 0.1.0 (git+https://github.com/servo/rust-stb-image)",

4
servo/ports/gonk/Cargo.lock сгенерированный
Просмотреть файл

@ -814,8 +814,6 @@ dependencies = [
"net_traits 0.0.1",
"openssl 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
"png 0.1.0 (git+https://github.com/servo/rust-png)",
"regex 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"regex_macros 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)",
"url 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
@ -833,6 +831,8 @@ dependencies = [
"log 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"msg 0.0.1",
"png 0.1.0 (git+https://github.com/servo/rust-png)",
"regex 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"regex_macros 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_macros 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"stb_image 0.1.0 (git+https://github.com/servo/rust-stb-image)",

Просмотреть файл

@ -39,13 +39,14 @@ extern crate layers;
extern crate egl;
extern crate url;
extern crate net;
extern crate net_traits;
extern crate env_logger;
#[link(name = "stlport")]
extern {}
use util::opts;
use net::resource_task;
use net_traits::hosts;
use servo::Browser;
use compositing::windowing::WindowEvent;
@ -64,7 +65,7 @@ fn main() {
// Parse the command line options and store them globally
opts::from_cmdline_args(env::args().collect::<Vec<_>>().as_slice());
resource_task::global_init();
hosts::global_init();
let window = if opts::get().headless {
None

Просмотреть файл

@ -3,8 +3,9 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use ipc_channel::ipc;
use net::resource_task::{new_resource_task, parse_hostsfile, replace_hosts};
use net::resource_task::new_resource_task;
use net_traits::{ControlMsg, LoadData, LoadConsumer};
use net_traits::hosts::{parse_hostsfile, host_replacement};
use net_traits::ProgressMsg;
use std::borrow::ToOwned;
use std::collections::HashMap;
@ -163,22 +164,12 @@ fn test_replace_hosts() {
let host_table: *mut HashMap<String, String> = Box::into_raw(host_table_box);
//Start the TCP server
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let url = Url::parse("http://foo.bar.com:8000/foo").unwrap();
assert_eq!(host_replacement(host_table, &url).domain().unwrap(), "127.0.0.1");
//Start the resource task and make a request to our TCP server
let resource_task = new_resource_task(None, None);
let (start_chan, _start_port) = ipc::channel().unwrap();
let url = Url::parse(&format!("http://foo.bar.com:{}", port)).unwrap();
let msg = ControlMsg::Load(replace_hosts(LoadData::new(url, None), host_table),
LoadConsumer::Channel(start_chan));
resource_task.send(msg).unwrap();
let url = Url::parse("http://servo.test.server").unwrap();
assert_eq!(host_replacement(host_table, &url).domain().unwrap(), "127.0.0.2");
match listener.accept() {
Ok(..) => assert!(true, "received request"),
Err(_) => assert!(false, "error")
}
resource_task.send(ControlMsg::Exit).unwrap();
let url = Url::parse("http://a.foo.bar.com").unwrap();
assert_eq!(host_replacement(host_table, &url).domain().unwrap(), "a.foo.bar.com");
}