servo: Merge #11468 - Implement meta referrer policy delivery (3) (from rebstar6:refPol4); r=nox

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes fix #10311 (github issue number if applicable).

<!-- Either: -->
- [X] There are tests for these changes OR
- [ ] These changes do not require tests because _____

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

Source-Repo: https://github.com/servo/servo
Source-Revision: 530b5a649eb7284c0ff4e316b8eabd5cc62c1e80

servo/components/msg/constellation_msg.rs

@@ -358,7 +358,7 @@ pub enum FrameType {
 
 /// [Policies](https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states)
 /// for providing a referrer header for a request
-#[derive(HeapSizeOf, Clone, Deserialize, Serialize)]
+#[derive(Clone, Copy, Debug, Deserialize, HeapSizeOf, Serialize)]
 pub enum ReferrerPolicy {
     NoReferrer,
     NoRefWhenDowngrade,

servo/components/net/http_loader.rs

@@ -596,7 +596,8 @@ pub fn modify_request_headers(headers: &mut Headers,
                               cookie_jar: &Arc<RwLock<CookieStorage>>,
                               auth_cache: &Arc<RwLock<AuthCache>>,
                               load_data: &LoadData,
-                              block_cookies: bool) {
+                              block_cookies: bool,
+                              referrer_url: &mut Option<Url>) {
     // Ensure that the host header is set from the original url
     let host = Host {
         hostname: url.host_str().unwrap().to_owned(),
@@ -617,10 +618,12 @@ pub fn modify_request_headers(headers: &mut Headers,
     set_default_accept(headers);
     set_default_accept_encoding(headers);
 
-    if let Some(referer_val) = determine_request_referrer(headers,
-                                                          load_data.referrer_policy.clone(),
-                                                          load_data.referrer_url.clone(),
-                                                          url.clone()) {
+    *referrer_url = determine_request_referrer(headers,
+                                               load_data.referrer_policy.clone(),
+                                               referrer_url.clone(),
+                                               url.clone());
+
+    if let Some(referer_val) = referrer_url.clone() {
         headers.set(Referer(referer_val.into_string()));
     }
 
@@ -808,6 +811,8 @@ pub fn load<A, B>(load_data: &LoadData,
     let mut doc_url = load_data.url.clone();
     let mut redirected_to = HashSet::new();
     let mut method = load_data.method.clone();
+    // URL of referrer - to be updated with redirects
+    let mut referrer_url = load_data.referrer_url.clone();
 
     let mut new_auth_header: Option<Authorization<Basic>> = None;
 
@@ -901,7 +906,8 @@ pub fn load<A, B>(load_data: &LoadData,
 
         modify_request_headers(&mut request_headers, &doc_url,
                                &user_agent, &http_state.cookie_jar,
-                               &http_state.auth_cache, &load_data, block_cookies);
+                               &http_state.auth_cache, &load_data,
+                               block_cookies, &mut referrer_url);
 
         //if there is a new auth header then set the request headers with it
         if let Some(ref auth_header) = new_auth_header {

servo/components/net_traits/lib.rs

@@ -140,7 +140,7 @@ impl LoadData {
             credentials_flag: true,
             context: context,
             referrer_policy: load_origin.referrer_policy(),
-            referrer_url: load_origin.referrer_url(),
+            referrer_url: load_origin.referrer_url().clone(),
             source: load_origin.request_source()
         }
     }

servo/components/script/dom/document.rs

@@ -237,7 +237,7 @@ pub struct Document {
     /// The document's origin.
     origin: Origin,
     ///  https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states
-    referrer_policy: Option<ReferrerPolicy>,
+    referrer_policy: Cell<Option<ReferrerPolicy>>,
 }
 
 #[derive(JSTraceable, HeapSizeOf)]
@@ -1697,7 +1697,7 @@ impl Document {
             touchpad_pressure_phase: Cell::new(TouchpadPressurePhase::BeforeClick),
             origin: origin,
             //TODO - setting this for now so no Referer header set
-            referrer_policy: Some(ReferrerPolicy::NoReferrer),
+            referrer_policy: Cell::new(Some(ReferrerPolicy::NoReferrer)),
         }
     }
 
@@ -1827,9 +1827,13 @@ impl Document {
         }
     }
 
-    //TODO - for now, returns no-referrer for all until reading in the value
+    pub fn set_referrer_policy(&self, policy: Option<ReferrerPolicy>) {
+        self.referrer_policy.set(policy);
+    }
+
+    //TODO - default still at no-referrer
     pub fn get_referrer_policy(&self) -> Option<ReferrerPolicy> {
-        return self.referrer_policy.clone();
+        return self.referrer_policy.get();
     }
 }
 
@@ -2800,6 +2804,20 @@ fn update_with_current_time_ms(marker: &Cell<u64>) {
     }
 }
 
+/// https://w3c.github.io/webappsec-referrer-policy/#determine-policy-for-token
+pub fn determine_policy_for_token(token: &str) -> Option<ReferrerPolicy> {
+    let lower = token.to_lowercase();
+    return match lower.as_ref() {
+        "never" | "no-referrer" => Some(ReferrerPolicy::NoReferrer),
+        "default" | "no-referrer-when-downgrade" => Some(ReferrerPolicy::NoRefWhenDowngrade),
+        "origin" => Some(ReferrerPolicy::OriginOnly),
+        "origin-when-cross-origin" => Some(ReferrerPolicy::OriginWhenCrossOrigin),
+        "always" | "unsafe-url" => Some(ReferrerPolicy::UnsafeUrl),
+        "" => Some(ReferrerPolicy::NoReferrer),
+        _ => None,
+    }
+}
+
 pub struct DocumentProgressHandler {
     addr: Trusted<Document>
 }

servo/components/script/dom/htmlheadelement.rs

@@ -2,13 +2,16 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+use dom::bindings::codegen::Bindings::DocumentBinding::DocumentMethods;
 use dom::bindings::codegen::Bindings::HTMLHeadElementBinding;
 use dom::bindings::inheritance::Castable;
-use dom::bindings::js::Root;
+use dom::bindings::js::{Root, RootedReference};
 use dom::bindings::str::DOMString;
-use dom::document::Document;
+use dom::document::{Document, determine_policy_for_token};
+use dom::element::Element;
 use dom::htmlelement::HTMLElement;
-use dom::node::Node;
+use dom::htmlmetaelement::HTMLMetaElement;
+use dom::node::{Node, document_from_node};
 use dom::userscripts::load_script;
 use dom::virtualmethods::VirtualMethods;
 use string_cache::Atom;
@@ -34,6 +37,33 @@ impl HTMLHeadElement {
         let element = HTMLHeadElement::new_inherited(localName, prefix, document);
         Node::reflect_node(box element, document, HTMLHeadElementBinding::Wrap)
     }
+
+    /// https://html.spec.whatwg.org/multipage/#meta-referrer
+    pub fn set_document_referrer(&self) {
+        let doc = document_from_node(self);
+
+        if doc.GetHead().r() != Some(self) {
+            return;
+        }
+
+        let node = self.upcast::<Node>();
+        let candidates = node.traverse_preorder()
+                             .filter_map(Root::downcast::<Element>)
+                             .filter(|elem| elem.is::<HTMLMetaElement>())
+                             .filter(|elem| elem.get_string_attribute(&atom!("name")) == "referrer")
+                             .filter(|elem| elem.get_attribute(&ns!(), &atom!("content")).is_some());
+
+        for meta in candidates {
+            if let Some(content) = meta.get_attribute(&ns!(), &atom!("content")).r() {
+                let content = content.value();
+                let content_val = content.trim();
+                if !content_val.is_empty() {
+                    doc.set_referrer_policy(determine_policy_for_token(content_val));
+                    return;
+                }
+            }
+        }
+    }
 }
 
 impl VirtualMethods for HTMLHeadElement {

servo/components/script/dom/htmliframeelement.rs

@@ -146,8 +146,9 @@ impl HTMLIFrameElement {
     pub fn process_the_iframe_attributes(&self) {
         let url = self.get_url();
 
-        // TODO - loaddata here should have referrer info (not None, None)
-        self.navigate_or_reload_child_browsing_context(Some(LoadData::new(url, None, None)));
+        let document = document_from_node(self);
+        self.navigate_or_reload_child_browsing_context(
+            Some(LoadData::new(url, document.get_referrer_policy(), Some(document.url().clone()))));
     }
 
     #[allow(unsafe_code)]

servo/components/script/dom/htmlmetaelement.rs

@@ -2,17 +2,19 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use dom::attr::AttrValue;
+use dom::attr::{Attr, AttrValue};
 use dom::bindings::cell::DOMRefCell;
 use dom::bindings::codegen::Bindings::HTMLMetaElementBinding;
 use dom::bindings::codegen::Bindings::HTMLMetaElementBinding::HTMLMetaElementMethods;
+use dom::bindings::codegen::Bindings::NodeBinding::NodeMethods;
 use dom::bindings::inheritance::Castable;
 use dom::bindings::js::{Root, RootedReference};
 use dom::bindings::str::DOMString;
 use dom::document::Document;
-use dom::element::Element;
+use dom::element::{AttributeMutation, Element};
 use dom::htmlelement::HTMLElement;
-use dom::node::{Node, document_from_node};
+use dom::htmlheadelement::HTMLHeadElement;
+use dom::node::{Node, UnbindContext, document_from_node};
 use dom::virtualmethods::VirtualMethods;
 use std::ascii::AsciiExt;
 use std::sync::Arc;
@@ -59,6 +61,10 @@ impl HTMLMetaElement {
             if name == "viewport" {
                 self.apply_viewport();
             }
+
+            if name == "referrer" {
+                self.apply_referrer();
+            }
         }
     }
 
@@ -85,6 +91,27 @@ impl HTMLMetaElement {
             }
         }
     }
+
+    fn process_referrer_attribute(&self) {
+        let element = self.upcast::<Element>();
+        if let Some(name) = element.get_attribute(&ns!(), &atom!("name")).r() {
+            let name = name.value().to_ascii_lowercase();
+            let name = name.trim_matches(HTML_SPACE_CHARACTERS);
+
+            if name == "referrer" {
+                self.apply_referrer();
+            }
+        }
+    }
+
+    /// https://html.spec.whatwg.org/multipage/#meta-referrer
+    fn apply_referrer(&self) {
+        if let Some(parent) = self.upcast::<Node>().GetParentElement() {
+            if let Some(head) = parent.downcast::<HTMLHeadElement>() {
+                head.set_document_referrer();
+            }
+        }
+    }
 }
 
 impl HTMLMetaElementMethods for HTMLMetaElement {
@@ -122,4 +149,22 @@ impl VirtualMethods for HTMLMetaElement {
             _ => self.super_type().unwrap().parse_plain_attribute(name, value),
         }
     }
+
+    fn attribute_mutated(&self, attr: &Attr, mutation: AttributeMutation) {
+        if let Some(s) = self.super_type() {
+            s.attribute_mutated(attr, mutation);
+        }
+
+        self.process_referrer_attribute();
+    }
+
+    fn unbind_from_tree(&self, context: &UnbindContext) {
+        if let Some(ref s) = self.super_type() {
+            s.unbind_from_tree(context);
+        }
+
+        if context.tree_in_doc {
+            self.process_referrer_attribute();
+        }
+    }
 }

servo/components/script/dom/xmlhttprequest.rs

@@ -148,10 +148,20 @@ pub struct XMLHttpRequest {
     fetch_time: Cell<i64>,
     generation_id: Cell<GenerationId>,
     response_status: Cell<Result<(), ()>>,
+    referrer_url: Option<Url>,
+    referrer_policy: Option<ReferrerPolicy>,
 }
 
 impl XMLHttpRequest {
     fn new_inherited(global: GlobalRef) -> XMLHttpRequest {
+        //TODO - update this when referrer policy implemented for workers
+        let (referrer_url, referrer_policy) = if let GlobalRef::Window(window) = global {
+            let document = window.Document();
+            (Some(document.url().clone()), document.get_referrer_policy())
+        } else {
+            (None, None)
+        };
+
         XMLHttpRequest {
             eventtarget: XMLHttpRequestEventTarget::new_inherited(),
             ready_state: Cell::new(XMLHttpRequestState::Unsent),
@@ -183,6 +193,8 @@ impl XMLHttpRequest {
             fetch_time: Cell::new(0),
             generation_id: Cell::new(GenerationId(0)),
             response_status: Cell::new(Ok(())),
+            referrer_url: referrer_url,
+            referrer_policy: referrer_policy,
         }
     }
     pub fn new(global: GlobalRef) -> Root<XMLHttpRequest> {
@@ -297,10 +309,10 @@ impl XMLHttpRequest {
 
 impl LoadOrigin for XMLHttpRequest {
     fn referrer_url(&self) -> Option<Url> {
-        None
+        return self.referrer_url.clone();
     }
     fn referrer_policy(&self) -> Option<ReferrerPolicy> {
-        None
+        return self.referrer_policy;
     }
     fn request_source(&self) -> RequestSource {
         if self.sync.get() {
@@ -592,11 +604,12 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
 
         // Step 5
         let global = self.global();
-        //TODO - set referrer_policy/referrer_url in load_data
+
         let mut load_data =
             LoadData::new(LoadContext::Browsing,
                           self.request_url.borrow().clone().unwrap(),
                           self);
+
         if load_data.url.origin().ne(&global.r().get_url().origin()) {
             load_data.credentials_flag = self.WithCredentials();
         }