servo: Merge #12780 - Add XHR support for FormData and related fixes (from izgzhen:xhr-file-upload); r=Manishearth

+ Update IDL for `FormData`, and related fixes
+ Reuse `htmlformelement::FormDatum` for `formdata.rs` for a uniform representation, and related fixes
+ Add support for `FormData` branch of XHR `BodyInit`.
+ Fixes a lot of related WPT tests

---
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] There are tests for these changes OR

Source-Repo: https://github.com/servo/servo
Source-Revision: ddf4e68c91bbab1a456b0eb86c2a7f0c3268d575

servo/components/script/dom/formdata.rs

@@ -5,40 +5,41 @@
 use dom::bindings::cell::DOMRefCell;
 use dom::bindings::codegen::Bindings::FormDataBinding;
 use dom::bindings::codegen::Bindings::FormDataBinding::FormDataMethods;
-use dom::bindings::codegen::UnionTypes::BlobOrUSVString;
+use dom::bindings::codegen::UnionTypes::FileOrUSVString;
 use dom::bindings::error::Fallible;
 use dom::bindings::global::GlobalRef;
-use dom::bindings::js::{JS, Root};
+use dom::bindings::js::Root;
 use dom::bindings::reflector::{Reflectable, Reflector, reflect_dom_object};
 use dom::bindings::str::{DOMString, USVString};
 use dom::blob::{Blob, BlobImpl};
 use dom::file::File;
-use dom::htmlformelement::HTMLFormElement;
+use dom::htmlformelement::{HTMLFormElement, FormDatumValue, FormDatum};
 use std::collections::HashMap;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
 use string_cache::Atom;
 
-#[derive(JSTraceable, Clone)]
-#[must_root]
-#[derive(HeapSizeOf)]
-pub enum FormDatum {
-    StringData(String),
-    BlobData(JS<Blob>)
-}
-
 #[dom_struct]
 pub struct FormData {
     reflector_: Reflector,
     data: DOMRefCell<HashMap<Atom, Vec<FormDatum>>>,
-    form: Option<JS<HTMLFormElement>>
 }
 
 impl FormData {
-    fn new_inherited(form: Option<&HTMLFormElement>) -> FormData {
+    fn new_inherited(opt_form: Option<&HTMLFormElement>) -> FormData {
+        let mut hashmap: HashMap<Atom, Vec<FormDatum>> = HashMap::new();
+
+        if let Some(form) = opt_form {
+            for datum in form.get_form_dataset(None) {
+                match hashmap.entry(Atom::from(datum.name.as_ref())) {
+                    Occupied(entry) => entry.into_mut().push(datum),
+                    Vacant(entry) => { entry.insert(vec!(datum)); }
+                }
+            }
+        }
+
         FormData {
             reflector_: Reflector::new(),
-            data: DOMRefCell::new(HashMap::new()),
-            form: form.map(|f| JS::from_ref(f)),
+            data: DOMRefCell::new(hashmap),
         }
     }
 
@@ -55,24 +56,34 @@ impl FormData {
 
 impl FormDataMethods for FormData {
     // https://xhr.spec.whatwg.org/#dom-formdata-append
-    fn Append(&self, name: USVString, value: USVString) {
+    fn Append(&self, name: USVString, str_value: USVString) {
+        let datum = FormDatum {
+            ty: DOMString::from("string"),
+            name: DOMString::from(name.0.clone()),
+            value: FormDatumValue::String(DOMString::from(str_value.0)),
+        };
+
         let mut data = self.data.borrow_mut();
         match data.entry(Atom::from(name.0)) {
-            Occupied(entry) => entry.into_mut().push(FormDatum::StringData(value.0)),
-            Vacant  (entry) => { entry.insert(vec!(FormDatum::StringData(value.0))); }
+            Occupied(entry) => entry.into_mut().push(datum),
+            Vacant(entry) => { entry.insert(vec!(datum)); }
         }
     }
 
     #[allow(unrooted_must_root)]
     // https://xhr.spec.whatwg.org/#dom-formdata-append
-    fn Append_(&self, name: USVString, value: &Blob, filename: Option<USVString>) {
-        let blob = FormDatum::BlobData(JS::from_ref(&*self.get_file_or_blob(value, filename)));
+    fn Append_(&self, name: USVString, blob: &Blob, filename: Option<USVString>) {
+        let datum = FormDatum {
+            ty: DOMString::from("file"),
+            name: DOMString::from(name.0.clone()),
+            value: FormDatumValue::File(Root::from_ref(&*self.get_file(blob, filename))),
+        };
+
         let mut data = self.data.borrow_mut();
+
         match data.entry(Atom::from(name.0)) {
-            Occupied(entry) => entry.into_mut().push(blob),
-            Vacant(entry) => {
-                entry.insert(vec!(blob));
-            }
+            Occupied(entry) => entry.into_mut().push(datum),
+            Vacant(entry) => { entry.insert(vec!(datum)); },
         }
     }
 
@@ -82,23 +93,23 @@ impl FormDataMethods for FormData {
     }
 
     // https://xhr.spec.whatwg.org/#dom-formdata-get
-    fn Get(&self, name: USVString) -> Option<BlobOrUSVString> {
+    fn Get(&self, name: USVString) -> Option<FileOrUSVString> {
         self.data.borrow()
                  .get(&Atom::from(name.0))
-                 .map(|entry| match entry[0] {
-                     FormDatum::StringData(ref s) => BlobOrUSVString::USVString(USVString(s.clone())),
-                     FormDatum::BlobData(ref b) => BlobOrUSVString::Blob(Root::from_ref(&*b)),
+                 .map(|entry| match entry[0].value {
+                     FormDatumValue::String(ref s) => FileOrUSVString::USVString(USVString(s.to_string())),
+                     FormDatumValue::File(ref b) => FileOrUSVString::File(Root::from_ref(&*b)),
                  })
     }
 
     // https://xhr.spec.whatwg.org/#dom-formdata-getall
-    fn GetAll(&self, name: USVString) -> Vec<BlobOrUSVString> {
+    fn GetAll(&self, name: USVString) -> Vec<FileOrUSVString> {
         self.data.borrow()
                  .get(&Atom::from(name.0))
                  .map_or(vec![], |data|
-                    data.iter().map(|item| match *item {
-                        FormDatum::StringData(ref s) => BlobOrUSVString::USVString(USVString(s.clone())),
-                        FormDatum::BlobData(ref b) => BlobOrUSVString::Blob(Root::from_ref(&*b)),
+                    data.iter().map(|item| match item.value {
+                        FormDatumValue::String(ref s) => FileOrUSVString::USVString(USVString(s.to_string())),
+                        FormDatumValue::File(ref b) => FileOrUSVString::File(Root::from_ref(&*b)),
                     }).collect()
                  )
     }
@@ -108,29 +119,48 @@ impl FormDataMethods for FormData {
         self.data.borrow().contains_key(&Atom::from(name.0))
     }
 
+    // https://xhr.spec.whatwg.org/#dom-formdata-set
+    fn Set(&self, name: USVString, str_value: USVString) {
+        self.data.borrow_mut().insert(Atom::from(name.0.clone()), vec![FormDatum {
+            ty: DOMString::from("string"),
+            name: DOMString::from(name.0),
+            value: FormDatumValue::String(DOMString::from(str_value.0)),
+        }]);
+    }
+
     #[allow(unrooted_must_root)]
     // https://xhr.spec.whatwg.org/#dom-formdata-set
-    fn Set(&self, name: USVString, value: BlobOrUSVString) {
-        let val = match value {
-            BlobOrUSVString::USVString(s) => FormDatum::StringData(s.0),
-            BlobOrUSVString::Blob(b) => FormDatum::BlobData(JS::from_ref(&*b))
-        };
-        self.data.borrow_mut().insert(Atom::from(name.0), vec!(val));
+    fn Set_(&self, name: USVString, blob: &Blob, filename: Option<USVString>) {
+        self.data.borrow_mut().insert(Atom::from(name.0.clone()), vec![FormDatum {
+            ty: DOMString::from("file"),
+            name: DOMString::from(name.0),
+            value: FormDatumValue::File(Root::from_ref(&*self.get_file(blob, filename))),
+        }]);
     }
+
 }
 
 
 impl FormData {
-    fn get_file_or_blob(&self, blob: &Blob, filename: Option<USVString>) -> Root<Blob> {
-        match filename {
-            Some(fname) => {
-                let global = self.global();
-                let name = DOMString::from(fname.0);
-                let bytes = blob.get_bytes().unwrap_or(vec![]);
+    fn get_file(&self, blob: &Blob, opt_filename: Option<USVString>) -> Root<File> {
+        let global = self.global();
 
-                Root::upcast(File::new(global.r(), BlobImpl::new_from_bytes(bytes), name, None, ""))
-            }
-            None => Root::from_ref(blob)
+        let name = match opt_filename {
+            Some(filename) => DOMString::from(filename.0),
+            None => DOMString::from(""),
+        };
+
+        let bytes = blob.get_bytes().unwrap_or(vec![]);
+
+        File::new(global.r(), BlobImpl::new_from_bytes(bytes), name, None, "")
+    }
+
+    pub fn datums(&self) -> Vec<FormDatum> {
+        let mut ret = vec![];
+        for values in self.data.borrow().values() {
+            ret.append(&mut values.clone());
         }
+
+        ret
     }
 }

servo/components/script/dom/htmlformelement.rs

@@ -250,13 +250,6 @@ pub enum ResetFrom {
 
 
 impl HTMLFormElement {
-    fn generate_boundary(&self) -> String {
-        let i1 = random::<u32>();
-        let i2 = random::<u32>();
-
-        format!("---------------------------{0}{1}", i1, i2)
-    }
-
     // https://html.spec.whatwg.org/multipage/#picking-an-encoding-for-the-form
     fn pick_encoding(&self) -> EncodingRef {
         // Step 2
@@ -275,66 +268,6 @@ impl HTMLFormElement {
         document_from_node(self).encoding()
     }
 
-    // https://html.spec.whatwg.org/multipage/#multipart/form-data-encoding-algorithm
-    fn encode_multipart_form_data(&self, form_data: &mut Vec<FormDatum>,
-                                  boundary: String, encoding: EncodingRef) -> Vec<u8> {
-        // Step 1
-        let mut result = vec![];
-
-        // Step 2
-        let charset = &*encoding.whatwg_name().unwrap_or("UTF-8");
-
-        // Step 3
-        for entry in form_data.iter_mut() {
-            // 3.1
-            if entry.name == "_charset_" && entry.ty == "hidden" {
-                entry.value = FormDatumValue::String(DOMString::from(charset.clone()));
-            }
-            // TODO: 3.2
-
-            // Step 4
-            // https://tools.ietf.org/html/rfc7578#section-4
-            // NOTE(izgzhen): The encoding here expected by most servers seems different from
-            // what spec says (that it should start with a '\r\n').
-            let mut boundary_bytes = format!("--{}\r\n", boundary).into_bytes();
-            result.append(&mut boundary_bytes);
-            let mut content_disposition = ContentDisposition {
-                disposition: DispositionType::Ext("form-data".to_owned()),
-                parameters: vec![DispositionParam::Ext("name".to_owned(), String::from(entry.name.clone()))]
-            };
-
-            match entry.value {
-                FormDatumValue::String(ref s) => {
-                    let mut bytes = format!("Content-Disposition: {}\r\n\r\n{}",
-                                            content_disposition, s).into_bytes();
-                    result.append(&mut bytes);
-                }
-                FormDatumValue::File(ref f) => {
-                    content_disposition.parameters.push(
-                        DispositionParam::Filename(Charset::Ext(String::from(charset.clone())),
-                                                   None,
-                                                   f.name().clone().into()));
-                    // https://tools.ietf.org/html/rfc7578#section-4.4
-                    let content_type = ContentType(f.upcast::<Blob>().Type()
-                                                    .parse().unwrap_or(mime!(Text / Plain)));
-                    let mut type_bytes = format!("Content-Disposition: {}\r\n{}\r\n\r\n",
-                                                 content_disposition,
-                                                 content_type).into_bytes();
-                    result.append(&mut type_bytes);
-
-                    let mut bytes = f.upcast::<Blob>().get_bytes().unwrap_or(vec![]);
-
-                    result.append(&mut bytes);
-                }
-            }
-        }
-
-        let mut boundary_bytes = format!("\r\n--{}--", boundary).into_bytes();
-        result.append(&mut boundary_bytes);
-
-        result
-    }
-
     // https://html.spec.whatwg.org/multipage/#text/plain-encoding-algorithm
     fn encode_plaintext(&self, form_data: &mut Vec<FormDatum>) -> String {
         // Step 1
@@ -459,7 +392,7 @@ impl HTMLFormElement {
     // https://html.spec.whatwg.org/multipage/#submit-body
     fn submit_entity_body(&self, form_data: &mut Vec<FormDatum>, mut load_data: LoadData,
                           enctype: FormEncType, encoding: EncodingRef) {
-        let boundary = self.generate_boundary();
+        let boundary = generate_boundary();
         let bytes = match enctype {
             FormEncType::UrlEncoded => {
                 let mut url = load_data.url.clone();
@@ -476,7 +409,7 @@ impl HTMLFormElement {
             FormEncType::FormDataEncoded => {
                 let mime = mime!(Multipart / FormData; Boundary =(&boundary));
                 load_data.headers.set(ContentType(mime));
-                self.encode_multipart_form_data(form_data, boundary, encoding)
+                encode_multipart_form_data(form_data, boundary, encoding)
             }
             FormEncType::TextPlainEncoded => {
                 load_data.headers.set(ContentType(mime!(Text / Plain)));
@@ -718,12 +651,13 @@ impl HTMLFormElement {
 
 }
 
+#[derive(JSTraceable, HeapSizeOf, Clone)]
 pub enum FormDatumValue {
     File(Root<File>),
     String(DOMString)
 }
 
-// #[derive(HeapSizeOf)]
+#[derive(HeapSizeOf, JSTraceable, Clone)]
 pub struct FormDatum {
     pub ty: DOMString,
     pub name: DOMString,
@@ -972,3 +906,72 @@ impl Runnable for PlannedNavigation {
         }
     }
 }
+
+
+// https://html.spec.whatwg.org/multipage/#multipart/form-data-encoding-algorithm
+pub fn encode_multipart_form_data(form_data: &mut Vec<FormDatum>,
+                                  boundary: String, encoding: EncodingRef) -> Vec<u8> {
+    // Step 1
+    let mut result = vec![];
+
+    // Step 2
+    let charset = &*encoding.whatwg_name().unwrap_or("UTF-8");
+
+    // Step 3
+    for entry in form_data.iter_mut() {
+        // 3.1
+        if entry.name == "_charset_" && entry.ty == "hidden" {
+            entry.value = FormDatumValue::String(DOMString::from(charset.clone()));
+        }
+        // TODO: 3.2
+
+        // Step 4
+        // https://tools.ietf.org/html/rfc7578#section-4
+        // NOTE(izgzhen): The encoding here expected by most servers seems different from
+        // what spec says (that it should start with a '\r\n').
+        let mut boundary_bytes = format!("--{}\r\n", boundary).into_bytes();
+        result.append(&mut boundary_bytes);
+        let mut content_disposition = ContentDisposition {
+            disposition: DispositionType::Ext("form-data".to_owned()),
+            parameters: vec![DispositionParam::Ext("name".to_owned(), String::from(entry.name.clone()))]
+        };
+
+        match entry.value {
+            FormDatumValue::String(ref s) => {
+                let mut bytes = format!("Content-Disposition: {}\r\n\r\n{}",
+                                        content_disposition, s).into_bytes();
+                result.append(&mut bytes);
+            }
+            FormDatumValue::File(ref f) => {
+                content_disposition.parameters.push(
+                    DispositionParam::Filename(Charset::Ext(String::from(charset.clone())),
+                                               None,
+                                               f.name().clone().into()));
+                // https://tools.ietf.org/html/rfc7578#section-4.4
+                let content_type = ContentType(f.upcast::<Blob>().Type()
+                                                .parse().unwrap_or(mime!(Text / Plain)));
+                let mut type_bytes = format!("Content-Disposition: {}\r\ncontent-type: {}\r\n\r\n",
+                                             content_disposition,
+                                             content_type).into_bytes();
+                result.append(&mut type_bytes);
+
+                let mut bytes = f.upcast::<Blob>().get_bytes().unwrap_or(vec![]);
+
+                result.append(&mut bytes);
+            }
+        }
+    }
+
+    let mut boundary_bytes = format!("\r\n--{}--", boundary).into_bytes();
+    result.append(&mut boundary_bytes);
+
+    result
+}
+
+// https://tools.ietf.org/html/rfc7578#section-4.1
+pub fn generate_boundary() -> String {
+    let i1 = random::<u32>();
+    let i2 = random::<u32>();
+
+    format!("---------------------------{0}{1}", i1, i2)
+}

servo/components/script/dom/webidls/FormData.webidl

@@ -6,7 +6,7 @@
  * https://xhr.spec.whatwg.org/#interface-formdata
  */
 
-typedef (Blob or USVString) FormDataEntryValue;
+typedef (File or USVString) FormDataEntryValue;
 
 [Constructor(optional HTMLFormElement form),
  Exposed=(Window,Worker)]
@@ -17,6 +17,7 @@ interface FormData {
   FormDataEntryValue? get(USVString name);
   sequence<FormDataEntryValue> getAll(USVString name);
   boolean has(USVString name);
-  void set(USVString name, FormDataEntryValue value);
+  void set(USVString name, USVString value);
+  void set(USVString name, Blob value, optional USVString filename);
   // iterable<USVString, FormDataEntryValue>;
 };

servo/components/script/dom/webidls/XMLHttpRequest.webidl

@@ -13,7 +13,7 @@
  */
 
 // https://fetch.spec.whatwg.org/#bodyinit
-typedef (Blob or /*BufferSource or FormData or */DOMString or URLSearchParams) BodyInit;
+typedef (Blob or /*BufferSource or */ FormData or DOMString or URLSearchParams) BodyInit;
 
 enum XMLHttpRequestResponseType {
   "",

servo/components/script/dom/xmlhttprequest.rs

@@ -26,6 +26,7 @@ use dom::document::{Document, IsHTMLDocument};
 use dom::event::{Event, EventBubbles, EventCancelable};
 use dom::eventtarget::EventTarget;
 use dom::headers::is_forbidden_header_name;
+use dom::htmlformelement::{encode_multipart_form_data, generate_boundary};
 use dom::progressevent::ProgressEvent;
 use dom::xmlhttprequesteventtarget::XMLHttpRequestEventTarget;
 use dom::xmlhttprequestupload::XMLHttpRequestUpload;
@@ -1355,12 +1356,12 @@ impl Extractable for BodyInit {
                 let encoding = UTF_8 as EncodingRef;
                 (encoding.encode(s, EncoderTrap::Replace).unwrap(),
                     Some(DOMString::from("text/plain;charset=UTF-8")))
-            },
+            }
             BodyInit::URLSearchParams(ref usp) => {
                 // Default encoding is UTF-8.
                 (usp.serialize(None).into_bytes(),
                     Some(DOMString::from("application/x-www-form-urlencoded;charset=UTF-8")))
-            },
+            }
             BodyInit::Blob(ref b) => {
                 let content_type = if b.Type().as_ref().is_empty() {
                     None
@@ -1369,7 +1370,13 @@ impl Extractable for BodyInit {
                 };
                 let bytes = b.get_bytes().unwrap_or(vec![]);
                 (bytes, content_type)
-            },
+            }
+            BodyInit::FormData(ref formdata) => {
+                let boundary = generate_boundary();
+                let bytes = encode_multipart_form_data(&mut formdata.datums(), boundary.clone(),
+                                                       UTF_8 as EncodingRef);
+                (bytes, Some(DOMString::from(format!("multipart/form-data;boundary={}", boundary))))
+            }
         }
     }
 }