Auto merge of #8828 - Ronak6892:docResp, r=eefriedman

M1503/M1504 : Support for missing XMLHttpRequestAPIs and Document response handling

Hi,
This is for NCSU Fall 15 project M1503 and M1504:
M1503: Integrate XML Parser (Step 3 of 4): Support for Document response handling and calling the appropriate parser.
M1504: Support for missing XMLHttpRequest APIs
https://github.com/servo/servo/wiki/Integrate-xml5ever
It includes changes to handle document response for XMLHttpRequest.
Kindly review.

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/servo/servo/8828)
<!-- Reviewable:end -->

components/script/dom/webidls/XMLHttpRequest.webidl

@@ -72,5 +72,6 @@ interface XMLHttpRequest : XMLHttpRequestEventTarget {
   readonly attribute any response;
   [Throws]
   readonly attribute DOMString responseText;
+  [Throws]
   /*[Exposed=Window]*/ readonly attribute Document? responseXML;
 };

components/script/dom/xmlhttprequest.rs

@@ -4,8 +4,10 @@
 
 use cors::CORSResponse;
 use cors::{AsyncCORSResponseListener, CORSRequest, RequestMode, allow_cross_origin_request};
+use document_loader::DocumentLoader;
 use dom::bindings::cell::DOMRefCell;
 use dom::bindings::codegen::Bindings::EventHandlerBinding::EventHandlerNonNull;
+use dom::bindings::codegen::Bindings::WindowBinding::WindowMethods;
 use dom::bindings::codegen::Bindings::XMLHttpRequestBinding;
 use dom::bindings::codegen::Bindings::XMLHttpRequestBinding::XMLHttpRequestMethods;
 use dom::bindings::codegen::Bindings::XMLHttpRequestBinding::XMLHttpRequestResponseType;
@@ -16,12 +18,13 @@ use dom::bindings::conversions::{ToJSValConvertible};
 use dom::bindings::error::{Error, ErrorResult, Fallible};
 use dom::bindings::global::{GlobalField, GlobalRef, GlobalRoot};
 use dom::bindings::inheritance::Castable;
-use dom::bindings::js::Root;
 use dom::bindings::js::{JS, MutNullableHeap};
+use dom::bindings::js::{Root, RootedReference};
 use dom::bindings::refcounted::Trusted;
 use dom::bindings::reflector::{Reflectable, reflect_dom_object};
 use dom::bindings::str::ByteString;
-use dom::document::Document;
+use dom::document::DocumentSource;
+use dom::document::{Document, IsHTMLDocument};
 use dom::event::{Event, EventBubbles, EventCancelable};
 use dom::eventtarget::EventTarget;
 use dom::progressevent::ProgressEvent;
@@ -45,6 +48,8 @@ use net_traits::ControlMsg::Load;
 use net_traits::{AsyncResponseListener, AsyncResponseTarget, Metadata};
 use net_traits::{LoadConsumer, LoadData, ResourceCORSData, ResourceTask};
 use network_listener::{NetworkListener, PreInvoke};
+use parse::html::{ParseContext, parse_html};
+use parse::xml::{self, parse_xml};
 use script_task::{ScriptChan, ScriptPort};
 use std::ascii::AsciiExt;
 use std::borrow::ToOwned;
@@ -701,6 +706,14 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
                 _ if self.ready_state.get() != XMLHttpRequestState::Done => {
                     return NullValue()
                 },
+                XMLHttpRequestResponseType::Document => {
+                    let op_doc = self.GetResponseXML();
+                    if let Ok(Some(doc)) = op_doc {
+                        doc.to_jsval(cx, rval.handle_mut());
+                    } else {
+                        return NullValue();
+                    }
+                },
                 Json => {
                     let decoded = UTF_8.decode(&self.response.borrow(), DecoderTrap::Replace).unwrap().to_owned();
                     let decoded: Vec<u16> = decoded.utf16_units().collect();
@@ -735,8 +748,25 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
     }
 
     // https://xhr.spec.whatwg.org/#the-responsexml-attribute
-    fn GetResponseXML(&self) -> Option<Root<Document>> {
+    fn GetResponseXML(&self) -> Fallible<Option<Root<Document>>> {
-        self.response_xml.get()
+        match self.response_type.get() {
+            _empty | XMLHttpRequestResponseType::Document => {
+                match self.ready_state.get() {
+                    XMLHttpRequestState::Done => {
+                        match self.response_xml.get() {
+                            Some(response) => Ok(Some(response)),
+                            None => {
+                                let response = self.document_response();
+                                self.response_xml.set(response.r());
+                                Ok(response)
+                            }
+                        }
+                    },
+                    _ => Ok(None)
+                }
+            },
+            _ => { Err(Error::InvalidState) }
+        }
     }
 }
 
@@ -1006,6 +1036,87 @@ impl XMLHttpRequest {
         // the result should be fine. XXXManishearth have a closer look at this later
         encoding.decode(&self.response.borrow(), DecoderTrap::Replace).unwrap().to_owned()
     }
+
+    fn document_response(&self) -> Option<Root<Document>> {
+        let mime_type = self.final_mime_type();
+        //TODO: prescan the response to determine encoding if final charset is null
+        let charset = self.final_charset().unwrap_or(UTF_8);
+        let temp_doc: Root<Document>;
+        match mime_type {
+            Some(Mime(mime::TopLevel::Text, mime::SubLevel::Html, _)) => {
+                if self.response_type.get() == XMLHttpRequestResponseType::_empty {
+                    return None;
+                }
+                else {
+                    temp_doc = self.document_text_html();
+                }
+            },
+            Some(Mime(mime::TopLevel::Text, mime::SubLevel::Xml, _)) |
+            Some(Mime(mime::TopLevel::Application, mime::SubLevel::Xml, _)) |
+            None => {
+                temp_doc = self.handle_xml();
+            },
+            Some(Mime(_, mime::SubLevel::Ext(sub), _)) => {
+                if sub.ends_with("+xml") {
+                    temp_doc = self.handle_xml();
+                }
+                else {
+                    return None;
+                }
+            },
+            _ => { return None; }
+        }
+        temp_doc.set_encoding_name(DOMString::from(charset.name()));
+        Some(temp_doc)
+    }
+
+    fn document_text_html(&self) -> Root<Document>{
+        let charset = self.final_charset().unwrap_or(UTF_8);
+        let wr = self.global.root();
+        let wr = wr.r();
+        let decoded = charset.decode(&self.response.borrow(), DecoderTrap::Replace).unwrap().to_owned();
+        let document = self.new_doc(IsHTMLDocument::HTMLDocument);
+        // TODO: Disable scripting while parsing
+        parse_html(document.r(), DOMString::from(decoded), wr.get_url(), ParseContext::Owner(Some(wr.pipeline())));
+        document
+    }
+
+    fn handle_xml(&self) -> Root<Document> {
+        let charset = self.final_charset().unwrap_or(UTF_8);
+        let wr = self.global.root();
+        let wr = wr.r();
+        let decoded = charset.decode(&self.response.borrow(), DecoderTrap::Replace).unwrap().to_owned();
+        let document = self.new_doc(IsHTMLDocument::NonHTMLDocument);
+        // TODO: Disable scripting while parsing
+        parse_xml(document.r(), DOMString::from(decoded), wr.get_url(), xml::ParseContext::Owner(Some(wr.pipeline())));
+        document
+    }
+
+    fn new_doc(&self, is_html_document: IsHTMLDocument) -> Root<Document> {
+        let wr = self.global.root();
+        let wr = wr.r();
+        let win = wr.as_window();
+        let doc = win.Document();
+        let doc = doc.r();
+        let docloader = DocumentLoader::new(&*doc.loader());
+        let base = self.global.root().r().get_url();
+        let parsed_url = match UrlParser::new().base_url(&base).parse(&self.ResponseURL()) {
+            Ok(parsed) => Some(parsed),
+            Err(_) => None // Step 7
+        };
+        let mime_type = self.final_mime_type();
+        let content_type = mime_type.map(|mime|{
+            DOMString::from(format!("{}", mime))
+        });
+        let document = Document::new(win,
+                                     parsed_url,
+                                     is_html_document,
+                                     content_type,
+                                     None,
+                                     DocumentSource::FromParser, docloader);
+       document
+    }
+
     fn filter_response_headers(&self) -> Headers {
         // https://fetch.spec.whatwg.org/#concept-response-header-list
         use hyper::error::Result;

tests/wpt/metadata/XMLHttpRequest/abort-after-send.htm.ini

@@ -0,0 +1,5 @@
+[abort-after-send.htm]
+  type: testharness
+  [XMLHttpRequest: abort() after send()]
+    expected: FAIL
+

tests/wpt/metadata/XMLHttpRequest/overridemimetype-open-state-force-xml.htm.ini

@@ -1,6 +0,0 @@
-[overridemimetype-open-state-force-xml.htm]
-  type: testharness
-  expected: TIMEOUT
-  [XMLHttpRequest: overrideMimeType() in open state, XML MIME type with UTF-8 charset]
-    expected: TIMEOUT
-

tests/wpt/metadata/XMLHttpRequest/responsexml-basic.htm.ini

@@ -3,3 +3,6 @@
   [XMLHttpRequest: responseXML basic test]
     expected: FAIL
 
+  [responseXML on empty response documents]
+    expected: FAIL
+

tests/wpt/metadata/XMLHttpRequest/responsexml-document-properties.htm.ini

@@ -12,12 +12,6 @@
   [referrer]
     expected: FAIL
 
-  [title]
-    expected: FAIL
-
-  [contentType]
-    expected: FAIL
-
   [readyState]
     expected: FAIL
 
@@ -33,30 +27,15 @@
   [images]
     expected: FAIL
 
-  [doctype]
-    expected: FAIL
-
   [forms]
     expected: FAIL
 
-  [all]
-    expected: FAIL
-
   [links]
     expected: FAIL
 
-  [cookie]
-    expected: FAIL
-
   [lastModified set according to HTTP header]
     expected: FAIL
 
-  [cookie (after setting it)]
-    expected: FAIL
-
   [styleSheets]
     expected: FAIL
 
-  [implementation]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/responsexml-media-type.htm.ini

@@ -1,26 +0,0 @@
-[responsexml-media-type.htm]
-  type: testharness
-  [XMLHttpRequest: responseXML MIME type tests ('', should  parse)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML MIME type tests ('bogus', should  parse)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML MIME type tests ('bogus+xml', should  parse)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML MIME type tests ('video/x-awesome+xml', should  parse)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML MIME type tests ('text/xml', should  parse)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML MIME type tests ('application', should  parse)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML MIME type tests ('application/xhtml+xml', should  parse)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML MIME type tests ('image/svg+xml', should  parse)]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/responsexml-non-document-types.htm.ini

@@ -1,14 +0,0 @@
-[responsexml-non-document-types.htm]
-  type: testharness
-  [XMLHttpRequest: responseXML/responseText on other responseType (arraybuffer)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML/responseText on other responseType (blob)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML/responseText on other responseType (json)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseXML/responseText on other responseType (text)]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/responsexml-non-well-formed.htm.ini

@@ -0,0 +1,23 @@
+[responsexml-non-well-formed.htm]
+  type: testharness
+  [XMLHttpRequest: responseXML non well-formed tests]
+    expected: FAIL
+
+  [XMLHttpRequest: responseXML non well-formed tests 1]
+    expected: FAIL
+
+  [XMLHttpRequest: responseXML non well-formed tests 2]
+    expected: FAIL
+
+  [XMLHttpRequest: responseXML non well-formed tests 3]
+    expected: FAIL
+
+  [XMLHttpRequest: responseXML non well-formed tests 4]
+    expected: FAIL
+
+  [XMLHttpRequest: responseXML non well-formed tests 5]
+    expected: FAIL
+
+  [XMLHttpRequest: responseXML non well-formed tests 6]
+    expected: FAIL
+

tests/wpt/metadata/XMLHttpRequest/send-redirect-no-location.htm.ini

@@ -1,14 +0,0 @@
-[send-redirect-no-location.htm]
-  type: testharness
-  [XMLHttpRequest: send() - Redirects (no Location header) (301)]
-    expected: FAIL
-
-  [XMLHttpRequest: send() - Redirects (no Location header) (302)]
-    expected: FAIL
-
-  [XMLHttpRequest: send() - Redirects (no Location header) (303)]
-    expected: FAIL
-
-  [XMLHttpRequest: send() - Redirects (no Location header) (307)]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/status-async.htm.ini

@@ -1,8 +0,0 @@
-[status-async.htm]
-  type: testharness
-  [XMLHttpRequest: status/statusText - various responses 7 (GET 402)]
-    expected: FAIL
-
-  [XMLHttpRequest: status/statusText - various responses 9 (CHICKEN 402)]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/status-basic.htm.ini

@@ -1,8 +0,0 @@
-[status-basic.htm]
-  type: testharness
-  [XMLHttpRequest: status/statusText - various responses 7 (GET 402)]
-    expected: FAIL
-
-  [XMLHttpRequest: status/statusText - various responses 9 (CHICKEN 402)]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/xmlhttprequest-network-error-sync.htm.ini

@@ -0,0 +1,5 @@
+[xmlhttprequest-network-error-sync.htm]
+  type: testharness
+  [XMLHttpRequest: members during network errors (sync)]
+    expected: FAIL
+

tests/wpt/metadata/XMLHttpRequest/xmlhttprequest-network-error.htm.ini

@@ -0,0 +1,5 @@
+[xmlhttprequest-network-error.htm]
+  type: testharness
+  [XMLHttpRequest: members during network errors]
+    expected: FAIL
+

tests/wpt/metadata/dom/nodes/Document-contentType/contentType/xhr_responseType_document.html.ini

@@ -1,5 +0,0 @@
-[xhr_responseType_document.html]
-  type: testharness
-  [XHR - retrieve HTML document: document.contentType === 'application/xml']
-    expected: FAIL
-

tests/wpt/metadata/workers/semantics/xhr/001.html.ini

@@ -0,0 +1,3 @@
+[001.html]
+  type: testharness
+  expected: CRASH

tests/wpt/metadata/workers/semantics/xhr/002.html.ini

@@ -0,0 +1,6 @@
+[002.html]
+  type: testharness
+  expected: TIMEOUT
+  [sync XMLHttpRequest in dedicated worker]
+    expected: TIMEOUT
+