Auto merge of #12467 - jeenalee:jeena-headersAPI, r=jdm

Add the append method for the Headers API

<!-- Please describe your changes on the following line: -->
This commit adds the append method for the Headers API. @malisas and I are both contributors.

There are a few TODOs related:
- The script needs to parse the header value for certain header names to decide the header group it belongs
- There are possible spec bugs that could change what a valid header value looks like (related: [issue page](https://github.com/whatwg/fetch/issues/332))

There are WPT tests already written for the Headers API, but they will fail as the Headers API is not fully implemented.

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [ ] These changes fix #__ (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [X] These changes do not require tests because tests for the Headers API already exists, but this commit does not implement the interface fully. The tests will fail.

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/12467)
<!-- Reviewable:end -->

components/script/dom/bindings/str.rs

@@ -15,7 +15,7 @@ use std::str::{Bytes, FromStr};
 use string_cache::Atom;
 
 /// Encapsulates the IDL `ByteString` type.
-#[derive(JSTraceable, Clone, Eq, PartialEq, HeapSizeOf)]
+#[derive(JSTraceable, Clone, Eq, PartialEq, HeapSizeOf, Debug)]
 pub struct ByteString(Vec<u8>);
 
 impl ByteString {

components/script/dom/headers.rs

@@ -0,0 +1,251 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use dom::bindings::cell::DOMRefCell;
+use dom::bindings::codegen::Bindings::HeadersBinding;
+use dom::bindings::codegen::Bindings::HeadersBinding::HeadersMethods;
+use dom::bindings::error::Error;
+use dom::bindings::global::GlobalRef;
+use dom::bindings::js::Root;
+use dom::bindings::reflector::{Reflector, reflect_dom_object};
+use dom::bindings::str::{ByteString, is_token};
+use hyper;
+use std::result::Result;
+
+#[dom_struct]
+pub struct Headers {
+    reflector_: Reflector,
+    guard: Guard,
+    #[ignore_heap_size_of = "Defined in hyper"]
+    header_list: DOMRefCell<hyper::header::Headers>
+}
+
+// https://fetch.spec.whatwg.org/#concept-headers-guard
+#[derive(JSTraceable, HeapSizeOf, PartialEq)]
+pub enum Guard {
+    Immutable,
+    Request,
+    RequestNoCors,
+    Response,
+    None,
+}
+
+impl Headers {
+    pub fn new_inherited() -> Headers {
+        Headers {
+            reflector_: Reflector::new(),
+            guard: Guard::None,
+            header_list: DOMRefCell::new(hyper::header::Headers::new()),
+        }
+    }
+
+    pub fn new(global: GlobalRef) -> Root<Headers> {
+        reflect_dom_object(box Headers::new_inherited(), global, HeadersBinding::Wrap)
+    }
+}
+
+impl HeadersMethods for Headers {
+    // https://fetch.spec.whatwg.org/#concept-headers-append
+    fn Append(&self, name: ByteString, value: ByteString) -> Result<(), Error> {
+        // Step 1
+        let value = normalize_value(value);
+
+        // Step 2
+        let (valid_name, valid_value) = try!(validate_name_and_value(name, value));
+        // Step 3
+        if self.guard == Guard::Immutable {
+            return Err(Error::Type("Guard is immutable".to_string()));
+        }
+
+        // Step 4
+        if self.guard == Guard::Request && is_forbidden_header_name(&valid_name) {
+            return Ok(());
+        }
+
+        // Step 5
+        if self.guard == Guard::RequestNoCors && !is_cors_safelisted_request_header(&valid_name) {
+            return Ok(());
+        }
+
+        // Step 6
+        if self.guard == Guard::Response && is_forbidden_response_header(&valid_name) {
+            return Ok(());
+        }
+
+        // Step 7
+        self.header_list.borrow_mut().set_raw(valid_name, vec![valid_value]);
+        return Ok(());
+    }
+}
+
+// TODO
+// "Content-Type" once parsed, the value should be
+// `application/x-www-form-urlencoded`, `multipart/form-data`,
+// or `text/plain`.
+// "DPR", "Downlink", "Save-Data", "Viewport-Width", "Width":
+// once parsed, the value should not be failure.
+// https://fetch.spec.whatwg.org/#cors-safelisted-request-header
+fn is_cors_safelisted_request_header(name: &str) -> bool {
+    match name {
+        "accept" |
+        "accept-language" |
+        "content-language" => true,
+        _ => false,
+    }
+}
+
+// https://fetch.spec.whatwg.org/#forbidden-response-header-name
+fn is_forbidden_response_header(name: &str) -> bool {
+    match name {
+        "set-cookie" |
+        "set-cookie2"  => true,
+        _ => false,
+    }
+}
+
+// https://fetch.spec.whatwg.org/#forbidden-header-name
+pub fn is_forbidden_header_name(name: &str) -> bool {
+    let disallowed_headers =
+        ["accept-charset", "accept-encoding",
+         "access-control-request-headers",
+         "access-control-request-method",
+         "connection", "content-length",
+         "cookie", "cookie2", "date", "dnt",
+         "expect", "host", "keep-alive", "origin",
+         "referer", "te", "trailer", "transfer-encoding",
+         "upgrade", "via"];
+
+    let disallowed_header_prefixes = ["sec-", "proxy-"];
+
+    disallowed_headers.iter().any(|header| *header == name) ||
+        disallowed_header_prefixes.iter().any(|prefix| name.starts_with(prefix))
+}
+
+// There is some unresolved confusion over the definition of a name and a value.
+// The fetch spec [1] defines a name as "a case-insensitive byte
+// sequence that matches the field-name token production. The token
+// productions are viewable in [2]." A field-name is defined as a
+// token, which is defined in [3].
+// ISSUE 1:
+// It defines a value as "a byte sequence that matches the field-content token production."
+// To note, there is a difference between field-content and
+// field-value (which is made up of fied-content and obs-fold). The
+// current definition does not allow for obs-fold (which are white
+// space and newlines) in values. So perhaps a value should be defined
+// as "a byte sequence that matches the field-value token production."
+// However, this would then allow values made up entirely of white space and newlines.
+// RELATED ISSUE 2:
+// According to a previously filed Errata ID: 4189 in [4], "the
+// specified field-value rule does not allow single field-vchar
+// surrounded by whitespace anywhere". They provided a fix for the
+// field-content production, but ISSUE 1 has still not been resolved.
+// The production definitions likely need to be re-written.
+// [1] https://fetch.spec.whatwg.org/#concept-header-value
+// [2] https://tools.ietf.org/html/rfc7230#section-3.2
+// [3] https://tools.ietf.org/html/rfc7230#section-3.2.6
+// [4] https://www.rfc-editor.org/errata_search.php?rfc=7230
+fn validate_name_and_value(name: ByteString, value: ByteString)
+                           -> Result<(String, Vec<u8>), Error> {
+    if !is_field_name(&name) {
+        return Err(Error::Type("Name is not valid".to_string()));
+    }
+    if !is_field_content(&value) {
+        return Err(Error::Type("Value is not valid".to_string()));
+    }
+    match String::from_utf8(name.into()) {
+        Ok(ns) => Ok((ns, value.into())),
+        _ => Err(Error::Type("Non-UTF8 header name found".to_string())),
+    }
+}
+
+// Removes trailing and leading HTTP whitespace bytes.
+// https://fetch.spec.whatwg.org/#concept-header-value-normalize
+pub fn normalize_value(value: ByteString) -> ByteString {
+    match (index_of_first_non_whitespace(&value), index_of_last_non_whitespace(&value)) {
+        (Some(begin), Some(end)) => ByteString::new(value[begin..end + 1].to_owned()),
+        _ => ByteString::new(vec![]),
+    }
+}
+
+fn is_HTTP_whitespace(byte: u8) -> bool {
+    byte == b'\t' || byte == b'\n' || byte == b'\r' || byte == b' '
+}
+
+fn index_of_first_non_whitespace(value: &ByteString) -> Option<usize> {
+    for (index, &byte) in value.iter().enumerate() {
+        if !is_HTTP_whitespace(byte) {
+            return Some(index);
+        }
+    }
+    None
+}
+
+fn index_of_last_non_whitespace(value: &ByteString) -> Option<usize> {
+    for (index, &byte) in value.iter().enumerate().rev() {
+        if !is_HTTP_whitespace(byte) {
+            return Some(index);
+        }
+    }
+    None
+}
+
+// http://tools.ietf.org/html/rfc7230#section-3.2
+fn is_field_name(name: &ByteString) -> bool {
+    is_token(&*name)
+}
+
+// https://tools.ietf.org/html/rfc7230#section-3.2
+// http://www.rfc-editor.org/errata_search.php?rfc=7230
+// Errata ID: 4189
+// field-content = field-vchar [ 1*( SP / HTAB / field-vchar )
+//                               field-vchar ]
+fn is_field_content(value: &ByteString) -> bool {
+    if value.len() == 0 {
+        return false;
+    }
+    if !is_field_vchar(value[0]) {
+        return false;
+    }
+
+    for &ch in &value[1..value.len() - 1] {
+        if !is_field_vchar(ch) || !is_space(ch) || !is_htab(ch) {
+            return false;
+        }
+    }
+
+    if !is_field_vchar(value[value.len() - 1]) {
+        return false;
+    }
+
+    return true;
+}
+
+fn is_space(x: u8) -> bool {
+    x == b' '
+}
+
+fn is_htab(x: u8) -> bool {
+    x == b'\t'
+}
+
+// https://tools.ietf.org/html/rfc7230#section-3.2
+fn is_field_vchar(x: u8) -> bool {
+    is_vchar(x) || is_obs_text(x)
+}
+
+// https://tools.ietf.org/html/rfc5234#appendix-B.1
+fn is_vchar(x: u8) -> bool {
+    match x {
+        0x21...0x7E => true,
+        _ => false,
+    }
+}
+
+// http://tools.ietf.org/html/rfc7230#section-3.2.6
+fn is_obs_text(x: u8) -> bool {
+    match x {
+        0x80...0xFF => true,
+        _ => false,
+    }
+}

components/script/dom/mod.rs

@@ -269,6 +269,7 @@ pub mod focusevent;
 pub mod forcetouchevent;
 pub mod formdata;
 pub mod hashchangeevent;
+pub mod headers;
 pub mod htmlanchorelement;
 pub mod htmlappletelement;
 pub mod htmlareaelement;

components/script/dom/webidls/Headers.webidl

@@ -0,0 +1,22 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// https://fetch.spec.whatwg.org/#headers-class
+
+/* typedef (Headers or sequence<sequence<ByteString>>) HeadersInit; */
+
+/* [Constructor(optional HeadersInit init),*/
+ [Exposed=(Window,Worker)]
+
+interface Headers {
+  [Throws]
+  void append(ByteString name, ByteString value);
+};
+
+/* void delete(ByteString name);
+ * ByteString? get(ByteString name);
+ * boolean has(ByteString name);
+ * void set(ByteString name, ByteString value);
+ * iterable<ByteString, ByteString>;
+ * }; */

components/script/dom/xmlhttprequest.rs

@@ -25,6 +25,7 @@ use dom::document::DocumentSource;
 use dom::document::{Document, IsHTMLDocument};
 use dom::event::{Event, EventBubbles, EventCancelable};
 use dom::eventtarget::EventTarget;
+use dom::headers::is_forbidden_header_name;
 use dom::progressevent::ProgressEvent;
 use dom::xmlhttprequesteventtarget::XMLHttpRequestEventTarget;
 use dom::xmlhttprequestupload::XMLHttpRequestUpload;
@@ -409,21 +410,8 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
                 // Step 5
                 // Disallowed headers and header prefixes:
                 // https://fetch.spec.whatwg.org/#forbidden-header-name
-                let disallowedHeaders =
-                    ["accept-charset", "accept-encoding",
-                    "access-control-request-headers",
-                    "access-control-request-method",
-                    "connection", "content-length",
-                    "cookie", "cookie2", "date", "dnt",
-                    "expect", "host", "keep-alive", "origin",
-                    "referer", "te", "trailer", "transfer-encoding",
-                    "upgrade", "via"];
-
-                let disallowedHeaderPrefixes = ["sec-", "proxy-"];
-
-                if disallowedHeaders.iter().any(|header| *header == s) ||
-                   disallowedHeaderPrefixes.iter().any(|prefix| s.starts_with(prefix)) {
-                    return Ok(())
+                if is_forbidden_header_name(s) {
+                    return Ok(());
                 } else {
                     s
                 }

python/tidy/servo_tidy/tidy.py

@@ -88,6 +88,7 @@ WEBIDL_STANDARDS = [
     "//drafts.csswg.org/cssom",
     "//drafts.fxtf.org",
     "//encoding.spec.whatwg.org",
+    "//fetch.spec.whatwg.org",
     "//html.spec.whatwg.org",
     "//url.spec.whatwg.org",
     "//xhr.spec.whatwg.org",

tests/unit/script/headers.rs

@@ -0,0 +1,74 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use script::dom::bindings::str::ByteString;
+use script::dom::headers;
+
+#[test]
+fn test_normalize_empty_bytestring() {
+    // empty ByteString test
+    let empty_bytestring = ByteString::new(vec![]);
+    let actual = headers::normalize_value(empty_bytestring);
+    let expected = ByteString::new(vec![]);
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn test_normalize_all_whitespace_bytestring() {
+    // All whitespace test. A horizontal tab, a line feed, a carriage return , and a space
+    let all_whitespace_bytestring = ByteString::new(vec![b'\t', b'\n', b'\r', b' ']);
+    let actual = headers::normalize_value(all_whitespace_bytestring);
+    let expected = ByteString::new(vec![]);
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn test_normalize_non_empty_no_whitespace_bytestring() {
+    // Non-empty, no whitespace ByteString test
+    let no_whitespace_bytestring = ByteString::new(vec![b'S', b'!']);
+    let actual = headers::normalize_value(no_whitespace_bytestring);
+    let expected = ByteString::new(vec![b'S', b'!']);
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn test_normalize_non_empty_leading_whitespace_bytestring() {
+    // Non-empty, leading whitespace, no trailing whitespace ByteString test
+    let leading_whitespace_bytestring = ByteString::new(vec![b'\t', b'\n', b' ', b'\r', b'S', b'!']);
+    let actual = headers::normalize_value(leading_whitespace_bytestring);
+    let expected = ByteString::new(vec![b'S', b'!']);
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn test_normalize_non_empty_no_leading_whitespace_trailing_whitespace_bytestring() {
+    // Non-empty, no leading whitespace, but with trailing whitespace ByteString test
+    let trailing_whitespace_bytestring = ByteString::new(vec![b'S', b'!', b'\t', b'\n', b' ', b'\r']);
+    let actual = headers::normalize_value(trailing_whitespace_bytestring);
+    let expected = ByteString::new(vec![b'S', b'!']);
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn test_normalize_non_empty_leading_and_trailing_whitespace_bytestring() {
+    // Non-empty, leading whitespace, and trailing whitespace ByteString test
+    let whitespace_sandwich_bytestring =
+        ByteString::new(vec![b'\t', b'\n', b' ', b'\r', b'S', b'!', b'\t', b'\n', b' ', b'\r']);
+    let actual = headers::normalize_value(whitespace_sandwich_bytestring);
+    let expected = ByteString::new(vec![b'S', b'!']);
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn test_normalize_non_empty_leading_trailing_and_internal_whitespace_bytestring() {
+    // Non-empty, leading whitespace, trailing whitespace,
+    // and internal whitespace ByteString test
+    let whitespace_bigmac_bytestring =
+        ByteString::new(vec![b'\t', b'\n', b' ', b'\r', b'S',
+                             b'\t', b'\n', b' ', b'\r', b'!',
+                             b'\t', b'\n', b' ', b'\r']);
+    let actual = headers::normalize_value(whitespace_bigmac_bytestring);
+    let expected = ByteString::new(vec![b'S', b'\t', b'\n', b' ', b'\r', b'!']);
+    assert_eq!(actual, expected);
+}

tests/unit/script/lib.rs

@@ -12,3 +12,4 @@ extern crate url;
 #[cfg(test)] mod origin;
 #[cfg(all(test, target_pointer_width = "64"))] mod size_of;
 #[cfg(test)] mod textinput;
+#[cfg(test)] mod headers;

tests/wpt/mozilla/tests/mozilla/interfaces.html

@@ -48,6 +48,7 @@ test_interfaces([
   "FocusEvent",
   "FormData",
   "HashChangeEvent",
+  "Headers",
   "HTMLAnchorElement",
   "HTMLAppletElement",
   "HTMLAreaElement",

tests/wpt/mozilla/tests/mozilla/interfaces.worker.js

@@ -44,6 +44,7 @@ test_interfaces([
   "FocusEvent",
   "FormData",
   "HashChangeEvent",
+  "Headers",  
   "HTMLAnchorElement",
   "HTMLAppletElement",
   "HTMLAreaElement",