Auto merge of #9768 - jdm:add_disallowed_prefixes, r=jdm

disallow restricted XMLHttpRequest header prefixes

Rebased from #9376.

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.svg" height="40" alt="Review on Reviewable"/>](https://reviewable.io/reviews/servo/servo/9768)
<!-- Reviewable:end -->

components/net/http_loader.rs

@@ -523,7 +523,16 @@ pub fn modify_request_headers(headers: &mut Headers,
         port: doc_url.port_or_default()
     };
     headers.set(host);
-    headers.set(UserAgent(user_agent.to_owned()));
+
+    // If the user-agent has not already been set, then use the
+    // browser's default user-agent or the user-agent override
+    // from the command line. If the user-agent is set, don't
+    // modify it, as setting of the user-agent by the user is
+    // allowed.
+    // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 8
+    if !headers.has::<UserAgent>() {
+        headers.set(UserAgent(user_agent.to_owned()));
+    }
 
     set_default_accept(headers);
     set_default_accept_encoding(headers);

components/script/dom/xmlhttprequest.rs

@@ -423,20 +423,26 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
         let name_lower = name.to_lower();
         let name_str = match name_lower.as_str() {
             Some(s) => {
-                match s {
+                // Step 5
-                    // Step 5
+                // Disallowed headers and header prefixes:
-                    // Disallowed headers
+                // https://fetch.spec.whatwg.org/#forbidden-header-name
-                    "accept-charset" | "accept-encoding" |
+                let disallowedHeaders =
-                    "access-control-request-headers" |
+                    ["accept-charset", "accept-encoding",
-                    "access-control-request-method" |
+                    "access-control-request-headers",
-                    "connection" | "content-length" |
+                    "access-control-request-method",
-                    "cookie" | "cookie2" | "date" |"dnt" |
+                    "connection", "content-length",
-                    "expect" | "host" | "keep-alive" | "origin" |
+                    "cookie", "cookie2", "date", "dnt",
-                    "referer" | "te" | "trailer" | "transfer-encoding" |
+                    "expect", "host", "keep-alive", "origin",
-                    "upgrade" | "user-agent" | "via" => {
+                    "referer", "te", "trailer", "transfer-encoding",
-                        return Ok(());
+                    "upgrade", "via"];
-                    },
+
-                    _ => s
+                let disallowedHeaderPrefixes = ["sec-", "proxy-"];
+
+                if disallowedHeaders.iter().any(|header| *header == s) ||
+                   disallowedHeaderPrefixes.iter().any(|prefix| s.starts_with(prefix)) {
+                    return Ok(())
+                } else {
+                    s
                 }
             },
             None => unreachable!()

tests/wpt/metadata/XMLHttpRequest/preserve-ua-header-on-redirect.htm.ini

@@ -1,5 +0,0 @@
-[preserve-ua-header-on-redirect.htm]
-  type: testharness
-  [XMLHttpRequest: User-Agent header is preserved on redirect 1]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/setrequestheader-header-allowed.htm.ini

@@ -1,5 +0,0 @@
-[setrequestheader-header-allowed.htm]
-  type: testharness
-  [XMLHttpRequest: setRequestHeader() - headers that are allowed (User-Agent)]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/setrequestheader-header-forbidden.htm.ini

@@ -1,5 +0,0 @@
-[setrequestheader-header-forbidden.htm]
-  type: testharness
-  [XMLHttpRequest: setRequestHeader() - headers that are forbidden]
-    expected: FAIL
-

tests/wpt/metadata/websockets/security/002.html.ini

@@ -1,5 +0,0 @@
-[002.html]
-  type: testharness
-  [WebSockets: check Sec-WebSocket-Key]
-    expected: FAIL
-