auto merge of #1377 : SimonSapin/servo/encoding, r=kmcallister

Depends on https://github.com/mozilla-servo/rust-cssparser/pull/33 being merged.

Also (unrelated) pass around a base URL for stylesheets.

Adds a new dependency:
Upstream: https://github.com/lifthrasiir/rust-encoding
Servo’s fork: https://github.com/mozilla-servo/rust-encoding

As of this writing, upstream’s master branch targets Rust 0.8, and its rust-0.9-pre branch targets Rust master. Servo uses a Rust version in-between those. I pushed a rust-servo branch to our fork that backports from rust-0.9-pre, and also included some changes that have yet to be merged in upstream pull requests.

.gitmodules

@@ -106,3 +106,7 @@
 [submodule "src/support/http/rust-http"]
 	path = src/support/http/rust-http
 	url = https://github.com/mozilla-servo/rust-http
+[submodule "src/support/encoding/rust-encoding"]
+	path = src/support/encoding/rust-encoding
+	url = https://github.com/mozilla-servo/rust-encoding.git
+	branch = rust-servo

configure

@@ -498,6 +498,7 @@ CFG_SUBMODULES="\
     support/alert/rust-alert \
     support/azure/rust-azure \
     support/css/rust-cssparser \
+    support/encoding/rust-encoding \
     support/harfbuzz/rust-harfbuzz \
     support/http/rust-http \
     support/hubbub/libhubbub \

mk/sub.mk

@@ -13,7 +13,6 @@ NO_TESTS += \
 
 # These submodules will not be cleaned by the `make clean-fast` target.
 SLOW_BUILDS += \
-	libcss \
 	libparserutils \
 	mozjs \
 	sharegl \
@@ -22,7 +21,6 @@ SLOW_BUILDS += \
 
 # Builds that do not require rustc
 NATIVE_BUILDS += \
-	libcss \
 	libhubbub \
 	libparserutils \
 	mozjs \
@@ -52,6 +50,15 @@ DEPS_rust-azure += \
 	rust \
 	$(NULL)
 
+DEPS_rust-cssparser += \
+	rust-encoding \
+	rust \
+	$(NULL)
+
+DEPS_rust-encoding += \
+	rust \
+	$(NULL)
+
 DEPS_glfw-rs += \
 	glfw \
 	rust \

src/components/main/css/select.rs

@@ -3,12 +3,16 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 use style::{Stylesheet, Stylist, UserAgentOrigin, with_errors_silenced};
+use extra::url;
 
 
 pub fn new_stylist() -> Stylist {
     let mut stylist = Stylist::new();
-    let ua_stylesheet = with_errors_silenced(
+    let ua_stylesheet = with_errors_silenced(|| Stylesheet::from_bytes(
-        || Stylesheet::from_str(include_str!("user-agent.css")));
+        include_bin!("user-agent.css"),
+        url::from_str("chrome:///user-agent.css").unwrap(),
+        None,
+        None));
     stylist.add_stylesheet(ua_stylesheet, UserAgentOrigin);
     stylist
 }

src/components/script/html/cssparse.rs

@@ -8,8 +8,10 @@ use std::cell::Cell;
 use std::comm;
 use std::comm::Port;
 use std::task;
+use encoding::EncodingRef;
+use encoding::all::UTF_8;
 use style::Stylesheet;
-use servo_net::resource_task::{Load, ProgressMsg, Payload, Done, ResourceTask};
+use servo_net::resource_task::{Load, LoadResponse, ProgressMsg, Payload, Done, ResourceTask};
 use extra::url::Url;
 
 /// Where a style sheet comes from.
@@ -23,6 +25,9 @@ pub fn spawn_css_parser(provenance: StylesheetProvenance,
                      -> Port<Stylesheet> {
     let (result_port, result_chan) = comm::stream();
 
+    // TODO: Get the actual value. http://dev.w3.org/csswg/css-syntax/#environment-encoding
+    let environment_encoding = UTF_8 as EncodingRef;
+
     let provenance_cell = Cell::new(provenance);
     do task::spawn {
         // TODO: CSS parsing should take a base URL.
@@ -38,12 +43,16 @@ pub fn spawn_css_parser(provenance: StylesheetProvenance,
                 debug!("cssparse: loading style sheet at {:s}", url.to_str());
                 let (input_port, input_chan) = comm::stream();
                 resource_task.send(Load(url, input_chan));
-                Stylesheet::from_iter(ProgressMsgPortIterator {
+                let LoadResponse { metadata: metadata, progress_port: progress_port }
-                    progress_port: input_port.recv().progress_port
+                    = input_port.recv();
-                })
+                let protocol_encoding_label = metadata.charset.as_ref().map(|s| s.as_slice());
+                let iter = ProgressMsgPortIterator { progress_port: progress_port };
+                Stylesheet::from_bytes_iter(
+                    iter, metadata.final_url,
+                    protocol_encoding_label, Some(environment_encoding))
             }
-            InlineProvenance(_, data) => {
+            InlineProvenance(base_url, data) => {
-                Stylesheet::from_str(data)
+                Stylesheet::from_str(data, base_url, environment_encoding)
             }
         };
         result_chan.send(sheet);

src/components/script/script.rc

@@ -15,6 +15,7 @@
 
 extern mod geom = "rust-geom";
 extern mod hubbub;
+extern mod encoding;
 extern mod js;
 extern mod servo_net (name = "net");
 extern mod servo_util (name = "util");

src/components/style/selector_matching.rs

@@ -713,28 +713,27 @@ fn match_attribute<N: TreeNode<T>, T: TreeNodeRefAsElement<N, E>, E: ElementLike
         }
     }
 }
-fn get_rules(css_string: &str) -> ~[~[Rule]] {
+
-    let device = &Device { media_type: Screen };
-    let sheet = Stylesheet::from_str(css_string);
-    let mut index = 0u;
-    let mut results = ~[];
-    do iter_style_rules(sheet.rules.as_slice(), device) |style_rule| {
-        results.push(style_rule.selectors.iter().map(|s| Rule {
-                    selector: Arc::new(s.clone()),
-                    declarations: style_rule.declarations.normal.clone(),
-                    index: index,
-                    stylesheet_index: 0u,
-                }).collect());
-        index += 1u;
-    }
-    results
-}
 
 /// Helper method to get some Rules from selector strings.
 /// Each sublist of the result contains the Rules for one StyleRule.
 fn get_mock_rules(css_selectors: &[&str]) -> ~[~[Rule]] {
-    let css_string = css_selectors.map(|s| s + " { color: red; } ").concat();
+    use namespaces::NamespaceMap;
-    get_rules(css_string)
+    use selectors::parse_selector_list;
+    use cssparser::tokenize;
+
+    let namespaces = NamespaceMap::new();
+    css_selectors.iter().enumerate().map(|(i, selectors)| {
+        parse_selector_list(tokenize(*selectors).map(|(c, _)| c).to_owned_vec(), &namespaces)
+        .unwrap().move_iter().map(|s| {
+            Rule {
+                selector: Arc::new(s),
+                declarations: Arc::new(~[]),
+                index: i,
+                stylesheet_index: 0u,
+            }
+        }).to_owned_vec()
+    }).to_owned_vec()
 }
 
 #[test]

src/components/style/style.rc

@@ -15,6 +15,7 @@
 
 extern mod extra;
 extern mod cssparser;
+extern mod encoding;
 extern mod servo_util (name = "util");
 
 
@@ -33,6 +34,3 @@ mod properties;
 mod namespaces;
 mod media_queries;
 mod parsing_utils;
-
-#[cfg(test)]
-mod tests;

src/components/style/stylesheets.rs

@@ -2,10 +2,13 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use std::str;
 use std::iter::Iterator;
 use std::ascii::StrAsciiExt;
-use cssparser::{tokenize, parse_stylesheet_rules, ToCss};
+use extra::url::Url;
+
+use encoding::EncodingRef;
+
+use cssparser::{decode_stylesheet_bytes, tokenize, parse_stylesheet_rules, ToCss};
 use cssparser::ast::*;
 use selectors;
 use properties;
@@ -20,6 +23,8 @@ pub struct Stylesheet {
     /// cascading order)
     rules: ~[CSSRule],
     namespaces: NamespaceMap,
+    encoding: EncodingRef,
+    base_url: Url,
 }
 
 
@@ -36,19 +41,26 @@ pub struct StyleRule {
 
 
 impl Stylesheet {
-    pub fn from_iter<I: Iterator<~[u8]>>(input: I) -> Stylesheet {
+    pub fn from_bytes_iter<I: Iterator<~[u8]>>(
-        let mut string = ~"";
+            mut input: I, base_url: Url, protocol_encoding_label: Option<&str>,
-        let mut input = input;
+            environment_encoding: Option<EncodingRef>) -> Stylesheet {
-        // TODO: incremental tokinization/parsing
+        let mut bytes = ~[];
+        // TODO: incremental decoding and tokinization/parsing
         for chunk in input {
-            // Assume UTF-8. This fails on invalid UTF-8
+            bytes.push_all(chunk)
-            // TODO: support character encodings (use rust-encodings in rust-cssparser)
-            string.push_str(str::from_utf8_owned(chunk))
         }
-        Stylesheet::from_str(string)
+        Stylesheet::from_bytes(bytes, base_url, protocol_encoding_label, environment_encoding)
     }
 
-    pub fn from_str(css: &str) -> Stylesheet {
+    pub fn from_bytes(
+            bytes: &[u8], base_url: Url, protocol_encoding_label: Option<&str>,
+            environment_encoding: Option<EncodingRef>) -> Stylesheet {
+        let (string, used_encoding) = decode_stylesheet_bytes(
+            bytes, protocol_encoding_label, environment_encoding);
+        Stylesheet::from_str(string, base_url, used_encoding)
+    }
+
+    pub fn from_str(css: &str, base_url: Url, encoding: EncodingRef) -> Stylesheet {
         static STATE_CHARSET: uint = 1;
         static STATE_IMPORTS: uint = 2;
         static STATE_NAMESPACES: uint = 3;
@@ -107,7 +119,7 @@ impl Stylesheet {
             }
             state = next_state;
         }
-        Stylesheet{ rules: rules, namespaces: namespaces }
+        Stylesheet{ rules: rules, namespaces: namespaces, encoding: encoding, base_url: base_url }
     }
 }
 

src/components/style/tests/mod.rs

@@ -1,12 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-use super::stylesheets::Stylesheet;
-
-#[test]
-fn test_bootstrap() {
-    // Test that parsing bootstrap does not trigger an assertion or otherwise fail.
-    let stylesheet = Stylesheet::from_str(include_str!("bootstrap-v3.0.0.css"));
-    assert!(stylesheet.rules.len() > 100);  // This depends on whet selectors are supported.
-}

src/support/css/rust-cssparser

@@ -1 +1 @@
-Subproject commit 1a7b4ca6d368f6035a4135504f89d66c5e225d09
+Subproject commit 5178431109f4a51f38c257e36640702320c54958

src/support/encoding/rust-encoding

@@ -0,0 +1 @@
+Subproject commit 1a6c011b697fe6d51a1d92a2504d6fac1b67d2dc