Responding to review comments

components/net/resource_thread.rs

@@ -25,6 +25,7 @@ use malloc_size_of::{MallocSizeOf, MallocSizeOfOps};
 use net_traits::request::{Destination, RequestBuilder};
 use net_traits::response::{Response, ResponseInit};
 use net_traits::storage_thread::StorageThreadMsg;
+use net_traits::DiscardFetch;
 use net_traits::FetchTaskTarget;
 use net_traits::WebSocketNetworkEvent;
 use net_traits::{CookieSource, CoreResourceMsg, CoreResourceThread};
@@ -248,7 +249,7 @@ impl ResourceChannelManager {
                 ),
                 FetchChannels::Prefetch => {
                     self.resource_manager
-                        .fetch(req_init, None, (), http_state, None)
+                        .fetch(req_init, None, DiscardFetch, http_state, None)
                 },
             },
             CoreResourceMsg::DeleteCookies(request) => {

components/net_traits/lib.rs

@@ -237,7 +237,12 @@ impl FetchTaskTarget for IpcSender<FetchResponseMsg> {
     }
 }
 
-impl FetchTaskTarget for () {
+/// A fetch task that discards all data it's sent,
+/// useful when speculatively prefetching data that we don't need right
+/// now, but might need in the future.
+pub struct DiscardFetch;
+
+impl FetchTaskTarget for DiscardFetch {
     fn process_request_body(&mut self, _: &Request) {}
 
     fn process_request_eof(&mut self, _: &Request) {}

components/script/dom/servoparser/mod.rs

@@ -102,7 +102,9 @@ pub struct ServoParser {
     aborted: Cell<bool>,
     /// <https://html.spec.whatwg.org/multipage/#script-created-parser>
     script_created_parser: bool,
-    /// We do a quick-and-dirty parse of the input looking for resources to prefetch
+    /// We do a quick-and-dirty parse of the input looking for resources to prefetch.
+    // TODO: if we had speculative parsing, we could do this when speculatively
+    // building the DOM. https://github.com/servo/servo/pull/19203
     prefetch_tokenizer: DomRefCell<prefetch::Tokenizer>,
     #[ignore_malloc_size_of = "Defined in html5ever"]
     prefetch_input: DomRefCell<BufferQueue>,
@@ -433,7 +435,9 @@ impl ServoParser {
     }
 
     fn push_tendril_input_chunk(&self, chunk: StrTendril) {
-        if !chunk.is_empty() {
+        if chunk.is_empty() {
+            return;
+        }
         // Per https://github.com/whatwg/html/issues/1495
         // stylesheets should not be loaded for documents
         // without browsing contexts.
@@ -449,7 +453,7 @@ impl ServoParser {
             // have been wasted, but in most cases it won't.
             let mut prefetch_input = self.prefetch_input.borrow_mut();
             prefetch_input.push_back(chunk.clone());
-                let _ = self.prefetch_tokenizer
+            self.prefetch_tokenizer
                 .borrow_mut()
                 .feed(&mut *prefetch_input);
         }
@@ -457,7 +461,6 @@ impl ServoParser {
         // which is tokenized lazily.
         self.network_input.borrow_mut().push_back(chunk);
     }
-    }
 
     fn push_bytes_input_chunk(&self, chunk: Vec<u8>) {
         // For byte input, we convert it to text using the network decoder.
@@ -471,7 +474,6 @@ impl ServoParser {
     }
 
     fn push_string_input_chunk(&self, chunk: String) {
-        // Convert the chunk to a tendril so cloning it isn't expensive.
         // The input has already been decoded as a string, so doesn't need
         // to be decoded by the network decoder again.
         let chunk = StrTendril::from(chunk);

components/script/dom/servoparser/prefetch.rs

@@ -9,6 +9,7 @@ use crate::dom::htmlimageelement::image_fetch_request;
 use crate::dom::htmlscriptelement::script_fetch_request;
 use crate::stylesheet_loader::stylesheet_fetch_request;
 use html5ever::buffer_queue::BufferQueue;
+use html5ever::tokenizer::states::RawKind;
 use html5ever::tokenizer::Tag;
 use html5ever::tokenizer::TagKind;
 use html5ever::tokenizer::Token;
@@ -63,8 +64,8 @@ impl Tokenizer {
         Tokenizer { inner }
     }
 
-    pub fn feed(&mut self, input: &mut BufferQueue) -> TokenizerResult<()> {
+    pub fn feed(&mut self, input: &mut BufferQueue) {
-        self.inner.feed(input)
+        while let TokenizerResult::Script(PrefetchHandle) = self.inner.feed(input) {}
     }
 }
 
@@ -79,10 +80,20 @@ struct PrefetchSink {
     prefetching: bool,
 }
 
+/// The prefetch tokenizer produces trivial results
+struct PrefetchHandle;
+
 impl TokenSink for PrefetchSink {
-    type Handle = ();
+    type Handle = PrefetchHandle;
-    fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+    fn process_token(
-        if let Token::TagToken(ref tag) = token {
+        &mut self,
+        token: Token,
+        _line_number: u64,
+    ) -> TokenSinkResult<PrefetchHandle> {
+        let tag = match token {
+            Token::TagToken(ref tag) => tag,
+            _ => return TokenSinkResult::Continue,
+        };
         match (tag.kind, &tag.name) {
             (TagKind::StartTag, local_name!("script")) if self.prefetching => {
                 if let Some(url) = self.get_url(tag, local_name!("src")) {
@@ -105,18 +116,17 @@ impl TokenSink for PrefetchSink {
                         .resource_threads
                         .send(CoreResourceMsg::Fetch(request, FetchChannels::Prefetch));
                 }
-                    // Don't prefetch inside script
+                TokenSinkResult::RawData(RawKind::ScriptData)
-                    self.prefetching = false;
             },
             (TagKind::StartTag, local_name!("img")) if self.prefetching => {
                 if let Some(url) = self.get_url(tag, local_name!("src")) {
                     debug!("Prefetch {} {}", tag.name, url);
-                        let request =
+                    let request = image_fetch_request(url, self.origin.clone(), self.pipeline_id);
-                            image_fetch_request(url, self.origin.clone(), self.pipeline_id);
                     let _ = self
                         .resource_threads
                         .send(CoreResourceMsg::Fetch(request, FetchChannels::Prefetch));
                 }
+                TokenSinkResult::Continue
             },
             (TagKind::StartTag, local_name!("link")) if self.prefetching => {
                 if let Some(rel) = self.get_attr(tag, local_name!("rel")) {
@@ -144,21 +154,25 @@ impl TokenSink for PrefetchSink {
                         }
                     }
                 }
+                TokenSinkResult::Continue
+            },
+            (TagKind::StartTag, local_name!("script")) => {
+                TokenSinkResult::RawData(RawKind::ScriptData)
             },
             (TagKind::EndTag, local_name!("script")) => {
                 // After the first script tag, the main parser is blocked, so it's worth prefetching.
                 self.prefetching = true;
+                TokenSinkResult::Script(PrefetchHandle)
             },
             (TagKind::StartTag, local_name!("base")) => {
                 if let Some(url) = self.get_url(tag, local_name!("href")) {
                     debug!("Setting base {}", url);
                     self.base = url;
                 }
-                },
-                _ => {},
-            }
-        }
                 TokenSinkResult::Continue
+            },
+            _ => TokenSinkResult::Continue,
+        }
     }
 }