DevTools: sources for HTML files should be the whole HTML file (#37456)

To show the contents of inline scripts in the Sources panel, we need to
send the whole HTML file from script to devtools, not just the script
code. This is trickier than the external script case, but we can look to
[how Firefox does
it](https://servo.zulipchat.com/#narrow/channel/263398-general/topic/Getting.20the.20original.20page.20HTML.20from.20script/near/524392861)
for some inspiration. The process is as follows:

- when we execute a script
  - notify devtools to create the source actor
- if it’s an external script, send the script code to the devtools
server
  - if it’s an inline script, don’t send any source contents yet
  - devtools stores the contents in the source actor
- while loading a new document
  - buffer the markup, so we can send it to devtools
- when we finish loading a new document
  - send the buffered markup to the devtools server
- devtools stores the contents in any source actors with no contents yet
- when a source actor gets a `source` request
  - if we have the contents, send those contents to the client
- if we don’t have the contents (inline script that loaded while
devtools was closed)
    - FUTURE: try to fetch the markup out of cache
    - otherwise send `<!-- not available; please reload! -->`

Testing: Several tests added to test the changes, also updates an
existing test with correct assertion
Fixes: https://github.com/servo/servo/issues/36874

---------

Signed-off-by: atbrakhi <atbrakhi@igalia.com>
Signed-off-by: Delan Azabani <dazabani@igalia.com>
Co-authored-by: Delan Azabani <dazabani@igalia.com>

components/script/dom/dedicatedworkerglobalscope.rs

@@ -488,10 +488,10 @@ impl DedicatedWorkerGlobalScope {
                         url: metadata.final_url,
                         external: true, // Worker scripts are always external.
                         worker_id: Some(global.upcast::<WorkerGlobalScope>().get_worker_id()),
-                        content: source.to_string(),
+                        content: Some(source.to_string()),
                         content_type: metadata.content_type.map(|c_type| c_type.0.to_string()),
                     };
-                    let _ = chan.send(ScriptToDevtoolsControlMsg::ScriptSourceLoaded(
+                    let _ = chan.send(ScriptToDevtoolsControlMsg::CreateSourceActor(
                         pipeline_id,
                         source_info,
                     ));

components/script/dom/htmlscriptelement.rs

@@ -16,6 +16,7 @@ use content_security_policy as csp;
 use devtools_traits::{ScriptToDevtoolsControlMsg, SourceInfo};
 use dom_struct::dom_struct;
 use encoding_rs::Encoding;
+use html5ever::serialize::TraversalScope;
 use html5ever::{LocalName, Prefix, local_name, namespace_url, ns};
 use ipc_channel::ipc;
 use js::jsval::UndefinedValue;
@@ -27,7 +28,7 @@ use net_traits::request::{
     RequestBuilder, RequestId,
 };
 use net_traits::{
-    FetchMetadata, FetchResponseListener, Metadata, NetworkError, ResourceFetchTiming,
+    FetchMetadata, FetchResponseListener, IpcSend, Metadata, NetworkError, ResourceFetchTiming,
     ResourceTimingType,
 };
 use servo_config::pref;
@@ -72,7 +73,7 @@ use crate::dom::trustedscript::TrustedScript;
 use crate::dom::trustedscripturl::TrustedScriptURL;
 use crate::dom::virtualmethods::VirtualMethods;
 use crate::dom::window::Window;
-use crate::fetch::create_a_potential_cors_request;
+use crate::fetch::{create_a_potential_cors_request, load_whole_resource};
 use crate::network_listener::{self, NetworkListener, PreInvoke, ResourceTimingListener};
 use crate::realms::enter_realm;
 use crate::script_module::{
@@ -1085,23 +1086,32 @@ impl HTMLScriptElement {
         if let Some(chan) = self.global().devtools_chan() {
             let pipeline_id = self.global().pipeline_id();
 
-            // TODO: https://github.com/servo/servo/issues/36874
-            let content = match &script.code {
-                SourceCode::Text(text) => text.to_string(),
-                SourceCode::Compiled(compiled) => compiled.original_text.to_string(),
-            };
+            let (url, content, content_type, is_external) = if script.external {
+                let content = match &script.code {
+                    SourceCode::Text(text) => text.to_string(),
+                    SourceCode::Compiled(compiled) => compiled.original_text.to_string(),
+                };
 
-            // https://html.spec.whatwg.org/multipage/#scriptingLanguages
-            let content_type = Some("text/javascript".to_string());
+                // content_type: https://html.spec.whatwg.org/multipage/#scriptingLanguages
+                (script.url.clone(), Some(content), "text/javascript", true)
+            } else {
+                // TODO: if needed, fetch the page again, in the same way as in the original request.
+                // Fetch it from cache, even if the original request was non-idempotent (e.g. POST).
+                // If we can’t fetch it from cache, we should probably give up, because with a real
+                // fetch, the server could return a different response.
+
+                // TODO: handle cases where Content-Type is not text/html.
+                (doc.url(), None, "text/html", false)
+            };
 
             let source_info = SourceInfo {
-                url: script.url.clone(),
-                external: script.external,
+                url,
+                external: is_external,
                 worker_id: None,
                 content,
-                content_type,
+                content_type: Some(content_type.to_string()),
             };
-            let _ = chan.send(ScriptToDevtoolsControlMsg::ScriptSourceLoaded(
+            let _ = chan.send(ScriptToDevtoolsControlMsg::CreateSourceActor(
                 pipeline_id,
                 source_info,
             ));

components/script/dom/servoparser/mod.rs

@@ -10,6 +10,7 @@ use base::id::PipelineId;
 use base64::Engine as _;
 use base64::engine::general_purpose;
 use content_security_policy as csp;
+use devtools_traits::ScriptToDevtoolsControlMsg;
 use dom_struct::dom_struct;
 use embedder_traits::resources::{self, Resource};
 use encoding_rs::Encoding;
@@ -137,6 +138,9 @@ pub(crate) struct ServoParser {
     #[ignore_malloc_size_of = "Defined in html5ever"]
     #[no_trace]
     prefetch_input: BufferQueue,
+    // The whole input as a string, if needed for the devtools Sources panel.
+    // TODO: use a faster type for concatenating strings?
+    content_for_devtools: Option<DomRefCell<String>>,
 }
 
 pub(crate) struct ElementAttribute {
@@ -457,6 +461,13 @@ impl ServoParser {
 
     #[cfg_attr(crown, allow(crown::unrooted_must_root))]
     fn new_inherited(document: &Document, tokenizer: Tokenizer, kind: ParserKind) -> Self {
+        // Store the whole input for the devtools Sources panel, if the devtools server is running
+        // and we are parsing for a document load (not just things like innerHTML).
+        // TODO: check if a devtools client is actually connected and/or wants the sources?
+        let content_for_devtools = (document.global().devtools_chan().is_some() &&
+            document.has_browsing_context())
+        .then_some(DomRefCell::new(String::new()));
+
         ServoParser {
             reflector: Reflector::new(),
             document: Dom::from_ref(document),
@@ -472,6 +483,7 @@ impl ServoParser {
             script_created_parser: kind == ParserKind::ScriptCreated,
             prefetch_tokenizer: prefetch::Tokenizer::new(document),
             prefetch_input: BufferQueue::default(),
+            content_for_devtools,
         }
     }
 
@@ -490,6 +502,15 @@ impl ServoParser {
     }
 
     fn push_tendril_input_chunk(&self, chunk: StrTendril) {
+        if let Some(mut content_for_devtools) = self
+            .content_for_devtools
+            .as_ref()
+            .map(|content| content.borrow_mut())
+        {
+            // TODO: append these chunks more efficiently
+            content_for_devtools.push_str(chunk.as_ref());
+        }
+
         if chunk.is_empty() {
             return;
         }
@@ -687,6 +708,21 @@ impl ServoParser {
         // Steps 3-12 are in another castle, namely finish_load.
         let url = self.tokenizer.url().clone();
         self.document.finish_load(LoadType::PageSource(url), can_gc);
+
+        // Send the source contents to devtools, if needed.
+        if let Some(content_for_devtools) = self
+            .content_for_devtools
+            .as_ref()
+            .map(|content| content.take())
+        {
+            let global = self.document.global();
+            let chan = global.devtools_chan().expect("Guaranteed by new");
+            let pipeline_id = self.document.global().pipeline_id();
+            let _ = chan.send(ScriptToDevtoolsControlMsg::UpdateSourceContent(
+                pipeline_id,
+                content_for_devtools,
+            ));
+        }
     }
 }