net: Start reducing number of IPCs channels used for fetch with a FetchThread (#33863)

Instead of creating a `ROUTER` for each fetch, create a fetch thread
which handles all incoming and outcoming fetch requests. Now messages
involving fetches carry a "request id" which indicates which fetch is
being addressed by the message. This greatly reduces the number of file
descriptors used by fetch.

In addition, the interface for kicking off fetches is simplified when
using the `Listener` with `Document`s and the `GlobalScope`.

This does not fix all leaked file descriptors / mach ports, but greatly
eliminates the number used. Now tests can be run without limiting
procesess on modern macOS systems.

Followup work:

1. There are more instances where fetch is done using the old method.
   Some of these require more changes in order to be converted to the
   `FetchThread` approach.
2. Eliminate usage of IPC channels when doing redirects.
3. Also eliminate the IPC channel used for cancel handling.
4. This change opens up the possiblity of controlling the priority of
   fetch requests.

Fixes #29834.

Signed-off-by: Martin Robinson <mrobinson@igalia.com>
This commit is contained in:
Martin Robinson 2024-10-16 09:53:24 -07:00 committed by GitHub
parent 2115267328
commit 036e74524a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 761 additions and 766 deletions

View file

@ -465,7 +465,7 @@ pub async fn main_fetch(
let mut response_loaded = false;
let mut response = if !response.is_network_error() && !request.integrity_metadata.is_empty() {
// Step 19.1.
wait_for_response(&mut response, target, done_chan).await;
wait_for_response(request, &mut response, target, done_chan).await;
response_loaded = true;
// Step 19.2.
@ -487,12 +487,12 @@ pub async fn main_fetch(
if request.synchronous {
// process_response is not supposed to be used
// by sync fetch, but we overload it here for simplicity
target.process_response(&response);
target.process_response(request, &response);
if !response_loaded {
wait_for_response(&mut response, target, done_chan).await;
wait_for_response(request, &mut response, target, done_chan).await;
}
// overloaded similarly to process_response
target.process_response_eof(&response);
target.process_response_eof(request, &response);
return response;
}
@ -507,15 +507,15 @@ pub async fn main_fetch(
}
// Step 22.
target.process_response(&response);
target.process_response(request, &response);
// Step 23.
if !response_loaded {
wait_for_response(&mut response, target, done_chan).await;
wait_for_response(request, &mut response, target, done_chan).await;
}
// Step 24.
target.process_response_eof(&response);
target.process_response_eof(request, &response);
if let Ok(http_cache) = context.state.http_cache.write() {
http_cache.update_awaiting_consumers(request, &response);
@ -527,6 +527,7 @@ pub async fn main_fetch(
}
async fn wait_for_response(
request: &Request,
response: &mut Response,
target: Target<'_>,
done_chan: &mut DoneChannel,
@ -535,7 +536,7 @@ async fn wait_for_response(
loop {
match ch.1.recv().await {
Some(Data::Payload(vec)) => {
target.process_response_chunk(vec);
target.process_response_chunk(request, vec);
},
Some(Data::Done) => {
break;
@ -555,7 +556,7 @@ async fn wait_for_response(
// in case there was no channel to wait for, the body was
// obtained synchronously via scheme_fetch for data/file/about/etc
// We should still send the body across as a chunk
target.process_response_chunk(vec.clone());
target.process_response_chunk(request, vec.clone());
} else {
assert_eq!(*body, ResponseBody::Empty)
}

View file

@ -581,9 +581,9 @@ impl ImageCache for ImageCacheImpl {
/// Inform the image cache about a response for a pending request.
fn notify_pending_response(&self, id: PendingImageId, action: FetchResponseMsg) {
match (action, id) {
(FetchResponseMsg::ProcessRequestBody, _) |
(FetchResponseMsg::ProcessRequestEOF, _) => (),
(FetchResponseMsg::ProcessResponse(response), _) => {
(FetchResponseMsg::ProcessRequestBody(..), _) |
(FetchResponseMsg::ProcessRequestEOF(..), _) => (),
(FetchResponseMsg::ProcessResponse(_, response), _) => {
debug!("Received {:?} for {:?}", response.as_ref().map(|_| ()), id);
let mut store = self.store.lock().unwrap();
let pending_load = store.pending_loads.get_by_key_mut(&id).unwrap();
@ -608,7 +608,7 @@ impl ImageCache for ImageCacheImpl {
pending_load.final_url = final_url;
pending_load.cors_status = cors_status;
},
(FetchResponseMsg::ProcessResponseChunk(data), _) => {
(FetchResponseMsg::ProcessResponseChunk(_, data), _) => {
debug!("Got some data for {:?}", id);
let mut store = self.store.lock().unwrap();
let pending_load = store.pending_loads.get_by_key_mut(&id).unwrap();
@ -627,7 +627,7 @@ impl ImageCache for ImageCacheImpl {
pending_load.metadata = Some(img_metadata);
}
},
(FetchResponseMsg::ProcessResponseEOF(result), key) => {
(FetchResponseMsg::ProcessResponseEOF(_, result), key) => {
debug!("Received EOF for {:?}", key);
match result {
Ok(_) => {

View file

@ -7,8 +7,8 @@ use std::ops::Deref;
use headers::{ContentType, HeaderMapExt};
use hyper_serde::Serde;
use mime::{self, Mime};
use net_traits::request::{Origin, Referrer, Request};
use net_traits::response::{HttpsState, ResponseBody};
use net_traits::request::Referrer;
use net_traits::response::ResponseBody;
use net_traits::{FetchMetadata, FilteredMetadata, NetworkError};
use servo_url::ServoUrl;
@ -21,15 +21,13 @@ fn assert_parse(
charset: Option<&str>,
data: Option<&[u8]>,
) {
use net_traits::request::RequestBuilder;
let url = ServoUrl::parse(url).unwrap();
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.pipeline_id(None)
.build();
let response = fetch(&mut request, None);

View file

@ -33,15 +33,15 @@ use net::test::HttpState;
use net_traits::filemanager_thread::FileTokenCheck;
use net_traits::http_status::HttpStatus;
use net_traits::request::{
Destination, Origin, RedirectMode, Referrer, Request, RequestBuilder, RequestMode,
Destination, RedirectMode, Referrer, Request, RequestBuilder, RequestMode,
};
use net_traits::response::{CacheState, HttpsState, Response, ResponseBody, ResponseType};
use net_traits::response::{CacheState, Response, ResponseBody, ResponseType};
use net_traits::{
FetchTaskTarget, IncludeSubdomains, NetworkError, ReferrerPolicy, ResourceFetchTiming,
ResourceTimingType,
};
use servo_arc::Arc as ServoArc;
use servo_url::{ImmutableOrigin, ServoUrl};
use servo_url::ServoUrl;
use tokio_test::block_on;
use uuid::Uuid;
@ -61,14 +61,9 @@ fn test_fetch_response_is_not_network_error() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -80,14 +75,9 @@ fn test_fetch_response_is_not_network_error() {
#[test]
fn test_fetch_on_bad_port_is_network_error() {
let url = ServoUrl::parse("http://www.example.org:6667").unwrap();
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
assert!(fetch_response.is_network_error());
let fetch_error = fetch_response.get_network_error().unwrap();
@ -105,14 +95,9 @@ fn test_fetch_response_body_matches_const_message() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -130,14 +115,9 @@ fn test_fetch_response_body_matches_const_message() {
#[test]
fn test_fetch_aboutblank() {
let url = ServoUrl::parse("about:blank").unwrap();
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
// We should see an opaque-filtered response.
@ -168,12 +148,12 @@ fn test_fetch_blob() {
impl FetchTaskTarget for FetchResponseCollector {
fn process_request_body(&mut self, _: &Request) {}
fn process_request_eof(&mut self, _: &Request) {}
fn process_response(&mut self, _: &Response) {}
fn process_response_chunk(&mut self, chunk: Vec<u8>) {
fn process_response(&mut self, _: &Request, _: &Response) {}
fn process_response_chunk(&mut self, _: &Request, chunk: Vec<u8>) {
self.buffer.extend_from_slice(chunk.as_slice());
}
/// Fired when the response is fully fetched
fn process_response_eof(&mut self, response: &Response) {
fn process_response_eof(&mut self, _: &Request, response: &Response) {
assert_eq!(self.buffer, self.expected);
let _ = self.sender.send(response.clone());
}
@ -200,13 +180,9 @@ fn test_fetch_blob() {
);
let url = ServoUrl::parse(&format!("blob:{}{}", origin.as_str(), id.simple())).unwrap();
let mut request = Request::new(
url,
Some(Origin::Origin(origin.origin())),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(origin.origin())
.build();
let (sender, receiver) = unbounded();
@ -246,14 +222,9 @@ fn test_file() {
.unwrap();
let url = ServoUrl::from_file_path(path.clone()).unwrap();
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let pool = CoreResourceThreadPool::new(1);
let pool_handle = Arc::new(pool);
@ -294,14 +265,9 @@ fn test_file() {
#[test]
fn test_fetch_ftp() {
let url = ServoUrl::parse("ftp://not-supported").unwrap();
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
assert!(fetch_response.is_network_error());
}
@ -309,14 +275,9 @@ fn test_fetch_ftp() {
#[test]
fn test_fetch_bogus_scheme() {
let url = ServoUrl::parse("bogus://whatever").unwrap();
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
assert!(fetch_response.is_network_error());
}
@ -360,15 +321,7 @@ fn test_cors_preflight_fetch() {
let (server, url) = make_server(handler);
let target_url = url.clone().join("a.html").unwrap();
let origin = Origin::Origin(ImmutableOrigin::new_opaque());
let mut request = Request::new(
url.clone(),
Some(origin),
Referrer::ReferrerUrl(target_url),
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url, Referrer::ReferrerUrl(target_url)).build();
request.referrer_policy = Some(ReferrerPolicy::Origin);
request.use_cors_preflight = true;
request.mode = RequestMode::CorsMode;
@ -418,14 +371,7 @@ fn test_cors_preflight_cache_fetch() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(ImmutableOrigin::new_opaque());
let mut request = Request::new(
url.clone(),
Some(origin.clone()),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build();
request.use_cors_preflight = true;
request.mode = RequestMode::CorsMode;
let mut wrapped_request0 = request.clone();
@ -485,14 +431,7 @@ fn test_cors_preflight_fetch_network_error() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(ImmutableOrigin::new_opaque());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build();
request.method = Method::from_bytes(b"CHICKEN").unwrap();
request.use_cors_preflight = true;
request.mode = RequestMode::CorsMode;
@ -519,14 +458,9 @@ fn test_fetch_response_is_basic_filtered() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -587,14 +521,7 @@ fn test_fetch_response_is_cors_filtered() {
let (server, url) = make_server(handler);
// an origin mis-match will stop it from defaulting to a basic filtered response
let origin = Origin::Origin(ImmutableOrigin::new_opaque());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build();
request.mode = RequestMode::CorsMode;
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -626,14 +553,7 @@ fn test_fetch_response_is_opaque_filtered() {
let (server, url) = make_server(handler);
// an origin mis-match will fall through to an Opaque filtered response
let origin = Origin::Origin(ImmutableOrigin::new_opaque());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -679,14 +599,9 @@ fn test_fetch_response_is_opaque_redirect_filtered() {
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
request.redirect_mode = RedirectMode::Manual;
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -718,14 +633,9 @@ fn test_fetch_with_local_urls_only() {
let (server, server_url) = make_server(handler);
let do_fetch = |url: ServoUrl| {
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
// Set the flag.
request.local_urls_only = true;
@ -786,14 +696,9 @@ fn test_fetch_with_hsts() {
HstsEntry::new("localhost".to_owned(), IncludeSubdomains::NotIncluded, None).unwrap(),
);
}
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
// Set the flag.
request.local_urls_only = false;
let response = fetch_with_context(&mut request, &mut context);
@ -935,14 +840,9 @@ fn test_fetch_with_sri_network_error() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
// To calulate hash use :
// echo -n "alert('Hello, Network Error');" | openssl dgst -sha384 -binary | openssl base64 -A
request.integrity_metadata =
@ -964,14 +864,9 @@ fn test_fetch_with_sri_sucess() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
// To calulate hash use :
// echo -n "alert('Hello, Network Error');" | openssl dgst -sha384 -binary | openssl base64 -A
request.integrity_metadata =
@ -1009,15 +904,10 @@ fn test_fetch_blocked_nosniff() {
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
request.destination = destination;
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.destination(destination)
.build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -1059,14 +949,9 @@ fn setup_server_and_fetch(message: &'static [u8], redirect_cap: u32) -> Response
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
fetch_response
@ -1152,15 +1037,10 @@ fn test_fetch_redirect_updates_method_runner(
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
request.method = method;
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.method(method)
.build();
let _ = fetch(&mut request, None);
let _ = server.close();
@ -1240,15 +1120,9 @@ fn test_fetch_async_returns_complete_response() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -1264,15 +1138,7 @@ fn test_opaque_filtered_fetch_async_returns_complete_response() {
let (server, url) = make_server(handler);
// an origin mis-match will fall through to an Opaque filtered response
let origin = Origin::Origin(ImmutableOrigin::new_opaque());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build();
let fetch_response = fetch(&mut request, None);
let _ = server.close();
@ -1304,16 +1170,10 @@ fn test_opaque_redirect_filtered_fetch_async_returns_complete_response() {
};
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url,
Some(origin),
Referrer::NoReferrer,
None,
HttpsState::None,
);
request.redirect_mode = RedirectMode::Manual;
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.redirect_mode(RedirectMode::Manual)
.build();
let fetch_response = fetch(&mut request, None);
@ -1333,14 +1193,11 @@ fn test_fetch_with_devtools() {
let (server, url) = make_server(handler);
let origin = Origin::Origin(url.origin());
let mut request = Request::new(
url.clone(),
Some(origin),
Referrer::NoReferrer,
Some(TEST_PIPELINE_ID),
HttpsState::None,
);
let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.origin(url.origin())
.redirect_mode(RedirectMode::Manual)
.pipeline_id(Some(TEST_PIPELINE_ID))
.build();
let (devtools_chan, devtools_port) = unbounded();

View file

@ -6,8 +6,8 @@ use base::id::TEST_PIPELINE_ID;
use http::header::{HeaderValue, EXPIRES};
use http::StatusCode;
use net::http_cache::HttpCache;
use net_traits::request::{Origin, Referrer, Request};
use net_traits::response::{HttpsState, Response, ResponseBody};
use net_traits::request::{Referrer, RequestBuilder};
use net_traits::response::{Response, ResponseBody};
use net_traits::{ResourceFetchTiming, ResourceTimingType};
use servo_url::ServoUrl;
use tokio::sync::mpsc::unbounded_channel as unbounded;
@ -20,13 +20,10 @@ fn test_refreshing_resource_sets_done_chan_the_appropriate_value() {
ResponseBody::Done(vec![]),
];
let url = ServoUrl::parse("https://servo.org").unwrap();
let request = Request::new(
url.clone(),
Some(Origin::Origin(url.clone().origin())),
Referrer::NoReferrer,
Some(TEST_PIPELINE_ID),
HttpsState::None,
);
let request = RequestBuilder::new(url.clone(), Referrer::NoReferrer)
.pipeline_id(Some(TEST_PIPELINE_ID))
.origin(url.origin())
.build();
let timing = ResourceFetchTiming::new(ResourceTimingType::Navigation);
let mut response = Response::new(url.clone(), timing);
// Expires header makes the response cacheable.

View file

@ -1426,12 +1426,12 @@ fn test_fetch_compressed_response_update_count() {
impl FetchTaskTarget for FetchResponseCollector {
fn process_request_body(&mut self, _: &Request) {}
fn process_request_eof(&mut self, _: &Request) {}
fn process_response(&mut self, _: &Response) {}
fn process_response_chunk(&mut self, _: Vec<u8>) {
fn process_response(&mut self, _: &Request, _: &Response) {}
fn process_response_chunk(&mut self, _: &Request, _: Vec<u8>) {
self.update_count += 1;
}
/// Fired when the response is fully fetched
fn process_response_eof(&mut self, _: &Response) {
fn process_response_eof(&mut self, _: &Request, _: &Response) {
let _ = self.sender.send(self.update_count);
}
}

View file

@ -121,10 +121,10 @@ fn new_fetch_context(
impl FetchTaskTarget for FetchResponseCollector {
fn process_request_body(&mut self, _: &Request) {}
fn process_request_eof(&mut self, _: &Request) {}
fn process_response(&mut self, _: &Response) {}
fn process_response_chunk(&mut self, _: Vec<u8>) {}
fn process_response(&mut self, _: &Request, _: &Response) {}
fn process_response_chunk(&mut self, _: &Request, _: Vec<u8>) {}
/// Fired when the response is fully fetched
fn process_response_eof(&mut self, response: &Response) {
fn process_response_eof(&mut self, _: &Request, response: &Response) {
let _ = self.sender.send(response.clone());
}
}