Split out shared networking code into net_traits crate

Fixes #4476
This commit is contained in:
Gilles Leblanc 2015-03-23 21:19:55 -04:00
parent 74f8c0eeb7
commit ba36a108c1
56 changed files with 674 additions and 518 deletions

View file

@ -0,0 +1,27 @@
[package]
name = "net_traits"
version = "0.0.1"
authors = ["The Servo Project Developers"]
[lib]
name = "net_traits"
path = "lib.rs"
[dependencies.geom]
git = "https://github.com/servo/rust-geom"
[dependencies.png]
git = "https://github.com/servo/rust-png"
[dependencies.profile]
path = "../profile"
[dependencies.util]
path = "../util"
[dependencies.stb_image]
git = "https://github.com/servo/rust-stb-image"
[dependencies]
url = "0.2.16"
hyper = "0.3"

View file

@ -0,0 +1,86 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use png;
use stb_image::image as stb_image2;
use std::iter::range_step;
use util::vec::byte_swap;
// FIXME: Images must not be copied every frame. Instead we should atomically
// reference count them.
pub type Image = png::Image;
// TODO(pcwalton): Speed up with SIMD, or better yet, find some way to not do this.
fn byte_swap_and_premultiply(data: &mut [u8]) {
let length = data.len();
for i in range_step(0, length, 4) {
let r = data[i + 2];
let g = data[i + 1];
let b = data[i + 0];
let a = data[i + 3];
data[i + 0] = ((r as u32) * (a as u32) / 255) as u8;
data[i + 1] = ((g as u32) * (a as u32) / 255) as u8;
data[i + 2] = ((b as u32) * (a as u32) / 255) as u8;
}
}
pub fn load_from_memory(buffer: &[u8]) -> Option<Image> {
if buffer.len() == 0 {
return None;
}
if png::is_png(buffer) {
match png::load_png_from_memory(buffer) {
Ok(mut png_image) => {
match png_image.pixels {
png::PixelsByColorType::RGB8(ref mut data) => byte_swap(data.as_mut_slice()),
png::PixelsByColorType::RGBA8(ref mut data) => {
byte_swap_and_premultiply(data.as_mut_slice())
}
_ => {}
}
Some(png_image)
}
Err(_err) => None,
}
} else {
// For non-png images, we use stb_image
// Can't remember why we do this. Maybe it's what cairo wants
static FORCE_DEPTH: uint = 4;
match stb_image2::load_from_memory_with_depth(buffer, FORCE_DEPTH, true) {
stb_image2::LoadResult::ImageU8(mut image) => {
assert!(image.depth == 4);
// handle gif separately because the alpha-channel has to be premultiplied
if is_gif(buffer) {
byte_swap_and_premultiply(image.data.as_mut_slice());
} else {
byte_swap(image.data.as_mut_slice());
}
Some(png::Image {
width: image.width as u32,
height: image.height as u32,
pixels: png::PixelsByColorType::RGBA8(image.data)
})
}
stb_image2::LoadResult::ImageF32(_image) => {
error!("HDR images not implemented");
None
}
stb_image2::LoadResult::Error(e) => {
error!("stb_image failed: {}", e);
None
}
}
}
}
fn is_gif(buffer: &[u8]) -> bool {
match buffer {
[b'G',b'I',b'F',b'8', n, b'a', ..] if n == b'7' || n == b'9' => true,
_ => false
}
}

View file

@ -0,0 +1,101 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use image::base::Image;
use image_cache_task::ImageResponseMsg;
use local_image_cache::LocalImageCache;
use geom::size::Size2D;
use std::sync::{Arc, Mutex};
use url::Url;
// FIXME: Nasty coupling here This will be a problem if we want to factor out image handling from
// the network stack. This should probably be factored out into an interface and use dependency
// injection.
/// A struct to store image data. The image will be loaded once the first time it is requested,
/// and an Arc will be stored. Clones of this Arc are given out on demand.
#[derive(Clone)]
pub struct ImageHolder<NodeAddress> {
url: Url,
image: Option<Arc<Box<Image>>>,
cached_size: Size2D<int>,
local_image_cache: Arc<Mutex<LocalImageCache<NodeAddress>>>,
}
impl<NodeAddress: Send + 'static> ImageHolder<NodeAddress> {
pub fn new(url: Url, local_image_cache: Arc<Mutex<LocalImageCache<NodeAddress>>>)
-> ImageHolder<NodeAddress> {
debug!("ImageHolder::new() {}", url.serialize());
let holder = ImageHolder {
url: url,
image: None,
cached_size: Size2D(0,0),
local_image_cache: local_image_cache.clone(),
};
// Tell the image cache we're going to be interested in this url
// FIXME: These two messages must be sent to prep an image for use
// but they are intended to be spread out in time. Ideally prefetch
// should be done as early as possible and decode only once we
// are sure that the image will be used.
{
let val = holder.local_image_cache.lock().unwrap();
let mut local_image_cache = val;
local_image_cache.prefetch(&holder.url);
local_image_cache.decode(&holder.url);
}
holder
}
/// This version doesn't perform any computation, but may be stale w.r.t. newly-available image
/// data that determines size.
///
/// The intent is that the impure version is used during layout when dimensions are used for
/// computing layout.
pub fn size(&self) -> Size2D<int> {
self.cached_size
}
/// Query and update the current image size.
pub fn get_size(&mut self, node_address: NodeAddress) -> Option<Size2D<int>> {
debug!("get_size() {}", self.url.serialize());
self.get_image(node_address).map(|img| {
self.cached_size = Size2D(img.width as int,
img.height as int);
self.cached_size.clone()
})
}
pub fn get_image_if_present(&self) -> Option<Arc<Box<Image>>> {
debug!("get_image_if_present() {}", self.url.serialize());
self.image.clone()
}
pub fn get_image(&mut self, node_address: NodeAddress) -> Option<Arc<Box<Image>>> {
debug!("get_image() {}", self.url.serialize());
// If this is the first time we've called this function, load
// the image and store it for the future
if self.image.is_none() {
let port = {
let val = self.local_image_cache.lock().unwrap();
let mut local_image_cache = val;
local_image_cache.get_image(node_address, &self.url)
};
match port.recv().unwrap() {
ImageResponseMsg::ImageReady(image) => self.image = Some(image),
ImageResponseMsg::ImageNotReady => debug!("image not ready for {}", self.url.serialize()),
ImageResponseMsg::ImageFailed => debug!("image decoding failed for {}", self.url.serialize()),
}
}
return self.image.clone();
}
pub fn url(&self) -> &Url {
&self.url
}
}

View file

@ -0,0 +1,106 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use image::base::Image;
use {ControlMsg, LoadData, ProgressMsg, ResourceTask};
use url::Url;
use std::sync::Arc;
use std::sync::mpsc::{channel, Sender};
pub enum Msg {
/// Tell the cache that we may need a particular image soon. Must be posted
/// before Decode
Prefetch(Url),
/// Tell the cache to decode an image. Must be posted before GetImage/WaitForImage
Decode(Url),
/// Request an Image object for a URL. If the image is not is not immediately
/// available then ImageNotReady is returned.
GetImage(Url, Sender<ImageResponseMsg>),
/// Wait for an image to become available (or fail to load).
WaitForImage(Url, Sender<ImageResponseMsg>),
/// Clients must wait for a response before shutting down the ResourceTask
Exit(Sender<()>),
/// Used by the prefetch tasks to post back image binaries
StorePrefetchedImageData(Url, Result<Vec<u8>, ()>),
/// Used by the decoder tasks to post decoded images back to the cache
StoreImage(Url, Option<Arc<Box<Image>>>),
/// For testing
WaitForStore(Sender<()>),
/// For testing
WaitForStorePrefetched(Sender<()>),
}
#[derive(Clone)]
pub enum ImageResponseMsg {
ImageReady(Arc<Box<Image>>),
ImageNotReady,
ImageFailed
}
impl PartialEq for ImageResponseMsg {
fn eq(&self, other: &ImageResponseMsg) -> bool {
match (self, other) {
(&ImageResponseMsg::ImageReady(..), &ImageResponseMsg::ImageReady(..)) => panic!("unimplemented comparison"),
(&ImageResponseMsg::ImageNotReady, &ImageResponseMsg::ImageNotReady) => true,
(&ImageResponseMsg::ImageFailed, &ImageResponseMsg::ImageFailed) => true,
(&ImageResponseMsg::ImageReady(..), _) | (&ImageResponseMsg::ImageNotReady, _) | (&ImageResponseMsg::ImageFailed, _) => false
}
}
}
#[derive(Clone)]
pub struct ImageCacheTask {
pub chan: Sender<Msg>,
}
impl ImageCacheTask {
pub fn send(&self, msg: Msg) {
self.chan.send(msg).unwrap();
}
}
pub fn load_image_data(url: Url, resource_task: ResourceTask, placeholder: &[u8]) -> Result<Vec<u8>, ()> {
let (response_chan, response_port) = channel();
resource_task.send(ControlMsg::Load(LoadData::new(url.clone(), response_chan))).unwrap();
let mut image_data = vec!();
let progress_port = response_port.recv().unwrap().progress_port;
loop {
match progress_port.recv().unwrap() {
ProgressMsg::Payload(data) => {
image_data.push_all(&data);
}
ProgressMsg::Done(Ok(..)) => {
return Ok(image_data);
}
ProgressMsg::Done(Err(..)) => {
// Failure to load the requested image will return the
// placeholder instead. In case it failed to load at init(),
// we still recover and return Err() but nothing will be drawn.
if placeholder.len() != 0 {
debug!("image_cache_task: failed to load {:?}, use placeholder instead.", url);
// Clean in case there was an error after started loading the image.
image_data.clear();
image_data.push_all(&placeholder);
return Ok(image_data);
} else {
debug!("image_cache_task: invalid placeholder.");
return Err(());
}
}
}
}
}

View file

@ -0,0 +1,218 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#![feature(box_syntax)]
#![feature(collections)]
#![feature(core)]
#![feature(int_uint)]
#![feature(rustc_private)]
#![feature(std_misc)]
extern crate geom;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate png;
extern crate profile;
extern crate stb_image;
extern crate url;
extern crate util;
use hyper::header::Headers;
use hyper::http::RawStatus;
use hyper::method::Method;
use hyper::mime::{Mime, Attr};
use url::Url;
use std::borrow::IntoCow;
use std::sync::mpsc::{channel, Receiver, Sender};
pub mod image_cache_task;
pub mod local_image_cache;
pub mod storage_task;
/// Image handling.
///
/// It may be surprising that this goes in the network crate as opposed to the graphics crate.
/// However, image handling is generally very integrated with the network stack (especially where
/// caching is involved) and as a result it must live in here.
pub mod image {
pub mod base;
pub mod holder;
}
#[derive(Clone)]
pub struct LoadData {
pub url: Url,
pub method: Method,
/// Headers that will apply to the initial request only
pub headers: Headers,
/// Headers that will apply to the initial request and any redirects
pub preserved_headers: Headers,
pub data: Option<Vec<u8>>,
pub cors: Option<ResourceCORSData>,
pub consumer: Sender<LoadResponse>,
}
impl LoadData {
pub fn new(url: Url, consumer: Sender<LoadResponse>) -> LoadData {
LoadData {
url: url,
method: Method::Get,
headers: Headers::new(),
preserved_headers: Headers::new(),
data: None,
cors: None,
consumer: consumer,
}
}
}
/// Handle to a resource task
pub type ResourceTask = Sender<ControlMsg>;
pub enum ControlMsg {
/// Request the data associated with a particular URL
Load(LoadData),
/// Store a set of cookies for a given originating URL
SetCookiesForUrl(Url, String, CookieSource),
/// Retrieve the stored cookies for a given URL
GetCookiesForUrl(Url, Sender<Option<String>>, CookieSource),
Exit
}
/// Message sent in response to `Load`. Contains metadata, and a port
/// for receiving the data.
///
/// Even if loading fails immediately, we send one of these and the
/// progress_port will provide the error.
pub struct LoadResponse {
/// Metadata, such as from HTTP headers.
pub metadata: Metadata,
/// Port for reading data.
pub progress_port: Receiver<ProgressMsg>,
}
#[derive(Clone)]
pub struct ResourceCORSData {
/// CORS Preflight flag
pub preflight: bool,
/// Origin of CORS Request
pub origin: Url
}
/// Metadata about a loaded resource, such as is obtained from HTTP headers.
#[derive(Clone)]
pub struct Metadata {
/// Final URL after redirects.
pub final_url: Url,
/// MIME type / subtype.
pub content_type: Option<(String, String)>,
/// Character set.
pub charset: Option<String>,
/// Headers
pub headers: Option<Headers>,
/// HTTP Status
pub status: Option<RawStatus>,
}
impl Metadata {
/// Metadata with defaults for everything optional.
pub fn default(url: Url) -> Self {
Metadata {
final_url: url,
content_type: None,
charset: None,
headers: None,
// http://fetch.spec.whatwg.org/#concept-response-status-message
status: Some(RawStatus(200, "OK".into_cow())),
}
}
/// Extract the parts of a Mime that we care about.
pub fn set_content_type(&mut self, content_type: Option<&Mime>) {
match content_type {
None => (),
Some(&Mime(ref type_, ref subtype, ref parameters)) => {
self.content_type = Some((type_.to_string(), subtype.to_string()));
for &(ref k, ref v) in parameters.iter() {
if &Attr::Charset == k {
self.charset = Some(v.to_string());
}
}
}
}
}
}
/// The creator of a given cookie
#[derive(PartialEq, Copy)]
pub enum CookieSource {
/// An HTTP API
HTTP,
/// A non-HTTP API
NonHTTP,
}
/// Messages sent in response to a `Load` message
#[derive(PartialEq,Debug)]
pub enum ProgressMsg {
/// Binary data - there may be multiple of these
Payload(Vec<u8>),
/// Indicates loading is complete, either successfully or not
Done(Result<(), String>)
}
/// Convenience function for synchronously loading a whole resource.
pub fn load_whole_resource(resource_task: &ResourceTask, url: Url)
-> Result<(Metadata, Vec<u8>), String> {
let (start_chan, start_port) = channel();
resource_task.send(ControlMsg::Load(LoadData::new(url, start_chan))).unwrap();
let response = start_port.recv().unwrap();
let mut buf = vec!();
loop {
match response.progress_port.recv().unwrap() {
ProgressMsg::Payload(data) => buf.push_all(&data),
ProgressMsg::Done(Ok(())) => return Ok((response.metadata, buf)),
ProgressMsg::Done(Err(e)) => return Err(e)
}
}
}
/// Load a URL asynchronously and iterate over chunks of bytes from the response.
pub fn load_bytes_iter(resource_task: &ResourceTask, url: Url) -> (Metadata, ProgressMsgPortIterator) {
let (input_chan, input_port) = channel();
resource_task.send(ControlMsg::Load(LoadData::new(url, input_chan))).unwrap();
let response = input_port.recv().unwrap();
let iter = ProgressMsgPortIterator { progress_port: response.progress_port };
(response.metadata, iter)
}
/// Iterator that reads chunks of bytes from a ProgressMsg port
pub struct ProgressMsgPortIterator {
progress_port: Receiver<ProgressMsg>
}
impl Iterator for ProgressMsgPortIterator {
type Item = Vec<u8>;
fn next(&mut self) -> Option<Vec<u8>> {
match self.progress_port.recv().unwrap() {
ProgressMsg::Payload(data) => Some(data),
ProgressMsg::Done(Ok(())) => None,
ProgressMsg::Done(Err(e)) => {
error!("error receiving bytes: {}", e);
None
}
}
}
}

View file

@ -0,0 +1,170 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/*!
An adapter for ImageCacheTask that does local caching to avoid
extra message traffic, it also avoids waiting on the same image
multiple times and thus triggering reflows multiple times.
*/
use image_cache_task::{ImageResponseMsg, ImageCacheTask, Msg};
use url::Url;
use std::borrow::ToOwned;
use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::sync::mpsc::{channel, Receiver};
use util::task::spawn_named;
pub trait ImageResponder<NodeAddress: Send> {
fn respond(&self) -> Box<Fn(ImageResponseMsg, NodeAddress)+Send>;
}
pub struct LocalImageCache<NodeAddress> {
image_cache_task: ImageCacheTask,
round_number: uint,
on_image_available: Option<Box<ImageResponder<NodeAddress>+Send>>,
state_map: HashMap<Url, ImageState>
}
impl<NodeAddress: Send> LocalImageCache<NodeAddress> {
pub fn new(image_cache_task: ImageCacheTask) -> LocalImageCache<NodeAddress> {
LocalImageCache {
image_cache_task: image_cache_task,
round_number: 1,
on_image_available: None,
state_map: HashMap::new()
}
}
}
#[derive(Clone)]
struct ImageState {
prefetched: bool,
decoded: bool,
last_request_round: uint,
last_response: ImageResponseMsg
}
impl<NodeAddress: Send + 'static> LocalImageCache<NodeAddress> {
/// The local cache will only do a single remote request for a given
/// URL in each 'round'. Layout should call this each time it begins
pub fn next_round(&mut self, on_image_available: Box<ImageResponder<NodeAddress> + Send>) {
self.round_number += 1;
self.on_image_available = Some(on_image_available);
}
pub fn prefetch(&mut self, url: &Url) {
{
let state = self.get_state(url);
if state.prefetched {
return
}
state.prefetched = true;
}
self.image_cache_task.send(Msg::Prefetch((*url).clone()));
}
pub fn decode(&mut self, url: &Url) {
{
let state = self.get_state(url);
if state.decoded {
return
}
state.decoded = true;
}
self.image_cache_task.send(Msg::Decode((*url).clone()));
}
// FIXME: Should return a Future
pub fn get_image(&mut self, node_address: NodeAddress, url: &Url) -> Receiver<ImageResponseMsg> {
{
let round_number = self.round_number;
let state = self.get_state(url);
// Save the previous round number for comparison
let last_round = state.last_request_round;
// Set the current round number for this image
state.last_request_round = round_number;
match state.last_response {
ImageResponseMsg::ImageReady(ref image) => {
let (chan, port) = channel();
chan.send(ImageResponseMsg::ImageReady(image.clone())).unwrap();
return port;
}
ImageResponseMsg::ImageNotReady => {
if last_round == round_number {
let (chan, port) = channel();
chan.send(ImageResponseMsg::ImageNotReady).unwrap();
return port;
} else {
// We haven't requested the image from the
// remote cache this round
}
}
ImageResponseMsg::ImageFailed => {
let (chan, port) = channel();
chan.send(ImageResponseMsg::ImageFailed).unwrap();
return port;
}
}
}
let (response_chan, response_port) = channel();
self.image_cache_task.send(Msg::GetImage((*url).clone(), response_chan));
let response = response_port.recv().unwrap();
match response {
ImageResponseMsg::ImageNotReady => {
// Need to reflow when the image is available
// FIXME: Instead we should be just passing a Future
// to the caller, then to the display list. Finally,
// the compositor should be responsible for waiting
// on the image to load and triggering layout
let image_cache_task = self.image_cache_task.clone();
assert!(self.on_image_available.is_some());
let on_image_available =
self.on_image_available.as_ref().unwrap().respond();
let url = (*url).clone();
spawn_named("LocalImageCache".to_owned(), move || {
let (response_chan, response_port) = channel();
image_cache_task.send(Msg::WaitForImage(url, response_chan));
on_image_available(response_port.recv().unwrap(), node_address);
});
}
_ => ()
}
// Put a copy of the response in the cache
let response_copy = match response {
ImageResponseMsg::ImageReady(ref image) => ImageResponseMsg::ImageReady(image.clone()),
ImageResponseMsg::ImageNotReady => ImageResponseMsg::ImageNotReady,
ImageResponseMsg::ImageFailed => ImageResponseMsg::ImageFailed
};
self.get_state(url).last_response = response_copy;
let (chan, port) = channel();
chan.send(response).unwrap();
return port;
}
fn get_state<'a>(&'a mut self, url: &Url) -> &'a mut ImageState {
match self.state_map.entry((*url).clone()) {
Occupied(entry) => entry.into_mut(),
Vacant(entry) =>
entry.insert(ImageState {
prefetched: false,
decoded: false,
last_request_round: 0,
last_response: ImageResponseMsg::ImageNotReady,
})
}
}
}

View file

@ -0,0 +1,44 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use std::sync::mpsc::Sender;
use url::Url;
use util::str::DOMString;
#[derive(Copy)]
pub enum StorageType {
Session,
Local
}
/// Request operations on the storage data associated with a particular url
pub enum StorageTaskMsg {
/// gets the number of key/value pairs present in the associated storage data
Length(Sender<u32>, Url, StorageType),
/// gets the name of the key at the specified index in the associated storage data
Key(Sender<Option<DOMString>>, Url, StorageType, u32),
/// gets the value associated with the given key in the associated storage data
GetItem(Sender<Option<DOMString>>, Url, StorageType, DOMString),
/// sets the value of the given key in the associated storage data
/// TODO throw QuotaExceededError in case of error
SetItem(Sender<(bool, Option<DOMString>)>, Url, StorageType, DOMString, DOMString),
/// removes the key/value pair for the given key in the associated storage data
RemoveItem(Sender<Option<DOMString>>, Url, StorageType, DOMString),
/// clears the associated storage data by removing all the key/value pairs
Clear(Sender<bool>, Url, StorageType),
/// shut down this task
Exit
}
/// Handle to a storage task
pub type StorageTask = Sender<StorageTaskMsg>;