From 14d68968edc936fe67a226840af4c10ff0aea350 Mon Sep 17 00:00:00 2001 From: Zhen Zhang Date: Sat, 18 Jun 2016 18:14:40 +0800 Subject: [PATCH] Integration and improvements of File API backends 1. More complete origin check in FileManagerThreadMsg 2. Add reference counting logic to file manage store and script API 3. Integrate the support of slicing --- components/net/blob_loader.rs | 54 +--- components/net/filemanager_thread.rs | 325 ++++++++++++++------ components/net_traits/Cargo.toml | 1 + components/net_traits/blob_url_store.rs | 19 +- components/net_traits/filemanager_thread.rs | 117 ++++++- components/net_traits/lib.rs | 1 + components/script/dom/bindings/trace.rs | 3 +- components/script/dom/blob.rs | 206 ++++++++++--- components/script/dom/file.rs | 2 + components/script/dom/htmlinputelement.rs | 5 +- components/script/dom/url.rs | 40 +-- components/servo/Cargo.lock | 1 + ports/cef/Cargo.lock | 1 + tests/unit/net/filemanager_thread.rs | 12 +- 14 files changed, 543 insertions(+), 244 deletions(-) diff --git a/components/net/blob_loader.rs b/components/net/blob_loader.rs index 37665da86e2..1008b405e3e 100644 --- a/components/net/blob_loader.rs +++ b/components/net/blob_loader.rs @@ -2,60 +2,32 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use filemanager_thread::BlobURLStore; use hyper::header::{DispositionType, ContentDisposition, DispositionParam}; use hyper::header::{Headers, ContentType, ContentLength, Charset}; use hyper::http::RawStatus; use mime::{Mime, Attr}; use mime_classifier::MimeClassifier; use net_traits::ProgressMsg::Done; -use net_traits::blob_url_store::{parse_blob_url, BlobURLStoreEntry, BlobURLStoreError}; +use net_traits::blob_url_store::BlobURLStoreEntry; +use net_traits::filemanager_thread::RelativePos; use net_traits::response::HttpsState; -use net_traits::{LoadConsumer, LoadData, Metadata, NetworkError}; -use resource_thread::{send_error, start_sending_sniffed_opt}; -use std::str; -use std::sync::{Arc, RwLock}; - +use net_traits::{LoadConsumer, LoadData, Metadata}; +use resource_thread::start_sending_sniffed_opt; +use std::ops::Index; +use std::sync::Arc; // TODO: Check on GET // https://w3c.github.io/FileAPI/#requestResponseModel -pub fn load(load_data: LoadData, consumer: LoadConsumer, - blob_url_store: Arc>, - classifier: Arc) { // XXX: Move it into net process later - - match parse_blob_url(&load_data.url) { - None => { - let format_err = NetworkError::Internal(format!("Invalid blob URL format {:?}", load_data.url)); - send_error(load_data.url.clone(), format_err, consumer); - } - Some((uuid, _fragment)) => { - match blob_url_store.read().unwrap().request(uuid, &load_data.url.origin()) { - Ok(entry) => load_blob(&load_data, consumer, classifier, entry), - Err(e) => { - let err = match e { - BlobURLStoreError::InvalidKey => - format!("Invalid blob URL key {:?}", uuid.simple().to_string()), - BlobURLStoreError::InvalidOrigin => - format!("Invalid blob URL origin {:?}", load_data.url.origin()), - }; - send_error(load_data.url.clone(), NetworkError::Internal(err), consumer); - } - } - } - } -} - -fn load_blob(load_data: &LoadData, - start_chan: LoadConsumer, - classifier: Arc, - entry: &BlobURLStoreEntry) { +pub fn load_blob(load_data: &LoadData, start_chan: LoadConsumer, + classifier: Arc, opt_filename: Option, + rel_pos: &RelativePos, entry: &BlobURLStoreEntry) { let content_type: Mime = entry.type_string.parse().unwrap_or(mime!(Text / Plain)); let charset = content_type.get_param(Attr::Charset); let mut headers = Headers::new(); - if let Some(ref name) = entry.filename { + if let Some(name) = opt_filename { let charset = charset.and_then(|c| c.as_str().parse().ok()); headers.set(ContentDisposition { disposition: DispositionType::Inline, @@ -66,8 +38,10 @@ fn load_blob(load_data: &LoadData, }); } + let range = rel_pos.to_abs_range(entry.size as usize); + headers.set(ContentType(content_type.clone())); - headers.set(ContentLength(entry.size)); + headers.set(ContentLength(range.len() as u64)); let metadata = Metadata { final_url: load_data.url.clone(), @@ -81,7 +55,7 @@ fn load_blob(load_data: &LoadData, if let Ok(chan) = start_sending_sniffed_opt(start_chan, metadata, classifier, - &entry.bytes, load_data.context.clone()) { + &entry.bytes.index(range), load_data.context.clone()) { let _ = chan.send(Done(Ok(()))); } } diff --git a/components/net/filemanager_thread.rs b/components/net/filemanager_thread.rs index 246d3254e42..469cd36c3a7 100644 --- a/components/net/filemanager_thread.rs +++ b/components/net/filemanager_thread.rs @@ -2,21 +2,24 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use blob_loader; +use blob_loader::load_blob; use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use mime_classifier::MimeClassifier; use mime_guess::guess_mime_type_opt; -use net_traits::blob_url_store::{BlobURLStoreEntry, BlobURLStoreError, BlobURLStoreMsg}; -use net_traits::filemanager_thread::{FileManagerThreadMsg, FileManagerResult, FilterPattern}; -use net_traits::filemanager_thread::{SelectedFile, FileManagerThreadError, SelectedFileId}; +use net_traits::blob_url_store::{BlobURLStoreEntry, BlobURLStoreError, parse_blob_url}; +use net_traits::filemanager_thread::{FileManagerThreadMsg, FileManagerResult, FilterPattern, FileOrigin}; +use net_traits::filemanager_thread::{SelectedFile, RelativePos, FileManagerThreadError, SelectedFileId}; +use net_traits::{LoadConsumer, LoadData, NetworkError}; +use resource_thread::send_error; +use std::cell::Cell; use std::collections::HashMap; use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; #[cfg(any(target_os = "macos", target_os = "linux"))] use tinyfiledialogs; -use url::{Url, Origin}; +use url::Url; use util::thread::spawn_named; use uuid::Uuid; @@ -87,11 +90,26 @@ impl FileManagerThreadFactory for IpcSender, +} + +/// File backend implementation +enum FileImpl { + PathOnly(PathBuf), + Memory(BlobURLStoreEntry), + Sliced(Uuid, RelativePos), +} + struct FileManager { receiver: IpcReceiver, - idmap: HashMap, + store: HashMap, classifier: Arc, - blob_url_store: Arc>, ui: &'static UI, } @@ -99,10 +117,9 @@ impl FileManager { fn new(recv: IpcReceiver, ui: &'static UI) -> FileManager { FileManager { receiver: recv, - idmap: HashMap::new(), + store: HashMap::new(), classifier: Arc::new(MimeClassifier::new()), - blob_url_store: Arc::new(RwLock::new(BlobURLStore::new())), - ui: ui + ui: ui, } } @@ -110,33 +127,97 @@ impl FileManager { fn start(&mut self) { loop { match self.receiver.recv().unwrap() { - FileManagerThreadMsg::SelectFile(filter, sender) => self.select_file(filter, sender), - FileManagerThreadMsg::SelectFiles(filter, sender) => self.select_files(filter, sender), - FileManagerThreadMsg::ReadFile(sender, id) => { - match self.try_read_file(id) { + FileManagerThreadMsg::SelectFile(filter, sender, origin) => self.select_file(filter, sender, origin), + FileManagerThreadMsg::SelectFiles(filter, sender, origin) => self.select_files(filter, sender, origin), + FileManagerThreadMsg::ReadFile(sender, id, origin) => { + match self.try_read_file(id, origin) { Ok(buffer) => { let _ = sender.send(Ok(buffer)); } Err(_) => { let _ = sender.send(Err(FileManagerThreadError::ReadFileError)); } } } - FileManagerThreadMsg::DeleteFileID(id) => self.delete_fileid(id), - FileManagerThreadMsg::BlobURLStoreMsg(msg) => self.blob_url_store.write().unwrap().process(msg), + FileManagerThreadMsg::TransferMemory(entry, rel_pos, sender, origin) => + self.transfer_memory(entry, rel_pos, sender, origin), + FileManagerThreadMsg::AddSlicedEntry(id, rel_pos, sender, origin) => + self.add_sliced_entry(id, rel_pos, sender, origin), FileManagerThreadMsg::LoadBlob(load_data, consumer) => { - blob_loader::load(load_data, consumer, - self.blob_url_store.clone(), - self.classifier.clone()); + match parse_blob_url(&load_data.url) { + None => { + let e = format!("Invalid blob URL format {:?}", load_data.url); + let format_err = NetworkError::Internal(e); + send_error(load_data.url.clone(), format_err, consumer); + } + Some((id, _fragment)) => { + self.process_request(&load_data, consumer, &RelativePos::full_range(), &id); + } + } }, + FileManagerThreadMsg::DecRef(id, origin) => { + if let Ok(id) = Uuid::parse_str(&id.0) { + self.dec_ref(id, origin); + } + } + FileManagerThreadMsg::IncRef(id, origin) => { + if let Ok(id) = Uuid::parse_str(&id.0) { + self.inc_ref(id, origin); + } + } FileManagerThreadMsg::Exit => break, }; } } + fn inc_ref(&mut self, id: Uuid, origin_in: FileOrigin) { + match self.store.get(&id) { + Some(entry) => { + if entry.origin == origin_in { + entry.refs.set(entry.refs.get() + 1); + } + } + None => return, // Invalid UUID + } + } + + fn add_sliced_entry(&mut self, id: SelectedFileId, rel_pos: RelativePos, + sender: IpcSender>, + origin_in: FileOrigin) { + if let Ok(id) = Uuid::parse_str(&id.0) { + match self.store.get(&id) { + Some(entry) => { + if entry.origin == origin_in { + // inc_ref on parent entry + entry.refs.set(entry.refs.get() + 1); + } else { + let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin)); + return; + } + }, + None => { + let _ = sender.send(Err(BlobURLStoreError::InvalidFileID)); + return; + } + }; + + let new_id = Uuid::new_v4(); + self.store.insert(new_id, FileStoreEntry { + origin: origin_in.clone(), + file_impl: FileImpl::Sliced(id, rel_pos), + refs: Cell::new(1), + }); + + let _ = sender.send(Ok(SelectedFileId(new_id.simple().to_string()))); + } else { + let _ = sender.send(Err(BlobURLStoreError::InvalidFileID)); + } + } + fn select_file(&mut self, patterns: Vec, - sender: IpcSender>) { + sender: IpcSender>, + origin: FileOrigin) { match self.ui.open_file_dialog("", patterns) { Some(s) => { let selected_path = Path::new(&s); - match self.create_entry(selected_path) { + match self.create_entry(selected_path, &origin) { Some(triple) => { let _ = sender.send(Ok(triple)); } None => { let _ = sender.send(Err(FileManagerThreadError::InvalidSelection)); } }; @@ -149,7 +230,8 @@ impl FileManager { } fn select_files(&mut self, patterns: Vec, - sender: IpcSender>>) { + sender: IpcSender>>, + origin: FileOrigin) { match self.ui.open_file_dialog_multi("", patterns) { Some(v) => { let mut selected_paths = vec![]; @@ -161,7 +243,7 @@ impl FileManager { let mut replies = vec![]; for path in selected_paths { - match self.create_entry(path) { + match self.create_entry(path, &origin) { Some(triple) => replies.push(triple), None => { let _ = sender.send(Err(FileManagerThreadError::InvalidSelection)); } }; @@ -176,11 +258,17 @@ impl FileManager { } } - fn create_entry(&mut self, file_path: &Path) -> Option { + fn create_entry(&mut self, file_path: &Path, origin: &str) -> Option { match File::open(file_path) { Ok(handler) => { let id = Uuid::new_v4(); - self.idmap.insert(id, file_path.to_path_buf()); + let file_impl = FileImpl::PathOnly(file_path.to_path_buf()); + + self.store.insert(id, FileStoreEntry { + origin: origin.to_string(), + file_impl: file_impl, + refs: Cell::new(1), + }); // Unix Epoch: https://doc.servo.org/std/time/constant.UNIX_EPOCH.html let epoch = handler.metadata().and_then(|metadata| metadata.modified()).map_err(|_| ()) @@ -215,79 +303,138 @@ impl FileManager { } } - fn try_read_file(&mut self, id: SelectedFileId) -> Result, ()> { + fn try_read_file(&self, id: SelectedFileId, origin_in: String) -> Result, ()> { let id = try!(Uuid::parse_str(&id.0).map_err(|_| ())); - match self.idmap.get(&id) { - Some(filepath) => { - let mut buffer = vec![]; - let mut handler = try!(File::open(&filepath).map_err(|_| ())); - try!(handler.read_to_end(&mut buffer).map_err(|_| ())); - Ok(buffer) - }, - None => Err(()) - } - } - - fn delete_fileid(&mut self, id: SelectedFileId) { - if let Ok(id) = Uuid::parse_str(&id.0) { - self.idmap.remove(&id); - } - } -} - -pub struct BlobURLStore { - entries: HashMap, -} - -impl BlobURLStore { - pub fn new() -> BlobURLStore { - BlobURLStore { - entries: HashMap::new(), - } - } - - fn process(&mut self, msg: BlobURLStoreMsg) { - match msg { - BlobURLStoreMsg::AddEntry(entry, origin_str, sender) => { - match Url::parse(&origin_str) { - Ok(base_url) => { - let id = Uuid::new_v4(); - self.add_entry(id, base_url.origin(), entry); - - let _ = sender.send(Ok(id.simple().to_string())); + match self.store.get(&id) { + Some(entry) => { + match entry.file_impl { + FileImpl::PathOnly(ref filepath) => { + if *entry.origin == origin_in { + let mut buffer = vec![]; + let mut handler = try!(File::open(filepath).map_err(|_| ())); + try!(handler.read_to_end(&mut buffer).map_err(|_| ())); + Ok(buffer) + } else { + Err(()) + } + }, + FileImpl::Memory(ref buffered) => { + Ok(buffered.bytes.clone()) + }, + FileImpl::Sliced(ref id, ref _rel_pos) => { + self.try_read_file(SelectedFileId(id.simple().to_string()), origin_in) } - Err(_) => { - let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin)); + } + }, + None => Err(()), + } + } + + fn dec_ref(&mut self, id: Uuid, origin_in: FileOrigin) { + let (is_last_ref, opt_parent_id) = match self.store.get(&id) { + Some(entry) => { + if *entry.origin == origin_in { + let r = entry.refs.get(); + + if r > 1 { + entry.refs.set(r - 1); + (false, None) + } else { + if let FileImpl::Sliced(ref parent_id, _) = entry.file_impl { + // if it has a reference to parent id, dec_ref on parent later + (true, Some(parent_id.clone())) + } else { + (true, None) + } + } + } else { // Invalid origin + return; + } + } + None => return, // Invalid UUID + }; + + if is_last_ref { + self.store.remove(&id); + + if let Some(parent_id) = opt_parent_id { + self.dec_ref(parent_id, origin_in); + } + } + } + + fn process_request(&self, load_data: &LoadData, consumer: LoadConsumer, + rel_pos: &RelativePos, id: &Uuid) { + let origin_in = load_data.url.origin().unicode_serialization(); + match self.store.get(id) { + Some(entry) => { + match entry.file_impl { + FileImpl::Memory(ref buffered) => { + if *entry.origin == origin_in { + load_blob(&load_data, consumer, self.classifier.clone(), + None, rel_pos, buffered); + } else { + let e = format!("Invalid blob URL origin {:?}", origin_in); + send_error(load_data.url.clone(), NetworkError::Internal(e), consumer); + } + }, + FileImpl::PathOnly(ref filepath) => { + let opt_filename = filepath.file_name() + .and_then(|osstr| osstr.to_str()) + .map(|s| s.to_string()); + + if *entry.origin == origin_in { + let mut bytes = vec![]; + let mut handler = File::open(filepath).unwrap(); + let mime = guess_mime_type_opt(filepath); + let size = handler.read_to_end(&mut bytes).unwrap(); + + let entry = BlobURLStoreEntry { + type_string: match mime { + Some(x) => format!("{}", x), + None => "".to_string(), + }, + size: size as u64, + bytes: bytes, + }; + + load_blob(&load_data, consumer, self.classifier.clone(), + opt_filename, rel_pos, &entry); + } else { + let e = format!("Invalid blob URL origin {:?}", origin_in); + send_error(load_data.url.clone(), NetworkError::Internal(e), consumer); + } + }, + FileImpl::Sliced(ref id, ref rel_pos) => { + self.process_request(load_data, consumer, rel_pos, id); } } } - BlobURLStoreMsg::DeleteEntry(id) => { - if let Ok(id) = Uuid::parse_str(&id) { - self.delete_entry(id); - } - }, - } - } - - pub fn request(&self, id: Uuid, origin: &Origin) -> Result<&BlobURLStoreEntry, BlobURLStoreError> { - match self.entries.get(&id) { - Some(ref pair) => { - if pair.0 == *origin { - Ok(&pair.1) - } else { - Err(BlobURLStoreError::InvalidOrigin) - } + _ => { + let e = format!("Invalid blob URL key {:?}", id.simple().to_string()); + send_error(load_data.url.clone(), NetworkError::Internal(e), consumer); } - None => Err(BlobURLStoreError::InvalidKey) } } - pub fn add_entry(&mut self, id: Uuid, origin: Origin, blob: BlobURLStoreEntry) { - self.entries.insert(id, (origin, blob)); - } + fn transfer_memory(&mut self, entry: BlobURLStoreEntry, rel_pos: RelativePos, + sender: IpcSender>, origin: FileOrigin) { + match Url::parse(&origin) { // parse to check sanity + Ok(_) => { + let id = Uuid::new_v4(); + self.store.insert(id, FileStoreEntry { + origin: origin.clone(), + file_impl: FileImpl::Memory(entry), + refs: Cell::new(1), + }); + let sliced_id = SelectedFileId(id.simple().to_string()); - pub fn delete_entry(&mut self, id: Uuid) { - self.entries.remove(&id); + self.add_sliced_entry(sliced_id, rel_pos, sender, origin); + } + Err(_) => { + let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin)); + } + } } } diff --git a/components/net_traits/Cargo.toml b/components/net_traits/Cargo.toml index b7f99dbf6f1..ed75fabb7f4 100644 --- a/components/net_traits/Cargo.toml +++ b/components/net_traits/Cargo.toml @@ -18,6 +18,7 @@ hyper = { version = "0.9.9", features = [ "serde-serialization" ] } image = "0.10" lazy_static = "0.2" log = "0.3.5" +num-traits = "0.1.32" serde = "0.7.11" serde_macros = "0.7.11" url = {version = "1.0.0", features = ["heap_size"]} diff --git a/components/net_traits/blob_url_store.rs b/components/net_traits/blob_url_store.rs index 34f609ab10e..6959f658a2f 100644 --- a/components/net_traits/blob_url_store.rs +++ b/components/net_traits/blob_url_store.rs @@ -2,37 +2,24 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use ipc_channel::ipc::IpcSender; use std::str::FromStr; use url::Url; use uuid::Uuid; /// Errors returns to BlobURLStoreMsg::Request -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub enum BlobURLStoreError { - /// Invalid UUID key - InvalidKey, + /// Invalid File UUID + InvalidFileID, /// Invalid URL origin InvalidOrigin, } -#[derive(Serialize, Deserialize)] -pub enum BlobURLStoreMsg { - /// Add an entry and send back the associated uuid - /// XXX: Second field is an unicode-serialized Origin, it is a temporary workaround - /// and should not be trusted. See issue https://github.com/servo/servo/issues/11722 - AddEntry(BlobURLStoreEntry, String, IpcSender>), - /// Delete an entry by uuid - DeleteEntry(String), -} - /// Blob URL store entry, a packaged form of Blob DOM object #[derive(Clone, Serialize, Deserialize)] pub struct BlobURLStoreEntry { /// MIME type string pub type_string: String, - /// Some filename if the backend of Blob is a file - pub filename: Option, /// Size of content in bytes pub size: u64, /// Content of blob diff --git a/components/net_traits/filemanager_thread.rs b/components/net_traits/filemanager_thread.rs index 1650853bf15..0df9d45ba5f 100644 --- a/components/net_traits/filemanager_thread.rs +++ b/components/net_traits/filemanager_thread.rs @@ -2,11 +2,102 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use blob_url_store::BlobURLStoreMsg; +use blob_url_store::{BlobURLStoreEntry, BlobURLStoreError}; use ipc_channel::ipc::IpcSender; +use num_traits::ToPrimitive; +use std::cmp::{max, min}; +use std::ops::Range; use std::path::PathBuf; use super::{LoadConsumer, LoadData}; +// HACK: We should send Origin directly instead of this in future, blocked on #11722 +/// File manager store entry's origin +pub type FileOrigin = String; + +/// Relative slice positions of a sequence, +/// whose semantic should be consistent with (start, end) parameters in +/// https://w3c.github.io/FileAPI/#dfn-slice +#[derive(Clone, Deserialize, Serialize)] +pub struct RelativePos { + /// Relative to first byte if non-negative, + /// relative to one past last byte if negative, + pub start: i64, + /// Relative offset from first byte if Some(non-negative), + /// relative to one past last byte if Some(negative), + /// None if one past last byte + pub end: Option, +} + +impl RelativePos { + /// Full range from start to end + pub fn full_range() -> RelativePos { + RelativePos { + start: 0, + end: Some(0), + } + } + + /// Instantiate optional slice position parameters + pub fn from_opts(start: Option, end: Option) -> RelativePos { + RelativePos { + start: start.unwrap_or(0), + end: end, + } + } + + /// Slice the inner sliced range by repositioning + pub fn slice_inner(&self, rel_pos: &RelativePos) -> RelativePos { + RelativePos { + start: self.start + rel_pos.start, + end: match (self.end, rel_pos.end) { + (Some(old_end), Some(rel_end)) => Some(old_end + rel_end), + (old, None) => old, + (None, rel) => rel, + } + } + } + + /// Compute absolute range by giving the total size + /// https://w3c.github.io/FileAPI/#slice-method-algo + pub fn to_abs_range(&self, size: usize) -> Range { + let size = size as i64; + + let start = { + if self.start < 0 { + max(size + self.start, 0) + } else { + min(self.start, size) + } + }; + + let end = match self.end { + Some(rel_end) => { + if rel_end < 0 { + max(size + rel_end, 0) + } else { + min(rel_end, size) + } + } + None => size, + }; + + let span: i64 = max(end - start, 0); + + Range { + start: start.to_usize().unwrap(), + end: (start + span).to_usize().unwrap(), + } + } + + /// Inverse operation of to_abs_range + pub fn from_abs_range(range: Range, size: usize) -> RelativePos { + RelativePos { + start: range.start as i64, + end: Some(size as i64 - range.end as i64), + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct SelectedFileId(pub String); @@ -27,23 +118,29 @@ pub struct FilterPattern(pub String); #[derive(Deserialize, Serialize)] pub enum FileManagerThreadMsg { /// Select a single file, return triple (FileID, FileName, lastModified) - SelectFile(Vec, IpcSender>), + SelectFile(Vec, IpcSender>, FileOrigin), /// Select multiple files, return a vector of triples - SelectFiles(Vec, IpcSender>>), + SelectFiles(Vec, IpcSender>>, FileOrigin), /// Read file, return the bytes - ReadFile(IpcSender>>, SelectedFileId), - - /// Delete the FileID entry - DeleteFileID(SelectedFileId), - - // Blob URL message - BlobURLStoreMsg(BlobURLStoreMsg), + ReadFile(IpcSender>>, SelectedFileId, FileOrigin), /// Load resource by Blob URL LoadBlob(LoadData, LoadConsumer), + /// Add an entry and send back the associated uuid + TransferMemory(BlobURLStoreEntry, RelativePos, IpcSender>, FileOrigin), + + /// Add a sliced entry pointing to the parent id with a relative slicing positing + AddSlicedEntry(SelectedFileId, RelativePos, IpcSender>, FileOrigin), + + /// Decrease reference count + DecRef(SelectedFileId, FileOrigin), + + /// Increase reference count + IncRef(SelectedFileId, FileOrigin), + /// Shut down this thread Exit, } diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index d4da6d0d058..60163bb090a 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -23,6 +23,7 @@ extern crate lazy_static; #[macro_use] extern crate log; extern crate msg; +extern crate num_traits; extern crate serde; extern crate url; extern crate util; diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 1d1433b2693..a0813a047df 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -57,7 +57,7 @@ use js::jsval::JSVal; use js::rust::Runtime; use libc; use msg::constellation_msg::{FrameType, PipelineId, SubpageId, WindowSizeData, WindowSizeType, ReferrerPolicy}; -use net_traits::filemanager_thread::SelectedFileId; +use net_traits::filemanager_thread::{SelectedFileId, RelativePos}; use net_traits::image::base::{Image, ImageMetadata}; use net_traits::image_cache_thread::{ImageCacheChan, ImageCacheThread}; use net_traits::response::HttpsState; @@ -331,6 +331,7 @@ no_jsmanaged_fields!(ReferrerPolicy); no_jsmanaged_fields!(ResourceThreads); no_jsmanaged_fields!(SystemTime); no_jsmanaged_fields!(SelectedFileId); +no_jsmanaged_fields!(RelativePos); no_jsmanaged_fields!(OpaqueStyleAndLayoutData); no_jsmanaged_fields!(CSSErrorReporter); no_jsmanaged_fields!(WebGLBufferId); diff --git a/components/script/dom/blob.rs b/components/script/dom/blob.rs index dcb4c68f34a..1bc0498fb35 100644 --- a/components/script/dom/blob.rs +++ b/components/script/dom/blob.rs @@ -8,17 +8,18 @@ use dom::bindings::codegen::Bindings::BlobBinding::BlobMethods; use dom::bindings::codegen::UnionTypes::BlobOrString; use dom::bindings::error::{Error, Fallible}; use dom::bindings::global::GlobalRef; -use dom::bindings::js::Root; +use dom::bindings::js::{JS, Root}; use dom::bindings::reflector::{Reflectable, Reflector, reflect_dom_object}; use dom::bindings::str::DOMString; use encoding::all::UTF_8; use encoding::types::{EncoderTrap, Encoding}; use ipc_channel::ipc; -use net_traits::filemanager_thread::{FileManagerThreadMsg, SelectedFileId}; -use num_traits::ToPrimitive; +use net_traits::IpcSend; +use net_traits::blob_url_store::BlobURLStoreEntry; +use net_traits::filemanager_thread::{FileManagerThreadMsg, SelectedFileId, RelativePos}; use std::ascii::AsciiExt; use std::cell::Cell; -use std::cmp::{max, min}; +use std::ops::Range; use std::sync::Arc; #[derive(Clone, JSTraceable)] @@ -31,36 +32,12 @@ pub struct DataSlice { impl DataSlice { /// Construct DataSlice from reference counted bytes pub fn new(bytes: Arc>, start: Option, end: Option) -> DataSlice { - let size = bytes.len() as i64; - let relativeStart: i64 = match start { - None => 0, - Some(start) => { - if start < 0 { - max(size + start, 0) - } else { - min(start, size) - } - } - }; - let relativeEnd: i64 = match end { - None => size, - Some(end) => { - if end < 0 { - max(size + end, 0) - } else { - min(end, size) - } - } - }; - - let span: i64 = max(relativeEnd - relativeStart, 0); - let start = relativeStart.to_usize().unwrap(); - let end = (relativeStart + span).to_usize().unwrap(); + let range = RelativePos::from_opts(start, end).to_abs_range(bytes.len()); DataSlice { bytes: bytes, - bytes_start: start, - bytes_end: end + bytes_start: range.start, + bytes_end: range.end, } } @@ -87,15 +64,30 @@ impl DataSlice { pub fn size(&self) -> u64 { (self.bytes_end as u64) - (self.bytes_start as u64) } + + /// Further adjust the slice range based on passed-in relative positions + pub fn slice(&self, pos: &RelativePos) -> DataSlice { + let old_size = self.size(); + let range = pos.to_abs_range(old_size as usize); + DataSlice { + bytes: self.bytes.clone(), + bytes_start: self.bytes_start + range.start, + bytes_end: self.bytes_start + range.end, + } + } } - -#[derive(Clone, JSTraceable)] +#[must_root] +#[derive(JSTraceable)] pub enum BlobImpl { - /// File-based, cached backend + /// File-based blob, including id and possibly cached content File(SelectedFileId, DOMRefCell>), - /// Memory-based backend + /// Memory-based blob Memory(DataSlice), + /// Sliced blob, including parent blob and + /// relative positions representing current slicing range, + /// it is leaf of a two-layer fat tree + Sliced(JS, RelativePos), } impl BlobImpl { @@ -120,26 +112,58 @@ impl BlobImpl { pub struct Blob { reflector_: Reflector, #[ignore_heap_size_of = "No clear owner"] - blob_impl: BlobImpl, + blob_impl: DOMRefCell, typeString: String, isClosed_: Cell, } impl Blob { + #[allow(unrooted_must_root)] pub fn new(global: GlobalRef, blob_impl: BlobImpl, typeString: String) -> Root { let boxed_blob = box Blob::new_inherited(blob_impl, typeString); reflect_dom_object(boxed_blob, global, BlobBinding::Wrap) } + #[allow(unrooted_must_root)] pub fn new_inherited(blob_impl: BlobImpl, typeString: String) -> Blob { Blob { reflector_: Reflector::new(), - blob_impl: blob_impl, + blob_impl: DOMRefCell::new(blob_impl), typeString: typeString, isClosed_: Cell::new(false), } } + #[allow(unrooted_must_root)] + fn new_sliced(parent: &Blob, rel_pos: RelativePos, + relativeContentType: DOMString) -> Root { + let global = parent.global(); + let blob_impl = match *parent.blob_impl.borrow() { + BlobImpl::File(ref id, _) => { + inc_ref_id(global.r(), id.clone()); + + // Create new parent node + BlobImpl::Sliced(JS::from_ref(parent), rel_pos) + } + BlobImpl::Memory(_) => { + // Create new parent node + BlobImpl::Sliced(JS::from_ref(parent), rel_pos) + } + BlobImpl::Sliced(ref grandparent, ref old_rel_pos) => { + // Adjust the slicing position, using same parent + let new_rel_pos = old_rel_pos.slice_inner(&rel_pos); + + if let BlobImpl::File(ref id, _) = *grandparent.blob_impl.borrow() { + inc_ref_id(global.r(), id.clone()); + } + + BlobImpl::Sliced(grandparent.clone(), new_rel_pos) + } + }; + + Blob::new(global.r(), blob_impl, relativeContentType.into()) + } + // https://w3c.github.io/FileAPI/#constructorBlob pub fn Constructor(global: GlobalRef, blobParts: Option>, @@ -160,19 +184,29 @@ impl Blob { /// Get a slice to inner data, this might incur synchronous read and caching pub fn get_slice(&self) -> Result { - match self.blob_impl { - BlobImpl::File(ref id, ref slice) => { - match *slice.borrow() { + match *self.blob_impl.borrow() { + BlobImpl::File(ref id, ref cached) => { + let buffer = match *cached.borrow() { Some(ref s) => Ok(s.clone()), None => { let global = self.global(); let s = read_file(global.r(), id.clone())?; - *slice.borrow_mut() = Some(s.clone()); // Cached Ok(s) } + }; + + // Cache + if let Ok(buf) = buffer.clone() { + *cached.borrow_mut() = Some(buf); } + + buffer + } + BlobImpl::Memory(ref s) => Ok(s.clone()), + BlobImpl::Sliced(ref parent, ref rel_pos) => { + let dataslice = parent.get_slice_or_empty(); + Ok(dataslice.slice(rel_pos)) } - BlobImpl::Memory(ref s) => Ok(s.clone()) } } @@ -180,12 +214,83 @@ impl Blob { pub fn get_slice_or_empty(&self) -> DataSlice { self.get_slice().unwrap_or(DataSlice::empty()) } + + pub fn get_id(&self) -> SelectedFileId { + match *self.blob_impl.borrow() { + BlobImpl::File(ref id, _) => id.clone(), + BlobImpl::Memory(ref slice) => self.promote_to_file(slice), + BlobImpl::Sliced(ref parent, ref rel_pos) => { + match *parent.blob_impl.borrow() { + BlobImpl::Sliced(_, _) => { + debug!("Sliced can't have a sliced parent"); + // Return dummy id + SelectedFileId("".to_string()) + } + BlobImpl::File(ref parent_id, _) => + self.create_sliced_id(parent_id, rel_pos), + BlobImpl::Memory(ref parent_slice) => { + let parent_id = parent.promote_to_file(parent_slice); + *self.blob_impl.borrow_mut() = BlobImpl::Sliced(parent.clone(), rel_pos.clone()); + self.create_sliced_id(&parent_id, rel_pos) + } + } + } + } + } + + /// Promite memory-based Blob to file-based, + /// The bytes in data slice will be transferred to file manager thread + fn promote_to_file(&self, self_slice: &DataSlice) -> SelectedFileId { + let global = self.global(); + let origin = global.r().get_url().origin().unicode_serialization(); + let filemanager = global.r().resource_threads().sender(); + let bytes = self_slice.get_bytes(); + let rel_pos = RelativePos::from_abs_range(Range { + start: self_slice.bytes_start, + end: self_slice.bytes_end, + }, self_slice.bytes.len()); + + let entry = BlobURLStoreEntry { + type_string: self.typeString.clone(), + size: self.Size(), + bytes: bytes.to_vec(), + }; + + let (tx, rx) = ipc::channel().unwrap(); + let _ = filemanager.send(FileManagerThreadMsg::TransferMemory(entry, rel_pos, tx, origin.clone())); + + match rx.recv().unwrap() { + Ok(new_id) => SelectedFileId(new_id.0), + // Dummy id + Err(_) => SelectedFileId("".to_string()), + } + } + + fn create_sliced_id(&self, parent_id: &SelectedFileId, + rel_pos: &RelativePos) -> SelectedFileId { + let global = self.global(); + + let origin = global.r().get_url().origin().unicode_serialization(); + + let filemanager = global.r().resource_threads().sender(); + let (tx, rx) = ipc::channel().unwrap(); + let msg = FileManagerThreadMsg::AddSlicedEntry(parent_id.clone(), + rel_pos.clone(), + tx, origin.clone()); + let _ = filemanager.send(msg); + let new_id = rx.recv().unwrap().unwrap(); + + // Return the indirect id reference + SelectedFileId(new_id.0) + } } fn read_file(global: GlobalRef, id: SelectedFileId) -> Result { let file_manager = global.filemanager_thread(); let (chan, recv) = ipc::channel().map_err(|_|())?; - let _ = file_manager.send(FileManagerThreadMsg::ReadFile(chan, id)); + let origin = global.get_url().origin().unicode_serialization(); + let msg = FileManagerThreadMsg::ReadFile(chan, id, origin); + let _ = file_manager.send(msg); let result = match recv.recv() { Ok(ret) => ret, @@ -248,10 +353,8 @@ impl BlobMethods for Blob { } }; - let global = self.global(); - let bytes = self.get_slice_or_empty().bytes.clone(); - let slice = DataSlice::new(bytes, start, end); - Blob::new(global.r(), BlobImpl::new_from_slice(slice), relativeContentType.into()) + let rel_pos = RelativePos::from_opts(start, end); + Blob::new_sliced(self, rel_pos, relativeContentType) } // https://w3c.github.io/FileAPI/#dfn-isClosed @@ -274,7 +377,6 @@ impl BlobMethods for Blob { } } - impl BlobBinding::BlobPropertyBag { /// Get the normalized inner type string /// https://w3c.github.io/FileAPI/#dfn-type @@ -292,3 +394,11 @@ fn is_ascii_printable(string: &str) -> bool { // https://w3c.github.io/FileAPI/#constructorBlob string.chars().all(|c| c >= '\x20' && c <= '\x7E') } + +/// Bump the reference counter in file manager thread +fn inc_ref_id(global: GlobalRef, id: SelectedFileId) { + let file_manager = global.filemanager_thread(); + let origin = global.get_url().origin().unicode_serialization(); + let msg = FileManagerThreadMsg::IncRef(id, origin); + let _ = file_manager.send(msg); +} diff --git a/components/script/dom/file.rs b/components/script/dom/file.rs index 86e8888a88f..bfc6b886282 100644 --- a/components/script/dom/file.rs +++ b/components/script/dom/file.rs @@ -23,6 +23,7 @@ pub struct File { } impl File { + #[allow(unrooted_must_root)] fn new_inherited(blob_impl: BlobImpl, name: DOMString, modified: Option, typeString: &str) -> File { File { @@ -39,6 +40,7 @@ impl File { } } + #[allow(unrooted_must_root)] pub fn new(global: GlobalRef, blob_impl: BlobImpl, name: DOMString, modified: Option, typeString: &str) -> Root { reflect_dom_object(box File::new_inherited(blob_impl, name, modified, typeString), diff --git a/components/script/dom/htmlinputelement.rs b/components/script/dom/htmlinputelement.rs index 6c3b008bfc8..ed71c979ab7 100644 --- a/components/script/dom/htmlinputelement.rs +++ b/components/script/dom/htmlinputelement.rs @@ -1151,6 +1151,7 @@ impl Activatable for HTMLInputElement { }, InputType::InputFile => { let window = window_from_node(self); + let origin = window.get_url().origin().unicode_serialization(); let filemanager = window.resource_threads().sender(); let mut files: Vec> = vec![]; @@ -1160,7 +1161,7 @@ impl Activatable for HTMLInputElement { if self.Multiple() { let (chan, recv) = ipc::channel().expect("Error initializing channel"); - let msg = FileManagerThreadMsg::SelectFiles(filter, chan); + let msg = FileManagerThreadMsg::SelectFiles(filter, chan, origin); let _ = filemanager.send(msg).unwrap(); match recv.recv().expect("IpcSender side error") { @@ -1173,7 +1174,7 @@ impl Activatable for HTMLInputElement { }; } else { let (chan, recv) = ipc::channel().expect("Error initializing channel"); - let msg = FileManagerThreadMsg::SelectFile(filter, chan); + let msg = FileManagerThreadMsg::SelectFile(filter, chan, origin); let _ = filemanager.send(msg).unwrap(); match recv.recv().expect("IpcSender side error") { diff --git a/components/script/dom/url.rs b/components/script/dom/url.rs index 2a41f7c7bae..9a47be034f8 100644 --- a/components/script/dom/url.rs +++ b/components/script/dom/url.rs @@ -13,10 +13,9 @@ use dom::bindings::str::{DOMString, USVString}; use dom::blob::Blob; use dom::urlhelper::UrlHelper; use dom::urlsearchparams::URLSearchParams; -use ipc_channel::ipc; use net_traits::IpcSend; -use net_traits::blob_url_store::{BlobURLStoreEntry, BlobURLStoreMsg, parse_blob_url}; -use net_traits::filemanager_thread::FileManagerThreadMsg; +use net_traits::blob_url_store::parse_blob_url; +use net_traits::filemanager_thread::{SelectedFileId, FileManagerThreadMsg}; use std::borrow::ToOwned; use std::default::Default; use url::quirks::domain_to_unicode; @@ -125,34 +124,9 @@ impl URL { return DOMString::from(URL::unicode_serialization_blob_url(&origin, &id)); } - let filemanager = global.resource_threads().sender(); + let id = blob.get_id(); - let slice = blob.get_slice_or_empty(); - let bytes = slice.get_bytes(); - - let entry = BlobURLStoreEntry { - type_string: blob.Type().to_string(), - filename: None, // XXX: the filename is currently only in File object now - size: blob.Size(), - bytes: bytes.to_vec(), - }; - - let (tx, rx) = ipc::channel().unwrap(); - - let msg = BlobURLStoreMsg::AddEntry(entry, origin.clone(), tx); - - let _ = filemanager.send(FileManagerThreadMsg::BlobURLStoreMsg(msg)); - - match rx.recv().unwrap() { - Ok(id) => { - DOMString::from(URL::unicode_serialization_blob_url(&origin, &id)) - } - Err(_) => { - // Generate a dummy id - let id = Uuid::new_v4().simple().to_string(); - DOMString::from(URL::unicode_serialization_blob_url(&origin, &id)) - } - } + DOMString::from(URL::unicode_serialization_blob_url(&origin, &id.0)) } // https://w3c.github.io/FileAPI/#dfn-revokeObjectURL @@ -166,13 +140,15 @@ impl URL { NOTE: The first step is unnecessary, since closed blobs do not exist in the store */ + let origin = global.get_url().origin().unicode_serialization(); match Url::parse(&url) { Ok(url) => match parse_blob_url(&url) { Some((id, _)) => { let filemanager = global.resource_threads().sender(); - let msg = BlobURLStoreMsg::DeleteEntry(id.simple().to_string()); - let _ = filemanager.send(FileManagerThreadMsg::BlobURLStoreMsg(msg)); + let id = SelectedFileId(id.simple().to_string()); + let msg = FileManagerThreadMsg::DecRef(id, origin); + let _ = filemanager.send(msg); } None => {} }, diff --git a/components/servo/Cargo.lock b/components/servo/Cargo.lock index 7103bee63c9..24e2c5790c2 100644 --- a/components/servo/Cargo.lock +++ b/components/servo/Cargo.lock @@ -1470,6 +1470,7 @@ dependencies = [ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "msg 0.0.1", + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "serde_macros 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ports/cef/Cargo.lock b/ports/cef/Cargo.lock index 8f36c4363c7..6be02c19b6d 100644 --- a/ports/cef/Cargo.lock +++ b/ports/cef/Cargo.lock @@ -1351,6 +1351,7 @@ dependencies = [ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "msg 0.0.1", + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "serde_macros 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/tests/unit/net/filemanager_thread.rs b/tests/unit/net/filemanager_thread.rs index d5a2a1530b1..6a283669b70 100644 --- a/tests/unit/net/filemanager_thread.rs +++ b/tests/unit/net/filemanager_thread.rs @@ -35,12 +35,12 @@ fn test_filemanager() { .expect("Read tests/unit/net/test.txt error"); let patterns = vec![FilterPattern(".txt".to_string())]; - + let origin = "test.com".to_string(); { // Try to select a dummy file "tests/unit/net/test.txt" let (tx, rx) = ipc::channel().unwrap(); - chan.send(FileManagerThreadMsg::SelectFile(patterns.clone(), tx)).unwrap(); + chan.send(FileManagerThreadMsg::SelectFile(patterns.clone(), tx, origin.clone())).unwrap(); let selected = rx.recv().expect("File manager channel is broken") .expect("The file manager failed to find test.txt"); @@ -51,7 +51,7 @@ fn test_filemanager() { // Test by reading, expecting same content { let (tx2, rx2) = ipc::channel().unwrap(); - chan.send(FileManagerThreadMsg::ReadFile(tx2, selected.id.clone())).unwrap(); + chan.send(FileManagerThreadMsg::ReadFile(tx2, selected.id.clone(), origin.clone())).unwrap(); let msg = rx2.recv().expect("File manager channel is broken"); @@ -60,12 +60,12 @@ fn test_filemanager() { } // Delete the id - chan.send(FileManagerThreadMsg::DeleteFileID(selected.id.clone())).unwrap(); + chan.send(FileManagerThreadMsg::DecRef(selected.id.clone(), origin.clone())).unwrap(); // Test by reading again, expecting read error because we invalidated the id { let (tx2, rx2) = ipc::channel().unwrap(); - chan.send(FileManagerThreadMsg::ReadFile(tx2, selected.id.clone())).unwrap(); + chan.send(FileManagerThreadMsg::ReadFile(tx2, selected.id.clone(), origin.clone())).unwrap(); let msg = rx2.recv().expect("File manager channel is broken"); @@ -82,7 +82,7 @@ fn test_filemanager() { { let (tx, rx) = ipc::channel().unwrap(); - let _ = chan.send(FileManagerThreadMsg::SelectFile(patterns.clone(), tx)); + let _ = chan.send(FileManagerThreadMsg::SelectFile(patterns.clone(), tx, origin.clone())); assert!(rx.try_recv().is_err(), "The thread should not respond normally after exited"); }