Auto merge of #11875 - izgzhen:file-manager-backend, r=Manishearth

Integration and improvements of File API backends

Basically three major changes:

1. More complete origin check in `FileManagerThreadMsg`
2. Add reference counting logic to file manage store and script API
3. Integrate the support of slicing

r? @Manishearth

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [ ] These changes fix #__ (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [ ] These changes do not require tests because _____

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/11875)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2016-07-04 09:15:23 -07:00 committed by GitHub
commit 36974f0746
14 changed files with 543 additions and 244 deletions

View file

@ -2,60 +2,32 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use filemanager_thread::BlobURLStore;
use hyper::header::{DispositionType, ContentDisposition, DispositionParam};
use hyper::header::{Headers, ContentType, ContentLength, Charset};
use hyper::http::RawStatus;
use mime::{Mime, Attr};
use mime_classifier::MimeClassifier;
use net_traits::ProgressMsg::Done;
use net_traits::blob_url_store::{parse_blob_url, BlobURLStoreEntry, BlobURLStoreError};
use net_traits::blob_url_store::BlobURLStoreEntry;
use net_traits::filemanager_thread::RelativePos;
use net_traits::response::HttpsState;
use net_traits::{LoadConsumer, LoadData, Metadata, NetworkError};
use resource_thread::{send_error, start_sending_sniffed_opt};
use std::str;
use std::sync::{Arc, RwLock};
use net_traits::{LoadConsumer, LoadData, Metadata};
use resource_thread::start_sending_sniffed_opt;
use std::ops::Index;
use std::sync::Arc;
// TODO: Check on GET
// https://w3c.github.io/FileAPI/#requestResponseModel
pub fn load(load_data: LoadData, consumer: LoadConsumer,
blob_url_store: Arc<RwLock<BlobURLStore>>,
classifier: Arc<MimeClassifier>) { // XXX: Move it into net process later
match parse_blob_url(&load_data.url) {
None => {
let format_err = NetworkError::Internal(format!("Invalid blob URL format {:?}", load_data.url));
send_error(load_data.url.clone(), format_err, consumer);
}
Some((uuid, _fragment)) => {
match blob_url_store.read().unwrap().request(uuid, &load_data.url.origin()) {
Ok(entry) => load_blob(&load_data, consumer, classifier, entry),
Err(e) => {
let err = match e {
BlobURLStoreError::InvalidKey =>
format!("Invalid blob URL key {:?}", uuid.simple().to_string()),
BlobURLStoreError::InvalidOrigin =>
format!("Invalid blob URL origin {:?}", load_data.url.origin()),
};
send_error(load_data.url.clone(), NetworkError::Internal(err), consumer);
}
}
}
}
}
fn load_blob(load_data: &LoadData,
start_chan: LoadConsumer,
classifier: Arc<MimeClassifier>,
entry: &BlobURLStoreEntry) {
pub fn load_blob(load_data: &LoadData, start_chan: LoadConsumer,
classifier: Arc<MimeClassifier>, opt_filename: Option<String>,
rel_pos: &RelativePos, entry: &BlobURLStoreEntry) {
let content_type: Mime = entry.type_string.parse().unwrap_or(mime!(Text / Plain));
let charset = content_type.get_param(Attr::Charset);
let mut headers = Headers::new();
if let Some(ref name) = entry.filename {
if let Some(name) = opt_filename {
let charset = charset.and_then(|c| c.as_str().parse().ok());
headers.set(ContentDisposition {
disposition: DispositionType::Inline,
@ -66,8 +38,10 @@ fn load_blob(load_data: &LoadData,
});
}
let range = rel_pos.to_abs_range(entry.size as usize);
headers.set(ContentType(content_type.clone()));
headers.set(ContentLength(entry.size));
headers.set(ContentLength(range.len() as u64));
let metadata = Metadata {
final_url: load_data.url.clone(),
@ -81,7 +55,7 @@ fn load_blob(load_data: &LoadData,
if let Ok(chan) =
start_sending_sniffed_opt(start_chan, metadata, classifier,
&entry.bytes, load_data.context.clone()) {
&entry.bytes.index(range), load_data.context.clone()) {
let _ = chan.send(Done(Ok(())));
}
}

View file

@ -2,21 +2,24 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use blob_loader;
use blob_loader::load_blob;
use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
use mime_classifier::MimeClassifier;
use mime_guess::guess_mime_type_opt;
use net_traits::blob_url_store::{BlobURLStoreEntry, BlobURLStoreError, BlobURLStoreMsg};
use net_traits::filemanager_thread::{FileManagerThreadMsg, FileManagerResult, FilterPattern};
use net_traits::filemanager_thread::{SelectedFile, FileManagerThreadError, SelectedFileId};
use net_traits::blob_url_store::{BlobURLStoreEntry, BlobURLStoreError, parse_blob_url};
use net_traits::filemanager_thread::{FileManagerThreadMsg, FileManagerResult, FilterPattern, FileOrigin};
use net_traits::filemanager_thread::{SelectedFile, RelativePos, FileManagerThreadError, SelectedFileId};
use net_traits::{LoadConsumer, LoadData, NetworkError};
use resource_thread::send_error;
use std::cell::Cell;
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::sync::Arc;
#[cfg(any(target_os = "macos", target_os = "linux"))]
use tinyfiledialogs;
use url::{Url, Origin};
use url::Url;
use util::thread::spawn_named;
use uuid::Uuid;
@ -87,11 +90,26 @@ impl<UI: 'static + UIProvider> FileManagerThreadFactory<UI> for IpcSender<FileMa
}
}
struct FileStoreEntry {
/// Origin of the entry's "creator"
origin: FileOrigin,
/// Backend implementation
file_impl: FileImpl,
/// Reference counting
refs: Cell<usize>,
}
/// File backend implementation
enum FileImpl {
PathOnly(PathBuf),
Memory(BlobURLStoreEntry),
Sliced(Uuid, RelativePos),
}
struct FileManager<UI: 'static + UIProvider> {
receiver: IpcReceiver<FileManagerThreadMsg>,
idmap: HashMap<Uuid, PathBuf>,
store: HashMap<Uuid, FileStoreEntry>,
classifier: Arc<MimeClassifier>,
blob_url_store: Arc<RwLock<BlobURLStore>>,
ui: &'static UI,
}
@ -99,10 +117,9 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
fn new(recv: IpcReceiver<FileManagerThreadMsg>, ui: &'static UI) -> FileManager<UI> {
FileManager {
receiver: recv,
idmap: HashMap::new(),
store: HashMap::new(),
classifier: Arc::new(MimeClassifier::new()),
blob_url_store: Arc::new(RwLock::new(BlobURLStore::new())),
ui: ui
ui: ui,
}
}
@ -110,33 +127,97 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
fn start(&mut self) {
loop {
match self.receiver.recv().unwrap() {
FileManagerThreadMsg::SelectFile(filter, sender) => self.select_file(filter, sender),
FileManagerThreadMsg::SelectFiles(filter, sender) => self.select_files(filter, sender),
FileManagerThreadMsg::ReadFile(sender, id) => {
match self.try_read_file(id) {
FileManagerThreadMsg::SelectFile(filter, sender, origin) => self.select_file(filter, sender, origin),
FileManagerThreadMsg::SelectFiles(filter, sender, origin) => self.select_files(filter, sender, origin),
FileManagerThreadMsg::ReadFile(sender, id, origin) => {
match self.try_read_file(id, origin) {
Ok(buffer) => { let _ = sender.send(Ok(buffer)); }
Err(_) => { let _ = sender.send(Err(FileManagerThreadError::ReadFileError)); }
}
}
FileManagerThreadMsg::DeleteFileID(id) => self.delete_fileid(id),
FileManagerThreadMsg::BlobURLStoreMsg(msg) => self.blob_url_store.write().unwrap().process(msg),
FileManagerThreadMsg::TransferMemory(entry, rel_pos, sender, origin) =>
self.transfer_memory(entry, rel_pos, sender, origin),
FileManagerThreadMsg::AddSlicedEntry(id, rel_pos, sender, origin) =>
self.add_sliced_entry(id, rel_pos, sender, origin),
FileManagerThreadMsg::LoadBlob(load_data, consumer) => {
blob_loader::load(load_data, consumer,
self.blob_url_store.clone(),
self.classifier.clone());
match parse_blob_url(&load_data.url) {
None => {
let e = format!("Invalid blob URL format {:?}", load_data.url);
let format_err = NetworkError::Internal(e);
send_error(load_data.url.clone(), format_err, consumer);
}
Some((id, _fragment)) => {
self.process_request(&load_data, consumer, &RelativePos::full_range(), &id);
}
}
},
FileManagerThreadMsg::DecRef(id, origin) => {
if let Ok(id) = Uuid::parse_str(&id.0) {
self.dec_ref(id, origin);
}
}
FileManagerThreadMsg::IncRef(id, origin) => {
if let Ok(id) = Uuid::parse_str(&id.0) {
self.inc_ref(id, origin);
}
}
FileManagerThreadMsg::Exit => break,
};
}
}
fn inc_ref(&mut self, id: Uuid, origin_in: FileOrigin) {
match self.store.get(&id) {
Some(entry) => {
if entry.origin == origin_in {
entry.refs.set(entry.refs.get() + 1);
}
}
None => return, // Invalid UUID
}
}
fn add_sliced_entry(&mut self, id: SelectedFileId, rel_pos: RelativePos,
sender: IpcSender<Result<SelectedFileId, BlobURLStoreError>>,
origin_in: FileOrigin) {
if let Ok(id) = Uuid::parse_str(&id.0) {
match self.store.get(&id) {
Some(entry) => {
if entry.origin == origin_in {
// inc_ref on parent entry
entry.refs.set(entry.refs.get() + 1);
} else {
let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin));
return;
}
},
None => {
let _ = sender.send(Err(BlobURLStoreError::InvalidFileID));
return;
}
};
let new_id = Uuid::new_v4();
self.store.insert(new_id, FileStoreEntry {
origin: origin_in.clone(),
file_impl: FileImpl::Sliced(id, rel_pos),
refs: Cell::new(1),
});
let _ = sender.send(Ok(SelectedFileId(new_id.simple().to_string())));
} else {
let _ = sender.send(Err(BlobURLStoreError::InvalidFileID));
}
}
fn select_file(&mut self, patterns: Vec<FilterPattern>,
sender: IpcSender<FileManagerResult<SelectedFile>>) {
sender: IpcSender<FileManagerResult<SelectedFile>>,
origin: FileOrigin) {
match self.ui.open_file_dialog("", patterns) {
Some(s) => {
let selected_path = Path::new(&s);
match self.create_entry(selected_path) {
match self.create_entry(selected_path, &origin) {
Some(triple) => { let _ = sender.send(Ok(triple)); }
None => { let _ = sender.send(Err(FileManagerThreadError::InvalidSelection)); }
};
@ -149,7 +230,8 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
}
fn select_files(&mut self, patterns: Vec<FilterPattern>,
sender: IpcSender<FileManagerResult<Vec<SelectedFile>>>) {
sender: IpcSender<FileManagerResult<Vec<SelectedFile>>>,
origin: FileOrigin) {
match self.ui.open_file_dialog_multi("", patterns) {
Some(v) => {
let mut selected_paths = vec![];
@ -161,7 +243,7 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
let mut replies = vec![];
for path in selected_paths {
match self.create_entry(path) {
match self.create_entry(path, &origin) {
Some(triple) => replies.push(triple),
None => { let _ = sender.send(Err(FileManagerThreadError::InvalidSelection)); }
};
@ -176,11 +258,17 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
}
}
fn create_entry(&mut self, file_path: &Path) -> Option<SelectedFile> {
fn create_entry(&mut self, file_path: &Path, origin: &str) -> Option<SelectedFile> {
match File::open(file_path) {
Ok(handler) => {
let id = Uuid::new_v4();
self.idmap.insert(id, file_path.to_path_buf());
let file_impl = FileImpl::PathOnly(file_path.to_path_buf());
self.store.insert(id, FileStoreEntry {
origin: origin.to_string(),
file_impl: file_impl,
refs: Cell::new(1),
});
// Unix Epoch: https://doc.servo.org/std/time/constant.UNIX_EPOCH.html
let epoch = handler.metadata().and_then(|metadata| metadata.modified()).map_err(|_| ())
@ -215,79 +303,138 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
}
}
fn try_read_file(&mut self, id: SelectedFileId) -> Result<Vec<u8>, ()> {
fn try_read_file(&self, id: SelectedFileId, origin_in: String) -> Result<Vec<u8>, ()> {
let id = try!(Uuid::parse_str(&id.0).map_err(|_| ()));
match self.idmap.get(&id) {
Some(filepath) => {
let mut buffer = vec![];
let mut handler = try!(File::open(&filepath).map_err(|_| ()));
try!(handler.read_to_end(&mut buffer).map_err(|_| ()));
Ok(buffer)
},
None => Err(())
}
}
fn delete_fileid(&mut self, id: SelectedFileId) {
if let Ok(id) = Uuid::parse_str(&id.0) {
self.idmap.remove(&id);
}
}
}
pub struct BlobURLStore {
entries: HashMap<Uuid, (Origin, BlobURLStoreEntry)>,
}
impl BlobURLStore {
pub fn new() -> BlobURLStore {
BlobURLStore {
entries: HashMap::new(),
}
}
fn process(&mut self, msg: BlobURLStoreMsg) {
match msg {
BlobURLStoreMsg::AddEntry(entry, origin_str, sender) => {
match Url::parse(&origin_str) {
Ok(base_url) => {
let id = Uuid::new_v4();
self.add_entry(id, base_url.origin(), entry);
let _ = sender.send(Ok(id.simple().to_string()));
match self.store.get(&id) {
Some(entry) => {
match entry.file_impl {
FileImpl::PathOnly(ref filepath) => {
if *entry.origin == origin_in {
let mut buffer = vec![];
let mut handler = try!(File::open(filepath).map_err(|_| ()));
try!(handler.read_to_end(&mut buffer).map_err(|_| ()));
Ok(buffer)
} else {
Err(())
}
},
FileImpl::Memory(ref buffered) => {
Ok(buffered.bytes.clone())
},
FileImpl::Sliced(ref id, ref _rel_pos) => {
self.try_read_file(SelectedFileId(id.simple().to_string()), origin_in)
}
Err(_) => {
let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin));
}
},
None => Err(()),
}
}
fn dec_ref(&mut self, id: Uuid, origin_in: FileOrigin) {
let (is_last_ref, opt_parent_id) = match self.store.get(&id) {
Some(entry) => {
if *entry.origin == origin_in {
let r = entry.refs.get();
if r > 1 {
entry.refs.set(r - 1);
(false, None)
} else {
if let FileImpl::Sliced(ref parent_id, _) = entry.file_impl {
// if it has a reference to parent id, dec_ref on parent later
(true, Some(parent_id.clone()))
} else {
(true, None)
}
}
} else { // Invalid origin
return;
}
}
None => return, // Invalid UUID
};
if is_last_ref {
self.store.remove(&id);
if let Some(parent_id) = opt_parent_id {
self.dec_ref(parent_id, origin_in);
}
}
}
fn process_request(&self, load_data: &LoadData, consumer: LoadConsumer,
rel_pos: &RelativePos, id: &Uuid) {
let origin_in = load_data.url.origin().unicode_serialization();
match self.store.get(id) {
Some(entry) => {
match entry.file_impl {
FileImpl::Memory(ref buffered) => {
if *entry.origin == origin_in {
load_blob(&load_data, consumer, self.classifier.clone(),
None, rel_pos, buffered);
} else {
let e = format!("Invalid blob URL origin {:?}", origin_in);
send_error(load_data.url.clone(), NetworkError::Internal(e), consumer);
}
},
FileImpl::PathOnly(ref filepath) => {
let opt_filename = filepath.file_name()
.and_then(|osstr| osstr.to_str())
.map(|s| s.to_string());
if *entry.origin == origin_in {
let mut bytes = vec![];
let mut handler = File::open(filepath).unwrap();
let mime = guess_mime_type_opt(filepath);
let size = handler.read_to_end(&mut bytes).unwrap();
let entry = BlobURLStoreEntry {
type_string: match mime {
Some(x) => format!("{}", x),
None => "".to_string(),
},
size: size as u64,
bytes: bytes,
};
load_blob(&load_data, consumer, self.classifier.clone(),
opt_filename, rel_pos, &entry);
} else {
let e = format!("Invalid blob URL origin {:?}", origin_in);
send_error(load_data.url.clone(), NetworkError::Internal(e), consumer);
}
},
FileImpl::Sliced(ref id, ref rel_pos) => {
self.process_request(load_data, consumer, rel_pos, id);
}
}
}
BlobURLStoreMsg::DeleteEntry(id) => {
if let Ok(id) = Uuid::parse_str(&id) {
self.delete_entry(id);
}
},
}
}
pub fn request(&self, id: Uuid, origin: &Origin) -> Result<&BlobURLStoreEntry, BlobURLStoreError> {
match self.entries.get(&id) {
Some(ref pair) => {
if pair.0 == *origin {
Ok(&pair.1)
} else {
Err(BlobURLStoreError::InvalidOrigin)
}
_ => {
let e = format!("Invalid blob URL key {:?}", id.simple().to_string());
send_error(load_data.url.clone(), NetworkError::Internal(e), consumer);
}
None => Err(BlobURLStoreError::InvalidKey)
}
}
pub fn add_entry(&mut self, id: Uuid, origin: Origin, blob: BlobURLStoreEntry) {
self.entries.insert(id, (origin, blob));
}
fn transfer_memory(&mut self, entry: BlobURLStoreEntry, rel_pos: RelativePos,
sender: IpcSender<Result<SelectedFileId, BlobURLStoreError>>, origin: FileOrigin) {
match Url::parse(&origin) { // parse to check sanity
Ok(_) => {
let id = Uuid::new_v4();
self.store.insert(id, FileStoreEntry {
origin: origin.clone(),
file_impl: FileImpl::Memory(entry),
refs: Cell::new(1),
});
let sliced_id = SelectedFileId(id.simple().to_string());
pub fn delete_entry(&mut self, id: Uuid) {
self.entries.remove(&id);
self.add_sliced_entry(sliced_id, rel_pos, sender, origin);
}
Err(_) => {
let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin));
}
}
}
}