Integration and improvements of File API backends

1. More complete origin check in FileManagerThreadMsg
2. Add reference counting logic to file manage store and script API
3. Integrate the support of slicing

components/net/filemanager_thread.rs

@@ -2,21 +2,24 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use blob_loader;
+use blob_loader::load_blob;
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use mime_classifier::MimeClassifier;
 use mime_guess::guess_mime_type_opt;
-use net_traits::blob_url_store::{BlobURLStoreEntry, BlobURLStoreError, BlobURLStoreMsg};
-use net_traits::filemanager_thread::{FileManagerThreadMsg, FileManagerResult, FilterPattern};
-use net_traits::filemanager_thread::{SelectedFile, FileManagerThreadError, SelectedFileId};
+use net_traits::blob_url_store::{BlobURLStoreEntry, BlobURLStoreError, parse_blob_url};
+use net_traits::filemanager_thread::{FileManagerThreadMsg, FileManagerResult, FilterPattern, FileOrigin};
+use net_traits::filemanager_thread::{SelectedFile, RelativePos, FileManagerThreadError, SelectedFileId};
+use net_traits::{LoadConsumer, LoadData, NetworkError};
+use resource_thread::send_error;
+use std::cell::Cell;
 use std::collections::HashMap;
 use std::fs::File;
 use std::io::Read;
 use std::path::{Path, PathBuf};
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 #[cfg(any(target_os = "macos", target_os = "linux"))]
 use tinyfiledialogs;
-use url::{Url, Origin};
+use url::Url;
 use util::thread::spawn_named;
 use uuid::Uuid;
 
@@ -87,11 +90,26 @@ impl<UI: 'static + UIProvider> FileManagerThreadFactory<UI> for IpcSender<FileMa
     }
 }
 
+struct FileStoreEntry {
+    /// Origin of the entry's "creator"
+    origin: FileOrigin,
+    /// Backend implementation
+    file_impl: FileImpl,
+    /// Reference counting
+    refs: Cell<usize>,
+}
+
+/// File backend implementation
+enum FileImpl {
+    PathOnly(PathBuf),
+    Memory(BlobURLStoreEntry),
+    Sliced(Uuid, RelativePos),
+}
+
 struct FileManager<UI: 'static + UIProvider> {
     receiver: IpcReceiver<FileManagerThreadMsg>,
-    idmap: HashMap<Uuid, PathBuf>,
+    store: HashMap<Uuid, FileStoreEntry>,
     classifier: Arc<MimeClassifier>,
-    blob_url_store: Arc<RwLock<BlobURLStore>>,
     ui: &'static UI,
 }
 
@@ -99,10 +117,9 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
     fn new(recv: IpcReceiver<FileManagerThreadMsg>, ui: &'static UI) -> FileManager<UI> {
         FileManager {
             receiver: recv,
-            idmap: HashMap::new(),
+            store: HashMap::new(),
             classifier: Arc::new(MimeClassifier::new()),
-            blob_url_store: Arc::new(RwLock::new(BlobURLStore::new())),
-            ui: ui
+            ui: ui,
         }
     }
 
@@ -110,33 +127,97 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
     fn start(&mut self) {
         loop {
             match self.receiver.recv().unwrap() {
-                FileManagerThreadMsg::SelectFile(filter, sender) => self.select_file(filter, sender),
-                FileManagerThreadMsg::SelectFiles(filter, sender) => self.select_files(filter, sender),
-                FileManagerThreadMsg::ReadFile(sender, id) => {
-                    match self.try_read_file(id) {
+                FileManagerThreadMsg::SelectFile(filter, sender, origin) => self.select_file(filter, sender, origin),
+                FileManagerThreadMsg::SelectFiles(filter, sender, origin) => self.select_files(filter, sender, origin),
+                FileManagerThreadMsg::ReadFile(sender, id, origin) => {
+                    match self.try_read_file(id, origin) {
                         Ok(buffer) => { let _ = sender.send(Ok(buffer)); }
                         Err(_) => { let _ = sender.send(Err(FileManagerThreadError::ReadFileError)); }
                     }
                 }
-                FileManagerThreadMsg::DeleteFileID(id) => self.delete_fileid(id),
-                FileManagerThreadMsg::BlobURLStoreMsg(msg) => self.blob_url_store.write().unwrap().process(msg),
+                FileManagerThreadMsg::TransferMemory(entry, rel_pos, sender, origin) =>
+                    self.transfer_memory(entry, rel_pos, sender, origin),
+                FileManagerThreadMsg::AddSlicedEntry(id, rel_pos, sender, origin) =>
+                    self.add_sliced_entry(id, rel_pos, sender, origin),
                 FileManagerThreadMsg::LoadBlob(load_data, consumer) => {
-                    blob_loader::load(load_data, consumer,
-                                      self.blob_url_store.clone(),
-                                      self.classifier.clone());
+                    match parse_blob_url(&load_data.url) {
+                        None => {
+                            let e = format!("Invalid blob URL format {:?}", load_data.url);
+                            let format_err = NetworkError::Internal(e);
+                            send_error(load_data.url.clone(), format_err, consumer);
+                        }
+                        Some((id, _fragment)) => {
+                            self.process_request(&load_data, consumer, &RelativePos::full_range(), &id);
+                        }
+                    }
                 },
+                FileManagerThreadMsg::DecRef(id, origin) => {
+                    if let Ok(id) = Uuid::parse_str(&id.0) {
+                        self.dec_ref(id, origin);
+                    }
+                }
+                FileManagerThreadMsg::IncRef(id, origin) => {
+                    if let Ok(id) = Uuid::parse_str(&id.0) {
+                        self.inc_ref(id, origin);
+                    }
+                }
                 FileManagerThreadMsg::Exit => break,
             };
         }
     }
 
+    fn inc_ref(&mut self, id: Uuid, origin_in: FileOrigin) {
+        match self.store.get(&id) {
+            Some(entry) => {
+                if entry.origin == origin_in {
+                    entry.refs.set(entry.refs.get() + 1);
+                }
+            }
+            None => return, // Invalid UUID
+        }
+    }
+
+    fn add_sliced_entry(&mut self, id: SelectedFileId, rel_pos: RelativePos,
+                        sender: IpcSender<Result<SelectedFileId, BlobURLStoreError>>,
+                        origin_in: FileOrigin) {
+        if let Ok(id) = Uuid::parse_str(&id.0) {
+            match self.store.get(&id) {
+                Some(entry) => {
+                    if entry.origin == origin_in {
+                        // inc_ref on parent entry
+                        entry.refs.set(entry.refs.get() + 1);
+                    } else {
+                        let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin));
+                        return;
+                    }
+                },
+                None => {
+                    let _ = sender.send(Err(BlobURLStoreError::InvalidFileID));
+                    return;
+                }
+            };
+
+            let new_id = Uuid::new_v4();
+            self.store.insert(new_id, FileStoreEntry {
+                origin: origin_in.clone(),
+                file_impl: FileImpl::Sliced(id, rel_pos),
+                refs: Cell::new(1),
+            });
+
+            let _ = sender.send(Ok(SelectedFileId(new_id.simple().to_string())));
+        } else {
+            let _ = sender.send(Err(BlobURLStoreError::InvalidFileID));
+        }
+    }
+
     fn select_file(&mut self, patterns: Vec<FilterPattern>,
-                   sender: IpcSender<FileManagerResult<SelectedFile>>) {
+                   sender: IpcSender<FileManagerResult<SelectedFile>>,
+                   origin: FileOrigin) {
         match self.ui.open_file_dialog("", patterns) {
             Some(s) => {
                 let selected_path = Path::new(&s);
 
-                match self.create_entry(selected_path) {
+                match self.create_entry(selected_path, &origin) {
                     Some(triple) => { let _ = sender.send(Ok(triple)); }
                     None => { let _ = sender.send(Err(FileManagerThreadError::InvalidSelection)); }
                 };
@@ -149,7 +230,8 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
     }
 
     fn select_files(&mut self, patterns: Vec<FilterPattern>,
-                    sender: IpcSender<FileManagerResult<Vec<SelectedFile>>>) {
+                    sender: IpcSender<FileManagerResult<Vec<SelectedFile>>>,
+                    origin: FileOrigin) {
         match self.ui.open_file_dialog_multi("", patterns) {
             Some(v) => {
                 let mut selected_paths = vec![];
@@ -161,7 +243,7 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
                 let mut replies = vec![];
 
                 for path in selected_paths {
-                    match self.create_entry(path) {
+                    match self.create_entry(path, &origin) {
                         Some(triple) => replies.push(triple),
                         None => { let _ = sender.send(Err(FileManagerThreadError::InvalidSelection)); }
                     };
@@ -176,11 +258,17 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
         }
     }
 
-    fn create_entry(&mut self, file_path: &Path) -> Option<SelectedFile> {
+    fn create_entry(&mut self, file_path: &Path, origin: &str) -> Option<SelectedFile> {
         match File::open(file_path) {
             Ok(handler) => {
                 let id = Uuid::new_v4();
-                self.idmap.insert(id, file_path.to_path_buf());
+                let file_impl = FileImpl::PathOnly(file_path.to_path_buf());
+
+                self.store.insert(id, FileStoreEntry {
+                    origin: origin.to_string(),
+                    file_impl: file_impl,
+                    refs: Cell::new(1),
+                });
 
                 // Unix Epoch: https://doc.servo.org/std/time/constant.UNIX_EPOCH.html
                 let epoch = handler.metadata().and_then(|metadata| metadata.modified()).map_err(|_| ())
@@ -215,79 +303,138 @@ impl<UI: 'static + UIProvider> FileManager<UI> {
         }
     }
 
-    fn try_read_file(&mut self, id: SelectedFileId) -> Result<Vec<u8>, ()> {
+    fn try_read_file(&self, id: SelectedFileId, origin_in: String) -> Result<Vec<u8>, ()> {
         let id = try!(Uuid::parse_str(&id.0).map_err(|_| ()));
 
-        match self.idmap.get(&id) {
-            Some(filepath) => {
-                let mut buffer = vec![];
-                let mut handler = try!(File::open(&filepath).map_err(|_| ()));
-                try!(handler.read_to_end(&mut buffer).map_err(|_| ()));
-                Ok(buffer)
-            },
-            None => Err(())
-        }
-    }
-
-    fn delete_fileid(&mut self, id: SelectedFileId) {
-        if let Ok(id) = Uuid::parse_str(&id.0) {
-            self.idmap.remove(&id);
-        }
-    }
-}
-
-pub struct BlobURLStore {
-    entries: HashMap<Uuid, (Origin, BlobURLStoreEntry)>,
-}
-
-impl BlobURLStore {
-    pub fn new() -> BlobURLStore {
-        BlobURLStore {
-            entries: HashMap::new(),
-        }
-    }
-
-    fn process(&mut self, msg: BlobURLStoreMsg) {
-        match msg {
-            BlobURLStoreMsg::AddEntry(entry, origin_str, sender) => {
-                match Url::parse(&origin_str) {
-                    Ok(base_url) => {
-                        let id = Uuid::new_v4();
-                        self.add_entry(id, base_url.origin(), entry);
-
-                        let _ = sender.send(Ok(id.simple().to_string()));
+        match self.store.get(&id) {
+            Some(entry) => {
+                match entry.file_impl {
+                    FileImpl::PathOnly(ref filepath) => {
+                        if *entry.origin == origin_in {
+                            let mut buffer = vec![];
+                            let mut handler = try!(File::open(filepath).map_err(|_| ()));
+                            try!(handler.read_to_end(&mut buffer).map_err(|_| ()));
+                            Ok(buffer)
+                        } else {
+                            Err(())
+                        }
+                    },
+                    FileImpl::Memory(ref buffered) => {
+                        Ok(buffered.bytes.clone())
+                    },
+                    FileImpl::Sliced(ref id, ref _rel_pos) => {
+                        self.try_read_file(SelectedFileId(id.simple().to_string()), origin_in)
                     }
-                    Err(_) => {
-                        let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin));
+                }
+            },
+            None => Err(()),
+        }
+    }
+
+    fn dec_ref(&mut self, id: Uuid, origin_in: FileOrigin) {
+        let (is_last_ref, opt_parent_id) = match self.store.get(&id) {
+            Some(entry) => {
+                if *entry.origin == origin_in {
+                    let r = entry.refs.get();
+
+                    if r > 1 {
+                        entry.refs.set(r - 1);
+                        (false, None)
+                    } else {
+                        if let FileImpl::Sliced(ref parent_id, _) = entry.file_impl {
+                            // if it has a reference to parent id, dec_ref on parent later
+                            (true, Some(parent_id.clone()))
+                        } else {
+                            (true, None)
+                        }
+                    }
+                } else { // Invalid origin
+                    return;
+                }
+            }
+            None => return, // Invalid UUID
+        };
+
+        if is_last_ref {
+            self.store.remove(&id);
+
+            if let Some(parent_id) = opt_parent_id {
+                self.dec_ref(parent_id, origin_in);
+            }
+        }
+    }
+
+    fn process_request(&self, load_data: &LoadData, consumer: LoadConsumer,
+                       rel_pos: &RelativePos, id: &Uuid) {
+        let origin_in = load_data.url.origin().unicode_serialization();
+        match self.store.get(id) {
+            Some(entry) => {
+                match entry.file_impl {
+                    FileImpl::Memory(ref buffered) => {
+                        if *entry.origin == origin_in {
+                            load_blob(&load_data, consumer, self.classifier.clone(),
+                                      None, rel_pos, buffered);
+                        } else {
+                            let e = format!("Invalid blob URL origin {:?}", origin_in);
+                            send_error(load_data.url.clone(), NetworkError::Internal(e), consumer);
+                        }
+                    },
+                    FileImpl::PathOnly(ref filepath) => {
+                        let opt_filename = filepath.file_name()
+                                               .and_then(|osstr| osstr.to_str())
+                                               .map(|s| s.to_string());
+
+                        if *entry.origin == origin_in {
+                            let mut bytes = vec![];
+                            let mut handler = File::open(filepath).unwrap();
+                            let mime = guess_mime_type_opt(filepath);
+                            let size = handler.read_to_end(&mut bytes).unwrap();
+
+                            let entry = BlobURLStoreEntry {
+                                type_string: match mime {
+                                    Some(x) => format!("{}", x),
+                                    None    => "".to_string(),
+                                },
+                                size: size as u64,
+                                bytes: bytes,
+                            };
+
+                            load_blob(&load_data, consumer, self.classifier.clone(),
+                                      opt_filename, rel_pos, &entry);
+                        } else {
+                            let e = format!("Invalid blob URL origin {:?}", origin_in);
+                            send_error(load_data.url.clone(), NetworkError::Internal(e), consumer);
+                        }
+                    },
+                    FileImpl::Sliced(ref id, ref rel_pos) => {
+                        self.process_request(load_data, consumer, rel_pos, id);
                     }
                 }
             }
-            BlobURLStoreMsg::DeleteEntry(id) => {
-                if let Ok(id) = Uuid::parse_str(&id) {
-                    self.delete_entry(id);
-                }
-            },
-        }
-    }
-
-    pub fn request(&self, id: Uuid, origin: &Origin) -> Result<&BlobURLStoreEntry, BlobURLStoreError> {
-        match self.entries.get(&id) {
-            Some(ref pair) => {
-                if pair.0 == *origin {
-                    Ok(&pair.1)
-                } else {
-                    Err(BlobURLStoreError::InvalidOrigin)
-                }
+            _ => {
+                let e = format!("Invalid blob URL key {:?}", id.simple().to_string());
+                send_error(load_data.url.clone(), NetworkError::Internal(e), consumer);
             }
-            None => Err(BlobURLStoreError::InvalidKey)
         }
     }
 
-    pub fn add_entry(&mut self, id: Uuid, origin: Origin, blob: BlobURLStoreEntry) {
-        self.entries.insert(id, (origin, blob));
-    }
+    fn transfer_memory(&mut self, entry: BlobURLStoreEntry, rel_pos: RelativePos,
+                       sender: IpcSender<Result<SelectedFileId, BlobURLStoreError>>, origin: FileOrigin) {
+        match Url::parse(&origin) { // parse to check sanity
+            Ok(_) => {
+                let id = Uuid::new_v4();
+                self.store.insert(id, FileStoreEntry {
+                    origin: origin.clone(),
+                    file_impl: FileImpl::Memory(entry),
+                    refs: Cell::new(1),
+                });
+                let sliced_id = SelectedFileId(id.simple().to_string());
 
-    pub fn delete_entry(&mut self, id: Uuid) {
-        self.entries.remove(&id);
+                self.add_sliced_entry(sliced_id, rel_pos, sender, origin);
+            }
+            Err(_) => {
+                let _ = sender.send(Err(BlobURLStoreError::InvalidOrigin));
+            }
+        }
     }
 }