From b5d6555238d2f0068225f376d42759f47f50b8b1 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 5 Sep 2025 22:48:11 -0700 Subject: [PATCH] indexeddb: Implement getAll and getAllKeys (#38885) Implement getAll and getAllKeys for IDBObjectStore. Testing: WPT & Unit testing Fixes: Part of #6963. --------- Signed-off-by: Ashwin Naren --- components/net/indexeddb/engines/sqlite.rs | 80 +++++++++++++ components/net/tests/sqlite.rs | 102 +++++++++++++--- components/script/dom/idbobjectstore.rs | 112 +++++++++++++----- components/script/dom/idbrequest.rs | 49 +++++++- .../webidls/IDBObjectStore.webidl | 8 +- components/shared/net/indexeddb_thread.rs | 13 +- ...dbobjectstore-cross-realm-methods.html.ini | 6 - ...jectstore-query-exception-order.any.js.ini | 12 -- .../wpt/meta/IndexedDB/idlharness.any.js.ini | 12 -- 9 files changed, 308 insertions(+), 86 deletions(-) diff --git a/components/net/indexeddb/engines/sqlite.rs b/components/net/indexeddb/engines/sqlite.rs index 3086b848a6e..b5c66762b06 100644 --- a/components/net/indexeddb/engines/sqlite.rs +++ b/components/net/indexeddb/engines/sqlite.rs @@ -176,6 +176,55 @@ impl SqliteEngine { Self::get(connection, store, key_range).map(|opt| opt.map(|model| model.data)) } + fn get_all( + connection: &Connection, + store: object_store_model::Model, + key_range: IndexedDBKeyRange, + count: Option, + ) -> Result, Error> { + let query = range_to_query(key_range); + let mut sql_query = sea_query::Query::select(); + sql_query + .from(object_data_model::Column::Table) + .columns(vec![ + object_data_model::Column::ObjectStoreId, + object_data_model::Column::Key, + object_data_model::Column::Data, + ]) + .and_where(query.and(Expr::col(object_data_model::Column::ObjectStoreId).is(store.id))); + if let Some(count) = count { + sql_query.limit(count as u64); + } + let (sql, values) = sql_query.build_rusqlite(SqliteQueryBuilder); + let mut stmt = connection.prepare(&sql)?; + let models = stmt + .query_and_then(&*values.as_params(), |row| { + object_data_model::Model::try_from(row) + })? + .collect::, _>>()?; + Ok(models) + } + + fn get_all_keys( + connection: &Connection, + store: object_store_model::Model, + key_range: IndexedDBKeyRange, + count: Option, + ) -> Result>, Error> { + Self::get_all(connection, store, key_range, count) + .map(|models| models.into_iter().map(|m| m.key).collect()) + } + + fn get_all_items( + connection: &Connection, + store: object_store_model::Model, + key_range: IndexedDBKeyRange, + count: Option, + ) -> Result>, Error> { + Self::get_all(connection, store, key_range, count) + .map(|models| models.into_iter().map(|m| m.data).collect()) + } + fn put_item( connection: &Connection, store: object_store_model::Model, @@ -401,6 +450,37 @@ impl KvsEngine for SqliteEngine { .map_err(|e| BackendError::DbErr(format!("{:?}", e))), ); }, + AsyncOperation::ReadOnly(AsyncReadOnlyOperation::GetAllKeys { + sender, + key_range, + count, + }) => { + let Ok(object_store) = process_object_store(object_store, &sender) else { + continue; + }; + let _ = sender.send( + Self::get_all_keys(&connection, object_store, key_range, count) + .map(|keys| { + keys.into_iter() + .map(|k| bincode::deserialize(&k).unwrap()) + .collect() + }) + .map_err(|e| BackendError::DbErr(format!("{:?}", e))), + ); + }, + AsyncOperation::ReadOnly(AsyncReadOnlyOperation::GetAllItems { + sender, + key_range, + count, + }) => { + let Ok(object_store) = process_object_store(object_store, &sender) else { + continue; + }; + let _ = sender.send( + Self::get_all_items(&connection, object_store, key_range, count) + .map_err(|e| BackendError::DbErr(format!("{:?}", e))), + ); + }, AsyncOperation::ReadWrite(AsyncReadWriteOperation::RemoveItem { sender, key, diff --git a/components/net/tests/sqlite.rs b/components/net/tests/sqlite.rs index 1e0da5f71cf..29be2c4255e 100644 --- a/components/net/tests/sqlite.rs +++ b/components/net/tests/sqlite.rs @@ -9,8 +9,9 @@ use net::indexeddb::idb_thread::IndexedDBDescription; use net::resource_thread::CoreResourceThreadPool; use net_traits::indexeddb_thread::{ AsyncOperation, AsyncReadOnlyOperation, AsyncReadWriteOperation, CreateObjectResult, - IndexedDBKeyRange, IndexedDBKeyType, IndexedDBTxnMode, KeyPath, + IndexedDBKeyRange, IndexedDBKeyType, IndexedDBTxnMode, KeyPath, PutItemResult, }; +use serde::{Deserialize, Serialize}; use servo_url::ImmutableOrigin; use url::Host; @@ -183,6 +184,16 @@ fn test_delete_store() { #[test] fn test_async_operations() { + fn get_channel() -> ( + ipc_channel::ipc::IpcSender, + ipc_channel::ipc::IpcReceiver, + ) + where + T: for<'de> Deserialize<'de> + Serialize, + { + ipc_channel::ipc::channel().unwrap() + } + let base_dir = tempfile::tempdir().expect("Failed to create temp dir"); let thread_pool = get_pool(); let db = SqliteEngine::new( @@ -197,68 +208,121 @@ fn test_async_operations() { let store_name = "test_store"; db.create_store(store_name, None, false) .expect("Failed to create store"); - let channel = ipc_channel::ipc::channel().unwrap(); - let channel2 = ipc_channel::ipc::channel().unwrap(); - let channel3 = ipc_channel::ipc::channel().unwrap(); - let channel4 = ipc_channel::ipc::channel().unwrap(); - let channel5 = ipc_channel::ipc::channel().unwrap(); - let channel6 = ipc_channel::ipc::channel().unwrap(); + let put = get_channel(); + let put2 = get_channel(); + let put3 = get_channel(); + let put_dup = get_channel(); + let get_item_some = get_channel(); + let get_item_none = get_channel(); + let get_all_items = get_channel(); + let count = get_channel(); + let remove = get_channel(); + let clear = get_channel(); let rx = db.process_transaction(KvsTransaction { mode: IndexedDBTxnMode::Readwrite, requests: VecDeque::from(vec![ KvsOperation { store_name: store_name.to_owned(), operation: AsyncOperation::ReadWrite(AsyncReadWriteOperation::PutItem { - sender: channel.0, + sender: put.0, key: Some(IndexedDBKeyType::Number(1.0)), value: vec![1, 2, 3], should_overwrite: false, }), }, + KvsOperation { + store_name: store_name.to_owned(), + operation: AsyncOperation::ReadWrite(AsyncReadWriteOperation::PutItem { + sender: put2.0, + key: Some(IndexedDBKeyType::String("2.0".to_string())), + value: vec![4, 5, 6], + should_overwrite: false, + }), + }, + KvsOperation { + store_name: store_name.to_owned(), + operation: AsyncOperation::ReadWrite(AsyncReadWriteOperation::PutItem { + sender: put3.0, + key: Some(IndexedDBKeyType::Array(vec![ + IndexedDBKeyType::String("3".to_string()), + IndexedDBKeyType::Number(0.0), + ])), + value: vec![7, 8, 9], + should_overwrite: false, + }), + }, + // Try to put a duplicate key without overwrite + KvsOperation { + store_name: store_name.to_owned(), + operation: AsyncOperation::ReadWrite(AsyncReadWriteOperation::PutItem { + sender: put_dup.0, + key: Some(IndexedDBKeyType::Number(1.0)), + value: vec![10, 11, 12], + should_overwrite: false, + }), + }, KvsOperation { store_name: store_name.to_owned(), operation: AsyncOperation::ReadOnly(AsyncReadOnlyOperation::GetItem { - sender: channel2.0, + sender: get_item_some.0, key_range: IndexedDBKeyRange::only(IndexedDBKeyType::Number(1.0)), }), }, KvsOperation { store_name: store_name.to_owned(), operation: AsyncOperation::ReadOnly(AsyncReadOnlyOperation::GetItem { - sender: channel3.0, + sender: get_item_none.0, key_range: IndexedDBKeyRange::only(IndexedDBKeyType::Number(5.0)), }), }, + KvsOperation { + store_name: store_name.to_owned(), + operation: AsyncOperation::ReadOnly(AsyncReadOnlyOperation::GetAllItems { + sender: get_all_items.0, + key_range: IndexedDBKeyRange::lower_bound(IndexedDBKeyType::Number(0.0), false), + count: None, + }), + }, KvsOperation { store_name: store_name.to_owned(), operation: AsyncOperation::ReadOnly(AsyncReadOnlyOperation::Count { - sender: channel4.0, + sender: count.0, key_range: IndexedDBKeyRange::only(IndexedDBKeyType::Number(1.0)), }), }, KvsOperation { store_name: store_name.to_owned(), operation: AsyncOperation::ReadWrite(AsyncReadWriteOperation::RemoveItem { - sender: channel5.0, + sender: remove.0, key: IndexedDBKeyType::Number(1.0), }), }, KvsOperation { store_name: store_name.to_owned(), - operation: AsyncOperation::ReadWrite(AsyncReadWriteOperation::Clear(channel6.0)), + operation: AsyncOperation::ReadWrite(AsyncReadWriteOperation::Clear(clear.0)), }, ]), }); let _ = rx.blocking_recv().unwrap(); - channel.1.recv().unwrap().unwrap(); - let get_result = channel2.1.recv().unwrap(); + put.1.recv().unwrap().unwrap(); + put2.1.recv().unwrap().unwrap(); + put3.1.recv().unwrap().unwrap(); + let err = put_dup.1.recv().unwrap().unwrap(); + assert_eq!(err, PutItemResult::CannotOverwrite); + let get_result = get_item_some.1.recv().unwrap(); let value = get_result.unwrap(); assert_eq!(value, Some(vec![1, 2, 3])); - let get_result = channel3.1.recv().unwrap(); + let get_result = get_item_none.1.recv().unwrap(); let value = get_result.unwrap(); assert_eq!(value, None); - let amount = channel4.1.recv().unwrap().unwrap(); + let all_items = get_all_items.1.recv().unwrap().unwrap(); + assert_eq!(all_items.len(), 3); + // Check that all three items are present + assert!(all_items.contains(&vec![1, 2, 3])); + assert!(all_items.contains(&vec![4, 5, 6])); + assert!(all_items.contains(&vec![7, 8, 9])); + let amount = count.1.recv().unwrap().unwrap(); assert_eq!(amount, 1); - channel5.1.recv().unwrap().unwrap(); - channel6.1.recv().unwrap().unwrap(); + remove.1.recv().unwrap().unwrap(); + clear.1.recv().unwrap().unwrap(); } diff --git a/components/script/dom/idbobjectstore.rs b/components/script/dom/idbobjectstore.rs index 98cd11cb927..7f13b00fd46 100644 --- a/components/script/dom/idbobjectstore.rs +++ b/components/script/dom/idbobjectstore.rs @@ -283,7 +283,7 @@ impl IDBObjectStore { } impl IDBObjectStoreMethods for IDBObjectStore { - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-put + /// fn Put( &self, cx: SafeJSContext, @@ -293,7 +293,7 @@ impl IDBObjectStoreMethods for IDBObjectStore { self.put(cx, value, key, true, CanGc::note()) } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-add + /// fn Add( &self, cx: SafeJSContext, @@ -303,7 +303,7 @@ impl IDBObjectStoreMethods for IDBObjectStore { self.put(cx, value, key, false, CanGc::note()) } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-delete + /// fn Delete(&self, cx: SafeJSContext, query: HandleValue) -> Fallible> { // Step 1. Let transaction be this’s transaction. // Step 2. Let store be this's object store. @@ -331,7 +331,7 @@ impl IDBObjectStoreMethods for IDBObjectStore { }) } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-clear + /// fn Clear(&self) -> Fallible> { // Step 1. Let transaction be this’s transaction. // Step 2. Let store be this's object store. @@ -355,7 +355,7 @@ impl IDBObjectStoreMethods for IDBObjectStore { ) } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-get + /// fn Get(&self, cx: SafeJSContext, query: HandleValue) -> Fallible> { // Step 1. Let transaction be this’s transaction. // Step 2. Let store be this's object store. @@ -385,7 +385,7 @@ impl IDBObjectStoreMethods for IDBObjectStore { }) } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-getkey + /// fn GetKey(&self, cx: SafeJSContext, query: HandleValue) -> Result, Error> { // Step 1. Let transaction be this’s transaction. // Step 2. Let store be this's object store. @@ -416,27 +416,81 @@ impl IDBObjectStoreMethods for IDBObjectStore { }) } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-getall - // fn GetAll( - // &self, - // _cx: SafeJSContext, - // _query: HandleValue, - // _count: Option, - // ) -> DomRoot { - // unimplemented!(); - // } + /// + fn GetAll( + &self, + cx: SafeJSContext, + query: HandleValue, + count: Option, + ) -> Fallible> { + // Step 1. Let transaction be this’s transaction. + // Step 2. Let store be this's object store. + // Step 3. If store has been deleted, throw an "InvalidStateError" DOMException. + self.verify_not_deleted()?; - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-getallkeys - // fn GetAllKeys( - // &self, - // _cx: SafeJSContext, - // _query: HandleValue, - // _count: Option, - // ) -> DomRoot { - // unimplemented!(); - // } + // Step 4. If transaction’s state is not active, then throw a "TransactionInactiveError" DOMException. + self.check_transaction_active()?; - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-count + // Step 5. Let range be the result of running the steps to convert a value to a key range with query and null disallowed flag set. Rethrow any exceptions. + let serialized_query = convert_value_to_key_range(cx, query, None); + + // Step 6. Run the steps to asynchronously execute a request and return the IDBRequest created by these steps. + // The steps are run with this object store handle as source and the steps to retrieve a key from an object + // store as operation, using store and range. + let (sender, receiver) = indexed_db::create_channel(self.global()); + serialized_query.and_then(|q| { + IDBRequest::execute_async( + self, + AsyncOperation::ReadOnly(AsyncReadOnlyOperation::GetAllItems { + sender, + key_range: q, + count, + }), + receiver, + None, + CanGc::note(), + ) + }) + } + + /// + fn GetAllKeys( + &self, + cx: SafeJSContext, + query: HandleValue, + count: Option, + ) -> Fallible> { + // Step 1. Let transaction be this’s transaction. + // Step 2. Let store be this's object store. + // Step 3. If store has been deleted, throw an "InvalidStateError" DOMException. + self.verify_not_deleted()?; + + // Step 4. If transaction’s state is not active, then throw a "TransactionInactiveError" DOMException. + self.check_transaction_active()?; + + // Step 5. Let range be the result of running the steps to convert a value to a key range with query and null disallowed flag set. Rethrow any exceptions. + let serialized_query = convert_value_to_key_range(cx, query, None); + + // Step 6. Run the steps to asynchronously execute a request and return the IDBRequest created by these steps. + // The steps are run with this object store handle as source and the steps to retrieve a key from an object + // store as operation, using store and range. + let (sender, receiver) = indexed_db::create_channel(self.global()); + serialized_query.and_then(|q| { + IDBRequest::execute_async( + self, + AsyncOperation::ReadOnly(AsyncReadOnlyOperation::GetAllKeys { + sender, + key_range: q, + count, + }), + receiver, + None, + CanGc::note(), + ) + }) + } + + /// fn Count(&self, cx: SafeJSContext, query: HandleValue) -> Fallible> { // Step 1. Let transaction be this’s transaction. // Step 2. Let store be this's object store. @@ -466,12 +520,12 @@ impl IDBObjectStoreMethods for IDBObjectStore { }) } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-name + /// fn Name(&self) -> DOMString { self.name.borrow().clone() } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-setname + /// fn SetName(&self, value: DOMString) -> ErrorResult { // Step 2. Let transaction be this’s transaction. let transaction = &self.transaction; @@ -501,12 +555,12 @@ impl IDBObjectStoreMethods for IDBObjectStore { // unimplemented!(); // } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-transaction + /// fn Transaction(&self) -> DomRoot { self.transaction() } - // https://www.w3.org/TR/IndexedDB-2/#dom-idbobjectstore-autoincrement + /// fn AutoIncrement(&self) -> bool { self.has_key_generator() } diff --git a/components/script/dom/idbrequest.rs b/components/script/dom/idbrequest.rs index cec15a4e1c8..9c051cbd0ba 100644 --- a/components/script/dom/idbrequest.rs +++ b/components/script/dom/idbrequest.rs @@ -3,6 +3,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ use std::cell::Cell; +use std::iter::repeat_n; use dom_struct::dom_struct; use ipc_channel::router::ROUTER; @@ -15,6 +16,7 @@ use net_traits::indexeddb_thread::{ IndexedDBTxnMode, PutItemResult, }; use profile_traits::ipc::IpcReceiver; +use script_bindings::conversions::SafeToJSValConvertible; use serde::{Deserialize, Serialize}; use stylo_atoms::Atom; @@ -45,7 +47,9 @@ struct RequestListener { pub enum IdbResult { Key(IndexedDBKeyType), - Data(Vec), + Keys(Vec), + Value(Vec), + Values(Vec>), Count(u64), Error(Error), None, @@ -57,9 +61,21 @@ impl From for IdbResult { } } +impl From> for IdbResult { + fn from(value: Vec) -> Self { + IdbResult::Keys(value) + } +} + impl From> for IdbResult { fn from(value: Vec) -> Self { - IdbResult::Data(value) + IdbResult::Value(value) + } +} + +impl From>> for IdbResult { + fn from(value: Vec>) -> Self { + IdbResult::Values(value) } } @@ -115,7 +131,16 @@ impl RequestListener { IdbResult::Key(key) => { key_type_to_jsval(GlobalScope::get_cx(), &key, answer.handle_mut()) }, - IdbResult::Data(serialized_data) => { + IdbResult::Keys(keys) => { + rooted_vec!(let mut array <- repeat_n(UndefinedValue(), keys.len())); + for (count, key) in keys.into_iter().enumerate() { + rooted!(in(*cx) let mut val = UndefinedValue()); + key_type_to_jsval(GlobalScope::get_cx(), &key, val.handle_mut()); + array[count] = val.get(); + } + array.safe_to_jsval(cx, answer.handle_mut()); + }, + IdbResult::Value(serialized_data) => { let result = bincode::deserialize(&serialized_data) .map_err(|_| Error::Data) .and_then(|data| structuredclone::read(&global, data, answer.handle_mut())); @@ -125,6 +150,24 @@ impl RequestListener { return; }; }, + IdbResult::Values(serialized_values) => { + rooted_vec!(let mut values <- repeat_n(UndefinedValue(), serialized_values.len())); + for (count, serialized_data) in serialized_values.into_iter().enumerate() { + rooted!(in(*cx) let mut val = UndefinedValue()); + let result = bincode::deserialize(&serialized_data) + .map_err(|_| Error::Data) + .and_then(|data| { + structuredclone::read(&global, data, val.handle_mut()) + }); + if let Err(e) = result { + warn!("Error reading structuredclone data"); + Self::handle_async_request_error(&global, cx, request, e); + return; + }; + values[count] = val.get(); + } + values.safe_to_jsval(cx, answer.handle_mut()); + }, IdbResult::Count(count) => { answer.handle_mut().set(DoubleValue(count as f64)); }, diff --git a/components/script_bindings/webidls/IDBObjectStore.webidl b/components/script_bindings/webidls/IDBObjectStore.webidl index 5d88beef407..3c4fb41c638 100644 --- a/components/script_bindings/webidls/IDBObjectStore.webidl +++ b/components/script_bindings/webidls/IDBObjectStore.webidl @@ -22,10 +22,10 @@ interface IDBObjectStore { [NewObject, Throws] IDBRequest clear(); [NewObject, Throws] IDBRequest get(any query); [NewObject, Throws] IDBRequest getKey(any query); - // [NewObject] IDBRequest getAll(optional any query, - // optional [EnforceRange] unsigned long count); - // [NewObject] IDBRequest getAllKeys(optional any query, - // optional [EnforceRange] unsigned long count); + [NewObject, Throws] IDBRequest getAll(optional any query, + optional [EnforceRange] unsigned long count); + [NewObject, Throws] IDBRequest getAllKeys(optional any query, + optional [EnforceRange] unsigned long count); [NewObject, Throws] IDBRequest count(optional any query); // [NewObject] IDBRequest openCursor(optional any query, diff --git a/components/shared/net/indexeddb_thread.rs b/components/shared/net/indexeddb_thread.rs index 5b05ccf8ab4..8d5d721b536 100644 --- a/components/shared/net/indexeddb_thread.rs +++ b/components/shared/net/indexeddb_thread.rs @@ -248,7 +248,7 @@ fn test_as_singleton() { assert!(full_range.as_singleton().is_none()); } -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] pub enum PutItemResult { Success, CannotOverwrite, @@ -266,6 +266,17 @@ pub enum AsyncReadOnlyOperation { key_range: IndexedDBKeyRange, }, + GetAllKeys { + sender: IpcSender>>, + key_range: IndexedDBKeyRange, + count: Option, + }, + GetAllItems { + sender: IpcSender>>>, + key_range: IndexedDBKeyRange, + count: Option, + }, + Count { sender: IpcSender>, key_range: IndexedDBKeyRange, diff --git a/tests/wpt/meta/IndexedDB/idbobjectstore-cross-realm-methods.html.ini b/tests/wpt/meta/IndexedDB/idbobjectstore-cross-realm-methods.html.ini index 57cf0013a93..44e6fabbe39 100644 --- a/tests/wpt/meta/IndexedDB/idbobjectstore-cross-realm-methods.html.ini +++ b/tests/wpt/meta/IndexedDB/idbobjectstore-cross-realm-methods.html.ini @@ -12,12 +12,6 @@ [Cross-realm IDBObjectStore::clear() method from detached