Allow bulk updates with a limit and starting point

src/cli.rs

@@ -34,6 +34,10 @@ pub enum Subcommand {
         #[clap(long)]
         full: bool,
 
+        /// Only refresh this many entries.
+        #[clap(long)]
+        limit: Option<usize>,
+
         /// Do a partial update based on the wiki's recent changes.
         #[clap(long)]
         recent_changes: bool,
@@ -41,6 +45,16 @@ pub enum Subcommand {
         /// Do a partial update based on the wiki's game pages that are not yet cached.
         #[clap(long)]
         missing_pages: bool,
+
+        /// Refresh wiki entries starting from this article title.
+        /// This will enable full mode for wiki entries.
+        #[clap(long)]
+        wiki_from: Option<String>,
+
+        /// Refresh Steam entries starting from this app ID.
+        /// This will enable full mode for Steam entries.
+        #[clap(long)]
+        steam_from: Option<u32>,
     },
     /// Fetch a named subset of games.
     Solo {
@@ -78,23 +92,24 @@ pub async fn run(
     match sub {
         Subcommand::Bulk {
             full,
+            limit,
             recent_changes,
             missing_pages,
+            wiki_from,
+            steam_from,
         } => {
-            let outdated_only = !full;
+            let outdated_only = !full && wiki_from.is_none();
-
             if recent_changes {
                 wiki_cache.flag_recent_changes(wiki_meta_cache).await?;
             }
-
             if missing_pages {
                 wiki_cache.add_new_games().await?;
             }
+            wiki_cache.refresh(outdated_only, None, limit, wiki_from).await?;
 
-            wiki_cache.refresh(outdated_only, None).await?;
+            let outdated_only = !full && steam_from.is_none();
-
             steam_cache.transition_states_from(wiki_cache);
-            steam_cache.refresh(outdated_only, None)?;
+            steam_cache.refresh(outdated_only, None, limit, steam_from)?;
 
             manifest.refresh(manifest_override, wiki_cache, steam_cache, None)?;
             schema::validate_manifest(manifest)?;
@@ -103,7 +118,9 @@ pub async fn run(
             let outdated_only = false;
 
             if !local {
-                wiki_cache.refresh(outdated_only, Some(games.clone())).await?;
+                wiki_cache
+                    .refresh(outdated_only, Some(games.clone()), None, None)
+                    .await?;
 
                 let steam_ids: Vec<_> = games
                     .iter()
@@ -111,7 +128,7 @@ pub async fn run(
                     .collect();
 
                 steam_cache.transition_states_from(wiki_cache);
-                steam_cache.refresh(outdated_only, Some(steam_ids))?;
+                steam_cache.refresh(outdated_only, Some(steam_ids), None, None)?;
             }
 
             manifest.refresh(manifest_override, wiki_cache, steam_cache, Some(games))?;

src/steam.rs

@@ -12,12 +12,20 @@ impl ResourceFile for SteamCache {
 }
 
 impl SteamCache {
-    pub fn refresh(&mut self, outdated_only: bool, app_ids: Option<Vec<u32>>) -> Result<(), Error> {
+    pub fn refresh(
+        &mut self,
+        outdated_only: bool,
+        app_ids: Option<Vec<u32>>,
+        limit: Option<usize>,
+        from: Option<u32>,
+    ) -> Result<(), Error> {
         let mut i = 0;
         let app_ids: Vec<_> = app_ids.unwrap_or_else(|| {
             self.0
                 .iter()
                 .filter(|(_, v)| !outdated_only || v.state == State::Outdated)
+                .skip_while(|(k, _)| from.is_some_and(|from| &from != *k))
+                .take(limit.unwrap_or(usize::MAX))
                 .map(|(k, _)| *k)
                 .collect()
         });

src/wiki.rs

@@ -256,10 +256,23 @@ impl WikiCache {
         Ok(())
     }
 
-    pub async fn refresh(&mut self, outdated_only: bool, titles: Option<Vec<String>>) -> Result<(), Error> {
+    pub async fn refresh(
+        &mut self,
+        outdated_only: bool,
+        titles: Option<Vec<String>>,
+        limit: Option<usize>,
+        from: Option<String>,
+    ) -> Result<(), Error> {
         let mut i = 0;
         let solo = titles.is_some();
-        let titles: Vec<_> = titles.unwrap_or_else(|| self.0.keys().cloned().collect());
+        let titles: Vec<_> = titles.unwrap_or_else(|| {
+            self.0
+                .keys()
+                .skip_while(|x| from.as_ref().is_some_and(|from| from != *x))
+                .take(limit.unwrap_or(usize::MAX))
+                .cloned()
+                .collect()
+        });
 
         for title in &titles {
             let cached = self.0.get(title).cloned().unwrap_or_default();