Start saving raw templates to wiki cache for local reprocessing

data/wiki-game-cache.yaml

@@ -20563,6 +20563,19 @@ Celaria:
 Celeste:
   pageId: 61156
   revId: 1246544
+  templates:
+    - '{{Game data/config|Windows|{{P|game}}\Saves\settings.celeste}}'
+    - >-
+      {{Game data/config|Microsoft
+      Store|{{p|localappdata}}\Packages\MattMakesGamesInc.Celeste_79daxvg0dq3v6\SystemAppData\wgs\|{{P|localappdata}}\Packages\MattMakesGamesInc.Celeste_79daxvg0dq3v6\Settings\settings.dat}}
+    - '{{Game data/config|OS X|~/Library/Application Support/Celeste/Saves/settings.celeste}}'
+    - '{{Game data/config|Linux|{{P|xdgdatahome}}/Celeste/Saves/settings.celeste}}'
+    - '{{Game data/saves|Windows|{{P|game}}\Saves\*.celeste|{{P|game}}\Saves\debug.celeste}}'
+    - >-
+      {{Game data/saves|Microsoft
+      Store|{{p|localappdata}}\Packages\MattMakesGamesInc.Celeste_79daxvg0dq3v6\SystemAppData\wgs\}}
+    - '{{Game data/saves|OS X|~/Library/Application Support/Celeste/Saves}}'
+    - '{{Game data/saves|Linux|{{P|xdgdatahome}}/Celeste/Saves}}'
 Celestial:
   pageId: 144911
   revId: 1176987

package.json

@@ -28,6 +28,7 @@
     "moment": "^2.27.0",
     "nodemw": "^0.13.0",
     "steam-user": "^4.24.3",
-    "wikiapi": "^1.10.0"
+    "wikiapi": "^1.10.0",
+    "wikiparse": "^1.0.27"
   }
 }

src/bin.ts

@@ -1,5 +1,6 @@
 import * as minimist from "minimist";
 
+import { DEFAULT_GAME_LIMIT } from ".";
 import { ManifestFile } from "./manifest";
 import { SteamGameCacheFile, getSteamClient } from "./steam";
 import { WikiGameCacheFile, WikiMetaCacheFile } from "./wiki";
@@ -73,17 +74,18 @@ async function main() {
                 await wikiCache.flagRecentChanges(wikiMetaCache);
             } else {
                 await wikiCache.addNewGames();
+                await wikiCache.refresh(
+                    args.skipUntil,
+                    args.limit ?? DEFAULT_GAME_LIMIT,
+                );
             }
         }
 
         if (args.steam) {
             await steamCache.refresh(
-                {
+                args.skipUntil,
-                    all: args.all,
+                args.irregularPathUntagged,
-                    skipUntil: args.skipUntil,
+                args.limit ?? DEFAULT_GAME_LIMIT,
-                    irregularUntagged: args.irregularPathUntagged,
-                },
-                args.limit ?? 25,
             );
         }
 
@@ -106,7 +108,7 @@ async function main() {
                     games: args._,
                     recent: args.recent,
                 },
-                args.limit ?? 25,
+                args.limit ?? DEFAULT_GAME_LIMIT,
                 steamCache,
                 args.local,
             );

src/index.ts

@@ -4,6 +4,7 @@ import * as yaml from "js-yaml";
 
 export const REPO = pathMod.dirname(__dirname);
 export const DELAY_BETWEEN_GAMES_MS = 250;
+export const DEFAULT_GAME_LIMIT = 25;
 
 export class UnsupportedError extends Error {
     constructor(message?: string) {

src/manifest.ts

@@ -75,7 +75,7 @@ function doLaunchPathsMatch(fromSteam: string | undefined, fromManifest: string
 
 function integrateSteamData(game: Game, appInfo: SteamGameCache[""]) {
     if (appInfo.installDir !== undefined) {
-        game.installDir = {[appInfo.installDir]: {}};
+        game.installDir = { [appInfo.installDir]: {} };
     }
     if (appInfo.launch !== undefined) {
         delete game.launch;
@@ -100,7 +100,7 @@ function integrateSteamData(game: Game, appInfo: SteamGameCache[""]) {
                 "32": 32,
                 "64": 64,
             }[incoming.config?.osarch] as Bit;
-            const when: Constraint = {os, bit, store: "steam"};
+            const when: Constraint = { os, bit, store: "steam" };
             if (when.os === undefined) {
                 delete when.os;
             }
@@ -135,7 +135,7 @@ function integrateSteamData(game: Game, appInfo: SteamGameCache[""]) {
                     continue;
                 }
 
-                const candidate: Game["launch"][""][0] = {when: [when]};
+                const candidate: Game["launch"][""][0] = { when: [when] };
                 if (incoming.arguments !== undefined) {
                     candidate.arguments = incoming.arguments;
                 }

src/steam.ts

@@ -88,34 +88,24 @@ export class SteamGameCacheFile extends YamlFile<SteamGameCache> {
         return this.data[key];
     }
 
-    async refresh(filter: {all: boolean, skipUntil: string | undefined, irregularUntagged: boolean}, limit: number): Promise<void> {
+    async refresh(skipUntil: string | undefined, irregularUntagged: boolean, limit: number): Promise<void> {
         let i = 0;
         let foundSkipUntil = false;
         for (const appId of Object.keys(this.data)) {
-            if (filter.skipUntil && !foundSkipUntil) {
+            if (skipUntil && !foundSkipUntil) {
-                if (appId === filter.skipUntil) {
+                if (appId === skipUntil) {
                     foundSkipUntil = true;
                 } else {
                     continue;
                 }
             }
 
-            let check = false;
+            if (irregularUntagged) {
-            if (filter.all) {
+                const irregular = (this.data[appId].launch ?? []).some(x => this.hasIrregularKeys(x)) ||
-                check = true;
+                    this.isIrregularString(this.data[appId].installDir ?? "");
-            }
+                if (this.data[appId].irregular || !irregular) {
-            if (
+                    continue;
-                filter.irregularUntagged &&
+                }
-                !this.data[appId].irregular &&
-                (
-                    (this.data[appId].launch ?? []).some(x => this.hasIrregularKeys(x)) ||
-                    this.isIrregularString(this.data[appId].installDir ?? "")
-                )
-            ) {
-                check = true;
-            }
-            if (!check) {
-                continue;
             }
 
             console.log(`Refreshing Steam app ${appId}`)
@@ -126,7 +116,7 @@ export class SteamGameCacheFile extends YamlFile<SteamGameCache> {
                 break;
             }
 
-            // main() will save at the end, but we do a period save as well
+            // main() will save at the end, but we do a periodic save as well
             // in case something goes wrong or the script gets cancelled:
             if (i % 250 === 0) {
                 this.save();

src/wiki.ts

@@ -1,8 +1,16 @@
-import { REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from ".";
+import { DELAY_BETWEEN_GAMES_MS, REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from ".";
 import { Constraint, Game, Store, Tag, Os } from "./manifest";
 import * as moment from "moment";
 import * as NodeMw from "nodemw";
 import * as Wikiapi from "wikiapi";
+import { parse as parseWiki } from 'wikiparse';
+
+type Template = {
+    type: "template",
+    name: string,
+    parameters: {},
+    positionalParameters: Array<Template | string>,
+};
 
 export type WikiGameCache = {
     [title: string]: {
@@ -17,6 +25,7 @@ export type WikiGameCache = {
         recentlyChanged?: boolean,
         renamedFrom?: Array<string>,
         irregularPath?: boolean,
+        templates?: Array<string>,
     };
 };
 
@@ -57,6 +66,37 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
         };
     }
 
+    async refresh(skipUntil: string | undefined, limit: number): Promise<void> {
+        let i = 0;
+        let foundSkipUntil = false;
+        for (const pageTitle of Object.keys(this.data)) {
+            if (skipUntil && !foundSkipUntil) {
+                if (pageTitle === skipUntil) {
+                    foundSkipUntil = true;
+                } else {
+                    continue;
+                }
+            }
+
+            console.log(`Refreshing wiki page ${pageTitle}`)
+            await getGame(pageTitle, this.data);
+
+            i++;
+            if (limit > 0 && i >= limit) {
+                break;
+            }
+
+            // main() will save at the end, but we do a periodic save as well
+            // in case something goes wrong or the script gets cancelled:
+            if (i % 250 === 0) {
+                this.save();
+                console.log(":: saved");
+            }
+
+            await new Promise(resolve => setTimeout(resolve, DELAY_BETWEEN_GAMES_MS));
+        }
+    }
+
     async flagRecentChanges(metaCache: WikiMetaCacheFile): Promise<void> {
         const now = moment();
         const changes = await getRecentChanges(now.toDate(), moment(metaCache.data.lastCheckedRecentChanges).subtract(1, "minute").toDate());
@@ -431,7 +471,7 @@ function getConstraintFromSystem(system: string, path: string): Constraint {
 
     const storeFromPath = getStoreConstraintFromPath(path);
     if (storeFromPath !== undefined) {
-        constraint.store  = storeFromPath;
+        constraint.store = storeFromPath;
     }
 
     return constraint;
@@ -588,6 +628,7 @@ export async function getGame(pageTitle: string, cache: WikiGameCache): Promise<
     let unsupportedPath = 0;
     let tooBroad = 0;
     let irregularPath = 0;
+    delete cache[pageTitle].templates;
     page.parse().each("template", template => {
         if (template.name === "Infobox game") {
             const steamId = Number(template.parameters["steam appid"]);
@@ -595,6 +636,10 @@ export async function getGame(pageTitle: string, cache: WikiGameCache): Promise<
                 game.steam = { id: steamId };
             }
         } else if (template.name === "Game data/saves" || template.name === "Game data/config") {
+            if (cache[pageTitle].templates === undefined) {
+                cache[pageTitle].templates = [];
+            }
+            cache[pageTitle].templates.push(template.toString());
             // console.log("\n\n\n\n\n\n--------------------------------------------------------------------------")
             // console.log(template);
             for (const cellKey of Object.getOwnPropertyNames(template.parameters)) {