Incorporate the wiki's Recent Changes API and import data from the last 30 days

src/bin.ts

@@ -18,6 +18,7 @@ interface Cli {
     tooBroad?: boolean,
     tooBroadUntagged?: boolean,
     game?: string,
+    recent?: number,
     limit?: number,
 }
 
@@ -41,7 +42,11 @@ async function main() {
 
     try {
         if (args.cache) {
-            await wikiCache.addNewGames(manifest.data);
+            if (args.recent) {
+                await wikiCache.flagRecentChanges(args.recent);
+            } else {
+                await wikiCache.addNewGames();
+            }
         }
 
         if (args.manifest) {
@@ -57,6 +62,7 @@ async function main() {
                     tooBroad: args.tooBroad ?? false,
                     tooBroadUntagged: args.tooBroadUntagged ?? false,
                     game: args.game,
+                    recent: args.recent,
                 },
                 args.limit ?? 25,
                 steamCache,

src/manifest.ts

@@ -54,8 +54,9 @@ export class ManifestFile extends YamlFile<Manifest> {
             tooBroad: boolean,
             tooBroadUntagged: boolean,
             game: string | undefined,
+            recent: number | undefined,
         },
-        limit: number,
+        limit: number | undefined,
         steamCache: SteamGameCacheFile,
     ): Promise<void> {
         let i = 0;
@@ -88,16 +89,26 @@ export class ManifestFile extends YamlFile<Manifest> {
             if (filter.tooBroadUntagged && Object.keys(this.data[title]?.files ?? []).some(x => pathIsTooBroad(x))) {
                 check = true;
             }
+            if (filter.recent && wikiCache[title].recentlyChanged) {
+                check = true;
+            }
             if (!check) {
                 continue;
             }
 
             i++;
-            if (i > limit) {
+            if (limit > 0 && i > limit) {
                 break;
             }
 
+            if (info.renamedFrom) {
+                for (const oldName of info.renamedFrom) {
+                    delete this.data[oldName];
+                }
+            }
+
             const game = await getGame(title, wikiCache);
+            wikiCache[title].recentlyChanged = false;
             if (game.files === undefined && game.registry === undefined && game.steam?.id === undefined) {
                 delete this.data[title];
                 continue;

src/wiki.ts

@@ -1,6 +1,7 @@
 import { REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from ".";
-import { Manifest, Constraint, Game, Store, Tag, Os } from "./manifest";
+import { Constraint, Game, Store, Tag, Os } from "./manifest";
 import * as Wikiapi from "wikiapi";
+import * as NodeMw from "nodemw";
 
 export type WikiGameCache = {
     [title: string]: {
@@ -12,6 +13,8 @@ export type WikiGameCache = {
         unsupportedPath?: boolean,
         /** Whether an entry has a path that is too broad (e.g., the entirety of %WINDIR%). */
         tooBroad?: boolean,
+        recentlyChanged?: boolean,
+        renamedFrom?: Array<string>,
     };
 };
 
@@ -19,7 +22,7 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
     path = `${REPO}/data/wiki-game-cache.yaml`;
     defaultData = {};
 
-    async addNewGames(manifest: Manifest): Promise<void> {
+    async addNewGames(): Promise<void> {
         const wiki = makeApiClient();
         const pages: Array<{ pageid: number, title: string }> = JSON.parse(JSON.stringify(await wiki.categorymembers("Games")));
         for (const page of pages) {
@@ -31,6 +34,51 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
             }
         };
     }
+
+    async flagRecentChanges(days: number): Promise<void> {
+        const changes = await getRecentChanges(days);
+        const client = makeApiClient2();
+        for (const [recentName, recentInfo] of Object.entries(changes).sort((x, y) => x[0].localeCompare(y[0]))) {
+            if (this.data[recentName] !== undefined) {
+                // Existing entry has been edited.
+                console.log(`[E  ] ${recentName}`);
+                this.data[recentName].recentlyChanged = true;
+            } else {
+                // Check for a rename.
+                let renamed = false;
+                for (const [existingName, existingInfo] of Object.entries(this.data)) {
+                    if (existingInfo.pageId === recentInfo.pageId) {
+                        // We have a confirmed rename.
+                        console.log(`[ M ] ${recentName} <<< ${existingName}`);
+                        renamed = true;
+                        this.data[recentName] = {
+                            pageId: recentInfo.pageId,
+                            revId: existingInfo.revId,
+                            recentlyChanged: true,
+                            renamedFrom: [...(existingInfo.renamedFrom ?? []), existingName]
+                        };
+                        delete this.data[existingName];
+                        break;
+                    }
+                }
+                if (!renamed) {
+                    // Brand new page.
+                    const [data, _] = await callMw<Array<string>>(client, "getArticleCategories", recentName);
+                    if (data.includes("Category:Games")) {
+                        // It's a game, so add it to the cache.
+                        console.log(`[  C] ${recentName}`);
+                        this.data[recentName] = { pageId: recentInfo.pageId, revId: 0, recentlyChanged: true };
+                    }
+                }
+            }
+        }
+    }
+}
+
+interface RecentChanges {
+    [article: string]: {
+        pageId: number;
+    };
 }
 
 // This defines how {{P|game}} and such are converted.
@@ -263,10 +311,81 @@ function parseOs(os: string): Os {
     }
 }
 
+// Used for most functionality, but it seems like a less active project
+// and it's hard to figure out what functionality is available,
+// so we'll probably migrate to nodemw.
 function makeApiClient() {
     return new Wikiapi("https://www.pcgamingwiki.com/w");
 }
 
+// Used for the Recent Changes page and getting a single page's categories.
+// Will probably also migrate to this in general.
+function makeApiClient2(): any {
+    return new NodeMw({
+        protocol: "https",
+        server: "www.pcgamingwiki.com",
+        path: "/w",
+        debug: false,
+        userAgent: "ludusavi-manifest-importer/0.0.0",
+        concurrency: 1,
+    });
+}
+
+// Promise wrapper for nodemw.
+function callMw<T = any>(client, method: string, ...args: Array<any>): Promise<[T, any]> {
+    return new Promise((resolve, reject) => {
+        client[method](...args, (err: any, data: T, next: any) => {
+            if (err) {
+                reject(err);
+            } else {
+                resolve([data, next]);
+            }
+        });
+    });
+}
+
+export async function getRecentChanges(days: number): Promise<RecentChanges> {
+    const changes: RecentChanges = {};
+    const client = makeApiClient2();
+    const startTimestamp = new Date().toISOString();
+    const endTimestamp = new Date(new Date().setDate(new Date().getDate() - days)).toISOString();
+    let rccontinue: string | undefined = undefined;
+
+    while (true) {
+        const params = {
+            action: "query",
+            list: "recentchanges",
+            rcprop: "title|ids",
+            rcstart: startTimestamp,
+            rcend: endTimestamp,
+            rclimit: 500,
+            rcnamespace: 0,
+            rctype: "edit|new",
+            rccontinue,
+        };
+        if (params.rccontinue === undefined) {
+            delete params.rccontinue;
+        }
+        const [data, next] = await callMw<{ recentchanges: Array<{ title: string; pageid: number }> }>(
+            client.api, "call", params
+        );
+
+        for (const article of data.recentchanges) {
+            changes[article.title] = {
+                pageId: article.pageid,
+            };
+        }
+
+        if (next) {
+            rccontinue = next.rccontinue;
+        } else {
+            break;
+        }
+    }
+
+    return changes;
+}
+
 /**
  * https://www.pcgamingwiki.com/wiki/Template:Game_data
  */