Import recent changes and add cache file for last timestamp

src/bin.ts

@@ -2,7 +2,7 @@ import * as minimist from "minimist";
 
 import { ManifestFile } from "./manifest";
 import { SteamGameCacheFile, getSteamClient } from "./steam";
-import { WikiGameCacheFile } from "./wiki";
+import { WikiGameCacheFile, WikiMetaCacheFile } from "./wiki";
 import { saveMissingGames } from "./missing";
 
 interface Cli {
@@ -20,7 +20,7 @@ interface Cli {
     tooBroad?: boolean,
     tooBroadUntagged?: boolean,
     skipUntil?: string,
-    recent?: number,
+    recent?: boolean,
     limit?: number,
 }
 
@@ -45,6 +45,8 @@ async function main() {
 
     const wikiCache = new WikiGameCacheFile();
     wikiCache.load();
+    const wikiMetaCache = new WikiMetaCacheFile();
+    wikiMetaCache.load();
     const steamCache = new SteamGameCacheFile(await getSteamClient());
     steamCache.load();
     const manifest = new ManifestFile();
@@ -61,7 +63,7 @@ async function main() {
     try {
         if (args.cache) {
             if (args.recent) {
-                await wikiCache.flagRecentChanges(args.recent);
+                await wikiCache.flagRecentChanges(wikiMetaCache);
             } else {
                 await wikiCache.addNewGames();
             }
@@ -91,6 +93,7 @@ async function main() {
         }
 
         wikiCache.save();
+        wikiMetaCache.save();
         steamCache.save();
         manifest.save();
         saveMissingGames(wikiCache.data, manifest.data);
@@ -98,6 +101,7 @@ async function main() {
         process.exit(0);
     } catch (e) {
         wikiCache.save();
+        wikiMetaCache.save();
         steamCache.save();
         manifest.save();
         saveMissingGames(wikiCache.data, manifest.data);

src/manifest.ts

@@ -57,7 +57,7 @@ export class ManifestFile extends YamlFile<Manifest> {
             tooBroadUntagged: boolean,
             skipUntil: string | undefined,
             games: Array<string> | undefined,
-            recent: number | undefined,
+            recent: boolean | undefined,
         },
         limit: number | undefined,
         steamCache: SteamGameCacheFile,

src/wiki.ts

@@ -1,7 +1,8 @@
 import { REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from ".";
 import { Constraint, Game, Store, Tag, Os } from "./manifest";
-import * as Wikiapi from "wikiapi";
+import * as moment from "moment";
 import * as NodeMw from "nodemw";
+import * as Wikiapi from "wikiapi";
 
 export type WikiGameCache = {
     [title: string]: {
@@ -19,6 +20,10 @@ export type WikiGameCache = {
     };
 };
 
+export type WikiMetaCache = {
+    lastCheckedRecentChanges: string;
+};
+
 export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
     path = `${REPO}/data/wiki-game-cache.yaml`;
     defaultData = {};
@@ -36,9 +41,11 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
         };
     }
 
-    async flagRecentChanges(days: number): Promise<void> {
-        const changes = await getRecentChanges(days);
+    async flagRecentChanges(metaCache: WikiMetaCacheFile): Promise<void> {
+        const now = moment();
+        const changes = await getRecentChanges(now.toDate(), moment(metaCache.data.lastCheckedRecentChanges).subtract(1, "minute").toDate());
         const client = makeApiClient2();
+
         for (const [recentName, recentInfo] of Object.entries(changes).sort((x, y) => x[0].localeCompare(y[0]))) {
             if (this.data[recentName] !== undefined) {
                 // Existing entry has been edited.
@@ -73,9 +80,18 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
                 }
             }
         }
+
+        metaCache.data.lastCheckedRecentChanges = now.toISOString();
     }
 }
 
+export class WikiMetaCacheFile extends YamlFile<WikiMetaCache> {
+    path = `${REPO}/data/wiki-meta-cache.yaml`;
+    defaultData = {
+        lastCheckedRecentChanges: moment().subtract(7, "days").toISOString(),
+    };
+}
+
 interface RecentChanges {
     [article: string]: {
         pageId: number;
@@ -418,11 +434,11 @@ function callMw<T = any>(client, method: string, ...args: Array<any>): Promise<[
     });
 }
 
-export async function getRecentChanges(days: number): Promise<RecentChanges> {
+export async function getRecentChanges(newest: Date, oldest: Date): Promise<RecentChanges> {
     const changes: RecentChanges = {};
     const client = makeApiClient2();
-    const startTimestamp = new Date().toISOString();
-    const endTimestamp = new Date(new Date().setDate(new Date().getDate() - days)).toISOString();
+    const startTimestamp = newest.toISOString();
+    const endTimestamp = oldest.toISOString();
     let rccontinue: string | undefined = undefined;
 
     while (true) {