Conor425/ludusavi-manifest
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Minimize diffs by no longer storing revId in wiki cache

Browse source
This commit is contained in:
mtkennerly 2022-10-11 04:59:45 +08:00
parent 99bbc83494
commit 206b2f5ea9
No known key found for this signature in database
GPG key ID: E764BE00BE6E6408
2 changed files with 9 additions and 47744 deletions
Download patch file Download diff file Expand all files Collapse all files

47734
data/wiki-game-cache.yaml
View file

File diff suppressed because it is too large Load diff

19
src/wiki.ts
View file

@ -33,8 +33,7 @@ type WikiNode = string | Template | {
export type WikiGameCache = { export type WikiGameCache = {
[title: string]: { [title: string]: {
pageId: number, pageId: number,
revId: number | null, dirty?: boolean,
recentlyChanged?: boolean,
renamedFrom?: Array<string>, renamedFrom?: Array<string>,
templates?: Array<string>, templates?: Array<string>,
steam?: number, steam?: number,
@ -59,7 +58,7 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
if (v.pageId === page.pageid) { if (v.pageId === page.pageid) {
newGame = false; newGame = false;
this.data[page.title] = v; this.data[page.title] = v;
this.data[page.title].recentlyChanged = true; this.data[page.title].dirty = true;
if (!(v.renamedFrom ?? []).includes(k)) { if (!(v.renamedFrom ?? []).includes(k)) {
this.data[page.title].renamedFrom = [...(v.renamedFrom ?? []), k]; this.data[page.title].renamedFrom = [...(v.renamedFrom ?? []), k];
} }
@ -72,7 +71,7 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
} }
this.data[page.title] = { this.data[page.title] = {
pageId: page.pageid, pageId: page.pageid,
revId: null, dirty: true,
}; };
} }
}; };
@ -94,7 +93,7 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
if (!games.includes(pageTitle)) { if (!games.includes(pageTitle)) {
continue; continue;
} }
} else if (!all && this.data[pageTitle].revId !== null && !this.data[pageTitle].recentlyChanged) { } else if (!all && !this.data[pageTitle].dirty) {
continue; continue;
} }
@ -131,7 +130,7 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
} else if (this.data[recentName] !== undefined) { } else if (this.data[recentName] !== undefined) {
// Existing entry has been edited. // Existing entry has been edited.
console.log(`[E ] ${recentName}`); console.log(`[E ] ${recentName}`);
this.data[recentName].recentlyChanged = true; this.data[recentName].dirty = true;
} else { } else {
// Check for a rename. // Check for a rename.
let renamed = false; let renamed = false;
@ -143,7 +142,7 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
this.data[recentName] = { this.data[recentName] = {
...existingInfo, ...existingInfo,
pageId: recentInfo.pageId, pageId: recentInfo.pageId,
recentlyChanged: true, dirty: true,
renamedFrom: [...(existingInfo.renamedFrom ?? []), existingName] renamedFrom: [...(existingInfo.renamedFrom ?? []), existingName]
}; };
delete this.data[existingName]; delete this.data[existingName];
@ -156,7 +155,7 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
if (data.includes("Category:Games")) { if (data.includes("Category:Games")) {
// It's a game, so add it to the cache. // It's a game, so add it to the cache.
console.log(`[ C] ${recentName}`); console.log(`[ C] ${recentName}`);
this.data[recentName] = { pageId: recentInfo.pageId, revId: 0, recentlyChanged: true }; this.data[recentName] = { pageId: recentInfo.pageId, dirty: true };
} }
} }
} }
@ -496,6 +495,7 @@ function callMw<T = any>(client, method: string, ...args: Array<any>): Promise<[
} }
export async function getRecentChanges(newest: Date, oldest: Date): Promise<RecentChanges> { export async function getRecentChanges(newest: Date, oldest: Date): Promise<RecentChanges> {
console.log(`Getting recent changes from ${oldest.toISOString()} to ${newest.toISOString()}`);
const changes: RecentChanges = {}; const changes: RecentChanges = {};
const client = makeApiClient2(); const client = makeApiClient2();
const startTimestamp = newest.toISOString(); const startTimestamp = newest.toISOString();
@ -598,8 +598,7 @@ export async function getGame(pageTitle: string, cache: WikiGameCache, client: W
} }
}); });
cache[pageTitle].revId = page.revisions?.[0]?.revid ?? 0; delete cache[pageTitle].dirty;
delete cache[pageTitle].recentlyChanged;
return pageTitle; return pageTitle;
} }