Conor425/ludusavi-manifest
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Start saving raw templates to wiki cache for local reprocessing

Browse source
This commit is contained in:
mtkennerly 2022-06-28 16:04:39 +08:00
parent 1ad1824975
commit af8fbd1a42
No known key found for this signature in database
GPG key ID: E764BE00BE6E6408
8 changed files with 1765 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

13
data/wiki-game-cache.yaml
View file

@ -20563,6 +20563,19 @@ Celaria:
Celeste:
pageId: 61156
revId: 1246544
templates:
- '{{Game data/config|Windows|{{P|game}}\Saves\settings.celeste}}'
- >-
{{Game data/config|Microsoft
Store|{{p|localappdata}}\Packages\MattMakesGamesInc.Celeste_79daxvg0dq3v6\SystemAppData\wgs\|{{P|localappdata}}\Packages\MattMakesGamesInc.Celeste_79daxvg0dq3v6\Settings\settings.dat}}
- '{{Game data/config|OS X|~/Library/Application Support/Celeste/Saves/settings.celeste}}'
- '{{Game data/config|Linux|{{P|xdgdatahome}}/Celeste/Saves/settings.celeste}}'
- '{{Game data/saves|Windows|{{P|game}}\Saves\*.celeste|{{P|game}}\Saves\debug.celeste}}'
- >-
{{Game data/saves|Microsoft
Store|{{p|localappdata}}\Packages\MattMakesGamesInc.Celeste_79daxvg0dq3v6\SystemAppData\wgs\}}
- '{{Game data/saves|OS X|~/Library/Application Support/Celeste/Saves}}'
- '{{Game data/saves|Linux|{{P|xdgdatahome}}/Celeste/Saves}}'
Celestial:
pageId: 144911
revId: 1176987

1682
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

3
package.json
View file

@ -28,6 +28,7 @@
"moment": "^2.27.0",
"nodemw": "^0.13.0",
"steam-user": "^4.24.3",
"wikiapi": "^1.10.0"
"wikiapi": "^1.10.0",
"wikiparse": "^1.0.27"
}
}

16
src/bin.ts
View file

@ -1,5 +1,6 @@
import * as minimist from "minimist";
import { DEFAULT_GAME_LIMIT } from ".";
import { ManifestFile } from "./manifest";
import { SteamGameCacheFile, getSteamClient } from "./steam";
import { WikiGameCacheFile, WikiMetaCacheFile } from "./wiki";
@ -73,17 +74,18 @@ async function main() {
await wikiCache.flagRecentChanges(wikiMetaCache);
} else {
await wikiCache.addNewGames();
await wikiCache.refresh(
args.skipUntil,
args.limit ?? DEFAULT_GAME_LIMIT,
);
}
}
if (args.steam) {
await steamCache.refresh(
{
all: args.all,
skipUntil: args.skipUntil,
irregularUntagged: args.irregularPathUntagged,
},
args.limit ?? 25,
args.skipUntil,
args.irregularPathUntagged,
args.limit ?? DEFAULT_GAME_LIMIT,
);
}
@ -106,7 +108,7 @@ async function main() {
games: args._,
recent: args.recent,
},
args.limit ?? 25,
args.limit ?? DEFAULT_GAME_LIMIT,
steamCache,
args.local,
);

1
src/index.ts
View file

@ -4,6 +4,7 @@ import * as yaml from "js-yaml";
export const REPO = pathMod.dirname(__dirname);
export const DELAY_BETWEEN_GAMES_MS = 250;
export const DEFAULT_GAME_LIMIT = 25;
export class UnsupportedError extends Error {
constructor(message?: string) {

6
src/manifest.ts
View file

@ -75,7 +75,7 @@ function doLaunchPathsMatch(fromSteam: string | undefined, fromManifest: string
function integrateSteamData(game: Game, appInfo: SteamGameCache[""]) {
if (appInfo.installDir !== undefined) {
game.installDir = {[appInfo.installDir]: {}};
game.installDir = { [appInfo.installDir]: {} };
}
if (appInfo.launch !== undefined) {
delete game.launch;
@ -100,7 +100,7 @@ function integrateSteamData(game: Game, appInfo: SteamGameCache[""]) {
"32": 32,
"64": 64,
}[incoming.config?.osarch] as Bit;
const when: Constraint = {os, bit, store: "steam"};
const when: Constraint = { os, bit, store: "steam" };
if (when.os === undefined) {
delete when.os;
}
@ -135,7 +135,7 @@ function integrateSteamData(game: Game, appInfo: SteamGameCache[""]) {
continue;
}
const candidate: Game["launch"][""][0] = {when: [when]};
const candidate: Game["launch"][""][0] = { when: [when] };
if (incoming.arguments !== undefined) {
candidate.arguments = incoming.arguments;
}

30
src/steam.ts
View file

@ -88,34 +88,24 @@ export class SteamGameCacheFile extends YamlFile<SteamGameCache> {
return this.data[key];
}
async refresh(filter: {all: boolean, skipUntil: string | undefined, irregularUntagged: boolean}, limit: number): Promise<void> {
async refresh(skipUntil: string | undefined, irregularUntagged: boolean, limit: number): Promise<void> {
let i = 0;
let foundSkipUntil = false;
for (const appId of Object.keys(this.data)) {
if (filter.skipUntil && !foundSkipUntil) {
if (appId === filter.skipUntil) {
if (skipUntil && !foundSkipUntil) {
if (appId === skipUntil) {
foundSkipUntil = true;
} else {
continue;
}
}
let check = false;
if (filter.all) {
check = true;
}
if (
filter.irregularUntagged &&
!this.data[appId].irregular &&
(
(this.data[appId].launch ?? []).some(x => this.hasIrregularKeys(x)) ||
this.isIrregularString(this.data[appId].installDir ?? "")
)
) {
check = true;
}
if (!check) {
continue;
if (irregularUntagged) {
const irregular = (this.data[appId].launch ?? []).some(x => this.hasIrregularKeys(x)) ||
this.isIrregularString(this.data[appId].installDir ?? "");
if (this.data[appId].irregular || !irregular) {
continue;
}
}
console.log(`Refreshing Steam app ${appId}`)
@ -126,7 +116,7 @@ export class SteamGameCacheFile extends YamlFile<SteamGameCache> {
break;
}
// main() will save at the end, but we do a period save as well
// main() will save at the end, but we do a periodic save as well
// in case something goes wrong or the script gets cancelled:
if (i % 250 === 0) {
this.save();

49
src/wiki.ts
View file

@ -1,8 +1,16 @@
import { REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from ".";
import { DELAY_BETWEEN_GAMES_MS, REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from ".";
import { Constraint, Game, Store, Tag, Os } from "./manifest";
import * as moment from "moment";
import * as NodeMw from "nodemw";
import * as Wikiapi from "wikiapi";
import { parse as parseWiki } from 'wikiparse';
type Template = {
type: "template",
name: string,
parameters: {},
positionalParameters: Array<Template | string>,
};
export type WikiGameCache = {
[title: string]: {
@ -17,6 +25,7 @@ export type WikiGameCache = {
recentlyChanged?: boolean,
renamedFrom?: Array<string>,
irregularPath?: boolean,
templates?: Array<string>,
};
};
@ -57,6 +66,37 @@ export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
};
}
async refresh(skipUntil: string | undefined, limit: number): Promise<void> {
let i = 0;
let foundSkipUntil = false;
for (const pageTitle of Object.keys(this.data)) {
if (skipUntil && !foundSkipUntil) {
if (pageTitle === skipUntil) {
foundSkipUntil = true;
} else {
continue;
}
}
console.log(`Refreshing wiki page ${pageTitle}`)
await getGame(pageTitle, this.data);
i++;
if (limit > 0 && i >= limit) {
break;
}
// main() will save at the end, but we do a periodic save as well
// in case something goes wrong or the script gets cancelled:
if (i % 250 === 0) {
this.save();
console.log(":: saved");
}
await new Promise(resolve => setTimeout(resolve, DELAY_BETWEEN_GAMES_MS));
}
}
async flagRecentChanges(metaCache: WikiMetaCacheFile): Promise<void> {
const now = moment();
const changes = await getRecentChanges(now.toDate(), moment(metaCache.data.lastCheckedRecentChanges).subtract(1, "minute").toDate());
@ -431,7 +471,7 @@ function getConstraintFromSystem(system: string, path: string): Constraint {
const storeFromPath = getStoreConstraintFromPath(path);
if (storeFromPath !== undefined) {
constraint.store = storeFromPath;
constraint.store = storeFromPath;
}
return constraint;
@ -588,6 +628,7 @@ export async function getGame(pageTitle: string, cache: WikiGameCache): Promise<
let unsupportedPath = 0;
let tooBroad = 0;
let irregularPath = 0;
delete cache[pageTitle].templates;
page.parse().each("template", template => {
if (template.name === "Infobox game") {
const steamId = Number(template.parameters["steam appid"]);
@ -595,6 +636,10 @@ export async function getGame(pageTitle: string, cache: WikiGameCache): Promise<
game.steam = { id: steamId };
}
} else if (template.name === "Game data/saves" || template.name === "Game data/config") {
if (cache[pageTitle].templates === undefined) {
cache[pageTitle].templates = [];
}
cache[pageTitle].templates.push(template.toString());
// console.log("\n\n\n\n\n\n--------------------------------------------------------------------------")
// console.log(template);
for (const cellKey of Object.getOwnPropertyNames(template.parameters)) {