import { REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from "."; import { Constraint, Game, Store, Tag, Os } from "./manifest"; import * as Wikiapi from "wikiapi"; import * as NodeMw from "nodemw"; export type WikiGameCache = { [title: string]: { pageId: number, revId: number | null, /** Whether an entry on the page failed because of an unsupported OS. */ unsupportedOs?: boolean, /** Whether an entry on the page failed because of an unsupported Path template argument. */ unsupportedPath?: boolean, /** Whether an entry has a path that is too broad (e.g., the entirety of %WINDIR%). */ tooBroad?: boolean, recentlyChanged?: boolean, renamedFrom?: Array, irregularPath?: boolean, }; }; export class WikiGameCacheFile extends YamlFile { path = `${REPO}/data/wiki-game-cache.yaml`; defaultData = {}; async addNewGames(): Promise { const wiki = makeApiClient(); const pages: Array<{ pageid: number, title: string }> = JSON.parse(JSON.stringify(await wiki.categorymembers("Games"))); for (const page of pages) { if (!this.data.hasOwnProperty(page.title)) { this.data[page.title] = { pageId: page.pageid, revId: null, }; } }; } async flagRecentChanges(days: number): Promise { const changes = await getRecentChanges(days); const client = makeApiClient2(); for (const [recentName, recentInfo] of Object.entries(changes).sort((x, y) => x[0].localeCompare(y[0]))) { if (this.data[recentName] !== undefined) { // Existing entry has been edited. console.log(`[E ] ${recentName}`); this.data[recentName].recentlyChanged = true; } else { // Check for a rename. let renamed = false; for (const [existingName, existingInfo] of Object.entries(this.data)) { if (existingInfo.pageId === recentInfo.pageId) { // We have a confirmed rename. console.log(`[ M ] ${recentName} <<< ${existingName}`); renamed = true; this.data[recentName] = { pageId: recentInfo.pageId, revId: existingInfo.revId, recentlyChanged: true, renamedFrom: [...(existingInfo.renamedFrom ?? []), existingName] }; delete this.data[existingName]; break; } } if (!renamed) { // Brand new page. const [data, _] = await callMw>(client, "getArticleCategories", recentName); if (data.includes("Category:Games")) { // It's a game, so add it to the cache. console.log(`[ C] ${recentName}`); this.data[recentName] = { pageId: recentInfo.pageId, revId: 0, recentlyChanged: true }; } } } } } } interface RecentChanges { [article: string]: { pageId: number; }; } // This defines how {{P|game}} and such are converted. const PATH_ARGS: { [arg: string]: { mapped: string, when?: Constraint, registry?: boolean, ignored?: boolean } } = { game: { mapped: "", }, uid: { mapped: "", }, steam: { mapped: "", when: { store: "steam", }, }, uplay: { mapped: "", when: { store: "uplay" }, }, hkcu: { mapped: "HKEY_CURRENT_USER", when: { os: "windows" }, registry: true, }, hklm: { mapped: "HKEY_LOCAL_MACHINE", when: { os: "windows" }, registry: true, }, wow64: { mapped: "", when: { os: "windows" }, registry: true, ignored: true, }, username: { mapped: "", when: { os: "windows" }, }, userprofile: { mapped: "", when: { os: "windows" }, }, "userprofile\\documents": { mapped: "", when: { os: "windows" }, }, appdata: { mapped: "", when: { os: "windows" }, }, localappdata: { mapped: "", when: { os: "windows" }, }, public: { mapped: "", when: { os: "windows" }, }, allusersprofile: { mapped: "", when: { os: "windows" }, }, programdata: { mapped: "", when: { os: "windows" }, }, windir: { mapped: "", when: { os: "windows" }, }, syswow64: { mapped: "/SysWOW64", when: { os: "windows" }, }, osxhome: { mapped: "", when: { os: "mac" }, }, linuxhome: { mapped: "", when: { os: "linux" }, }, xdgdatahome: { mapped: "", when: { os: "linux" }, }, xdgconfighome: { mapped: "", when: { os: "linux" }, }, } function makePathArgRegex(arg: string): RegExp { const escaped = `{{P(ath)?|${arg}}}` .replace("\\", "\\\\") .replace("|", "\\|") .replace("{", "\\{") .replace("}", "\\}"); return new RegExp(escaped, "gi"); } interface PathCell { [index: number]: string | PathCell; type: "comment" | "transclusion" | "page_title" | "plain"; parameters: { [key: string]: any }; // keys are numbers as strings toString(): string; } function stringifyTransclusionSegment(segment: PathCell): [string, boolean] { const templateName = segment[0][0] as string; switch (templateName.toLowerCase()) { case "p": case "path": return [`{{${templateName}|${segment[1]}}}`, true]; case "code": case "file": return ["*", false]; case "localizedpath": return [segment[1] as string, false]; default: return ["", false]; } } function getRawPathFromCell(cell: string | PathCell): [string, boolean] { let composite = ""; let regular = true; if (typeof cell === "string") { composite += cell; } else if (cell.type === "transclusion") { const [stringified, segmentRegular] = stringifyTransclusionSegment(cell); if (!segmentRegular) { regular = false; } composite += stringified; } else if (cell.type === "plain") { for (let i = 0; i < 50; i++) { const segment = cell[i]; if (segment === undefined) { break; } if (typeof segment === "string") { composite += segment; } else if (segment.type === "transclusion") { const [stringified, segmentRegular] = stringifyTransclusionSegment(segment); if (!segmentRegular) { regular = false; } composite += stringified; } } } return [composite.replace(/.*?<\ref>/, "").trim(), regular]; } /** * https://www.pcgamingwiki.com/wiki/Template:Path */ function parsePath(path: string): [string, PathType] { const pathType = getPathType(path); for (const [arg, info] of Object.entries(PATH_ARGS)) { if (pathContainsArg(path, arg) && info.ignored) { throw new UnsupportedPathError(`Unsupported path argument: ${arg}`); } let limit = 100; let i = 0; while (pathContainsArg(path, arg)) { path = path.replace(makePathArgRegex(arg), info.mapped); i++; if (i >= limit) { throw new UnsupportedPathError(`Unable to resolve path arguments in: ${path}`); } } } return [ path .replace(/\\/g, "/") .replace(/\/(?=$)/g, "") .replace(/^~(?=($|\/))/, ""), pathType, ]; } export function pathIsTooBroad(path: string): boolean { if (Object.values(PATH_ARGS).map(x => x.mapped).includes(path)) { return true; } // TODO: These paths are present whether or not the game is installed. // If possible, they should be narrowed down on the wiki. if ([ "/Documents", "/Saved Games", "/config", "/win.ini", ].includes(path)) { return true; } return false; } function pathContainsArg(path: string, arg: string): boolean { return path.match(makePathArgRegex(arg)) !== null; } function getPathType(path: string): PathType { for (const [arg, info] of Object.entries(PATH_ARGS)) { if (info.registry && path.match(makePathArgRegex(arg)) !== null) { return PathType.Registry; } } return PathType.FileSystem; } function getOsConstraintFromPath(path: string): Os | undefined { for (const [arg, info] of Object.entries(PATH_ARGS)) { if (pathContainsArg(path, arg) && info?.when?.os) { return info?.when?.os; } } } function getStoreConstraintFromPath(path: string): Store | undefined { for (const [arg, info] of Object.entries(PATH_ARGS)) { if (pathContainsArg(path, arg) && info?.when?.store) { return info?.when?.store; } } } function getConstraintFromSystem(system: string, path: string): Constraint { const constraint: Constraint = {}; if (system.match(/steam/i)) { constraint.store = "steam"; } else if (system.match(/microsoft store/i)) { constraint.os = "windows"; constraint.store = "microsoft"; } else if (system.match(/gog\.com/i)) { constraint.store = "gog"; } else if (system.match(/epic games/i)) { constraint.store = "epic"; } else if (system.match(/uplay/i)) { constraint.store = "uplay"; } else { constraint.os = parseOs(system); constraint.store = getStoreConstraintFromPath(path); } return constraint; } function getTagFromTemplate(template: string): Tag | undefined { switch (template) { case "Game data/saves": return "save"; case "Game data/config": return "config"; default: return undefined; } } function parseOs(os: string): Os { switch (os) { case "Windows": return "windows"; case "OS X": return "mac"; case "Linux": return "linux"; case "DOS": return "dos"; default: throw new UnsupportedOsError(`Unsupported OS: ${os}`); } } // Used for most functionality, but it seems like a less active project // and it's hard to figure out what functionality is available, // so we'll probably migrate to nodemw. function makeApiClient() { return new Wikiapi("https://www.pcgamingwiki.com/w"); } // Used for the Recent Changes page and getting a single page's categories. // Will probably also migrate to this in general. function makeApiClient2(): any { return new NodeMw({ protocol: "https", server: "www.pcgamingwiki.com", path: "/w", debug: false, userAgent: "ludusavi-manifest-importer/0.0.0", concurrency: 1, }); } // Promise wrapper for nodemw. function callMw(client, method: string, ...args: Array): Promise<[T, any]> { return new Promise((resolve, reject) => { client[method](...args, (err: any, data: T, next: any) => { if (err) { reject(err); } else { resolve([data, next]); } }); }); } export async function getRecentChanges(days: number): Promise { const changes: RecentChanges = {}; const client = makeApiClient2(); const startTimestamp = new Date().toISOString(); const endTimestamp = new Date(new Date().setDate(new Date().getDate() - days)).toISOString(); let rccontinue: string | undefined = undefined; while (true) { const params = { action: "query", list: "recentchanges", rcprop: "title|ids", rcstart: startTimestamp, rcend: endTimestamp, rclimit: 500, rcnamespace: 0, rctype: "edit|new", rccontinue, }; if (params.rccontinue === undefined) { delete params.rccontinue; } const [data, next] = await callMw<{ recentchanges: Array<{ title: string; pageid: number }> }>( client.api, "call", params ); for (const article of data.recentchanges) { changes[article.title] = { pageId: article.pageid, }; } if (next) { rccontinue = next.rccontinue; } else { break; } } return changes; } /** * https://www.pcgamingwiki.com/wiki/Template:Game_data */ export async function getGame(pageTitle: string, cache: WikiGameCache): Promise { console.log(pageTitle); const wiki = makeApiClient(); const page = await wiki.page(pageTitle, { rvprop: "ids|content" }); const game: Game = { files: {}, registry: {}, }; let unsupportedOs = 0; let unsupportedPath = 0; let tooBroad = 0; let irregularPath = 0; page.parse().each("template", template => { if (template.name === "Infobox game") { const steamId = Number(template.parameters["steam appid"]); if (!isNaN(steamId) && steamId > 0) { game.steam = { id: steamId }; } } else if (template.name === "Game data/saves" || template.name === "Game data/config") { // console.log("\n\n\n\n\n\n--------------------------------------------------------------------------") // console.log(template); for (const cellKey of Object.getOwnPropertyNames(template.parameters)) { if (cellKey === "0" || cellKey === "1") { continue; } const cell = template.parameters[cellKey]; // console.log("======================================") // console.log(cell) const [rawPath, regular] = getRawPathFromCell(cell); // console.log("-----------------"); // console.log(`${regular} : ${rawPath}`); if (!regular) { irregularPath += 1; } if (rawPath.length === 0) { continue; } try { const [path, pathType] = parsePath(rawPath); if (pathIsTooBroad(path)) { tooBroad += 1; continue; } if (pathType === PathType.FileSystem) { const constraint = getConstraintFromSystem(template.parameters[1], rawPath); if (!game.files.hasOwnProperty(path)) { game.files[path] = { when: [], tags: [], }; } if (!game.files[path].when.some(x => x.os === constraint.os && x.store === constraint.store)) { if (constraint.os !== undefined && constraint.store !== undefined) { game.files[path].when.push(constraint); } else if (constraint.os !== undefined) { game.files[path].when.push({ os: constraint.os }); } else if (constraint.store !== undefined) { game.files[path].when.push({ store: constraint.store }); } } const tag = getTagFromTemplate(template.name); if (tag !== undefined && !game.files[path].tags.includes(tag)) { game.files[path].tags.push(tag); } } else if (pathType === PathType.Registry) { if (!game.registry.hasOwnProperty(path)) { game.registry[path] = { when: [], tags: [], }; } const store = getStoreConstraintFromPath(rawPath); if (store !== undefined && !game.registry[path].when.some(x => x.store === store)) { game.registry[path].when.push({ store }); } const tag = getTagFromTemplate(template.name); if (tag !== undefined && !game.registry[path].tags.includes(tag)) { game.registry[path].tags.push(tag); } } } catch (e) { console.log(` ${template.toString()}`); console.log(` ${e}`); if (e instanceof UnsupportedOsError) { unsupportedOs += 1; continue; } else if (e instanceof UnsupportedPathError) { unsupportedPath += 1; continue; } else { continue; } } } } }); if (Object.keys(game.files).length === 0) { delete game.files; } else { for (const path of Object.keys(game.files)) { if (game.files[path].when.length === 0) { delete game.files[path].when; } if (game.files[path].tags.length === 0) { delete game.files[path].tags; } } } if (Object.keys(game.registry).length === 0) { delete game.registry; } else { for (const path of Object.keys(game.registry)) { if (game.registry[path].when.length === 0) { delete game.registry[path].when; } if (game.registry[path].tags.length === 0) { delete game.registry[path].tags; } } } if (unsupportedOs > 0) { cache[pageTitle].unsupportedOs = true; } else { delete cache[pageTitle].unsupportedOs; } if (unsupportedPath > 0) { cache[pageTitle].unsupportedPath = true; } else { delete cache[pageTitle].unsupportedPath; } if (tooBroad > 0) { cache[pageTitle].tooBroad = true; } else { delete cache[pageTitle].tooBroad; } if (irregularPath > 0) { cache[pageTitle].irregularPath = true; } else { delete cache[pageTitle].irregularPath; } cache[pageTitle].revId = page.revisions?.[0]?.revid ?? 0; return game; }