Rewrite from TypeScript to Rust
This commit is contained in:
mtkennerly
parent
a8b8b549d0
commit
25d71ba0f2
29 changed files with 221685 additions and 232325 deletions
12
.github/workflows/import.yaml
vendored
12
.github/workflows/import.yaml
vendored
|
|
@ -12,19 +12,19 @@ jobs:
|
|||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
- uses: actions/setup-node@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
node-version: '16'
|
||||
python-version: '3.10'
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: data/wiki-meta-cache.yaml
|
||||
key: wiki-meta-cache-v2-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
wiki-meta-cache-v2
|
||||
- run: |
|
||||
npm install
|
||||
npm run recent
|
||||
npm run schema
|
||||
- run: cargo build
|
||||
- run: cargo run -- bulk --recent-changes
|
||||
- uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: Import recent changes from PCGamingWiki
|
||||
|
|
|
|||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -1,4 +1,5 @@
|
|||
node_modules/
|
||||
out/
|
||||
target/
|
||||
tmp/
|
||||
data/wiki-meta-cache.yaml
|
||||
.mypy_cache/
|
||||
|
|
|
|||
|
|
@ -8,11 +8,8 @@ repos:
|
|||
rev: v1.1.7
|
||||
hooks:
|
||||
- id: forbid-tabs
|
||||
- repo: local
|
||||
- repo: https://github.com/mtkennerly/pre-commit-hooks
|
||||
rev: v0.3.0
|
||||
hooks:
|
||||
- id: schema
|
||||
name: schema
|
||||
language: system
|
||||
entry: npm run schema
|
||||
types: [yaml]
|
||||
pass_filenames: false
|
||||
- id: cargo-fmt
|
||||
- id: cargo-clippy
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
## Development
|
||||
Requires Node.js.
|
||||
Use the latest version of Rust and Python.
|
||||
|
||||
Generally, you just need these commands:
|
||||
|
||||
* Add all recent changes (defaults to last 7 days, but then it remembers when you last ran it):
|
||||
* `npm run recent`
|
||||
* Validate the manifest against the schema:
|
||||
* `npm run schema`
|
||||
* Install script dependencies (one time):
|
||||
* `pip install "steam[client]"`
|
||||
* Add all recent changes (defaults to last day, but then it remembers when you last ran it):
|
||||
* `cargo run -- bulk --recent-changes`
|
||||
* List some stats about the data set:
|
||||
* `npm run stats`
|
||||
* `cargo run -- stats`
|
||||
* Activate pre-commit hooks (requires Python):
|
||||
```
|
||||
pip install --user pre-commit
|
||||
|
|
@ -18,10 +18,9 @@ Generally, you just need these commands:
|
|||
There are some lower-level commands for finer control or full imports:
|
||||
|
||||
* Add new games to wiki-game-cache.yaml (required in order to add them to the manifest):
|
||||
* `npm run cache`
|
||||
* Update the manifest with games from the cache:
|
||||
* All games in cache: `npm run manifest`
|
||||
* Specific games: `npm run manifest -- "Game 1" "Game 2"`
|
||||
* `cargo run -- bulk --missing-pages`
|
||||
* Validate schema:
|
||||
* `cargo run -- schema`
|
||||
|
||||
## API etiquette
|
||||
When running or modifying the importer script, please be mindful not to
|
||||
|
|
@ -34,4 +33,4 @@ suggest that:
|
|||
> to finish before sending a new request, should result in a safe request rate.
|
||||
|
||||
I am not sure about guidelines for the Steam API, but the cache file should mean
|
||||
that we only ever need to reach out to Steam once per game.
|
||||
that we are not making excessive requests.
|
||||
|
|
|
|||
2150
Cargo.lock
generated
Normal file
2150
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
24
Cargo.toml
Normal file
24
Cargo.toml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
[package]
|
||||
name = "ludusavi-manifest"
|
||||
version = "0.1.0"
|
||||
authors = ["mtkennerly <mtkennerly@gmail.com>"]
|
||||
edition = "2021"
|
||||
description = "Ludusavi manifest importer"
|
||||
repository = "https://github.com/mtkennerly/ludusavi-manifest"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
|
||||
[dependencies]
|
||||
chrono = { version = "0.4.31", features = ["serde"] }
|
||||
clap = { version = "4.4.8", features = ["derive"] }
|
||||
itertools = "0.12.0"
|
||||
jsonschema = "0.17.1"
|
||||
mediawiki = "0.2.11"
|
||||
once_cell = "1.18.0"
|
||||
regex = "1.10.2"
|
||||
serde = { version = "1.0.139", features = ["derive"] }
|
||||
serde_json = "1.0.108"
|
||||
serde_yaml = "0.8.25"
|
||||
thiserror = "1.0.50"
|
||||
tokio = { version = "1.34.0", features = ["full"] }
|
||||
wikitext-parser = "0.3.3"
|
||||
198455
data/manifest.yaml
198455
data/manifest.yaml
File diff suppressed because it is too large
Load diff
600
data/missing.md
600
data/missing.md
File diff suppressed because it is too large
Load diff
166590
data/steam-game-cache.yaml
166590
data/steam-game-cache.yaml
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
3286
package-lock.json
generated
3286
package-lock.json
generated
File diff suppressed because it is too large
Load diff
35
package.json
35
package.json
|
|
@ -1,35 +0,0 @@
|
|||
{
|
||||
"name": "ludusavi-manifest",
|
||||
"version": "0.0.0",
|
||||
"description": "Game data backup info",
|
||||
"author": "Matthew T. Kennerly <mtkennerly@gmail.com>",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"wiki": "ts-node ./src/bin.ts --wiki",
|
||||
"manifest": "ts-node ./src/bin.ts --manifest",
|
||||
"recent": "ts-node ./src/bin.ts --limit 0 --wiki --manifest --recent",
|
||||
"schema": "npm run schema:normal && npm run schema:strict",
|
||||
"schema:normal": "ajv validate -s ./data/schema.yaml -d ./data/manifest.yaml",
|
||||
"schema:strict": "ajv validate -s ./data/schema.strict.yaml -d ./data/manifest.yaml",
|
||||
"stats": "ts-node ./src/bin.ts --stats",
|
||||
"duplicates": "ts-node ./src/bin.ts --duplicates",
|
||||
"steam": "ts-node ./src/bin.ts --steam"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/js-yaml": "^3.12.4",
|
||||
"@types/minimist": "^1.2.2",
|
||||
"@types/node": "^20.3.1",
|
||||
"ajv-cli": "^5.0.0",
|
||||
"ts-node": "^10.9.1",
|
||||
"typescript": "^5.1.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"js-yaml": "^3.14.0",
|
||||
"minimist": "^1.2.8",
|
||||
"moment": "^2.29.4",
|
||||
"nodemw": "^0.19.0",
|
||||
"steam-user": "^4.28.6",
|
||||
"wikiapi": "^1.19.4",
|
||||
"wikiparse": "^1.0.27"
|
||||
}
|
||||
}
|
||||
1
rustfmt.toml
Normal file
1
rustfmt.toml
Normal file
|
|
@ -0,0 +1 @@
|
|||
max_width = 120
|
||||
18
scripts/get-steam-app-info.py
Normal file
18
scripts/get-steam-app-info.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import json
|
||||
import sys
|
||||
|
||||
from steam.client import SteamClient
|
||||
|
||||
|
||||
def main():
|
||||
app_id = int(sys.argv[1])
|
||||
|
||||
client = SteamClient()
|
||||
client.anonymous_login()
|
||||
|
||||
info = client.get_product_info(apps=[app_id])
|
||||
print(json.dumps(info, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
143
src/bin.ts
143
src/bin.ts
|
|
@ -1,143 +0,0 @@
|
|||
import minimist from "minimist";
|
||||
|
||||
import { DEFAULT_GAME_LIMIT } from ".";
|
||||
import { ManifestFile, ManifestOverrideFile } from "./manifest";
|
||||
import { SteamGameCacheFile, getSteamClient } from "./steam";
|
||||
import { WikiGameCacheFile, WikiMetaCacheFile } from "./wiki";
|
||||
import { saveMissingGames } from "./missing";
|
||||
|
||||
interface Cli {
|
||||
wiki?: boolean,
|
||||
manifest?: boolean,
|
||||
stats?: boolean,
|
||||
duplicates?: boolean,
|
||||
all?: boolean,
|
||||
irregular?: boolean,
|
||||
irregularPathUntagged?: boolean,
|
||||
skipUntil?: string,
|
||||
recent?: boolean,
|
||||
missing?: boolean,
|
||||
limit?: number,
|
||||
steam?: boolean,
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = minimist<Cli>(process.argv.slice(2), {
|
||||
boolean: [
|
||||
"wiki",
|
||||
"manifest",
|
||||
"stats",
|
||||
"all",
|
||||
"irregular",
|
||||
"irregularPathUntagged",
|
||||
"steam",
|
||||
"missing",
|
||||
],
|
||||
string: [
|
||||
"skipUntil",
|
||||
],
|
||||
});
|
||||
|
||||
const wikiCache = new WikiGameCacheFile();
|
||||
wikiCache.load();
|
||||
const wikiMetaCache = new WikiMetaCacheFile();
|
||||
wikiMetaCache.load();
|
||||
const steamCache = new SteamGameCacheFile(getSteamClient);
|
||||
steamCache.load();
|
||||
const manifest = new ManifestFile();
|
||||
manifest.load();
|
||||
const manifestOverride = new ManifestOverrideFile();
|
||||
manifestOverride.load();
|
||||
|
||||
if (args.stats) {
|
||||
console.log(`Total games in manifest: ${Object.keys(manifest.data).length}`);
|
||||
console.log(`Total games in manifest with files or registry: ${Object.values(manifest.data).filter(x => x.files !== undefined || x.registry !== undefined).length}`);
|
||||
console.log(`Total games in manifest without files and registry: ${Object.values(manifest.data).filter(x => x.files === undefined && x.registry === undefined).length}`);
|
||||
console.log(`Total games in wiki cache: ${Object.keys(wikiCache.data).length}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (args.duplicates) {
|
||||
const data: {[key: string]: Array<{ name: string; pageId: number }>} = {};
|
||||
for (const [name, info] of Object.entries(manifest.data)) {
|
||||
const key = JSON.stringify(info);
|
||||
let safe = false;
|
||||
for (const file of Object.keys(info.files ?? {})) {
|
||||
if (file.includes("<game>") || file.includes("<base>")) {
|
||||
safe = true;
|
||||
}
|
||||
}
|
||||
if (safe) {
|
||||
continue;
|
||||
}
|
||||
if (!(key in data)) {
|
||||
data[key] = [];
|
||||
}
|
||||
data[key].push({ name, pageId: wikiCache.data[name]?.pageId ?? 0 });
|
||||
}
|
||||
for (const games of Object.values(data)) {
|
||||
if (games.length > 1) {
|
||||
const lines = games.map(({ name, pageId }) => `[${pageId}] ${name}`);
|
||||
console.log(`\nSame manifest entry:\n - ${lines.join("\n - ")}`);
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
try {
|
||||
if (args.wiki) {
|
||||
if (args.recent) {
|
||||
await wikiCache.flagRecentChanges(wikiMetaCache);
|
||||
} else if (args.missing) {
|
||||
await wikiCache.addNewGames();
|
||||
}
|
||||
|
||||
await wikiCache.refresh(
|
||||
args.skipUntil,
|
||||
args.limit ?? DEFAULT_GAME_LIMIT,
|
||||
args.all ?? false,
|
||||
args._ ?? [],
|
||||
);
|
||||
}
|
||||
|
||||
if (args.steam) {
|
||||
await steamCache.refresh(
|
||||
args.skipUntil,
|
||||
args.irregular ?? false,
|
||||
args.irregularPathUntagged ?? false,
|
||||
args.limit ?? DEFAULT_GAME_LIMIT,
|
||||
args._.map(x => x.toString()) ?? [],
|
||||
);
|
||||
}
|
||||
|
||||
if (args.manifest) {
|
||||
await manifest.updateGames(
|
||||
wikiCache.data,
|
||||
args._ ?? [],
|
||||
steamCache,
|
||||
manifestOverride,
|
||||
);
|
||||
}
|
||||
|
||||
wikiCache.save();
|
||||
wikiMetaCache.save();
|
||||
steamCache.save();
|
||||
manifest.save();
|
||||
saveMissingGames(wikiCache.data, manifest.data, manifestOverride.data);
|
||||
if (steamCache.steamClient) {
|
||||
steamCache.steamClient.logOff();
|
||||
}
|
||||
process.exit(0);
|
||||
} catch (e) {
|
||||
wikiCache.save();
|
||||
steamCache.save();
|
||||
manifest.save();
|
||||
saveMissingGames(wikiCache.data, manifest.data, manifestOverride.data);
|
||||
if (steamCache.steamClient) {
|
||||
steamCache.steamClient.logOff();
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
164
src/cli.rs
Normal file
164
src/cli.rs
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use crate::{
|
||||
manifest::{placeholder, Manifest, ManifestOverride},
|
||||
schema,
|
||||
steam::SteamCache,
|
||||
wiki::{WikiCache, WikiMetaCache},
|
||||
Error,
|
||||
};
|
||||
|
||||
#[derive(clap::Parser, Clone, Debug, PartialEq, Eq)]
|
||||
#[clap(name = "ludusavi", version, term_width = 79)]
|
||||
pub struct Cli {
|
||||
#[clap(subcommand)]
|
||||
pub sub: Subcommand,
|
||||
}
|
||||
|
||||
#[derive(clap::Subcommand, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Subcommand {
|
||||
/// Fetch bulk updates from the data sources.
|
||||
/// By default, this only updates entries that are marked as outdated.
|
||||
Bulk {
|
||||
/// Do a full sync.
|
||||
#[clap(long)]
|
||||
full: bool,
|
||||
|
||||
/// Do a partial update based on the wiki's recent changes.
|
||||
#[clap(long)]
|
||||
recent_changes: bool,
|
||||
|
||||
/// Do a partial update based on the wiki's game pages that are not yet cached.
|
||||
#[clap(long)]
|
||||
missing_pages: bool,
|
||||
},
|
||||
/// Fetch a named subset of games.
|
||||
Solo {
|
||||
#[clap(long)]
|
||||
local: bool,
|
||||
|
||||
#[clap()]
|
||||
games: Vec<String>,
|
||||
},
|
||||
/// Validate the manifest against its schema.
|
||||
Schema,
|
||||
/// Display some stats about the manifest.
|
||||
Stats,
|
||||
/// Find duplicate manifest entries.
|
||||
Duplicates,
|
||||
}
|
||||
|
||||
pub fn parse() -> Cli {
|
||||
use clap::Parser;
|
||||
Cli::parse()
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
sub: Subcommand,
|
||||
manifest: &mut Manifest,
|
||||
manifest_override: &mut ManifestOverride,
|
||||
wiki_cache: &mut WikiCache,
|
||||
wiki_meta_cache: &mut WikiMetaCache,
|
||||
steam_cache: &mut SteamCache,
|
||||
) -> Result<(), Error> {
|
||||
match sub {
|
||||
Subcommand::Bulk {
|
||||
full,
|
||||
recent_changes,
|
||||
missing_pages,
|
||||
} => {
|
||||
let outdated_only = !full;
|
||||
|
||||
if recent_changes {
|
||||
wiki_cache.flag_recent_changes(wiki_meta_cache).await?;
|
||||
}
|
||||
|
||||
if missing_pages {
|
||||
wiki_cache.add_new_games().await?;
|
||||
}
|
||||
|
||||
wiki_cache.refresh(outdated_only, None).await?;
|
||||
|
||||
steam_cache.transition_states_from(wiki_cache);
|
||||
steam_cache.refresh(outdated_only, None)?;
|
||||
|
||||
manifest.refresh(manifest_override, wiki_cache, steam_cache, None)?;
|
||||
schema::validate_manifest(manifest)?;
|
||||
}
|
||||
Subcommand::Solo { local, games } => {
|
||||
let outdated_only = false;
|
||||
|
||||
if !local {
|
||||
wiki_cache.refresh(outdated_only, Some(games.clone())).await?;
|
||||
|
||||
let steam_ids: Vec<_> = games
|
||||
.iter()
|
||||
.filter_map(|x| wiki_cache.0.get(x).and_then(|x| x.steam))
|
||||
.collect();
|
||||
|
||||
steam_cache.transition_states_from(wiki_cache);
|
||||
steam_cache.refresh(outdated_only, Some(steam_ids))?;
|
||||
}
|
||||
|
||||
manifest.refresh(manifest_override, wiki_cache, steam_cache, Some(games))?;
|
||||
schema::validate_manifest(manifest)?;
|
||||
}
|
||||
Subcommand::Schema => {
|
||||
schema::validate_manifest(manifest)?;
|
||||
}
|
||||
Subcommand::Stats => {
|
||||
let games = manifest.0.keys().count();
|
||||
let files_or_registry = manifest
|
||||
.0
|
||||
.values()
|
||||
.filter(|x| !x.files.is_empty() || !x.registry.is_empty())
|
||||
.count();
|
||||
let no_files_or_registry = manifest
|
||||
.0
|
||||
.values()
|
||||
.filter(|x| x.files.is_empty() && x.registry.is_empty())
|
||||
.count();
|
||||
let in_wiki_cache = wiki_cache.0.keys().count();
|
||||
|
||||
println!("Total games in manifest: {}", games);
|
||||
println!("Total games in manifest with files or registry: {}", files_or_registry);
|
||||
println!(
|
||||
"Total games in manifest without files and registry: {}",
|
||||
no_files_or_registry
|
||||
);
|
||||
println!("Total games in wiki cache: {}", in_wiki_cache);
|
||||
}
|
||||
Subcommand::Duplicates => {
|
||||
struct Duplicate {
|
||||
name: String,
|
||||
page_id: u64,
|
||||
}
|
||||
let mut data = HashMap::<String, Vec<Duplicate>>::new();
|
||||
|
||||
'games: for (name, info) in &manifest.0 {
|
||||
for file in info.files.keys() {
|
||||
if file.contains(placeholder::GAME) || file.contains(placeholder::BASE) {
|
||||
continue 'games;
|
||||
}
|
||||
}
|
||||
let key = serde_json::to_string(info).unwrap();
|
||||
data.entry(key).or_default().push(Duplicate {
|
||||
name: name.clone(),
|
||||
page_id: wiki_cache.0.get(name).map(|x| x.page_id).unwrap_or(0),
|
||||
});
|
||||
}
|
||||
|
||||
for duplicates in data.values() {
|
||||
if duplicates.len() > 1 {
|
||||
let lines: Vec<_> = duplicates
|
||||
.iter()
|
||||
.map(|x| format!("[{}] {}", x.page_id, x.name))
|
||||
.collect();
|
||||
println!("\nSame manifest entry:\n - {}", lines.join("\n - "));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
62
src/index.ts
62
src/index.ts
|
|
@ -1,62 +0,0 @@
|
|||
import * as pathMod from "path";
|
||||
import * as fs from "fs";
|
||||
import * as yaml from "js-yaml";
|
||||
|
||||
export const REPO = pathMod.dirname(__dirname);
|
||||
export const DELAY_BETWEEN_GAMES_MS = 250;
|
||||
export const DEFAULT_GAME_LIMIT = 25;
|
||||
|
||||
export class UnsupportedError extends Error {
|
||||
constructor(message?: string) {
|
||||
super(message);
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
|
||||
export class UnsupportedOsError extends UnsupportedError {
|
||||
constructor(message?: string) {
|
||||
super(message);
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
|
||||
export class UnsupportedPathError extends UnsupportedError {
|
||||
constructor(message?: string) {
|
||||
super(message);
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
|
||||
export enum PathType {
|
||||
FileSystem,
|
||||
Registry,
|
||||
}
|
||||
|
||||
export abstract class YamlFile<T = object> {
|
||||
data: T;
|
||||
abstract path: string;
|
||||
abstract defaultData: T;
|
||||
|
||||
load(): void {
|
||||
if (fs.existsSync(this.path)) {
|
||||
this.data = yaml.safeLoad(fs.readFileSync(this.path, "utf8")) as T;
|
||||
} else {
|
||||
this.data = this.defaultData;
|
||||
}
|
||||
}
|
||||
|
||||
save(): void {
|
||||
fs.writeFileSync(
|
||||
this.path,
|
||||
yaml.safeDump(
|
||||
this.data,
|
||||
{
|
||||
sortKeys: true,
|
||||
indent: 2,
|
||||
skipInvalid: true,
|
||||
lineWidth: 120,
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
105
src/main.rs
Normal file
105
src/main.rs
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
mod cli;
|
||||
mod manifest;
|
||||
mod missing;
|
||||
mod resource;
|
||||
mod schema;
|
||||
mod steam;
|
||||
mod wiki;
|
||||
|
||||
use crate::{
|
||||
manifest::{Manifest, ManifestOverride},
|
||||
resource::ResourceFile,
|
||||
steam::SteamCache,
|
||||
wiki::{WikiCache, WikiMetaCache},
|
||||
};
|
||||
|
||||
pub const REPO: &str = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum State {
|
||||
/// This entry needs to be re-fetched from the data source.
|
||||
Outdated,
|
||||
/// This entry has been re-fetched, but is awaiting recognition by another step.
|
||||
Updated,
|
||||
/// This entry has been fully processed.
|
||||
#[default]
|
||||
Handled,
|
||||
}
|
||||
|
||||
impl State {
|
||||
pub fn is_handled(&self) -> bool {
|
||||
*self == Self::Handled
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("Wiki client: {0}")]
|
||||
WikiClient(#[from] mediawiki::media_wiki_error::MediaWikiError),
|
||||
#[error("Wiki data missing or malformed: {0}")]
|
||||
WikiData(&'static str),
|
||||
#[error("Unable to find page by title or ID")]
|
||||
PageMissing,
|
||||
#[error("Could not find product info")]
|
||||
SteamProductInfo,
|
||||
#[error("Schema validation failed for manifest")]
|
||||
ManifestSchema,
|
||||
#[error("Subprocess: {0}")]
|
||||
Subprocess(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn should_discard_work(&self) -> bool {
|
||||
match self {
|
||||
Error::WikiClient(_)
|
||||
| Error::WikiData(_)
|
||||
| Error::PageMissing
|
||||
| Error::SteamProductInfo
|
||||
| Error::Subprocess(_) => false,
|
||||
Error::ManifestSchema => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let cli = cli::parse();
|
||||
|
||||
let mut wiki_cache = WikiCache::load().unwrap();
|
||||
let mut wiki_meta_cache = WikiMetaCache::load().unwrap();
|
||||
let mut steam_cache = SteamCache::load().unwrap();
|
||||
let mut manifest = Manifest::load().unwrap();
|
||||
let mut manifest_override = ManifestOverride::load().unwrap();
|
||||
|
||||
let mut success = true;
|
||||
let mut discard = false;
|
||||
if let Err(e) = cli::run(
|
||||
cli.sub,
|
||||
&mut manifest,
|
||||
&mut manifest_override,
|
||||
&mut wiki_cache,
|
||||
&mut wiki_meta_cache,
|
||||
&mut steam_cache,
|
||||
)
|
||||
.await
|
||||
{
|
||||
eprintln!("{e}");
|
||||
success = false;
|
||||
discard = e.should_discard_work();
|
||||
}
|
||||
|
||||
if !discard {
|
||||
if success {
|
||||
wiki_meta_cache.save();
|
||||
}
|
||||
wiki_cache.save();
|
||||
steam_cache.save();
|
||||
manifest.save();
|
||||
missing::save_missing_games(&wiki_cache, &manifest, &manifest_override);
|
||||
}
|
||||
|
||||
if !success {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
458
src/manifest.rs
Normal file
458
src/manifest.rs
Normal file
|
|
@ -0,0 +1,458 @@
|
|||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crate::{
|
||||
resource::ResourceFile,
|
||||
steam::{SteamCache, SteamCacheEntry},
|
||||
wiki::{PathKind, WikiCache, WikiCacheEntry},
|
||||
Error,
|
||||
};
|
||||
|
||||
pub mod placeholder {
|
||||
pub const ROOT: &str = "<root>";
|
||||
pub const GAME: &str = "<game>";
|
||||
pub const BASE: &str = "<base>";
|
||||
pub const HOME: &str = "<home>";
|
||||
pub const STORE_USER_ID: &str = "<storeUserId>";
|
||||
pub const OS_USER_NAME: &str = "<osUserName>";
|
||||
pub const WIN_APP_DATA: &str = "<winAppData>";
|
||||
pub const WIN_LOCAL_APP_DATA: &str = "<winLocalAppData>";
|
||||
pub const WIN_DOCUMENTS: &str = "<winDocuments>";
|
||||
pub const WIN_PUBLIC: &str = "<winPublic>";
|
||||
pub const WIN_PROGRAM_DATA: &str = "<winProgramData>";
|
||||
pub const WIN_DIR: &str = "<winDir>";
|
||||
pub const XDG_DATA: &str = "<xdgData>";
|
||||
pub const XDG_CONFIG: &str = "<xdgConfig>";
|
||||
}
|
||||
|
||||
fn do_launch_paths_match(from_steam: Option<String>, from_manifest: Option<String>) -> bool {
|
||||
match (from_steam, from_manifest) {
|
||||
(None, None) => true,
|
||||
(Some(from_steam), from_manifest) => normalize_launch_path(&from_steam) == from_manifest,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_launch_path(raw: &str) -> Option<String> {
|
||||
if raw.contains("://") {
|
||||
return Some(raw.to_string());
|
||||
}
|
||||
|
||||
let standardized = raw.replace('\\', "/").replace("//", "/");
|
||||
let standardized = standardized
|
||||
.trim_end_matches('/')
|
||||
.trim_start_matches("./")
|
||||
.trim_start_matches('/');
|
||||
|
||||
if standardized.is_empty() || standardized == "." {
|
||||
None
|
||||
} else {
|
||||
Some(format!("{}/{}", placeholder::BASE, standardized))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Os {
|
||||
Dos,
|
||||
Windows,
|
||||
Mac,
|
||||
Linux,
|
||||
#[default]
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
impl From<&str> for Os {
|
||||
fn from(value: &str) -> Self {
|
||||
match value.to_lowercase().trim() {
|
||||
"windows" => Self::Windows,
|
||||
"linux" => Self::Linux,
|
||||
"mac" | "macos" => Self::Mac,
|
||||
"dos" => Self::Dos,
|
||||
_ => Self::Other,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Ord, PartialOrd, Hash, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Store {
|
||||
Ea,
|
||||
Epic,
|
||||
Gog,
|
||||
GogGalaxy,
|
||||
Heroic,
|
||||
Lutris,
|
||||
Microsoft,
|
||||
Origin,
|
||||
Prime,
|
||||
Steam,
|
||||
Uplay,
|
||||
OtherHome,
|
||||
OtherWine,
|
||||
#[default]
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Tag {
|
||||
Config,
|
||||
Save,
|
||||
#[default]
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Manifest(pub BTreeMap<String, Game>);
|
||||
|
||||
impl ResourceFile for Manifest {
|
||||
const FILE_NAME: &'static str = "data/manifest.yaml";
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
pub fn refresh(
|
||||
&mut self,
|
||||
overrides: &ManifestOverride,
|
||||
wiki_cache: &WikiCache,
|
||||
steam_cache: &SteamCache,
|
||||
games: Option<Vec<String>>,
|
||||
) -> Result<(), Error> {
|
||||
for (title, info) in &wiki_cache.0 {
|
||||
if let Some(games) = &games {
|
||||
if !games.contains(title) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if overrides.0.get(title).map(|x| x.omit).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut game = Game::default();
|
||||
game.integrate_wiki(info, title);
|
||||
if let Some(id) = game.steam.id {
|
||||
if let Some(info) = steam_cache.0.get(id.to_string().as_str()) {
|
||||
game.integrate_steam(info);
|
||||
}
|
||||
}
|
||||
if let Some(overridden) = overrides.0.get(title) {
|
||||
game.integrate_overrides(overridden);
|
||||
}
|
||||
if !game.usable() {
|
||||
continue;
|
||||
}
|
||||
|
||||
self.0.insert(title.to_string(), game);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct Game {
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub files: BTreeMap<String, GameFileEntry>,
|
||||
#[serde(skip_serializing_if = "GogMetadata::is_empty")]
|
||||
pub gog: GogMetadata,
|
||||
#[serde(skip_serializing_if = "IdMetadata::is_empty")]
|
||||
pub id: IdMetadata,
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub install_dir: BTreeMap<String, GameInstallDirEntry>,
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub launch: BTreeMap<String, Vec<LaunchEntry>>,
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub registry: BTreeMap<String, GameRegistryEntry>,
|
||||
#[serde(skip_serializing_if = "SteamMetadata::is_empty")]
|
||||
pub steam: SteamMetadata,
|
||||
}
|
||||
|
||||
impl Game {
|
||||
pub fn integrate_wiki(&mut self, cache: &WikiCacheEntry, title: &str) {
|
||||
self.steam = SteamMetadata { id: cache.steam };
|
||||
self.gog = GogMetadata { id: cache.gog };
|
||||
self.id = IdMetadata {
|
||||
flatpak: None,
|
||||
gog_extra: cache.gog_side.clone(),
|
||||
steam_extra: cache.steam_side.clone(),
|
||||
};
|
||||
|
||||
let paths = cache.parse_paths(title.to_string());
|
||||
for path in paths {
|
||||
let tags = path.tags.clone();
|
||||
let tags2 = path.tags.clone();
|
||||
|
||||
match path.kind {
|
||||
None | Some(PathKind::File) => {
|
||||
let constraint = GameFileConstraint {
|
||||
os: path.os,
|
||||
store: path.store,
|
||||
};
|
||||
let constraint2 = constraint.clone();
|
||||
|
||||
self.files
|
||||
.entry(path.composite)
|
||||
.and_modify(|x| {
|
||||
x.tags.extend(tags);
|
||||
if !constraint.is_empty() {
|
||||
x.when.insert(constraint);
|
||||
}
|
||||
})
|
||||
.or_insert_with(|| GameFileEntry {
|
||||
tags: tags2.into_iter().collect(),
|
||||
when: (if constraint2.is_empty() {
|
||||
vec![]
|
||||
} else {
|
||||
vec![constraint2]
|
||||
})
|
||||
.into_iter()
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
Some(PathKind::Registry) => {
|
||||
let constraint = GameRegistryConstraint { store: path.store };
|
||||
let constraint2 = constraint.clone();
|
||||
|
||||
self.registry
|
||||
.entry(path.composite)
|
||||
.and_modify(|x| {
|
||||
x.tags.extend(tags);
|
||||
if !constraint.is_empty() {
|
||||
x.when.insert(constraint);
|
||||
}
|
||||
})
|
||||
.or_insert_with(|| GameRegistryEntry {
|
||||
tags: tags2.into_iter().collect(),
|
||||
when: (if constraint2.is_empty() {
|
||||
vec![]
|
||||
} else {
|
||||
vec![constraint2]
|
||||
})
|
||||
.into_iter()
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn integrate_steam(&mut self, cache: &SteamCacheEntry) {
|
||||
if let Some(install_dir) = &cache.install_dir {
|
||||
self.install_dir.insert(install_dir.to_string(), GameInstallDirEntry {});
|
||||
}
|
||||
|
||||
for incoming in &cache.launch {
|
||||
if incoming.executable.is_none()
|
||||
|| incoming.executable.as_ref().map(|x| x.contains("://")).unwrap_or(false)
|
||||
|| !matches!(incoming.r#type.as_deref(), None | Some("default" | "none"))
|
||||
|| incoming.config.betakey.is_some()
|
||||
|| incoming.config.ownsdlc.is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let os = match incoming.config.oslist.as_deref() {
|
||||
Some("windows") => Some(Os::Windows),
|
||||
Some("macos" | "macosx") => Some(Os::Mac),
|
||||
Some("linux") => Some(Os::Linux),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let bit = match incoming.config.osarch.as_deref() {
|
||||
Some("32") => Some(32),
|
||||
Some("64") => Some(64),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let constraint = LaunchConstraint {
|
||||
bit,
|
||||
os,
|
||||
store: Some(Store::Steam),
|
||||
};
|
||||
|
||||
let mut found_existing = false;
|
||||
for (existing_executable, existing_options) in self.launch.iter_mut() {
|
||||
for existing in existing_options {
|
||||
if incoming.arguments == existing.arguments
|
||||
&& do_launch_paths_match(incoming.executable.clone(), Some(existing_executable.to_string()))
|
||||
&& do_launch_paths_match(incoming.workingdir.clone(), existing.working_dir.clone())
|
||||
{
|
||||
found_existing = true;
|
||||
existing.when.insert(constraint.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found_existing {
|
||||
let Some(key) = incoming.executable.as_ref().and_then(|x| normalize_launch_path(x)) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let candidate = LaunchEntry {
|
||||
arguments: incoming.arguments.clone(),
|
||||
when: vec![constraint.clone()].into_iter().collect(),
|
||||
working_dir: incoming.workingdir.as_ref().and_then(|x| normalize_launch_path(x)),
|
||||
};
|
||||
self.launch
|
||||
.entry(key)
|
||||
.and_modify(|x| x.push(candidate.clone()))
|
||||
.or_insert_with(|| vec![candidate]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn integrate_overrides(&mut self, overridden: &OverrideGame) {
|
||||
if let Some(id) = overridden.game.steam.id {
|
||||
self.steam.id = Some(id);
|
||||
}
|
||||
if let Some(id) = overridden.game.gog.id {
|
||||
self.gog.id = Some(id);
|
||||
}
|
||||
if let Some(flatpak) = overridden.game.id.flatpak.as_ref() {
|
||||
self.id.flatpak = Some(flatpak.clone());
|
||||
}
|
||||
self.install_dir.extend(overridden.game.install_dir.clone());
|
||||
}
|
||||
|
||||
pub fn usable(&self) -> bool {
|
||||
!(self.files.is_empty()
|
||||
&& self.registry.is_empty()
|
||||
&& self.steam.is_empty()
|
||||
&& self.gog.is_empty()
|
||||
&& self.id.is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct GameFileEntry {
|
||||
#[serde(skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub tags: BTreeSet<Tag>,
|
||||
#[serde(skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub when: BTreeSet<GameFileConstraint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct GameInstallDirEntry {}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct GameRegistryEntry {
|
||||
#[serde(skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub tags: BTreeSet<Tag>,
|
||||
#[serde(skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub when: BTreeSet<GameRegistryConstraint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct LaunchEntry {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub arguments: Option<String>,
|
||||
#[serde(skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub when: BTreeSet<LaunchConstraint>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub working_dir: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct GameFileConstraint {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub os: Option<Os>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub store: Option<Store>,
|
||||
}
|
||||
|
||||
impl GameFileConstraint {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.os.is_none() && self.store.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct GameRegistryConstraint {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub store: Option<Store>,
|
||||
}
|
||||
|
||||
impl GameRegistryConstraint {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.store.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct LaunchConstraint {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bit: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub os: Option<Os>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub store: Option<Store>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct SteamMetadata {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<u32>,
|
||||
}
|
||||
|
||||
impl SteamMetadata {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.id.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct GogMetadata {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<u64>,
|
||||
}
|
||||
|
||||
impl GogMetadata {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.id.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct IdMetadata {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub flatpak: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub gog_extra: BTreeSet<u64>,
|
||||
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub steam_extra: BTreeSet<u32>,
|
||||
}
|
||||
|
||||
impl IdMetadata {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.flatpak.is_none() && self.gog_extra.is_empty() && self.steam_extra.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct ManifestOverride(pub BTreeMap<String, OverrideGame>);
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct OverrideGame {
|
||||
pub omit: bool,
|
||||
#[serde(flatten)]
|
||||
pub game: Game,
|
||||
}
|
||||
|
||||
impl ResourceFile for ManifestOverride {
|
||||
const FILE_NAME: &'static str = "data/manifest-override.yaml";
|
||||
}
|
||||
292
src/manifest.ts
292
src/manifest.ts
|
|
@ -1,292 +0,0 @@
|
|||
import { REPO, YamlFile } from ".";
|
||||
import { SteamGameCache, SteamGameCacheFile } from "./steam";
|
||||
import { WikiGameCache, parseTemplates } from "./wiki";
|
||||
|
||||
const U32_MAX = 4_294_967_295;
|
||||
const U64_MAX = 18_446_744_073_709_551_615;
|
||||
|
||||
export type Os = "dos" | "linux" | "mac" | "windows";
|
||||
|
||||
export type Bit = 32 | 64;
|
||||
|
||||
export type Store = "epic" | "gog" | "microsoft" | "steam" | "uplay" | "origin";
|
||||
|
||||
export type Tag = "config" | "save";
|
||||
|
||||
export interface Manifest {
|
||||
[game: string]: Game;
|
||||
}
|
||||
|
||||
export interface Game {
|
||||
files?: {
|
||||
[path: string]: {
|
||||
when?: Array<Omit<Constraint, "bit">>,
|
||||
tags?: Array<Tag>,
|
||||
}
|
||||
};
|
||||
installDir?: {
|
||||
[name: string]: {}
|
||||
};
|
||||
launch?: {
|
||||
[path: string]: Array<{
|
||||
arguments?: string;
|
||||
workingDir?: string;
|
||||
when?: Array<Constraint>,
|
||||
}>
|
||||
},
|
||||
registry?: {
|
||||
[path: string]: {
|
||||
when?: Array<Omit<Constraint, "bit" | "os">>,
|
||||
tags?: Array<Tag>,
|
||||
}
|
||||
};
|
||||
steam?: {
|
||||
id?: number
|
||||
};
|
||||
gog?: {
|
||||
id?: number,
|
||||
};
|
||||
id?: {
|
||||
flatpak?: string,
|
||||
gogExtra?: Array<number>,
|
||||
steamExtra?: Array<number>,
|
||||
};
|
||||
}
|
||||
|
||||
type OverriddenGame = Game & { omit?: boolean };
|
||||
|
||||
export interface Constraint {
|
||||
os?: Os;
|
||||
bit?: Bit;
|
||||
store?: Store;
|
||||
}
|
||||
|
||||
function normalizeLaunchPath(raw: string): string | undefined {
|
||||
if (raw.includes("://")) {
|
||||
return raw;
|
||||
}
|
||||
const standardized = raw
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/\/\//g, "/")
|
||||
.replace(/\/(?=$)/g, "")
|
||||
.replace(/^.\//, "")
|
||||
.replace(/^\/+/, "")
|
||||
.trim();
|
||||
if (standardized.length === 0 || standardized === ".") {
|
||||
return undefined;
|
||||
}
|
||||
return `<base>/${standardized}`;
|
||||
}
|
||||
|
||||
function doLaunchPathsMatch(fromSteam: string | undefined, fromManifest: string | undefined): boolean {
|
||||
if (fromSteam === undefined) {
|
||||
return fromManifest === undefined;
|
||||
} else {
|
||||
return normalizeLaunchPath(fromSteam) === fromManifest;
|
||||
}
|
||||
}
|
||||
|
||||
function integrateWikiData(game: Game, cache: WikiGameCache[""]): void {
|
||||
game.id = {};
|
||||
if (cache.steam !== undefined && cache.steam <= U32_MAX) {
|
||||
game.steam = { id: cache.steam };
|
||||
}
|
||||
if (cache.steamSide !== undefined) {
|
||||
game.id.steamExtra = cache.steamSide.filter(x => x <= U32_MAX);
|
||||
}
|
||||
if (cache.gog !== undefined && cache.gog <= U64_MAX) {
|
||||
game.gog = { id: cache.gog };
|
||||
}
|
||||
if (cache.gogSide !== undefined) {
|
||||
game.id.gogExtra = cache.gogSide.filter(x => x <= U64_MAX);
|
||||
}
|
||||
const info = parseTemplates(cache.templates ?? []);
|
||||
game.files = info.files;
|
||||
game.registry = info.registry;
|
||||
|
||||
if (game.id.flatpak === undefined && game.id.gogExtra === undefined && game.id.steamExtra === undefined) {
|
||||
delete game.id;
|
||||
}
|
||||
}
|
||||
|
||||
function integrateSteamData(game: Game, appInfo: SteamGameCache[""] | undefined): void {
|
||||
if (appInfo === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (appInfo.installDir !== undefined) {
|
||||
game.installDir = { [appInfo.installDir]: {} };
|
||||
}
|
||||
if (appInfo.launch !== undefined) {
|
||||
delete game.launch;
|
||||
for (const incoming of appInfo.launch) {
|
||||
if (
|
||||
incoming.executable === undefined ||
|
||||
incoming.executable.includes("://") ||
|
||||
![undefined, "default", "none"].includes(incoming.type) ||
|
||||
incoming.config?.betakey !== undefined ||
|
||||
incoming.config?.ownsdlc !== undefined
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const os: Os | undefined = {
|
||||
"windows": "windows",
|
||||
"macos": "mac",
|
||||
"macosx": "mac",
|
||||
"linux": "linux",
|
||||
}[incoming.config?.oslist] as Os;
|
||||
const bit: Bit | undefined = {
|
||||
"32": 32,
|
||||
"64": 64,
|
||||
}[incoming.config?.osarch] as Bit;
|
||||
const when: Constraint = { os, bit, store: "steam" };
|
||||
if (when.os === undefined) {
|
||||
delete when.os;
|
||||
}
|
||||
if (when.bit === undefined) {
|
||||
delete when.bit;
|
||||
}
|
||||
|
||||
let foundExisting = false;
|
||||
for (const [existingExecutable, existingOptions] of Object.entries(game.launch ?? {})) {
|
||||
for (const existing of existingOptions) {
|
||||
if (
|
||||
incoming.arguments === existing.arguments &&
|
||||
doLaunchPathsMatch(incoming.executable, existingExecutable) &&
|
||||
doLaunchPathsMatch(incoming.workingdir, existing.workingDir)
|
||||
) {
|
||||
foundExisting = true;
|
||||
if (existing.when === undefined) {
|
||||
existing.when = [];
|
||||
}
|
||||
if (existing.when.every(x => x.os !== os && x.bit !== bit && x.store !== "steam")) {
|
||||
existing.when.push(when);
|
||||
}
|
||||
if (existing.when.length === 0) {
|
||||
delete existing.when;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!foundExisting) {
|
||||
const key = normalizeLaunchPath(incoming.executable);
|
||||
if (key === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidate: Game["launch"][""][0] = { when: [when] };
|
||||
if (incoming.arguments !== undefined) {
|
||||
candidate.arguments = incoming.arguments;
|
||||
}
|
||||
if (incoming.workingdir !== undefined) {
|
||||
const workingDir = normalizeLaunchPath(incoming.workingdir);
|
||||
if (workingDir !== undefined) {
|
||||
candidate.workingDir = workingDir;
|
||||
}
|
||||
}
|
||||
|
||||
if (game.launch === undefined) {
|
||||
game.launch = {};
|
||||
}
|
||||
if (game.launch[key] === undefined) {
|
||||
game.launch[key] = [];
|
||||
}
|
||||
game.launch[key].push(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function integrateOverriddenData(game: Game, override: OverriddenGame) {
|
||||
if (override.steam) {
|
||||
game.steam = override.steam;
|
||||
}
|
||||
|
||||
if (override.gog) {
|
||||
game.gog = override.gog;
|
||||
}
|
||||
|
||||
if (override.id) {
|
||||
if (game.id === undefined) {
|
||||
game.id = {};
|
||||
}
|
||||
|
||||
if (override.id.flatpak) {
|
||||
game.id.flatpak = override.id.flatpak;
|
||||
}
|
||||
}
|
||||
|
||||
if (override.installDir) {
|
||||
if (game.installDir === undefined) {
|
||||
game.installDir = {};
|
||||
}
|
||||
for (const key in override.installDir) {
|
||||
game.installDir[key] = {};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasAnyData(game: Game): boolean {
|
||||
return game.files !== undefined || game.registry !== undefined || game.steam?.id !== undefined || game.gog?.id !== undefined || game.id !== undefined;
|
||||
}
|
||||
|
||||
export class ManifestFile extends YamlFile<Manifest> {
|
||||
path = `${REPO}/data/manifest.yaml`;
|
||||
defaultData = {};
|
||||
|
||||
async updateGames(
|
||||
wikiCache: WikiGameCache,
|
||||
games: Array<string>,
|
||||
steamCache: SteamGameCacheFile,
|
||||
override: ManifestOverrideFile,
|
||||
): Promise<void> {
|
||||
this.data = {};
|
||||
|
||||
for (const [title, info] of Object.entries(wikiCache).sort()) {
|
||||
const overridden = override.data[title];
|
||||
if (overridden?.omit) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (games?.length > 0 && !games.includes(title)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const game: Game = {};
|
||||
integrateWikiData(game, info);
|
||||
|
||||
if (game.steam?.id !== undefined) {
|
||||
const appInfo = await steamCache.getAppInfo(game.steam.id);
|
||||
integrateSteamData(game, appInfo);
|
||||
}
|
||||
|
||||
if (overridden) {
|
||||
integrateOverriddenData(game, overridden);
|
||||
}
|
||||
|
||||
if (!hasAnyData(game)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.data[title] = game;
|
||||
}
|
||||
|
||||
for (const [title, info] of Object.entries(override.data).sort()) {
|
||||
if (title in this.data || info.omit) {
|
||||
continue;
|
||||
}
|
||||
delete info.omit;
|
||||
this.data[title] = info;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface ManifestOverride {
|
||||
[game: string]: OverriddenGame
|
||||
}
|
||||
|
||||
export class ManifestOverrideFile extends YamlFile<ManifestOverride> {
|
||||
path = `${REPO}/data/manifest-override.yaml`;
|
||||
defaultData = {};
|
||||
}
|
||||
26
src/missing.rs
Normal file
26
src/missing.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
use itertools::Itertools;
|
||||
|
||||
use crate::{
|
||||
manifest::{Manifest, ManifestOverride},
|
||||
wiki::WikiCache,
|
||||
REPO,
|
||||
};
|
||||
|
||||
pub fn save_missing_games(wiki_cache: &WikiCache, manifest: &Manifest, overrides: &ManifestOverride) {
|
||||
let lines: Vec<String> = wiki_cache
|
||||
.0
|
||||
.iter()
|
||||
.sorted_by(|(k1, _), (k2, _)| k1.to_lowercase().cmp(&k2.to_lowercase()))
|
||||
.filter(|(k, _)| {
|
||||
manifest
|
||||
.0
|
||||
.get(*k)
|
||||
.map(|x| x.files.is_empty() && x.registry.is_empty())
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.filter(|(k, _)| overrides.0.get(*k).map(|x| !x.omit).unwrap_or(true))
|
||||
.map(|(k, v)| format!("* [{}](https://www.pcgamingwiki.com/wiki/?curid={})", k, v.page_id))
|
||||
.collect();
|
||||
|
||||
_ = std::fs::write(format!("{}/data/missing.md", REPO), lines.join("\n") + "\n");
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
import * as fs from "fs";
|
||||
import { REPO } from ".";
|
||||
import { Manifest, ManifestOverride } from "./manifest";
|
||||
import { WikiGameCache } from "./wiki";
|
||||
|
||||
export function saveMissingGames(cache: WikiGameCache, manifest: Manifest, override: ManifestOverride): void {
|
||||
fs.writeFileSync(
|
||||
`${REPO}/data/missing.md`,
|
||||
Object.entries(cache)
|
||||
.sort((x, y) => x[0].localeCompare(y[0]))
|
||||
.filter(([k, _]) => (manifest[k]?.files ?? []).length === 0 && (manifest[k]?.registry ?? []).length === 0)
|
||||
.filter(([k, _]) => override[k]?.omit !== true)
|
||||
.map(([k, v]) => `* [${k}](https://www.pcgamingwiki.com/wiki/?curid=${v.pageId})`)
|
||||
.join("\n") + "\n",
|
||||
);
|
||||
}
|
||||
63
src/resource.rs
Normal file
63
src/resource.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
use crate::REPO;
|
||||
|
||||
pub type AnyError = Box<dyn std::error::Error>;
|
||||
|
||||
pub trait ResourceFile
|
||||
where
|
||||
Self: Default + serde::Serialize + serde::de::DeserializeOwned,
|
||||
{
|
||||
const FILE_NAME: &'static str;
|
||||
|
||||
fn path() -> std::path::PathBuf {
|
||||
let mut path = std::path::PathBuf::new();
|
||||
path.push(REPO);
|
||||
path.push(Self::FILE_NAME);
|
||||
path
|
||||
}
|
||||
|
||||
/// If the resource file does not exist, use default data and apply these modifications.
|
||||
fn initialize(self) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
/// Update any legacy settings on load.
|
||||
fn migrate(self) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
fn load() -> Result<Self, AnyError> {
|
||||
Self::load_from(&Self::path())
|
||||
}
|
||||
|
||||
fn load_from(path: &std::path::PathBuf) -> Result<Self, AnyError> {
|
||||
if !path.exists() {
|
||||
return Ok(Self::default().initialize());
|
||||
}
|
||||
let content = Self::load_raw(path)?;
|
||||
Self::load_from_string(&content)
|
||||
}
|
||||
|
||||
fn load_raw(path: &std::path::PathBuf) -> Result<String, AnyError> {
|
||||
Ok(std::fs::read_to_string(path)?)
|
||||
}
|
||||
|
||||
fn load_from_string(content: &str) -> Result<Self, AnyError> {
|
||||
Ok(ResourceFile::migrate(serde_yaml::from_str(content)?))
|
||||
}
|
||||
|
||||
fn serialize(&self) -> String {
|
||||
serde_yaml::to_string(&self).unwrap()
|
||||
}
|
||||
|
||||
fn save(&self) {
|
||||
let new_content = serde_yaml::to_string(&self).unwrap();
|
||||
|
||||
if let Ok(old_content) = Self::load_raw(&Self::path()) {
|
||||
if old_content == new_content {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let _ = std::fs::write(Self::path(), new_content.as_bytes());
|
||||
}
|
||||
}
|
||||
32
src/schema.rs
Normal file
32
src/schema.rs
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
use crate::{manifest::Manifest, resource::ResourceFile, Error, REPO};
|
||||
|
||||
pub fn validate_manifest(manifest: &Manifest) -> Result<(), Error> {
|
||||
let manifest: serde_json::Value = serde_yaml::from_str(&manifest.serialize()).unwrap();
|
||||
|
||||
let normal: serde_json::Value = serde_yaml::from_str(&read_data("schema.yaml")).unwrap();
|
||||
let strict: serde_json::Value = serde_yaml::from_str(&read_data("schema.strict.yaml")).unwrap();
|
||||
|
||||
for schema in [normal, strict] {
|
||||
if !check(&schema, &manifest) {
|
||||
return Err(Error::ManifestSchema);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_data(file: &str) -> String {
|
||||
std::fs::read_to_string(format!("{}/data/{}", REPO, file)).unwrap()
|
||||
}
|
||||
|
||||
fn check(schema: &serde_json::Value, instance: &serde_json::Value) -> bool {
|
||||
let mut valid = true;
|
||||
let compiled = jsonschema::JSONSchema::compile(schema).unwrap();
|
||||
if let Err(errors) = compiled.validate(instance) {
|
||||
valid = false;
|
||||
for error in errors {
|
||||
println!("Schema error: {} | {}", error, error.instance_path);
|
||||
}
|
||||
}
|
||||
valid
|
||||
}
|
||||
218
src/steam.rs
Normal file
218
src/steam.rs
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
use std::{collections::BTreeMap, process::Command};
|
||||
|
||||
use crate::{resource::ResourceFile, wiki::WikiCache, Error, State, REPO};
|
||||
|
||||
const SAVE_INTERVAL: u32 = 100;
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct SteamCache(pub BTreeMap<String, SteamCacheEntry>);
|
||||
|
||||
impl ResourceFile for SteamCache {
|
||||
const FILE_NAME: &'static str = "data/steam-game-cache.yaml";
|
||||
}
|
||||
|
||||
impl SteamCache {
|
||||
pub fn refresh(&mut self, outdated_only: bool, app_ids: Option<Vec<u32>>) -> Result<(), Error> {
|
||||
let mut i = 0;
|
||||
let app_ids: Vec<_> = app_ids.unwrap_or_else(|| {
|
||||
self.0
|
||||
.iter()
|
||||
.filter(|(_, v)| !outdated_only || v.state == State::Outdated)
|
||||
.filter_map(|(k, _)| k.parse::<u32>().ok())
|
||||
.collect()
|
||||
});
|
||||
|
||||
for app_id in app_ids {
|
||||
let latest = SteamCacheEntry::fetch_from_id(app_id)?;
|
||||
self.0.insert(app_id.to_string(), latest);
|
||||
|
||||
i += 1;
|
||||
if i % SAVE_INTERVAL == 0 {
|
||||
self.save();
|
||||
println!("\n:: saved\n");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn transition_states_from(&mut self, wiki_cache: &mut WikiCache) {
|
||||
for wiki in wiki_cache.0.values_mut() {
|
||||
if wiki.state == State::Updated {
|
||||
if let Some(id) = wiki.steam {
|
||||
self.0
|
||||
.entry(id.to_string())
|
||||
.and_modify(|x| {
|
||||
x.state = State::Outdated;
|
||||
})
|
||||
.or_insert(SteamCacheEntry {
|
||||
state: State::Outdated,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
wiki.state = State::Handled;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct SteamCacheEntry {
|
||||
#[serde(skip_serializing_if = "State::is_handled")]
|
||||
pub state: State,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub install_dir: Option<String>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub launch: Vec<Launch>,
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub name_localized: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct Launch {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub arguments: Option<String>,
|
||||
#[serde(skip_serializing_if = "LaunchConfig::is_empty")]
|
||||
pub config: LaunchConfig,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub executable: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub r#type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub workingdir: Option<String>,
|
||||
}
|
||||
|
||||
impl Launch {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.arguments.is_none()
|
||||
&& self.config.is_empty()
|
||||
&& self.description.is_none()
|
||||
&& self.executable.is_none()
|
||||
&& self.r#type.is_none()
|
||||
&& self.workingdir.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct LaunchConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub betakey: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub osarch: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub oslist: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ownsdlc: Option<String>,
|
||||
}
|
||||
|
||||
impl LaunchConfig {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.betakey.is_none() && self.osarch.is_none() && self.oslist.is_none() && self.ownsdlc.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
mod product_info {
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct Response {
|
||||
pub apps: BTreeMap<String, App>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct App {
|
||||
pub common: AppCommon,
|
||||
pub config: AppConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AppCommon {
|
||||
pub name_localized: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AppConfig {
|
||||
pub installdir: Option<String>,
|
||||
pub launch: BTreeMap<String, AppLaunch>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AppLaunch {
|
||||
pub executable: Option<String>,
|
||||
pub arguments: Option<String>,
|
||||
pub workingdir: Option<String>,
|
||||
pub r#type: Option<String>,
|
||||
pub config: AppLaunchConfig,
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AppLaunchConfig {
|
||||
pub betakey: Option<String>,
|
||||
pub osarch: Option<String>,
|
||||
pub oslist: Option<String>,
|
||||
pub ownsdlc: Option<String>,
|
||||
}
|
||||
}
|
||||
|
||||
impl SteamCacheEntry {
|
||||
pub fn fetch_from_id(app_id: u32) -> Result<Self, Error> {
|
||||
println!("Steam: {}", app_id);
|
||||
|
||||
let mut cmd = Command::new("python");
|
||||
cmd.arg(format!("{}/scripts/get-steam-app-info.py", REPO));
|
||||
cmd.arg(app_id.to_string());
|
||||
let output = cmd.output()?;
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
eprintln!("Steam product info failure: {}", &stderr);
|
||||
return Err(Error::SteamProductInfo);
|
||||
}
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
|
||||
let response = serde_json::from_str::<product_info::Response>(&stdout).map_err(|_| Error::SteamProductInfo)?;
|
||||
let app = response
|
||||
.apps
|
||||
.get(&app_id.to_string())
|
||||
.ok_or(Error::SteamProductInfo)?
|
||||
.clone();
|
||||
|
||||
let launch: Vec<_> = app
|
||||
.config
|
||||
.launch
|
||||
.into_values()
|
||||
.map(|x| Launch {
|
||||
executable: x.executable,
|
||||
arguments: x.arguments,
|
||||
workingdir: x.workingdir,
|
||||
r#type: x.r#type,
|
||||
description: x.description,
|
||||
config: LaunchConfig {
|
||||
betakey: x.config.betakey,
|
||||
osarch: x.config.osarch,
|
||||
oslist: x.config.oslist,
|
||||
ownsdlc: x.config.ownsdlc,
|
||||
},
|
||||
})
|
||||
.filter(|x| !x.is_empty())
|
||||
.collect();
|
||||
|
||||
Ok(Self {
|
||||
state: State::Handled,
|
||||
install_dir: app.config.installdir,
|
||||
name_localized: app.common.name_localized,
|
||||
launch,
|
||||
})
|
||||
}
|
||||
}
|
||||
158
src/steam.ts
158
src/steam.ts
|
|
@ -1,158 +0,0 @@
|
|||
import { DELAY_BETWEEN_GAMES_MS, REPO, YamlFile } from ".";
|
||||
import SteamUser from "steam-user";
|
||||
|
||||
export type SteamGameCache = {
|
||||
[appId: string]: {
|
||||
installDir?: string,
|
||||
unknown?: boolean,
|
||||
nameLocalized?: Map<string, string>;
|
||||
launch?: Array<{
|
||||
executable?: string;
|
||||
arguments?: string;
|
||||
workingdir?: string;
|
||||
type?: string;
|
||||
config?: {
|
||||
betakey?: string;
|
||||
osarch?: string;
|
||||
oslist?: string;
|
||||
ownsdlc?: string;
|
||||
},
|
||||
}>;
|
||||
irregular?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export class SteamGameCacheFile extends YamlFile<SteamGameCache> {
|
||||
path = `${REPO}/data/steam-game-cache.yaml`;
|
||||
defaultData = {};
|
||||
steamClient: SteamUser | null = null;
|
||||
|
||||
constructor(private makeSteamClient: () => Promise<SteamUser>) {
|
||||
super();
|
||||
}
|
||||
|
||||
hasIrregularKeys(info: object): boolean {
|
||||
return Object.keys(info).some(x => x.endsWith('"'));
|
||||
}
|
||||
|
||||
isIrregularString(info: string): boolean {
|
||||
return info.includes('"\n\t');
|
||||
}
|
||||
|
||||
async getAppInfo(appId: number, update: boolean = false): Promise<SteamGameCache[""] | undefined> {
|
||||
const key = appId.toString();
|
||||
if (!update && this.data.hasOwnProperty(key)) {
|
||||
return this.data[key];
|
||||
}
|
||||
|
||||
console.log(`Steam: ${appId}`);
|
||||
|
||||
if (this.steamClient === null) {
|
||||
this.steamClient = await this.makeSteamClient();
|
||||
}
|
||||
|
||||
const info: SteamProductInfoResponse = await this.steamClient.getProductInfo([appId], []);
|
||||
|
||||
if (info.unknownApps.includes(appId)) {
|
||||
this.data[key] = { unknown: true };
|
||||
return undefined;
|
||||
}
|
||||
|
||||
this.data[key] = {};
|
||||
|
||||
const installDir = info.apps[key].appinfo.config?.installdir;
|
||||
if (installDir !== undefined) {
|
||||
this.data[key].installDir = installDir;
|
||||
}
|
||||
|
||||
const nameLocalized = info.apps[key].appinfo.common?.name_localized;
|
||||
if (nameLocalized !== undefined && Object.keys(nameLocalized).length > 0) {
|
||||
this.data[key].nameLocalized = nameLocalized;
|
||||
}
|
||||
|
||||
const launch = info.apps[key].appinfo.config?.launch;
|
||||
if (launch !== undefined) {
|
||||
const keys = Object.keys(launch).sort((x, y) => parseInt(x) - parseInt(y));
|
||||
const launchArray = keys.map(x => launch[x]);
|
||||
this.data[key].launch = launchArray;
|
||||
}
|
||||
|
||||
return this.data[key];
|
||||
}
|
||||
|
||||
async refresh(skipUntil: string | undefined, irregularTagged: boolean, irregularUntagged: boolean, limit: number, appIds: Array<string>): Promise<void> {
|
||||
let i = 0;
|
||||
let foundSkipUntil = false;
|
||||
for (const appId of Object.keys(this.data).sort()) {
|
||||
if (skipUntil && !foundSkipUntil) {
|
||||
if (appId === skipUntil) {
|
||||
foundSkipUntil = true;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (appIds.length > 0 && !appIds.includes(appId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (irregularTagged) {
|
||||
if (!this.data[appId].irregular) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (irregularUntagged) {
|
||||
const irregular = (this.data[appId].launch ?? []).some(x => this.hasIrregularKeys(x)) ||
|
||||
this.isIrregularString(this.data[appId].installDir ?? "");
|
||||
if (this.data[appId].irregular || !irregular) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// console.log(`Refreshing Steam app ${appId}`);
|
||||
await this.getAppInfo(parseInt(appId), true);
|
||||
|
||||
i++;
|
||||
if (limit > 0 && i >= limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
// main() will save at the end, but we do a periodic save as well
|
||||
// in case something goes wrong or the script gets cancelled:
|
||||
if (i % 250 === 0) {
|
||||
this.save();
|
||||
console.log(":: saved");
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, DELAY_BETWEEN_GAMES_MS));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface SteamProductInfoResponse {
|
||||
apps: {
|
||||
[appId: string]: {
|
||||
appinfo: {
|
||||
common?: {
|
||||
name_localized?: Map<string, string>,
|
||||
},
|
||||
config?: {
|
||||
installdir?: string,
|
||||
launch?: object,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
unknownApps: Array<number>,
|
||||
}
|
||||
|
||||
export async function getSteamClient(): Promise<SteamUser> {
|
||||
const client = new SteamUser();
|
||||
client.logOn({ anonymous: true });
|
||||
await new Promise<void>(resolve => {
|
||||
client.on("loggedOn", () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
return client;
|
||||
}
|
||||
937
src/wiki.rs
Normal file
937
src/wiki.rs
Normal file
|
|
@ -0,0 +1,937 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
use wikitext_parser::{Attribute, TextPiece};
|
||||
|
||||
use crate::{
|
||||
manifest::{placeholder, Os, Store, Tag},
|
||||
resource::ResourceFile,
|
||||
Error, State,
|
||||
};
|
||||
|
||||
const SAVE_INTERVAL: u32 = 100;
|
||||
|
||||
async fn make_client() -> Result<mediawiki::api::Api, Error> {
|
||||
mediawiki::api::Api::new("https://www.pcgamingwiki.com/w/api.php")
|
||||
.await
|
||||
.map_err(Error::WikiClient)
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct WikiCache(pub BTreeMap<String, WikiCacheEntry>);
|
||||
|
||||
impl ResourceFile for WikiCache {
|
||||
const FILE_NAME: &'static str = "data/wiki-game-cache.yaml";
|
||||
}
|
||||
|
||||
async fn get_page_title(id: u64) -> Result<Option<String>, Error> {
|
||||
let wiki = make_client().await?;
|
||||
let params = wiki.params_into(&[("action", "query"), ("pageids", id.to_string().as_str())]);
|
||||
|
||||
let res = wiki.get_query_api_json_all(¶ms).await?;
|
||||
|
||||
for page in res["query"]["pages"]
|
||||
.as_object()
|
||||
.ok_or(Error::WikiData("query.pages"))?
|
||||
.values()
|
||||
{
|
||||
let found_id = page["pageid"].as_u64().ok_or(Error::WikiData("query.pages[].pageid"))?;
|
||||
if found_id == id {
|
||||
let title = page["title"].as_str();
|
||||
return Ok(title.map(|x| x.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn is_game_article(query: &str) -> Result<bool, Error> {
|
||||
let wiki = make_client().await?;
|
||||
let params = wiki.params_into(&[("action", "query"), ("prop", "categories"), ("titles", query)]);
|
||||
|
||||
let res = wiki.get_query_api_json_all(¶ms).await?;
|
||||
|
||||
for page in res["query"]["pages"]
|
||||
.as_object()
|
||||
.ok_or(Error::WikiData("query.pages"))?
|
||||
.values()
|
||||
{
|
||||
let title = page["title"].as_str().ok_or(Error::WikiData("query.pages[].title"))?;
|
||||
if title == query {
|
||||
for category in page["categories"]
|
||||
.as_array()
|
||||
.ok_or(Error::WikiData("query.pages[].categories"))?
|
||||
{
|
||||
let category_name = category["title"]
|
||||
.as_str()
|
||||
.ok_or(Error::WikiData("query.pages[].categories[].title"))?;
|
||||
if category_name == "Category:Games" {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
impl WikiCache {
|
||||
pub async fn flag_recent_changes(&mut self, meta: &mut WikiMetaCache) -> Result<(), Error> {
|
||||
struct RecentChange {
|
||||
page_id: u64,
|
||||
}
|
||||
|
||||
let start = meta.last_checked_recent_changes - chrono::Duration::minutes(1);
|
||||
let end = chrono::Utc::now();
|
||||
println!("Getting recent changes from {} to {}", start, end);
|
||||
|
||||
let wiki = make_client().await?;
|
||||
let params = wiki.params_into(&[
|
||||
("action", "query"),
|
||||
("list", "recentchanges"),
|
||||
("rcprop", "title|ids|redirect"),
|
||||
("rcdir", "newer"),
|
||||
("rcstart", &start.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)),
|
||||
("rcend", &end.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)),
|
||||
("rclimit", "500"),
|
||||
("rcnamespace", "0"),
|
||||
("rctype", "edit|new"),
|
||||
]);
|
||||
|
||||
let res = wiki.get_query_api_json_all(¶ms).await?;
|
||||
|
||||
let mut changes = BTreeMap::<String, RecentChange>::new();
|
||||
for change in res["query"]["recentchanges"]
|
||||
.as_array()
|
||||
.ok_or(Error::WikiData("query.recentchanges"))?
|
||||
{
|
||||
let title = change["title"]
|
||||
.as_str()
|
||||
.ok_or(Error::WikiData("query.recentchanges[].title"))?
|
||||
.to_string();
|
||||
let page_id = change["pageid"]
|
||||
.as_u64()
|
||||
.ok_or(Error::WikiData("query.recentchanges[].pageid"))?;
|
||||
let redirect = change["redirect"].is_string();
|
||||
|
||||
if !redirect {
|
||||
// We don't need the entries for the redirect pages themselves.
|
||||
// We'll update our data when we get to the entry for the new page name.
|
||||
changes.insert(title, RecentChange { page_id });
|
||||
}
|
||||
}
|
||||
|
||||
for (title, RecentChange { page_id }) in changes {
|
||||
if self.0.contains_key(&title) {
|
||||
// Existing entry has been edited.
|
||||
println!("[E ] {}", &title);
|
||||
self.0
|
||||
.entry(title.to_string())
|
||||
.and_modify(|x| x.state = State::Outdated);
|
||||
} else {
|
||||
// Check for a rename.
|
||||
let mut old_name = None;
|
||||
for (existing_name, existing_info) in &self.0 {
|
||||
if existing_info.page_id == page_id {
|
||||
// We have a confirmed rename.
|
||||
println!("[ M ] {} <<< {}", &title, existing_name);
|
||||
old_name = Some(existing_name.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
match old_name {
|
||||
None => {
|
||||
// Brand new page.
|
||||
match is_game_article(&title).await {
|
||||
Ok(true) => {
|
||||
// It's a game, so add it to the cache.
|
||||
println!("[ C] {}", &title);
|
||||
self.0.insert(
|
||||
title.to_string(),
|
||||
WikiCacheEntry {
|
||||
page_id,
|
||||
state: State::Outdated,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(false) => {
|
||||
// Ignore since it's not relevant.
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Unable to check if article is for a game: {} | {}", &title, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(old_name) => {
|
||||
self.0.entry(title.to_string()).and_modify(|x| {
|
||||
x.page_id = page_id;
|
||||
x.state = State::Outdated;
|
||||
x.renamed_from.push(old_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
meta.last_checked_recent_changes = end;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn add_new_games(&mut self) -> Result<(), Error> {
|
||||
let wiki = make_client().await?;
|
||||
let params = wiki.params_into(&[
|
||||
("action", "query"),
|
||||
("list", "categorymembers"),
|
||||
("cmtitle", "Category:Games"),
|
||||
("cmlimit", "500"),
|
||||
]);
|
||||
|
||||
let res = wiki.get_query_api_json_all(¶ms).await?;
|
||||
|
||||
for page in res["query"]["categorymembers"]
|
||||
.as_array()
|
||||
.ok_or(Error::WikiData("query.categorymembers"))?
|
||||
{
|
||||
let title = page["title"]
|
||||
.as_str()
|
||||
.ok_or(Error::WikiData("query.categorymembers[].title"))?;
|
||||
let page_id = page["pageid"]
|
||||
.as_u64()
|
||||
.ok_or(Error::WikiData("query.categorymembers[].pageid"))?;
|
||||
|
||||
if self.0.contains_key(title) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut old_name = None;
|
||||
for (existing_name, existing_info) in &self.0 {
|
||||
if existing_info.page_id == page_id {
|
||||
old_name = Some(existing_name.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
match old_name {
|
||||
None => {
|
||||
self.0.insert(
|
||||
title.to_string(),
|
||||
WikiCacheEntry {
|
||||
page_id,
|
||||
state: State::Outdated,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
Some(old_name) => {
|
||||
let mut data = self.0[&old_name].clone();
|
||||
data.state = State::Outdated;
|
||||
if !data.renamed_from.contains(&old_name) {
|
||||
data.renamed_from.push(old_name.clone());
|
||||
}
|
||||
|
||||
self.0.insert(title.to_string(), data);
|
||||
self.0.remove(&old_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn refresh(&mut self, outdated_only: bool, titles: Option<Vec<String>>) -> Result<(), Error> {
|
||||
let mut i = 0;
|
||||
let solo = titles.is_some();
|
||||
let titles: Vec<_> = titles.unwrap_or_else(|| self.0.keys().cloned().collect());
|
||||
|
||||
for title in &titles {
|
||||
let cached = self.0.get(title).cloned().unwrap_or_default();
|
||||
if outdated_only && cached.state != State::Outdated {
|
||||
continue;
|
||||
}
|
||||
|
||||
println!("Wiki: {}", title);
|
||||
let latest = WikiCacheEntry::fetch_from_page(title.clone()).await;
|
||||
match latest {
|
||||
Ok(mut latest) => {
|
||||
latest.renamed_from = cached.renamed_from.clone();
|
||||
self.0.insert(title.to_string(), latest);
|
||||
}
|
||||
Err(Error::PageMissing) => {
|
||||
if solo {
|
||||
return Err(Error::PageMissing);
|
||||
}
|
||||
|
||||
// Couldn't find it by name, so try again by ID.
|
||||
// This can happen for pages moved without leaving a redirect.
|
||||
// (If they have a redirect, then the recent changes code takes care of it.)
|
||||
let Some(new_title) = get_page_title(cached.page_id).await? else {
|
||||
return Err(Error::PageMissing);
|
||||
};
|
||||
println!(
|
||||
":: refresh: page {} called '{}' renamed to '{}'",
|
||||
cached.page_id, title, &new_title
|
||||
);
|
||||
|
||||
let mut data = self.0[title].clone();
|
||||
data.renamed_from.push(title.clone());
|
||||
self.0.insert(new_title.clone(), data);
|
||||
self.0.remove(title);
|
||||
|
||||
let mut latest = WikiCacheEntry::fetch_from_page(title.clone()).await?;
|
||||
latest.renamed_from = cached.renamed_from.clone();
|
||||
self.0.insert(new_title.clone(), latest);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
i += 1;
|
||||
if i % SAVE_INTERVAL == 0 {
|
||||
self.save();
|
||||
println!("\n:: saved\n");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct WikiCacheEntry {
|
||||
#[serde(skip_serializing_if = "State::is_handled")]
|
||||
pub state: State,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub gog: Option<u64>,
|
||||
#[serde(skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub gog_side: BTreeSet<u64>,
|
||||
pub page_id: u64,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub renamed_from: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub steam: Option<u32>,
|
||||
#[serde(skip_serializing_if = "BTreeSet::is_empty")]
|
||||
pub steam_side: BTreeSet<u32>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub templates: Vec<String>,
|
||||
}
|
||||
|
||||
impl WikiCacheEntry {
|
||||
pub async fn fetch_from_page(article: String) -> Result<Self, Error> {
|
||||
let mut out = WikiCacheEntry {
|
||||
state: State::Updated,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let wiki = make_client().await?;
|
||||
let params = wiki.params_into(&[("action", "parse"), ("prop", "wikitext"), ("page", &article)]);
|
||||
|
||||
let res = wiki
|
||||
.get_query_api_json_all(¶ms)
|
||||
.await
|
||||
.map_err(|_| Error::PageMissing)?;
|
||||
out.page_id = res["parse"]["pageid"].as_u64().ok_or(Error::WikiData("parse.pageid"))?;
|
||||
let raw_wikitext = res["parse"]["wikitext"]["*"]
|
||||
.as_str()
|
||||
.ok_or(Error::WikiData("parse.wikitext"))?;
|
||||
|
||||
let wikitext = wikitext_parser::parse_wikitext(raw_wikitext, article, |_| ());
|
||||
|
||||
for template in wikitext.list_double_brace_expressions() {
|
||||
if let wikitext_parser::TextPiece::DoubleBraceExpression { tag, attributes } = &template {
|
||||
match tag.to_string().to_lowercase().trim() {
|
||||
"infobox game" => {
|
||||
for attribute in attributes {
|
||||
match attribute.name.as_deref() {
|
||||
Some("steam appid") => {
|
||||
if let Ok(value) = attribute.value.to_string().parse::<u32>() {
|
||||
if value > 0 {
|
||||
out.steam = Some(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("steam appid side") => {
|
||||
out.steam_side = attribute
|
||||
.value
|
||||
.to_string()
|
||||
.split(',')
|
||||
.filter_map(|x| x.trim().parse::<u32>().ok())
|
||||
.filter(|x| *x > 0)
|
||||
.collect();
|
||||
}
|
||||
Some("gogcom id") => {
|
||||
if let Ok(value) = attribute.value.to_string().parse::<u64>() {
|
||||
if value > 0 {
|
||||
out.gog = Some(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("gogcom id side") => {
|
||||
out.gog_side = attribute
|
||||
.value
|
||||
.to_string()
|
||||
.split(',')
|
||||
.filter_map(|x| x.trim().parse::<u64>().ok())
|
||||
.filter(|x| *x > 0)
|
||||
.collect();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
"game data" => {
|
||||
for attribute in attributes {
|
||||
for template in &attribute.value.pieces {
|
||||
if let wikitext_parser::TextPiece::DoubleBraceExpression { tag, .. } = &template {
|
||||
let is_save = tag.to_string() == "Game data/saves";
|
||||
let is_config = tag.to_string() == "Game data/config";
|
||||
|
||||
if is_save || is_config {
|
||||
out.templates.push(template.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The parser does not handle HTML tags, so we remove some tags that are only used for annotations.
|
||||
/// Others, like `code` and `sup`, are used both for path segments and annotations,
|
||||
/// so we can't assume how to replace them properly.
|
||||
fn preprocess_template(template: &str) -> String {
|
||||
let mut out = template.to_string();
|
||||
|
||||
static HTML_COMMENT: Lazy<Regex> = Lazy::new(|| Regex::new(r"<!--.+?-->").unwrap());
|
||||
static HTML_REF: Lazy<Regex> = Lazy::new(|| Regex::new(r"<ref>.+?</ref>").unwrap());
|
||||
|
||||
for (pattern, replacement) in [(&HTML_COMMENT, ""), (&HTML_REF, "")] {
|
||||
out = pattern.replace_all(&out, replacement).to_string();
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
pub fn parse_paths(&self, article: String) -> Vec<WikiPath> {
|
||||
let mut out = vec![];
|
||||
|
||||
for raw in &self.templates {
|
||||
let preprocessed = Self::preprocess_template(raw);
|
||||
let parsed = wikitext_parser::parse_wikitext(&preprocessed, article.clone(), |_| ());
|
||||
for template in parsed.list_double_brace_expressions() {
|
||||
if let wikitext_parser::TextPiece::DoubleBraceExpression { tag, attributes } = &template {
|
||||
let is_save = tag.to_string() == "Game data/saves";
|
||||
let is_config = tag.to_string() == "Game data/config";
|
||||
|
||||
if (!is_save && !is_config) || attributes.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let platform = attributes[0].value.to_string();
|
||||
for attribute in attributes.iter().skip(1) {
|
||||
let info = flatten_path(attribute)
|
||||
.with_platform(&platform)
|
||||
.with_tags(is_save, is_config)
|
||||
.normalize();
|
||||
if info.usable() {
|
||||
out.push(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum PathKind {
|
||||
File,
|
||||
Registry,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct WikiPath {
|
||||
pub composite: String,
|
||||
pub irregular: bool,
|
||||
pub kind: Option<PathKind>,
|
||||
pub store: Option<Store>,
|
||||
pub os: Option<Os>,
|
||||
pub tags: BTreeSet<Tag>,
|
||||
}
|
||||
|
||||
impl WikiPath {
|
||||
fn incorporate(&mut self, other: Self) {
|
||||
if other.irregular {
|
||||
self.irregular = true;
|
||||
}
|
||||
|
||||
if other.kind.is_some() {
|
||||
self.kind = other.kind;
|
||||
}
|
||||
|
||||
if other.store.is_some() {
|
||||
self.store = other.store;
|
||||
}
|
||||
|
||||
if other.os.is_some() {
|
||||
self.os = other.os;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn incorporate_text(&mut self, text: &str) {
|
||||
if text.contains(['<', '>']) {
|
||||
self.irregular = true;
|
||||
} else {
|
||||
self.composite += text;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn incorporate_raw(&mut self, other: Self) {
|
||||
self.incorporate_text(&other.composite);
|
||||
self.incorporate(other)
|
||||
}
|
||||
|
||||
pub fn incorporate_path(&mut self, other: Self) {
|
||||
if let Some(mapped) = MAPPED_PATHS.get(other.composite.to_lowercase().as_str()) {
|
||||
self.composite += mapped.manifest;
|
||||
|
||||
if mapped.kind.is_some() {
|
||||
self.kind = mapped.kind;
|
||||
}
|
||||
|
||||
if mapped.store.is_some() {
|
||||
self.store = mapped.store;
|
||||
}
|
||||
|
||||
if mapped.os.is_some() {
|
||||
self.os = mapped.os;
|
||||
}
|
||||
}
|
||||
|
||||
self.incorporate(other)
|
||||
}
|
||||
|
||||
pub fn normalize(mut self) -> Self {
|
||||
self.composite = self.composite.trim().trim_end_matches(['/', '\\']).replace('\\', "/");
|
||||
|
||||
if self.composite == "~" || self.composite.starts_with("~/") {
|
||||
self.composite = self.composite.replacen('~', placeholder::HOME, 1);
|
||||
}
|
||||
|
||||
static CONSECUTIVE_SLASHES: Lazy<Regex> = Lazy::new(|| Regex::new(r"/{2,}").unwrap());
|
||||
static ENDING_WILDCARD: Lazy<Regex> = Lazy::new(|| Regex::new(r"(/\*)+$").unwrap());
|
||||
static APP_DATA: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?i)%appdata%").unwrap());
|
||||
static APP_DATA_ROAMING: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?i)%userprofile%/AppData/Roaming").unwrap());
|
||||
static APP_DATA_LOCAL: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?i)%localappdata%").unwrap());
|
||||
static APP_DATA_LOCAL_2: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?i)%userprofile%/AppData/Local/").unwrap());
|
||||
static USER_PROFILE: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?i)%userprofile%").unwrap());
|
||||
static DOCUMENTS: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?i)%userprofile%/Documents").unwrap());
|
||||
|
||||
for (pattern, replacement) in [
|
||||
(&CONSECUTIVE_SLASHES, "/"),
|
||||
(&ENDING_WILDCARD, ""),
|
||||
(&APP_DATA, placeholder::WIN_APP_DATA),
|
||||
(&APP_DATA_ROAMING, placeholder::WIN_APP_DATA),
|
||||
(&APP_DATA_LOCAL, placeholder::WIN_LOCAL_APP_DATA),
|
||||
(&APP_DATA_LOCAL_2, &format!("{}/", placeholder::WIN_LOCAL_APP_DATA)),
|
||||
(&USER_PROFILE, placeholder::HOME),
|
||||
(&DOCUMENTS, placeholder::WIN_DOCUMENTS),
|
||||
] {
|
||||
self.composite = pattern.replace_all(&self.composite, replacement).to_string();
|
||||
}
|
||||
|
||||
if self.kind.is_none() {
|
||||
self.kind = Some(PathKind::File);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_platform(mut self, platform: &str) -> Self {
|
||||
match platform.to_lowercase().trim() {
|
||||
"windows" => {
|
||||
self.os = Some(Os::Windows);
|
||||
}
|
||||
"os x" => {
|
||||
self.os = Some(Os::Mac);
|
||||
}
|
||||
"linux" => {
|
||||
self.os = Some(Os::Linux);
|
||||
}
|
||||
"dos" => {
|
||||
self.os = Some(Os::Dos);
|
||||
}
|
||||
"steam" => {
|
||||
self.store = Some(Store::Steam);
|
||||
}
|
||||
"microsoft store" => {
|
||||
self.os = Some(Os::Windows);
|
||||
self.store = Some(Store::Microsoft);
|
||||
}
|
||||
"gog.com" => {
|
||||
self.store = Some(Store::Gog);
|
||||
}
|
||||
"epic games" => {
|
||||
self.store = Some(Store::Epic);
|
||||
}
|
||||
"uplay" => {
|
||||
self.store = Some(Store::Uplay);
|
||||
}
|
||||
"origin" => {
|
||||
self.store = Some(Store::Origin);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_tags(mut self, save: bool, config: bool) -> Self {
|
||||
if save {
|
||||
self.tags.insert(Tag::Save);
|
||||
}
|
||||
if config {
|
||||
self.tags.insert(Tag::Config);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
fn too_broad(&self) -> bool {
|
||||
use placeholder::{BASE, HOME, ROOT, STORE_USER_ID, WIN_DIR, WIN_DOCUMENTS};
|
||||
|
||||
let placeholders: Vec<_> = MAPPED_PATHS.values().map(|x| x.manifest).collect();
|
||||
if placeholders.iter().any(|x| *x == self.composite) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// These paths are present whether or not the game is installed.
|
||||
// If possible, they should be narrowed down on the wiki.
|
||||
let broad = vec![
|
||||
format!("{BASE}/{STORE_USER_ID}"), // because `<storeUserId>` is handled as `*`
|
||||
format!("{HOME}/Documents"),
|
||||
format!("{HOME}/Saved Games"),
|
||||
format!("{HOME}/AppData"),
|
||||
format!("{HOME}/AppData/Local"),
|
||||
format!("{HOME}/AppData/Local/Packages"),
|
||||
format!("{HOME}/AppData/LocalLow"),
|
||||
format!("{HOME}/AppData/Roaming"),
|
||||
format!("{HOME}/Documents/My Games"),
|
||||
format!("{HOME}/Library/Application Support"),
|
||||
format!("{HOME}/Library/Preferences"),
|
||||
format!("{HOME}/Telltale Games"),
|
||||
format!("{ROOT}/config"),
|
||||
format!("{WIN_DIR}/win.ini"),
|
||||
format!("{WIN_DOCUMENTS}/My Games"),
|
||||
format!("{WIN_DOCUMENTS}/Telltale Games"),
|
||||
];
|
||||
if broad.iter().any(|x| *x == self.composite) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Several games/episodes are grouped together here.
|
||||
if self
|
||||
.composite
|
||||
.starts_with(&format!("{WIN_DOCUMENTS}/Telltale Games/*/"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Drive letters:
|
||||
static DRIVES: Lazy<Regex> = Lazy::new(|| Regex::new(r"^[a-zA-Z]:$").unwrap());
|
||||
if DRIVES.is_match(&self.composite) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Root:
|
||||
if self.composite == "/" {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Relative path wildcard:
|
||||
if self.composite.starts_with('*') {
|
||||
return true;
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
fn irregular(&self) -> bool {
|
||||
self.irregular || self.composite.contains("{{")
|
||||
}
|
||||
|
||||
pub fn usable(&self) -> bool {
|
||||
!self.composite.is_empty() && !self.irregular() && !self.too_broad()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MappedPath {
|
||||
pub manifest: &'static str,
|
||||
pub os: Option<Os>,
|
||||
pub store: Option<Store>,
|
||||
pub kind: Option<PathKind>,
|
||||
}
|
||||
|
||||
pub fn flatten_path(attribute: &Attribute) -> WikiPath {
|
||||
let mut out = WikiPath::default();
|
||||
|
||||
for piece in &attribute.value.pieces {
|
||||
match piece {
|
||||
TextPiece::Text { text, .. } => {
|
||||
out.incorporate_text(text);
|
||||
}
|
||||
TextPiece::DoubleBraceExpression { tag, attributes } => match tag.to_string().to_lowercase().trim() {
|
||||
"p" | "path" => {
|
||||
for attribute in attributes {
|
||||
let flat = flatten_path(attribute);
|
||||
out.incorporate_path(flat);
|
||||
}
|
||||
}
|
||||
"code" | "file" => {
|
||||
out.composite += "*";
|
||||
}
|
||||
"localizedpath" => {
|
||||
for attribute in attributes {
|
||||
let flat = flatten_path(attribute);
|
||||
out.incorporate_raw(flat);
|
||||
}
|
||||
}
|
||||
"note" => {
|
||||
// Ignored.
|
||||
}
|
||||
_ => {
|
||||
out.irregular = true;
|
||||
}
|
||||
},
|
||||
TextPiece::InternalLink { .. } => {}
|
||||
TextPiece::ListItem { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
static MAPPED_PATHS: Lazy<HashMap<&'static str, MappedPath>> = Lazy::new(|| {
|
||||
HashMap::from_iter([
|
||||
(
|
||||
"game",
|
||||
MappedPath {
|
||||
manifest: placeholder::BASE,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"uid",
|
||||
MappedPath {
|
||||
manifest: placeholder::STORE_USER_ID,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"steam",
|
||||
MappedPath {
|
||||
manifest: placeholder::ROOT,
|
||||
store: Some(Store::Steam),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"uplay",
|
||||
MappedPath {
|
||||
manifest: placeholder::ROOT,
|
||||
store: Some(Store::Uplay),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"ubisoftconnect",
|
||||
MappedPath {
|
||||
manifest: placeholder::ROOT,
|
||||
store: Some(Store::Uplay),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"hkcu",
|
||||
MappedPath {
|
||||
manifest: "HKEY_CURRENT_USER",
|
||||
os: Some(Os::Windows),
|
||||
kind: Some(PathKind::Registry),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"hklm",
|
||||
MappedPath {
|
||||
manifest: "HKEY_LOCAL_MACHINE",
|
||||
os: Some(Os::Windows),
|
||||
kind: Some(PathKind::Registry),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"wow64",
|
||||
MappedPath {
|
||||
manifest: "WOW6432Node",
|
||||
os: Some(Os::Windows),
|
||||
kind: Some(PathKind::Registry),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"username",
|
||||
MappedPath {
|
||||
manifest: placeholder::OS_USER_NAME,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"userprofile",
|
||||
MappedPath {
|
||||
manifest: placeholder::HOME,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"userprofile\\documents",
|
||||
MappedPath {
|
||||
manifest: placeholder::WIN_DOCUMENTS,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"userprofile\\appdata\\locallow",
|
||||
MappedPath {
|
||||
manifest: "<home>/AppData/LocalLow",
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"appdata",
|
||||
MappedPath {
|
||||
manifest: placeholder::WIN_APP_DATA,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"localappdata",
|
||||
MappedPath {
|
||||
manifest: placeholder::WIN_LOCAL_APP_DATA,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"public",
|
||||
MappedPath {
|
||||
manifest: placeholder::WIN_PUBLIC,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"allusersprofile",
|
||||
MappedPath {
|
||||
manifest: placeholder::WIN_PROGRAM_DATA,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"programdata",
|
||||
MappedPath {
|
||||
manifest: placeholder::WIN_PROGRAM_DATA,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"windir",
|
||||
MappedPath {
|
||||
manifest: placeholder::WIN_DIR,
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"syswow64",
|
||||
MappedPath {
|
||||
manifest: "<winDir>/SysWOW64",
|
||||
os: Some(Os::Windows),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"osxhome",
|
||||
MappedPath {
|
||||
manifest: placeholder::HOME,
|
||||
os: Some(Os::Mac),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"linuxhome",
|
||||
MappedPath {
|
||||
manifest: placeholder::HOME,
|
||||
os: Some(Os::Linux),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"xdgdatahome",
|
||||
MappedPath {
|
||||
manifest: placeholder::XDG_DATA,
|
||||
os: Some(Os::Linux),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"xdgconfighome",
|
||||
MappedPath {
|
||||
manifest: placeholder::XDG_CONFIG,
|
||||
os: Some(Os::Linux),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
])
|
||||
});
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct WikiMetaCache {
|
||||
pub last_checked_recent_changes: chrono::DateTime<chrono::Utc>,
|
||||
}
|
||||
|
||||
impl ResourceFile for WikiMetaCache {
|
||||
const FILE_NAME: &'static str = "data/wiki-meta-cache.yaml";
|
||||
|
||||
fn initialize(mut self) -> Self {
|
||||
self.last_checked_recent_changes = chrono::Utc::now() - chrono::Duration::days(1);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_is_game_article() {
|
||||
assert!(matches!(is_game_article("Celeste").await, Ok(true)));
|
||||
assert!(matches!(is_game_article("Template:Path").await, Ok(false)));
|
||||
}
|
||||
}
|
||||
818
src/wiki.ts
818
src/wiki.ts
|
|
@ -1,818 +0,0 @@
|
|||
import { DELAY_BETWEEN_GAMES_MS, REPO, PathType, UnsupportedOsError, UnsupportedPathError, YamlFile } from ".";
|
||||
import { Constraint, Game, Store, Tag, Os } from "./manifest";
|
||||
import moment from "moment";
|
||||
import NodeMw from "nodemw";
|
||||
import Wikiapi from "wikiapi";
|
||||
import { parse as parseWiki } from 'wikiparse';
|
||||
|
||||
interface Template {
|
||||
type: "template",
|
||||
name: string,
|
||||
parameters: {},
|
||||
positionalParameters: Array<Array<WikiNode>>,
|
||||
};
|
||||
|
||||
type WikiNode = string | Template | {
|
||||
type: "code" | "pre",
|
||||
content: Array<WikiNode>,
|
||||
attributes: { [key: string]: string },
|
||||
} | {
|
||||
type: "comment" | "bold" | "italic" | "ref",
|
||||
content: Array<WikiNode>,
|
||||
} | {
|
||||
type: "link",
|
||||
to: string,
|
||||
content: Array<WikiNode>,
|
||||
} | {
|
||||
type: "tag",
|
||||
content: Array<WikiNode>,
|
||||
attributes: { [key: string]: string },
|
||||
name: string,
|
||||
};
|
||||
|
||||
export type WikiGameCache = {
|
||||
[title: string]: {
|
||||
pageId: number,
|
||||
dirty?: boolean,
|
||||
renamedFrom?: Array<string>,
|
||||
templates?: Array<string>,
|
||||
steam?: number,
|
||||
steamSide?: Array<number>,
|
||||
gog?: number
|
||||
gogSide?: Array<number>,
|
||||
};
|
||||
};
|
||||
|
||||
export type WikiMetaCache = {
|
||||
lastCheckedRecentChanges: string;
|
||||
};
|
||||
|
||||
export class WikiGameCacheFile extends YamlFile<WikiGameCache> {
|
||||
path = `${REPO}/data/wiki-game-cache.yaml`;
|
||||
defaultData = {};
|
||||
|
||||
async addNewGames(): Promise<void> {
|
||||
const wiki = makeApiClient();
|
||||
const pages: Array<{ pageid: number, title: string }> = JSON.parse(JSON.stringify(await wiki.categorymembers("Games")));
|
||||
for (const page of pages) {
|
||||
if (!this.data.hasOwnProperty(page.title)) {
|
||||
let newGame = true;
|
||||
for (const [k, v] of Object.entries(this.data)) {
|
||||
if (v.pageId === page.pageid) {
|
||||
newGame = false;
|
||||
this.data[page.title] = v;
|
||||
this.data[page.title].dirty = true;
|
||||
if (!(v.renamedFrom ?? []).includes(k)) {
|
||||
this.data[page.title].renamedFrom = [...(v.renamedFrom ?? []), k];
|
||||
}
|
||||
delete this.data[k];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!newGame) {
|
||||
continue;
|
||||
}
|
||||
this.data[page.title] = {
|
||||
pageId: page.pageid,
|
||||
dirty: true,
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async refresh(skipUntil: string | undefined, limit: number, all: boolean, games: Array<string>): Promise<void> {
|
||||
let i = 0;
|
||||
let foundSkipUntil = false;
|
||||
const client = makeApiClient();
|
||||
for (const pageTitle of Object.keys(this.data).sort()) {
|
||||
if (skipUntil && !foundSkipUntil) {
|
||||
if (pageTitle === skipUntil) {
|
||||
foundSkipUntil = true;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (games.length > 0) {
|
||||
if (!games.includes(pageTitle)) {
|
||||
continue;
|
||||
}
|
||||
} else if (!all && !this.data[pageTitle].dirty) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// console.log(`Refreshing wiki page: ${pageTitle}`);
|
||||
await getGame(pageTitle, this.data, client);
|
||||
|
||||
i++;
|
||||
if (limit > 0 && i >= limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
// main() will save at the end, but we do a periodic save as well
|
||||
// in case something goes wrong or the script gets cancelled:
|
||||
if (i % 250 === 0) {
|
||||
this.save();
|
||||
console.log("\n:: saved\n");
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, DELAY_BETWEEN_GAMES_MS));
|
||||
}
|
||||
}
|
||||
|
||||
async flagRecentChanges(metaCache: WikiMetaCacheFile): Promise<void> {
|
||||
const now = moment();
|
||||
const changes = await getRecentChanges(now.toDate(), moment(metaCache.data.lastCheckedRecentChanges).subtract(1, "minute").toDate());
|
||||
const client = makeApiClient2();
|
||||
|
||||
for (const [recentName, recentInfo] of Object.entries(changes).sort((x, y) => x[0].localeCompare(y[0]))) {
|
||||
if (recentInfo.redirect) {
|
||||
// This is an entry for the redirect page itself. We can ignore
|
||||
// it, and we'll update our data when we get to the entry for
|
||||
// the new page name.
|
||||
continue;
|
||||
} else if (this.data[recentName] !== undefined) {
|
||||
// Existing entry has been edited.
|
||||
console.log(`[E ] ${recentName}`);
|
||||
this.data[recentName].dirty = true;
|
||||
} else {
|
||||
// Check for a rename.
|
||||
let renamed = false;
|
||||
for (const [existingName, existingInfo] of Object.entries(this.data)) {
|
||||
if (existingInfo.pageId === recentInfo.pageId) {
|
||||
// We have a confirmed rename.
|
||||
console.log(`[ M ] ${recentName} <<< ${existingName}`);
|
||||
renamed = true;
|
||||
this.data[recentName] = {
|
||||
...existingInfo,
|
||||
pageId: recentInfo.pageId,
|
||||
dirty: true,
|
||||
renamedFrom: [...(existingInfo.renamedFrom ?? []), existingName]
|
||||
};
|
||||
delete this.data[existingName];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!renamed) {
|
||||
// Brand new page.
|
||||
const [data, _] = await callMw<Array<string>>(client, "getArticleCategories", recentName);
|
||||
if (data.includes("Category:Games")) {
|
||||
// It's a game, so add it to the cache.
|
||||
console.log(`[ C] ${recentName}`);
|
||||
this.data[recentName] = { pageId: recentInfo.pageId, dirty: true };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metaCache.data.lastCheckedRecentChanges = now.toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
export class WikiMetaCacheFile extends YamlFile<WikiMetaCache> {
|
||||
path = `${REPO}/data/wiki-meta-cache.yaml`;
|
||||
defaultData = {
|
||||
lastCheckedRecentChanges: moment().subtract(1, "days").toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
interface RecentChanges {
|
||||
[article: string]: {
|
||||
pageId: number;
|
||||
redirect: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
// This defines how {{P|game}} and such are converted.
|
||||
const PATH_ARGS: { [arg: string]: { mapped: string, when?: Constraint, registry?: boolean, ignored?: boolean } } = {
|
||||
game: {
|
||||
mapped: "<base>",
|
||||
},
|
||||
uid: {
|
||||
mapped: "<storeUserId>",
|
||||
},
|
||||
steam: {
|
||||
mapped: "<root>",
|
||||
when: {
|
||||
store: "steam",
|
||||
},
|
||||
},
|
||||
uplay: {
|
||||
mapped: "<root>",
|
||||
when: {
|
||||
store: "uplay"
|
||||
},
|
||||
},
|
||||
ubisoftconnect: {
|
||||
mapped: "<root>",
|
||||
when: {
|
||||
store: "uplay"
|
||||
},
|
||||
},
|
||||
hkcu: {
|
||||
mapped: "HKEY_CURRENT_USER",
|
||||
when: { os: "windows" },
|
||||
registry: true,
|
||||
},
|
||||
hklm: {
|
||||
mapped: "HKEY_LOCAL_MACHINE",
|
||||
when: { os: "windows" },
|
||||
registry: true,
|
||||
},
|
||||
wow64: {
|
||||
mapped: "WOW6432Node",
|
||||
when: { os: "windows" },
|
||||
registry: true,
|
||||
},
|
||||
username: {
|
||||
mapped: "<osUserName>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
userprofile: {
|
||||
mapped: "<home>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
"userprofile\\documents": {
|
||||
mapped: "<winDocuments>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
"userprofile\\appdata\\locallow": {
|
||||
mapped: "<home>/AppData/LocalLow",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
appdata: {
|
||||
mapped: "<winAppData>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
localappdata: {
|
||||
mapped: "<winLocalAppData>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
public: {
|
||||
mapped: "<winPublic>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
allusersprofile: {
|
||||
mapped: "<winProgramData>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
programdata: {
|
||||
mapped: "<winProgramData>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
windir: {
|
||||
mapped: "<winDir>",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
syswow64: {
|
||||
mapped: "<winDir>/SysWOW64",
|
||||
when: { os: "windows" },
|
||||
},
|
||||
osxhome: {
|
||||
mapped: "<home>",
|
||||
when: { os: "mac" },
|
||||
},
|
||||
linuxhome: {
|
||||
mapped: "<home>",
|
||||
when: { os: "linux" },
|
||||
},
|
||||
xdgdatahome: {
|
||||
mapped: "<xdgData>",
|
||||
when: { os: "linux" },
|
||||
},
|
||||
xdgconfighome: {
|
||||
mapped: "<xdgConfig>",
|
||||
when: { os: "linux" },
|
||||
},
|
||||
}
|
||||
|
||||
function makePathArgRegex(arg: string): RegExp {
|
||||
const escaped = `{{P(ath)?|${arg}}}`
|
||||
.replace(/\\/g, "\\\\")
|
||||
.replace(/\|/g, "\\|")
|
||||
.replace(/\{/g, "\\{")
|
||||
.replace(/\}/g, "\\}");
|
||||
return new RegExp(escaped, "gi");
|
||||
}
|
||||
|
||||
function normalizePath(path: string): string {
|
||||
return path
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/\/{2,}/g, "/")
|
||||
.replace(/\/(?=$)/g, "")
|
||||
.replace(/^~(?=($|\/))/, "<home>");
|
||||
}
|
||||
|
||||
/**
|
||||
* https://www.pcgamingwiki.com/wiki/Template:Path
|
||||
*/
|
||||
function parsePath(path: string): [string, PathType] {
|
||||
const pathType = getPathType(path);
|
||||
|
||||
for (const [arg, info] of Object.entries(PATH_ARGS)) {
|
||||
if (pathContainsArg(path, arg) && info.ignored) {
|
||||
throw new UnsupportedPathError(`Unsupported path argument: ${arg}`);
|
||||
}
|
||||
|
||||
let limit = 100;
|
||||
let i = 0;
|
||||
while (pathContainsArg(path, arg)) {
|
||||
path = path.replace(makePathArgRegex(arg), info.mapped);
|
||||
i++;
|
||||
if (i >= limit) {
|
||||
throw new UnsupportedPathError(`Unable to resolve path arguments in: ${path}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
path = normalizePath(path)
|
||||
.replace(/%userprofile%\/AppData\/Roaming/i, "<winAppData>")
|
||||
.replace(/%userprofile%\/AppData\/Local(?!Low)/i, "<winLocalAppData>")
|
||||
.replace(/%userprofile%\/Documents/i, "<winDocuments>")
|
||||
.replace(/%userprofile%/i, "<home>")
|
||||
.replace(/%appdata%/i, "<winAppData>")
|
||||
.replace(/%localappdata%/i, "<winLocalAppData>");
|
||||
|
||||
while (path.endsWith("/*")) {
|
||||
path = path.slice(0, path.length - 2);
|
||||
}
|
||||
|
||||
return [path.trim(), pathType];
|
||||
}
|
||||
|
||||
export function pathIsTooBroad(path: string): boolean {
|
||||
if (Object.values(PATH_ARGS).map(x => x.mapped).includes(path)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// These paths are present whether or not the game is installed.
|
||||
// If possible, they should be narrowed down on the wiki.
|
||||
if ([
|
||||
"<base>/<storeUserId>", // because `<storeUserId>` is handled as `*`
|
||||
"<home>/Documents",
|
||||
"<home>/Saved Games",
|
||||
"<home>/AppData",
|
||||
"<home>/AppData/Local",
|
||||
"<home>/AppData/Local/Packages",
|
||||
"<home>/AppData/LocalLow",
|
||||
"<home>/AppData/Roaming",
|
||||
"<home>/Documents/My Games",
|
||||
"<home>/Library/Application Support",
|
||||
"<home>/Library/Preferences",
|
||||
"<home>/Telltale Games",
|
||||
"<root>/config",
|
||||
"<winDir>/win.ini",
|
||||
"<winDocuments>/My Games",
|
||||
"<winDocuments>/Telltale Games",
|
||||
].includes(path)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Several games/episodes are grouped together here.
|
||||
if (path.startsWith("<winDocuments>/Telltale Games/*/")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Drive letters:
|
||||
if (path.match(/^[a-zA-Z]:$/)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Root:
|
||||
if (path === "/") {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Relative path wildcard:
|
||||
if (path.startsWith("*")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function pathContainsArg(path: string, arg: string): boolean {
|
||||
return path.match(makePathArgRegex(arg)) !== null;
|
||||
}
|
||||
|
||||
function getPathType(path: string): PathType {
|
||||
for (const [arg, info] of Object.entries(PATH_ARGS)) {
|
||||
if (info.registry && path.match(makePathArgRegex(arg)) !== null) {
|
||||
return PathType.Registry;
|
||||
}
|
||||
}
|
||||
return PathType.FileSystem;
|
||||
}
|
||||
|
||||
function getOsConstraintFromPath(path: string): Os | undefined {
|
||||
for (const [arg, info] of Object.entries(PATH_ARGS)) {
|
||||
if (pathContainsArg(path, arg) && info?.when?.os) {
|
||||
return info?.when?.os;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getStoreConstraintFromPath(path: string): Store | undefined {
|
||||
for (const [arg, info] of Object.entries(PATH_ARGS)) {
|
||||
if (pathContainsArg(path, arg) && info?.when?.store) {
|
||||
return info?.when?.store;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getConstraintFromSystem(system: string, path: string): Constraint {
|
||||
const constraint: Constraint = {};
|
||||
|
||||
if (system.match(/steam/i)) {
|
||||
constraint.store = "steam";
|
||||
} else if (system.match(/microsoft store/i)) {
|
||||
constraint.os = "windows";
|
||||
constraint.store = "microsoft";
|
||||
} else if (system.match(/gog\.com/i)) {
|
||||
constraint.store = "gog";
|
||||
} else if (system.match(/epic games/i)) {
|
||||
constraint.store = "epic";
|
||||
} else if (system.match(/uplay/i)) {
|
||||
constraint.store = "uplay";
|
||||
} else if (system.match(/origin/i)) {
|
||||
constraint.store = "origin";
|
||||
} else {
|
||||
try {
|
||||
constraint.os = parseOs(system);
|
||||
} catch { }
|
||||
}
|
||||
|
||||
const storeFromPath = getStoreConstraintFromPath(path);
|
||||
if (storeFromPath !== undefined) {
|
||||
constraint.store = storeFromPath;
|
||||
}
|
||||
|
||||
return constraint;
|
||||
}
|
||||
|
||||
function getTagFromTemplate(template: string): Tag | undefined {
|
||||
switch (template.toLowerCase()) {
|
||||
case "game data/saves":
|
||||
return "save";
|
||||
case "game data/config":
|
||||
return "config";
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function parseOs(os: string): Os {
|
||||
// Others seen: "Mac OS", "PC booter", "Amazon Games", "Ubisoft Connect"
|
||||
switch (os) {
|
||||
case "Windows":
|
||||
return "windows";
|
||||
case "OS X":
|
||||
return "mac";
|
||||
case "Linux":
|
||||
return "linux";
|
||||
case "DOS":
|
||||
return "dos";
|
||||
default:
|
||||
throw new UnsupportedOsError(`Unsupported OS: ${os}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Used for most functionality, but it seems like a less active project
|
||||
// and it's hard to figure out what functionality is available,
|
||||
// so we'll probably migrate to nodemw.
|
||||
function makeApiClient() {
|
||||
return new Wikiapi("https://www.pcgamingwiki.com/w/api.php");
|
||||
}
|
||||
|
||||
// Used for the Recent Changes page and getting a single page's categories.
|
||||
// Will probably also migrate to this in general.
|
||||
function makeApiClient2(): any {
|
||||
return new NodeMw({
|
||||
protocol: "https",
|
||||
server: "www.pcgamingwiki.com",
|
||||
path: "/w/api.php",
|
||||
debug: false,
|
||||
userAgent: "ludusavi-manifest-importer/0.0.0",
|
||||
concurrency: 1,
|
||||
});
|
||||
}
|
||||
|
||||
// Promise wrapper for nodemw.
|
||||
function callMw<T = any>(client, method: string, ...args: Array<any>): Promise<[T, any]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
client[method](...args, (err: any, data: T, next: any) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve([data, next]);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function getRecentChanges(newest: Date, oldest: Date): Promise<RecentChanges> {
|
||||
console.log(`Getting recent changes from ${oldest.toISOString()} to ${newest.toISOString()}`);
|
||||
const changes: RecentChanges = {};
|
||||
const client = makeApiClient2();
|
||||
const startTimestamp = newest.toISOString();
|
||||
const endTimestamp = oldest.toISOString();
|
||||
let rccontinue: string | undefined = undefined;
|
||||
|
||||
while (true) {
|
||||
const params = {
|
||||
action: "query",
|
||||
list: "recentchanges",
|
||||
rcprop: "title|ids|redirect",
|
||||
rcstart: startTimestamp,
|
||||
rcend: endTimestamp,
|
||||
rclimit: 500,
|
||||
rcnamespace: 0,
|
||||
rctype: "edit|new",
|
||||
rccontinue,
|
||||
};
|
||||
if (params.rccontinue === undefined) {
|
||||
delete params.rccontinue;
|
||||
}
|
||||
const [data, next] = await callMw<{ recentchanges: Array<{ title: string; pageid: number, redirect?: string }> }>(
|
||||
client.api, "call", params
|
||||
);
|
||||
|
||||
for (const article of data.recentchanges) {
|
||||
changes[article.title] = {
|
||||
pageId: article.pageid,
|
||||
redirect: article.redirect !== undefined,
|
||||
};
|
||||
}
|
||||
|
||||
if (next) {
|
||||
rccontinue = next.rccontinue;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return changes;
|
||||
}
|
||||
|
||||
/**
|
||||
* https://www.pcgamingwiki.com/wiki/Template:Game_data
|
||||
*/
|
||||
export async function getGame(pageTitle: string, cache: WikiGameCache, client: Wikiapi = null): Promise<string> {
|
||||
console.log(`Wiki: ${pageTitle}`);
|
||||
const wiki = client === null ? makeApiClient() : client;
|
||||
let page = await wiki.page(pageTitle, { rvprop: "ids|content" });
|
||||
if (page.missing !== undefined) {
|
||||
// Couldn't find it by name, so try again by ID.
|
||||
// This can happen for pages moved without leaving a redirect.
|
||||
// (If they have a redirect, then the recent changes code takes care of it.)
|
||||
const pageId = cache[pageTitle].pageId;
|
||||
const client = makeApiClient2();
|
||||
const params = {
|
||||
action: "query",
|
||||
pageids: [pageId],
|
||||
};
|
||||
try {
|
||||
const [data, _] = await callMw<{ pages: { [id: string]: { title: string } } }>(
|
||||
client.api, "call", params
|
||||
);
|
||||
const newTitle = data.pages[pageId.toString()].title;
|
||||
if (newTitle === undefined) {
|
||||
// This happened once intermittently; the cause is unclear.
|
||||
throw new Error("Unable to retrieve page by ID");
|
||||
}
|
||||
console.log(`:: getGame: page ${pageId} called '${pageTitle}' renamed to '${newTitle}'`);
|
||||
cache[newTitle] = cache[pageTitle];
|
||||
delete cache[pageTitle];
|
||||
if (cache[newTitle].renamedFrom === undefined) {
|
||||
cache[newTitle].renamedFrom = [pageTitle];
|
||||
} else {
|
||||
cache[newTitle].renamedFrom.push(pageTitle);
|
||||
}
|
||||
page = await wiki.page(newTitle, { rvprop: "ids|content" });
|
||||
pageTitle = newTitle;
|
||||
} catch {
|
||||
console.log(`:: page ${pageId} called '${pageTitle}' no longer exists`);
|
||||
}
|
||||
}
|
||||
|
||||
delete cache[pageTitle].templates;
|
||||
page.parse().each("template", template => {
|
||||
const templateName = template.name.toLowerCase();
|
||||
if (templateName === "infobox game") {
|
||||
const steamId = Number(template.parameters["steam appid"]);
|
||||
if (!isNaN(steamId) && steamId > 0) {
|
||||
cache[pageTitle].steam = steamId;
|
||||
}
|
||||
|
||||
cache[pageTitle].steamSide = [];
|
||||
const steamSides = template.parameters["steam appid side"] as string | undefined;
|
||||
if (steamSides !== undefined && typeof(steamSides) === "string") {
|
||||
for (const side of steamSides.split(",").map(Number)) {
|
||||
if (!isNaN(side) && side > 0) {
|
||||
cache[pageTitle].steamSide.push(side);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cache[pageTitle].steamSide.length === 0) {
|
||||
delete cache[pageTitle].steamSide;
|
||||
}
|
||||
|
||||
const gogId = Number(template.parameters["gogcom id"]);
|
||||
if (!isNaN(gogId) && gogId > 0) {
|
||||
cache[pageTitle].gog = gogId
|
||||
}
|
||||
|
||||
cache[pageTitle].gogSide = [];
|
||||
const gogSides = template.parameters["gogcom id side"] as string | undefined;
|
||||
if (gogSides !== undefined && typeof(gogSides) === "string") {
|
||||
for (const side of gogSides.split(",").map(Number)) {
|
||||
if (!isNaN(side) && side > 0) {
|
||||
cache[pageTitle].gogSide.push(side);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cache[pageTitle].gogSide.length === 0) {
|
||||
delete cache[pageTitle].gogSide;
|
||||
}
|
||||
} else if (templateName === "game data/saves" || templateName === "game data/config") {
|
||||
const reparsed = parseWiki(template.toString());
|
||||
if (reparsed[0].positionalParameters[1]?.length > 0 ?? false) {
|
||||
if (cache[pageTitle].templates === undefined) {
|
||||
cache[pageTitle].templates = [];
|
||||
}
|
||||
cache[pageTitle].templates.push(template.toString());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
delete cache[pageTitle].dirty;
|
||||
return pageTitle;
|
||||
}
|
||||
|
||||
function flattenParameter(nodes: Array<WikiNode>): [string, boolean] {
|
||||
let composite = "";
|
||||
let regular = true;
|
||||
|
||||
for (const node of nodes) {
|
||||
if (typeof node === "string") {
|
||||
composite += node;
|
||||
} else switch (node.type) {
|
||||
case "code":
|
||||
case "pre":
|
||||
composite += "*";
|
||||
break;
|
||||
case "template":
|
||||
switch (node.name.toLowerCase()) {
|
||||
case "p":
|
||||
case "path":
|
||||
const [flatP, regularP] = flattenParameter(node.positionalParameters[0]);
|
||||
if (!regularP) {
|
||||
regular = false;
|
||||
}
|
||||
composite += `{{${node.name}|${flatP}}}`;
|
||||
break;
|
||||
case "code":
|
||||
case "file":
|
||||
composite += "*";
|
||||
break;
|
||||
case "localizedpath":
|
||||
const [flatC, regularC] = flattenParameter(node.positionalParameters[0]);
|
||||
if (!regularC) {
|
||||
regular = false;
|
||||
}
|
||||
composite += flatC;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case "comment":
|
||||
case "bold":
|
||||
case "italic":
|
||||
case "ref":
|
||||
break;
|
||||
case "tag":
|
||||
const [flatT, regularT] = flattenParameter(node.content);
|
||||
if (!regularT) {
|
||||
regular = false;
|
||||
}
|
||||
if (flatT.includes("/") || flatT.includes("\\")) {
|
||||
// This is probably an unclosed tag with more path content after it,
|
||||
// like `.../<game.version>/...`.
|
||||
composite += `*/${flatT}`;
|
||||
} else if (flatT.length > 0) {
|
||||
// This is probably a closed tag, like `.../<sup>user ID</sup>/...`.
|
||||
composite += "*";
|
||||
}
|
||||
break;
|
||||
default:
|
||||
regular = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return [composite.trim(), regular];
|
||||
}
|
||||
|
||||
export function parseTemplates(templates: Array<string>): Pick<Game, "files" | "registry"> {
|
||||
const game: Pick<Game, "files" | "registry"> = { files: {}, registry: {} };
|
||||
|
||||
for (const template of templates.flatMap(parseWiki) as Array<Template>) {
|
||||
if (template.type !== "template") {
|
||||
console.log(`WARNING: unknown template type '${template.type}' from: '${JSON.stringify(template)}'`);
|
||||
continue;
|
||||
}
|
||||
if (template.positionalParameters.length < 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const [system, _] = flattenParameter(template.positionalParameters[0]);
|
||||
|
||||
for (const [rawPath, regular] of template.positionalParameters.slice(1).map(flattenParameter)) {
|
||||
if (rawPath.length === 0 || !regular) {
|
||||
// console.log(`IRREGULAR: ${rawPath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const [pathUnsplit, pathType] = parsePath(rawPath);
|
||||
if (pathUnsplit.includes("{{")) {
|
||||
continue;
|
||||
}
|
||||
for (const path of pathUnsplit.split("|").map(normalizePath)) {
|
||||
if (pathIsTooBroad(path)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pathType === PathType.FileSystem) {
|
||||
const constraint = getConstraintFromSystem(system, rawPath);
|
||||
|
||||
if (!game.files.hasOwnProperty(path)) {
|
||||
game.files[path] = {
|
||||
when: [],
|
||||
tags: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (!game.files[path].when.some(x => x.os === constraint.os && x.store === constraint.store)) {
|
||||
if (constraint.os !== undefined && constraint.store !== undefined) {
|
||||
game.files[path].when.push(constraint);
|
||||
} else if (constraint.os !== undefined) {
|
||||
game.files[path].when.push({ os: constraint.os });
|
||||
} else if (constraint.store !== undefined) {
|
||||
game.files[path].when.push({ store: constraint.store });
|
||||
}
|
||||
}
|
||||
|
||||
const tag = getTagFromTemplate(template.name);
|
||||
if (tag !== undefined && !game.files[path].tags.includes(tag)) {
|
||||
game.files[path].tags.push(tag);
|
||||
}
|
||||
} else if (pathType === PathType.Registry) {
|
||||
if (!game.registry.hasOwnProperty(path)) {
|
||||
game.registry[path] = {
|
||||
when: [],
|
||||
tags: [],
|
||||
};
|
||||
}
|
||||
|
||||
const store = getStoreConstraintFromPath(rawPath);
|
||||
if (store !== undefined && !game.registry[path].when.some(x => x.store === store)) {
|
||||
game.registry[path].when.push({ store });
|
||||
}
|
||||
|
||||
const tag = getTagFromTemplate(template.name);
|
||||
if (tag !== undefined && !game.registry[path].tags.includes(tag)) {
|
||||
game.registry[path].tags.push(tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(game.files).length === 0) {
|
||||
delete game.files;
|
||||
} else {
|
||||
for (const path of Object.keys(game.files)) {
|
||||
if (game.files[path].when.length === 0) {
|
||||
delete game.files[path].when;
|
||||
}
|
||||
if (game.files[path].tags.length === 0) {
|
||||
delete game.files[path].tags;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(game.registry).length === 0) {
|
||||
delete game.registry;
|
||||
} else {
|
||||
for (const path of Object.keys(game.registry)) {
|
||||
if (game.registry[path].when.length === 0) {
|
||||
delete game.registry[path].when;
|
||||
}
|
||||
if (game.registry[path].tags.length === 0) {
|
||||
delete game.registry[path].tags;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return game;
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"target": "es5",
|
||||
"outDir": "out",
|
||||
"lib": [
|
||||
"es2019"
|
||||
],
|
||||
"esModuleInterop": true
|
||||
},
|
||||
"include": [
|
||||
"src/*.ts"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue