UmaViewer/tools/dump_meta.py

#!/usr/bin/env python3
"""Decrypt and query the game's `meta` database.

The meta DB maps an asset path (``cutt/cutt_son1157/type_01``) to the hashed
file name under ``dat/`` plus the per-bundle XOR key.  It is a SQLite database
encrypted with SQLite3 Multiple Ciphers (ChaCha20); see ``uma_common.py`` for
the key derivation.

Usage
-----
    PY=~/.venvs/umatools/bin/python

    # write a plaintext copy you can open with any sqlite client
    $PY tools/dump_meta.py plain                      # -> tools/out/meta_plain.db
    $PY tools/dump_meta.py plain -o /tmp/meta.db

    # search asset paths (SQL LIKE pattern, % is the wildcard)
    $PY tools/dump_meta.py find 'cutt/cutt_son1157/%'
    $PY tools/dump_meta.py find '%live1014%' --limit 20

    # everything known about one asset, including where it lives on disk
    $PY tools/dump_meta.py info cutt/cutt_son1157/type_01

    # list the live song ids that have cutt bundles
    $PY tools/dump_meta.py songs

Table ``a`` columns (see Assets/Scripts/UmaDatabase/UmaDatabaseEntry.cs):
    i  row id            n  asset path          d  ';'-joined dependencies
    g  ?                 l  length              c  checksum
    h  hashed filename   m  asset type          e  per-bundle cipher key
"""

from __future__ import annotations

import argparse
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

import uma_common as uc  # noqa: E402

DEFAULT_OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "out")


def cmd_plain(args) -> int:
    """Copy the decrypted meta DB into a plaintext SQLite file.

    ``sqlite3_backup`` and ``VACUUM INTO`` both refuse / re-encrypt here (the
    ChaCha20 codec reserves bytes per page, so source and target page layouts
    are incompatible), so the schema and rows are copied explicitly.
    """
    uc.require("apsw-sqlite3mc")
    import apsw

    out = args.output or os.path.join(DEFAULT_OUT_DIR, "meta_plain.db")
    os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True)
    for suffix in ("", "-journal", "-wal", "-shm"):
        if os.path.exists(out + suffix):
            os.remove(out + suffix)

    with uc.MetaDb(args.game_path, args.region) as db:
        dest = apsw.Connection(out)
        try:
            schema = list(
                db.con.execute(
                    "SELECT type,name,sql FROM sqlite_master "
                    "WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'"
                )
            )
            tables = [(n, sql) for t, n, sql in schema if t == "table"]
            indexes = [(n, sql) for t, n, sql in schema if t == "index"]
            total = 0
            for name, sql in tables:
                dest.execute(sql)
                rows = list(db.con.execute('SELECT * FROM "%s"' % name))
                if rows:
                    ph = ",".join("?" * len(rows[0]))
                    with dest:
                        dest.executemany('INSERT INTO "%s" VALUES (%s)' % (name, ph), rows)
                total += len(rows)
                print("  %-16s %8d rows" % (name, len(rows)))
            for name, sql in indexes:
                try:
                    dest.execute(sql)
                except Exception as exc:  # a stale index definition must not kill the dump
                    print("  index %s skipped (%s)" % (name, exc), file=sys.stderr)
        finally:
            dest.close()
    print("wrote %s (%d rows total, %.1f MB)" % (out, total, os.path.getsize(out) / 1048576.0))
    return 0


def cmd_find(args) -> int:
    pattern = args.pattern if "%" in args.pattern else "%" + args.pattern + "%"
    with uc.MetaDb(args.game_path, args.region) as db:
        rows = db.like(pattern)
        if args.limit:
            rows = rows[: args.limit]
        for e in rows:
            on_disk = "" if os.path.isfile(e.path(db.game_path)) else "  [NOT DOWNLOADED]"
            print("%-60s %-34s key=%-22d%s" % (e.name, e.url, e.key, on_disk))
        print("-- %d entr%s" % (len(rows), "y" if len(rows) == 1 else "ies"), file=sys.stderr)
    return 0


def cmd_info(args) -> int:
    with uc.MetaDb(args.game_path, args.region) as db:
        e = db.get(args.name)
        if e is None:
            print("no such asset: %s" % args.name, file=sys.stderr)
            return 1
        p = e.path(db.game_path)
        print("name         : %s" % e.name)
        print("type         : %s" % e.type)
        print("hash (url)   : %s" % e.url)
        print("path         : %s%s" % (p, "" if os.path.isfile(p) else "   [NOT DOWNLOADED]"))
        if os.path.isfile(p):
            print("size         : %d bytes" % os.path.getsize(p))
        print("checksum     : %s" % e.checksum)
        print("key          : %d (%s)" % (e.key, "encrypted" if e.is_encrypted else "plaintext"))
        if e.is_encrypted:
            print("fkey         : %s" % uc.build_fkey(e.key).hex())
        deps = e.deps()
        print("dependencies : %d" % len(deps))
        for d in deps:
            print("    %s" % d)
    return 0


def cmd_songs(args) -> int:
    with uc.MetaDb(args.game_path, args.region) as db:
        ids = db.song_ids()
        for sid in ids:
            entries = db.like("cutt/cutt_son%s/%%" % sid)
            missing = sum(0 if os.path.isfile(e.path(db.game_path)) else 1 for e in entries)
            print("son%-8s %2d bundles%s" % (sid, len(entries), "  (%d missing)" % missing if missing else ""))
        print("-- %d songs" % len(ids), file=sys.stderr)
    return 0


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = ap.add_subparsers(dest="cmd", required=True)

    p = sub.add_parser("plain", help="write a decrypted copy of the meta DB")
    p.add_argument("-o", "--output", default=None, help="output path (default: tools/out/meta_plain.db)")
    uc.add_common_args(p)
    p.set_defaults(func=cmd_plain)

    p = sub.add_parser("find", help="search asset paths with a LIKE pattern")
    p.add_argument("pattern")
    p.add_argument("--limit", type=int, default=0)
    uc.add_common_args(p)
    p.set_defaults(func=cmd_find)

    p = sub.add_parser("info", help="show one asset's meta row and disk location")
    p.add_argument("name")
    uc.add_common_args(p)
    p.set_defaults(func=cmd_info)

    p = sub.add_parser("songs", help="list live song ids that have cutt bundles")
    uc.add_common_args(p)
    p.set_defaults(func=cmd_songs)

    args = ap.parse_args(argv)
    return args.func(args)


if __name__ == "__main__":
    raise SystemExit(main())