climperor/fetch_pro_builds.py

"""Aggregate pro-player item + skill builds per hero (Climperor web only).

Uses OpenDota registered pros (/proPlayers) and league match rows from
data/hero_matches.json (run fetch_hero_matches.py --source league first).
Optionally backfill more league samples with --fetch.

Output: data/pro_builds.json — not used by recommend.

Usage:
  python fetch_hero_matches.py --source league --limit 12
  python fetch_pro_builds.py
  python fetch_pro_builds.py --fetch --heroes juggernaut --limit 20
"""

from __future__ import annotations

import argparse
import json
import time
import urllib.error
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path

from common import DATA
from fetch_hero_items import (
    BLESSING_TO_SCEPTER,
    TOP_N,
    is_core_finished,
    load_item_catalog,
)
from fetch_hero_matches import (
    extract_player_row,
    fetch_match,
    league_match_ids,
    load_ability_id_map,
)
from grid import hero_table
from http_utils import http_json

OPENDOTA = "https://api.opendota.com/api"
OUT = DATA / "pro_builds.json"
MATCHES_IN = DATA / "hero_matches.json"
PRO_MATCHES_IN = DATA / "pro_matches.json"
TOP_SKILL_ORDERS = 3


def fetch_pro_index() -> dict[int, dict]:
    """account_id → slim pro profile from OpenDota."""
    try:
        raw = http_json(f"{OPENDOTA}/proPlayers")
    except (
        urllib.error.HTTPError,
        urllib.error.URLError,
        TimeoutError,
        json.JSONDecodeError,
        OSError,
    ) as e:
        raise SystemExit(f"proPlayers fetch failed: {e}") from e
    if not isinstance(raw, list):
        raise SystemExit("proPlayers: unexpected payload")
    out: dict[int, dict] = {}
    for row in raw:
        if not isinstance(row, dict):
            continue
        try:
            aid = int(row.get("account_id") or 0)
        except (TypeError, ValueError):
            continue
        if aid <= 0:
            continue
        name = row.get("name")
        if isinstance(name, str):
            name = name.strip() or None
        else:
            name = None
        team_tag = row.get("team_tag")
        if isinstance(team_tag, str):
            team_tag = team_tag.strip() or None
        else:
            team_tag = None
        team_name = row.get("team_name")
        if isinstance(team_name, str):
            team_name = team_name.strip() or None
        else:
            team_name = None
        country = row.get("country_code") or row.get("loccountrycode")
        if isinstance(country, str):
            country = country.strip().lower() or None
        else:
            country = None
        last_match = row.get("last_match_time")
        if isinstance(last_match, str):
            last_match = last_match.strip() or None
        else:
            last_match = None
        out[aid] = {
            "account_id": aid,
            "name": name,
            "team_tag": team_tag,
            "team_name": team_name,
            "country_code": country,
            "last_match_time": last_match,
        }
    return out


def normalize_item_id(iid: int) -> int:
    return BLESSING_TO_SCEPTER.get(iid, iid)


def match_item_ids(row: dict) -> list[int]:
    ids: list[int] = []
    for key in ("items", "backpack"):
        part = row.get(key)
        if not isinstance(part, list):
            continue
        for raw in part:
            try:
                iid = int(raw or 0)
            except (TypeError, ValueError):
                continue
            if iid > 0:
                ids.append(normalize_item_id(iid))
    neut = row.get("item_neutral")
    try:
        ni = int(neut or 0)
    except (TypeError, ValueError):
        ni = 0
    if ni > 0:
        ids.append(ni)
    return ids


def is_pro_match(row: dict, pro_ids: set[int]) -> bool:
    origin = str(row.get("origin") or "")
    if origin not in ("league", "pro"):
        return False
    aid = row.get("account_id")
    try:
        aid_i = int(aid) if aid is not None else 0
    except (TypeError, ValueError):
        aid_i = 0
    if aid_i in pro_ids:
        return True
    # Registered pro name on league row (OpenDota match detail).
    name = row.get("name")
    return isinstance(name, str) and bool(name.strip())


def aggregate_items(
    rows: list[dict], catalog: dict[int, dict], n: int = TOP_N
) -> list[dict]:
    counts: Counter[int] = Counter()
    for row in rows:
        seen: set[int] = set()
        for iid in match_item_ids(row):
            if iid in seen:
                continue
            meta = catalog.get(iid)
            if meta is None or not is_core_finished(meta):
                continue
            seen.add(iid)
            counts[iid] += 1
    total = len(rows)
    ranked = sorted(counts.items(), key=lambda t: (-t[1], t[0]))
    out: list[dict] = []
    for iid, c in ranked[:n]:
        cell: dict = {"id": iid, "count": c}
        if total > 0:
            cell["pct"] = round(c / total * 1000) / 10
        out.append(cell)
    return out


def aggregate_skill_orders(
    rows: list[dict], *, top: int = TOP_SKILL_ORDERS
) -> list[dict]:
    counts: Counter[tuple[str, ...]] = Counter()
    for row in rows:
        ups = row.get("ability_upgrades")
        if not isinstance(ups, list) or not ups:
            continue
        seq = tuple(str(k) for k in ups if isinstance(k, str) and k.strip())
        if not seq:
            continue
        counts[seq] += 1
    total = sum(counts.values())
    ranked = sorted(counts.items(), key=lambda t: (-t[1], t[0]))
    out: list[dict] = []
    for seq, c in ranked[:top]:
        cell: dict = {"sequence": list(seq), "count": c}
        if total > 0:
            cell["pct"] = round(c / total * 1000) / 10
        out.append(cell)
    return out


def aggregate_pros(
    rows: list[dict], pro_index: dict[int, dict]
) -> list[dict]:
    by_aid: dict[int, dict] = {}
    for row in rows:
        aid = row.get("account_id")
        try:
            aid_i = int(aid) if aid is not None else 0
        except (TypeError, ValueError):
            aid_i = 0
        prof = pro_index.get(aid_i) if aid_i else None
        name = (prof or {}).get("name") or row.get("name") or row.get("display_name")
        if isinstance(name, str):
            name = name.strip() or None
        else:
            name = None
        if not name and not aid_i:
            continue
        key = aid_i if aid_i else hash(name or "")
        cell = by_aid.setdefault(
            key,
            {
                "account_id": aid_i or None,
                "name": name,
                "team_tag": (prof or {}).get("team_tag"),
                "games": 0,
                "wins": 0,
                "last_match_id": None,
                "last_start_time": None,
            },
        )
        cell["games"] += 1
        if row.get("won"):
            cell["wins"] += 1
        mid = row.get("match_id")
        st = row.get("start_time")
        try:
            st_i = int(st) if st is not None else 0
        except (TypeError, ValueError):
            st_i = 0
        prev = cell.get("last_start_time") or 0
        if st_i >= prev:
            cell["last_start_time"] = st_i or None
            cell["last_match_id"] = mid
    ranked = sorted(
        by_aid.values(),
        key=lambda r: (
            -(r.get("games") or 0),
            -(r.get("last_start_time") or 0),
            str(r.get("name") or ""),
        ),
    )
    for r in ranked:
        r.pop("last_start_time", None)
    return ranked[:12]


def load_pro_league_rows(path: Path) -> dict[str, list[dict]]:
    """League/pro rows from pro_matches.json by_hero or by_pro."""
    if not path.is_file():
        return {}
    try:
        raw = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return {}
    out: dict[str, list[dict]] = {}

    by_hero = raw.get("by_hero")
    if isinstance(by_hero, dict):
        for key, cell in by_hero.items():
            if not isinstance(cell, dict):
                continue
            matches = cell.get("matches")
            if isinstance(matches, list) and matches:
                out[str(key)] = [m for m in matches if isinstance(m, dict)]

    by_pro = raw.get("by_pro")
    if isinstance(by_pro, dict):
        for cell in by_pro.values():
            if not isinstance(cell, dict):
                continue
            for row in cell.get("matches") or []:
                if not isinstance(row, dict):
                    continue
                key = row.get("hero_key")
                if not key:
                    continue
                out.setdefault(str(key), []).append(row)

    deduped: dict[str, list[dict]] = {}
    for key, rows in out.items():
        seen: set[int] = set()
        merged: list[dict] = []
        for row in rows:
            try:
                mid = int(row.get("match_id") or 0)
            except (TypeError, ValueError):
                continue
            if mid in seen:
                continue
            seen.add(mid)
            merged.append(row)
        if merged:
            deduped[key] = merged
    return deduped


def load_league_rows(path: Path) -> dict[str, list[dict]]:
    if not path.is_file():
        return {}
    try:
        raw = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return {}
    by_hero = raw.get("by_hero")
    if not isinstance(by_hero, dict):
        return {}
    out: dict[str, list[dict]] = {}
    for key, cell in by_hero.items():
        if not isinstance(cell, dict):
            continue
        matches = cell.get("matches")
        if not isinstance(matches, list):
            continue
        league = [m for m in matches if isinstance(m, dict) and m.get("origin") == "league"]
        if league:
            out[str(key)] = league
    return out


def fetch_league_rows_for_hero(
    hero_id: int,
    *,
    limit: int,
    id_map: dict[int, str],
    catalog: dict[int, dict],
    delay: float,
) -> list[dict]:
    metas = league_match_ids(hero_id, limit)
    rows: list[dict] = []
    for meta in metas:
        try:
            mid = int(meta.get("match_id") or 0)
        except (TypeError, ValueError):
            continue
        if mid <= 0:
            continue
        detail = fetch_match(mid)
        if delay > 0:
            time.sleep(delay)
        if not detail:
            continue
        slim = extract_player_row(
            detail,
            hero_id,
            origin="league",
            id_map=id_map,
            item_catalog=catalog,
            list_meta=meta,
        )
        if slim:
            rows.append(slim)
    return rows


def build_payload(
    *,
    by_hero_rows: dict[str, list[dict]],
    pro_index: dict[int, dict],
    catalog: dict[int, dict],
    match_source: str,
) -> dict:
    pro_ids = set(pro_index)
    by_hero: dict[str, dict] = {}
    used_item_ids: set[int] = set()
    total_matches = 0

    for key, all_rows in sorted(by_hero_rows.items()):
        pro_rows = [r for r in all_rows if is_pro_match(r, pro_ids)]
        if not pro_rows:
            continue
        total_matches += len(pro_rows)
        items = aggregate_items(pro_rows, catalog)
        for ent in items:
            used_item_ids.add(int(ent["id"]))
        by_hero[key] = {
            "sample_count": len(pro_rows),
            "items": items,
            "skill_orders": aggregate_skill_orders(pro_rows),
            "pros": aggregate_pros(pro_rows, pro_index),
        }

    items_out = {
        str(iid): {
            "key": catalog[iid]["key"],
            "dname": catalog[iid]["dname"],
            "name_loc": catalog[iid].get("name_loc") or catalog[iid]["dname"],
        }
        for iid in sorted(used_item_ids)
        if iid in catalog
    }

    pros_out = {
        str(aid): prof
        for aid, prof in sorted(pro_index.items(), key=lambda t: t[0])
    }

    return {
        "meta": {
            "fetched_at": datetime.now(timezone.utc).isoformat(),
            "source": "opendota",
            "attribution": "https://www.opendota.com",
            "match_source": match_source,
            "pro_count": len(pro_index),
            "hero_count": len(by_hero),
            "match_count": total_matches,
            "note_zh": (
                "职业选手样本：OpenDota 注册选手 + 联赛对局终局出装/加点；"
                "出装频率 = 该装备出现在样本终局栏位的比例；"
                "加点为完整升级顺序的最常见方案。"
            ),
        },
        "pros": pros_out,
        "items": items_out,
        "by_hero": by_hero,
    }


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--out", type=Path, default=OUT)
    ap.add_argument(
        "--matches",
        type=Path,
        default=MATCHES_IN,
        help="Input hero_matches.json league rows (default: data/hero_matches.json)",
    )
    ap.add_argument(
        "--pro-matches",
        type=Path,
        default=PRO_MATCHES_IN,
        help="Merge pro player rows from this file (default: data/pro_matches.json)",
    )
    ap.add_argument(
        "--fetch",
        action="store_true",
        help="Fetch league matches from OpenDota instead of only reading --matches",
    )
    ap.add_argument(
        "--limit",
        type=int,
        default=12,
        help="League matches per hero when --fetch (default: 12)",
    )
    ap.add_argument(
        "--heroes",
        default="",
        help="Comma-separated hero keys (default: all when --fetch; else keys in matches file)",
    )
    ap.add_argument("--delay", type=float, default=0.35)
    args = ap.parse_args()

    print("loading pro players ...", flush=True)
    pro_index = fetch_pro_index()
    print(f"  {len(pro_index)} registered pros", flush=True)

    catalog, _ = load_item_catalog()
    id_map = load_ability_id_map()
    heroes = hero_table()
    by_key = {h["key"]: h for h in heroes}
    hero_filter = [k.strip() for k in args.heroes.split(",") if k.strip()]

    by_hero_rows: dict[str, list[dict]] = {}

    if args.fetch:
        keys = hero_filter or sorted(by_key)
        limit = max(1, int(args.limit))
        for i, key in enumerate(keys, 1):
            hero = by_key.get(key)
            if not hero:
                print(f"  skip unknown hero {key}", flush=True)
                continue
            hid = int(hero["id"])
            print(f"[{i}/{len(keys)}] fetch league {key} ...", flush=True)
            rows = fetch_league_rows_for_hero(
                hid,
                limit=limit,
                id_map=id_map,
                catalog=catalog,
                delay=args.delay,
            )
            if rows:
                by_hero_rows[key] = rows
    else:
        by_hero_rows = load_league_rows(args.matches)
        pro_path = args.pro_matches
        if pro_path and pro_path.is_file():
            pro_rows = load_pro_league_rows(pro_path)
            for key, rows in pro_rows.items():
                bucket = by_hero_rows.setdefault(key, [])
                seen = {int(r.get("match_id") or 0) for r in bucket if isinstance(r, dict)}
                for row in rows:
                    try:
                        mid = int(row.get("match_id") or 0)
                    except (TypeError, ValueError):
                        continue
                    if mid in seen:
                        continue
                    seen.add(mid)
                    bucket.append(row)
        if hero_filter:
            by_hero_rows = {k: v for k, v in by_hero_rows.items() if k in hero_filter}
        if not by_hero_rows:
            raise SystemExit(
                f"No league/pro rows in {args.matches} or {args.pro_matches}. "
                "Run: python fetch_pro_matches.py or fetch_hero_matches.py --source league"
            )

    sources = []
    if args.fetch:
        sources.append("fetch")
    else:
        sources.append(str(args.matches.name))
        if args.pro_matches and args.pro_matches.is_file():
            sources.append(str(args.pro_matches.name))
    match_source = "+".join(sources)
    payload = build_payload(
        by_hero_rows=by_hero_rows,
        pro_index=pro_index,
        catalog=catalog,
        match_source=match_source,
    )

    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(
        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
    )
    meta = payload["meta"]
    print(
        f"done heroes={meta['hero_count']} matches={meta['match_count']} → {args.out}",
        flush=True,
    )


if __name__ == "__main__":
    main()