climperor/web/fetch_pro_builds.py

"""Aggregate pro-player item + skill builds per hero (Climperor web only).

Uses OpenDota registered pros (/proPlayers) and league match rows from
data/hero_matches.json (run fetch_hero_matches.py --source league first).
Optionally backfill more league samples with --fetch.

Output: data/pro_builds.json — not used by recommend.

Usage:
  python fetch_hero_matches.py --source league --limit 12
  python fetch_pro_builds.py
  python fetch_pro_builds.py --fetch --heroes juggernaut --limit 20
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import argparse
import json
import time
import urllib.error
from collections import Counter, defaultdict
from datetime import datetime, timezone

from shared.grid import hero_table
from shared.http_utils import http_json
from shared.paths import DATA

from fetch_hero_items import (
    BLESSING_TO_SCEPTER,
    TOP_N,
    is_core_finished,
    load_item_catalog,
)
from fetch_hero_matches import (
    extract_player_row,
    fetch_match,
    league_match_ids,
    load_ability_id_map,
)

OPENDOTA = "https://api.opendota.com/api"
OUT = DATA / "pro_builds.json"
MATCHES_IN = DATA / "hero_matches.json"
PRO_MATCHES_IN = DATA / "pro_matches.json"
TOP_SKILL_ORDERS = 3


def fetch_pro_index() -> dict[int, dict]:
    """account_id → slim pro profile from OpenDota."""
    try:
        raw = http_json(f"{OPENDOTA}/proPlayers")
    except (
        urllib.error.HTTPError,
        urllib.error.URLError,
        TimeoutError,
        json.JSONDecodeError,
        OSError,
    ) as e:
        raise SystemExit(f"proPlayers fetch failed: {e}") from e
    if not isinstance(raw, list):
        raise SystemExit("proPlayers: unexpected payload")
    out: dict[int, dict] = {}
    for row in raw:
        if not isinstance(row, dict):
            continue
        try:
            aid = int(row.get("account_id") or 0)
        except (TypeError, ValueError):
            continue
        if aid <= 0:
            continue
        name = row.get("name")
        if isinstance(name, str):
            name = name.strip() or None
        else:
            name = None
        team_tag = row.get("team_tag")
        if isinstance(team_tag, str):
            team_tag = team_tag.strip() or None
        else:
            team_tag = None
        team_name = row.get("team_name")
        if isinstance(team_name, str):
            team_name = team_name.strip() or None
        else:
            team_name = None
        country = row.get("country_code") or row.get("loccountrycode")
        if isinstance(country, str):
            country = country.strip().lower() or None
        else:
            country = None
        last_match = row.get("last_match_time")
        if isinstance(last_match, str):
            last_match = last_match.strip() or None
        else:
            last_match = None
        out[aid] = {
            "account_id": aid,
            "name": name,
            "team_tag": team_tag,
            "team_name": team_name,
            "country_code": country,
            "last_match_time": last_match,
        }
    return out


def normalize_item_id(iid: int) -> int:
    return BLESSING_TO_SCEPTER.get(iid, iid)


def match_item_ids(row: dict) -> list[int]:
    ids: list[int] = []
    for key in ("items", "backpack"):
        part = row.get(key)
        if not isinstance(part, list):
            continue
        for raw in part:
            try:
                iid = int(raw or 0)
            except (TypeError, ValueError):
                continue
            if iid > 0:
                ids.append(normalize_item_id(iid))
    neut = row.get("item_neutral")
    try:
        ni = int(neut or 0)
    except (TypeError, ValueError):
        ni = 0
    if ni > 0:
        ids.append(ni)
    return ids


def is_pro_match(row: dict, pro_ids: set[int]) -> bool:
    origin = str(row.get("origin") or "")
    if origin not in ("league", "pro"):
        return False
    aid = row.get("account_id")
    try:
        aid_i = int(aid) if aid is not None else 0
    except (TypeError, ValueError):
        aid_i = 0
    if aid_i in pro_ids:
        return True
    # Registered pro name on league row (OpenDota match detail).
    name = row.get("name")
    return isinstance(name, str) and bool(name.strip())


def aggregate_items(
    rows: list[dict], catalog: dict[int, dict], n: int = TOP_N
) -> list[dict]:
    counts: Counter[int] = Counter()
    for row in rows:
        seen: set[int] = set()
        for iid in match_item_ids(row):
            if iid in seen:
                continue
            meta = catalog.get(iid)
            if meta is None or not is_core_finished(meta):
                continue
            seen.add(iid)
            counts[iid] += 1
    total = len(rows)
    ranked = sorted(counts.items(), key=lambda t: (-t[1], t[0]))
    out: list[dict] = []
    for iid, c in ranked[:n]:
        cell: dict = {"id": iid, "count": c}
        if total > 0:
            cell["pct"] = round(c / total * 1000) / 10
        out.append(cell)
    return out


def aggregate_skill_orders(
    rows: list[dict], *, top: int = TOP_SKILL_ORDERS
) -> list[dict]:
    counts: Counter[tuple[str, ...]] = Counter()
    for row in rows:
        ups = row.get("ability_upgrades")
        if not isinstance(ups, list) or not ups:
            continue
        seq = tuple(str(k) for k in ups if isinstance(k, str) and k.strip())
        if not seq:
            continue
        counts[seq] += 1
    total = sum(counts.values())
    ranked = sorted(counts.items(), key=lambda t: (-t[1], t[0]))
    out: list[dict] = []
    for seq, c in ranked[:top]:
        cell: dict = {"sequence": list(seq), "count": c}
        if total > 0:
            cell["pct"] = round(c / total * 1000) / 10
        out.append(cell)
    return out


def aggregate_pros(
    rows: list[dict], pro_index: dict[int, dict]
) -> list[dict]:
    by_aid: dict[int, dict] = {}
    for row in rows:
        aid = row.get("account_id")
        try:
            aid_i = int(aid) if aid is not None else 0
        except (TypeError, ValueError):
            aid_i = 0
        prof = pro_index.get(aid_i) if aid_i else None
        name = (prof or {}).get("name") or row.get("name") or row.get("display_name")
        if isinstance(name, str):
            name = name.strip() or None
        else:
            name = None
        if not name and not aid_i:
            continue
        key = aid_i if aid_i else hash(name or "")
        cell = by_aid.setdefault(
            key,
            {
                "account_id": aid_i or None,
                "name": name,
                "team_tag": (prof or {}).get("team_tag"),
                "games": 0,
                "wins": 0,
                "last_match_id": None,
                "last_start_time": None,
            },
        )
        cell["games"] += 1
        if row.get("won"):
            cell["wins"] += 1
        mid = row.get("match_id")
        st = row.get("start_time")
        try:
            st_i = int(st) if st is not None else 0
        except (TypeError, ValueError):
            st_i = 0
        prev = cell.get("last_start_time") or 0
        if st_i >= prev:
            cell["last_start_time"] = st_i or None
            cell["last_match_id"] = mid
    ranked = sorted(
        by_aid.values(),
        key=lambda r: (
            -(r.get("games") or 0),
            -(r.get("last_start_time") or 0),
            str(r.get("name") or ""),
        ),
    )
    for r in ranked:
        r.pop("last_start_time", None)
    return ranked[:12]


def load_pro_league_rows(path: Path) -> dict[str, list[dict]]:
    """League/pro rows from pro_matches.json by_hero or by_pro."""
    if not path.is_file():
        return {}
    try:
        raw = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return {}
    out: dict[str, list[dict]] = {}

    by_hero = raw.get("by_hero")
    if isinstance(by_hero, dict):
        for key, cell in by_hero.items():
            if not isinstance(cell, dict):
                continue
            matches = cell.get("matches")
            if isinstance(matches, list) and matches:
                out[str(key)] = [m for m in matches if isinstance(m, dict)]

    by_pro = raw.get("by_pro")
    if isinstance(by_pro, dict):
        for cell in by_pro.values():
            if not isinstance(cell, dict):
                continue
            for row in cell.get("matches") or []:
                if not isinstance(row, dict):
                    continue
                key = row.get("hero_key")
                if not key:
                    continue
                out.setdefault(str(key), []).append(row)

    deduped: dict[str, list[dict]] = {}
    for key, rows in out.items():
        seen: set[int] = set()
        merged: list[dict] = []
        for row in rows:
            try:
                mid = int(row.get("match_id") or 0)
            except (TypeError, ValueError):
                continue
            if mid in seen:
                continue
            seen.add(mid)
            merged.append(row)
        if merged:
            deduped[key] = merged
    return deduped


def load_league_rows(path: Path) -> dict[str, list[dict]]:
    if not path.is_file():
        return {}
    try:
        raw = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return {}
    by_hero = raw.get("by_hero")
    if not isinstance(by_hero, dict):
        return {}
    out: dict[str, list[dict]] = {}
    for key, cell in by_hero.items():
        if not isinstance(cell, dict):
            continue
        matches = cell.get("matches")
        if not isinstance(matches, list):
            continue
        league = [m for m in matches if isinstance(m, dict) and m.get("origin") == "league"]
        if league:
            out[str(key)] = league
    return out


def fetch_league_rows_for_hero(
    hero_id: int,
    *,
    limit: int,
    id_map: dict[int, str],
    catalog: dict[int, dict],
    delay: float,
) -> list[dict]:
    metas = league_match_ids(hero_id, limit)
    rows: list[dict] = []
    for meta in metas:
        try:
            mid = int(meta.get("match_id") or 0)
        except (TypeError, ValueError):
            continue
        if mid <= 0:
            continue
        detail = fetch_match(mid)
        if delay > 0:
            time.sleep(delay)
        if not detail:
            continue
        slim = extract_player_row(
            detail,
            hero_id,
            origin="league",
            id_map=id_map,
            item_catalog=catalog,
            list_meta=meta,
        )
        if slim:
            rows.append(slim)
    return rows


def build_payload(
    *,
    by_hero_rows: dict[str, list[dict]],
    pro_index: dict[int, dict],
    catalog: dict[int, dict],
    match_source: str,
) -> dict:
    pro_ids = set(pro_index)
    by_hero: dict[str, dict] = {}
    used_item_ids: set[int] = set()
    total_matches = 0

    for key, all_rows in sorted(by_hero_rows.items()):
        pro_rows = [r for r in all_rows if is_pro_match(r, pro_ids)]
        if not pro_rows:
            continue
        total_matches += len(pro_rows)
        items = aggregate_items(pro_rows, catalog)
        for ent in items:
            used_item_ids.add(int(ent["id"]))
        by_hero[key] = {
            "sample_count": len(pro_rows),
            "items": items,
            "skill_orders": aggregate_skill_orders(pro_rows),
            "pros": aggregate_pros(pro_rows, pro_index),
        }

    items_out = {
        str(iid): {
            "key": catalog[iid]["key"],
            "dname": catalog[iid]["dname"],
            "name_loc": catalog[iid].get("name_loc") or catalog[iid]["dname"],
        }
        for iid in sorted(used_item_ids)
        if iid in catalog
    }

    pros_out = {
        str(aid): prof
        for aid, prof in sorted(pro_index.items(), key=lambda t: t[0])
    }

    return {
        "meta": {
            "fetched_at": datetime.now(timezone.utc).isoformat(),
            "source": "opendota",
            "attribution": "https://www.opendota.com",
            "match_source": match_source,
            "pro_count": len(pro_index),
            "hero_count": len(by_hero),
            "match_count": total_matches,
            "note_zh": (
                "职业选手样本：OpenDota 注册选手 + 联赛对局终局出装/加点；"
                "出装频率 = 该装备出现在样本终局栏位的比例；"
                "加点为完整升级顺序的最常见方案。"
            ),
        },
        "pros": pros_out,
        "items": items_out,
        "by_hero": by_hero,
    }


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--out", type=Path, default=OUT)
    ap.add_argument(
        "--matches",
        type=Path,
        default=MATCHES_IN,
        help="Input hero_matches.json league rows (default: data/hero_matches.json)",
    )
    ap.add_argument(
        "--pro-matches",
        type=Path,
        default=PRO_MATCHES_IN,
        help="Merge pro player rows from this file (default: data/pro_matches.json)",
    )
    ap.add_argument(
        "--fetch",
        action="store_true",
        help="Fetch league matches from OpenDota instead of only reading --matches",
    )
    ap.add_argument(
        "--limit",
        type=int,
        default=12,
        help="League matches per hero when --fetch (default: 12)",
    )
    ap.add_argument(
        "--heroes",
        default="",
        help="Comma-separated hero keys (default: all when --fetch; else keys in matches file)",
    )
    ap.add_argument("--delay", type=float, default=0.35)
    args = ap.parse_args()

    print("loading pro players ...", flush=True)
    pro_index = fetch_pro_index()
    print(f"  {len(pro_index)} registered pros", flush=True)

    catalog, _ = load_item_catalog()
    id_map = load_ability_id_map()
    heroes = hero_table()
    by_key = {h["key"]: h for h in heroes}
    hero_filter = [k.strip() for k in args.heroes.split(",") if k.strip()]

    by_hero_rows: dict[str, list[dict]] = {}

    if args.fetch:
        keys = hero_filter or sorted(by_key)
        limit = max(1, int(args.limit))
        for i, key in enumerate(keys, 1):
            hero = by_key.get(key)
            if not hero:
                print(f"  skip unknown hero {key}", flush=True)
                continue
            hid = int(hero["id"])
            print(f"[{i}/{len(keys)}] fetch league {key} ...", flush=True)
            rows = fetch_league_rows_for_hero(
                hid,
                limit=limit,
                id_map=id_map,
                catalog=catalog,
                delay=args.delay,
            )
            if rows:
                by_hero_rows[key] = rows
    else:
        by_hero_rows = load_league_rows(args.matches)
        pro_path = args.pro_matches
        if pro_path and pro_path.is_file():
            pro_rows = load_pro_league_rows(pro_path)
            for key, rows in pro_rows.items():
                bucket = by_hero_rows.setdefault(key, [])
                seen = {int(r.get("match_id") or 0) for r in bucket if isinstance(r, dict)}
                for row in rows:
                    try:
                        mid = int(row.get("match_id") or 0)
                    except (TypeError, ValueError):
                        continue
                    if mid in seen:
                        continue
                    seen.add(mid)
                    bucket.append(row)
        if hero_filter:
            by_hero_rows = {k: v for k, v in by_hero_rows.items() if k in hero_filter}
        if not by_hero_rows:
            raise SystemExit(
                f"No league/pro rows in {args.matches} or {args.pro_matches}. "
                "Run: python fetch_pro_matches.py or fetch_hero_matches.py --source league"
            )

    sources = []
    if args.fetch:
        sources.append("fetch")
    else:
        sources.append(str(args.matches.name))
        if args.pro_matches and args.pro_matches.is_file():
            sources.append(str(args.pro_matches.name))
    match_source = "+".join(sources)
    payload = build_payload(
        by_hero_rows=by_hero_rows,
        pro_index=pro_index,
        catalog=catalog,
        match_source=match_source,
    )

    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(
        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
    )
    meta = payload["meta"]
    print(
        f"done heroes={meta['hero_count']} matches={meta['match_count']} → {args.out}",
        flush=True,
    )


if __name__ == "__main__":
    main()