v0.5.59: item counter evidence for fears, catch up Web features to site version.
Ship OpenDota counter-stats reordering for feared items, finalize SITE_VERSION/docs for rankings/streamers/trends/matches/mechanics and draft archetypes, and ignore regenerable Web data caches. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
voson
Cursor
@@ -0,0 +1,544 @@
|
||||
"""Aggregate pro-player item + skill builds per hero (Climperor web only).
|
||||
|
||||
Uses OpenDota registered pros (/proPlayers) and league match rows from
|
||||
data/hero_matches.json (run fetch_hero_matches.py --source league first).
|
||||
Optionally backfill more league samples with --fetch.
|
||||
|
||||
Output: data/pro_builds.json — not used by recommend.
|
||||
|
||||
Usage:
|
||||
python fetch_hero_matches.py --source league --limit 12
|
||||
python fetch_pro_builds.py
|
||||
python fetch_pro_builds.py --fetch --heroes juggernaut --limit 20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from common import DATA
|
||||
from fetch_hero_items import (
|
||||
BLESSING_TO_SCEPTER,
|
||||
TOP_N,
|
||||
is_core_finished,
|
||||
load_item_catalog,
|
||||
)
|
||||
from fetch_hero_matches import (
|
||||
extract_player_row,
|
||||
fetch_match,
|
||||
league_match_ids,
|
||||
load_ability_id_map,
|
||||
)
|
||||
from grid import hero_table
|
||||
from http_utils import http_json
|
||||
|
||||
OPENDOTA = "https://api.opendota.com/api"
|
||||
OUT = DATA / "pro_builds.json"
|
||||
MATCHES_IN = DATA / "hero_matches.json"
|
||||
PRO_MATCHES_IN = DATA / "pro_matches.json"
|
||||
TOP_SKILL_ORDERS = 3
|
||||
|
||||
|
||||
def fetch_pro_index() -> dict[int, dict]:
|
||||
"""account_id → slim pro profile from OpenDota."""
|
||||
try:
|
||||
raw = http_json(f"{OPENDOTA}/proPlayers")
|
||||
except (
|
||||
urllib.error.HTTPError,
|
||||
urllib.error.URLError,
|
||||
TimeoutError,
|
||||
json.JSONDecodeError,
|
||||
OSError,
|
||||
) as e:
|
||||
raise SystemExit(f"proPlayers fetch failed: {e}") from e
|
||||
if not isinstance(raw, list):
|
||||
raise SystemExit("proPlayers: unexpected payload")
|
||||
out: dict[int, dict] = {}
|
||||
for row in raw:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
try:
|
||||
aid = int(row.get("account_id") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if aid <= 0:
|
||||
continue
|
||||
name = row.get("name")
|
||||
if isinstance(name, str):
|
||||
name = name.strip() or None
|
||||
else:
|
||||
name = None
|
||||
team_tag = row.get("team_tag")
|
||||
if isinstance(team_tag, str):
|
||||
team_tag = team_tag.strip() or None
|
||||
else:
|
||||
team_tag = None
|
||||
team_name = row.get("team_name")
|
||||
if isinstance(team_name, str):
|
||||
team_name = team_name.strip() or None
|
||||
else:
|
||||
team_name = None
|
||||
country = row.get("country_code") or row.get("loccountrycode")
|
||||
if isinstance(country, str):
|
||||
country = country.strip().lower() or None
|
||||
else:
|
||||
country = None
|
||||
last_match = row.get("last_match_time")
|
||||
if isinstance(last_match, str):
|
||||
last_match = last_match.strip() or None
|
||||
else:
|
||||
last_match = None
|
||||
out[aid] = {
|
||||
"account_id": aid,
|
||||
"name": name,
|
||||
"team_tag": team_tag,
|
||||
"team_name": team_name,
|
||||
"country_code": country,
|
||||
"last_match_time": last_match,
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def normalize_item_id(iid: int) -> int:
|
||||
return BLESSING_TO_SCEPTER.get(iid, iid)
|
||||
|
||||
|
||||
def match_item_ids(row: dict) -> list[int]:
|
||||
ids: list[int] = []
|
||||
for key in ("items", "backpack"):
|
||||
part = row.get(key)
|
||||
if not isinstance(part, list):
|
||||
continue
|
||||
for raw in part:
|
||||
try:
|
||||
iid = int(raw or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if iid > 0:
|
||||
ids.append(normalize_item_id(iid))
|
||||
neut = row.get("item_neutral")
|
||||
try:
|
||||
ni = int(neut or 0)
|
||||
except (TypeError, ValueError):
|
||||
ni = 0
|
||||
if ni > 0:
|
||||
ids.append(ni)
|
||||
return ids
|
||||
|
||||
|
||||
def is_pro_match(row: dict, pro_ids: set[int]) -> bool:
|
||||
origin = str(row.get("origin") or "")
|
||||
if origin not in ("league", "pro"):
|
||||
return False
|
||||
aid = row.get("account_id")
|
||||
try:
|
||||
aid_i = int(aid) if aid is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
aid_i = 0
|
||||
if aid_i in pro_ids:
|
||||
return True
|
||||
# Registered pro name on league row (OpenDota match detail).
|
||||
name = row.get("name")
|
||||
return isinstance(name, str) and bool(name.strip())
|
||||
|
||||
|
||||
def aggregate_items(
|
||||
rows: list[dict], catalog: dict[int, dict], n: int = TOP_N
|
||||
) -> list[dict]:
|
||||
counts: Counter[int] = Counter()
|
||||
for row in rows:
|
||||
seen: set[int] = set()
|
||||
for iid in match_item_ids(row):
|
||||
if iid in seen:
|
||||
continue
|
||||
meta = catalog.get(iid)
|
||||
if meta is None or not is_core_finished(meta):
|
||||
continue
|
||||
seen.add(iid)
|
||||
counts[iid] += 1
|
||||
total = len(rows)
|
||||
ranked = sorted(counts.items(), key=lambda t: (-t[1], t[0]))
|
||||
out: list[dict] = []
|
||||
for iid, c in ranked[:n]:
|
||||
cell: dict = {"id": iid, "count": c}
|
||||
if total > 0:
|
||||
cell["pct"] = round(c / total * 1000) / 10
|
||||
out.append(cell)
|
||||
return out
|
||||
|
||||
|
||||
def aggregate_skill_orders(
|
||||
rows: list[dict], *, top: int = TOP_SKILL_ORDERS
|
||||
) -> list[dict]:
|
||||
counts: Counter[tuple[str, ...]] = Counter()
|
||||
for row in rows:
|
||||
ups = row.get("ability_upgrades")
|
||||
if not isinstance(ups, list) or not ups:
|
||||
continue
|
||||
seq = tuple(str(k) for k in ups if isinstance(k, str) and k.strip())
|
||||
if not seq:
|
||||
continue
|
||||
counts[seq] += 1
|
||||
total = sum(counts.values())
|
||||
ranked = sorted(counts.items(), key=lambda t: (-t[1], t[0]))
|
||||
out: list[dict] = []
|
||||
for seq, c in ranked[:top]:
|
||||
cell: dict = {"sequence": list(seq), "count": c}
|
||||
if total > 0:
|
||||
cell["pct"] = round(c / total * 1000) / 10
|
||||
out.append(cell)
|
||||
return out
|
||||
|
||||
|
||||
def aggregate_pros(
|
||||
rows: list[dict], pro_index: dict[int, dict]
|
||||
) -> list[dict]:
|
||||
by_aid: dict[int, dict] = {}
|
||||
for row in rows:
|
||||
aid = row.get("account_id")
|
||||
try:
|
||||
aid_i = int(aid) if aid is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
aid_i = 0
|
||||
prof = pro_index.get(aid_i) if aid_i else None
|
||||
name = (prof or {}).get("name") or row.get("name") or row.get("display_name")
|
||||
if isinstance(name, str):
|
||||
name = name.strip() or None
|
||||
else:
|
||||
name = None
|
||||
if not name and not aid_i:
|
||||
continue
|
||||
key = aid_i if aid_i else hash(name or "")
|
||||
cell = by_aid.setdefault(
|
||||
key,
|
||||
{
|
||||
"account_id": aid_i or None,
|
||||
"name": name,
|
||||
"team_tag": (prof or {}).get("team_tag"),
|
||||
"games": 0,
|
||||
"wins": 0,
|
||||
"last_match_id": None,
|
||||
"last_start_time": None,
|
||||
},
|
||||
)
|
||||
cell["games"] += 1
|
||||
if row.get("won"):
|
||||
cell["wins"] += 1
|
||||
mid = row.get("match_id")
|
||||
st = row.get("start_time")
|
||||
try:
|
||||
st_i = int(st) if st is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
st_i = 0
|
||||
prev = cell.get("last_start_time") or 0
|
||||
if st_i >= prev:
|
||||
cell["last_start_time"] = st_i or None
|
||||
cell["last_match_id"] = mid
|
||||
ranked = sorted(
|
||||
by_aid.values(),
|
||||
key=lambda r: (
|
||||
-(r.get("games") or 0),
|
||||
-(r.get("last_start_time") or 0),
|
||||
str(r.get("name") or ""),
|
||||
),
|
||||
)
|
||||
for r in ranked:
|
||||
r.pop("last_start_time", None)
|
||||
return ranked[:12]
|
||||
|
||||
|
||||
def load_pro_league_rows(path: Path) -> dict[str, list[dict]]:
|
||||
"""League/pro rows from pro_matches.json by_hero or by_pro."""
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
out: dict[str, list[dict]] = {}
|
||||
|
||||
by_hero = raw.get("by_hero")
|
||||
if isinstance(by_hero, dict):
|
||||
for key, cell in by_hero.items():
|
||||
if not isinstance(cell, dict):
|
||||
continue
|
||||
matches = cell.get("matches")
|
||||
if isinstance(matches, list) and matches:
|
||||
out[str(key)] = [m for m in matches if isinstance(m, dict)]
|
||||
|
||||
by_pro = raw.get("by_pro")
|
||||
if isinstance(by_pro, dict):
|
||||
for cell in by_pro.values():
|
||||
if not isinstance(cell, dict):
|
||||
continue
|
||||
for row in cell.get("matches") or []:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
key = row.get("hero_key")
|
||||
if not key:
|
||||
continue
|
||||
out.setdefault(str(key), []).append(row)
|
||||
|
||||
deduped: dict[str, list[dict]] = {}
|
||||
for key, rows in out.items():
|
||||
seen: set[int] = set()
|
||||
merged: list[dict] = []
|
||||
for row in rows:
|
||||
try:
|
||||
mid = int(row.get("match_id") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if mid in seen:
|
||||
continue
|
||||
seen.add(mid)
|
||||
merged.append(row)
|
||||
if merged:
|
||||
deduped[key] = merged
|
||||
return deduped
|
||||
|
||||
|
||||
def load_league_rows(path: Path) -> dict[str, list[dict]]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
by_hero = raw.get("by_hero")
|
||||
if not isinstance(by_hero, dict):
|
||||
return {}
|
||||
out: dict[str, list[dict]] = {}
|
||||
for key, cell in by_hero.items():
|
||||
if not isinstance(cell, dict):
|
||||
continue
|
||||
matches = cell.get("matches")
|
||||
if not isinstance(matches, list):
|
||||
continue
|
||||
league = [m for m in matches if isinstance(m, dict) and m.get("origin") == "league"]
|
||||
if league:
|
||||
out[str(key)] = league
|
||||
return out
|
||||
|
||||
|
||||
def fetch_league_rows_for_hero(
|
||||
hero_id: int,
|
||||
*,
|
||||
limit: int,
|
||||
id_map: dict[int, str],
|
||||
catalog: dict[int, dict],
|
||||
delay: float,
|
||||
) -> list[dict]:
|
||||
metas = league_match_ids(hero_id, limit)
|
||||
rows: list[dict] = []
|
||||
for meta in metas:
|
||||
try:
|
||||
mid = int(meta.get("match_id") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if mid <= 0:
|
||||
continue
|
||||
detail = fetch_match(mid)
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
if not detail:
|
||||
continue
|
||||
slim = extract_player_row(
|
||||
detail,
|
||||
hero_id,
|
||||
origin="league",
|
||||
id_map=id_map,
|
||||
item_catalog=catalog,
|
||||
list_meta=meta,
|
||||
)
|
||||
if slim:
|
||||
rows.append(slim)
|
||||
return rows
|
||||
|
||||
|
||||
def build_payload(
|
||||
*,
|
||||
by_hero_rows: dict[str, list[dict]],
|
||||
pro_index: dict[int, dict],
|
||||
catalog: dict[int, dict],
|
||||
match_source: str,
|
||||
) -> dict:
|
||||
pro_ids = set(pro_index)
|
||||
by_hero: dict[str, dict] = {}
|
||||
used_item_ids: set[int] = set()
|
||||
total_matches = 0
|
||||
|
||||
for key, all_rows in sorted(by_hero_rows.items()):
|
||||
pro_rows = [r for r in all_rows if is_pro_match(r, pro_ids)]
|
||||
if not pro_rows:
|
||||
continue
|
||||
total_matches += len(pro_rows)
|
||||
items = aggregate_items(pro_rows, catalog)
|
||||
for ent in items:
|
||||
used_item_ids.add(int(ent["id"]))
|
||||
by_hero[key] = {
|
||||
"sample_count": len(pro_rows),
|
||||
"items": items,
|
||||
"skill_orders": aggregate_skill_orders(pro_rows),
|
||||
"pros": aggregate_pros(pro_rows, pro_index),
|
||||
}
|
||||
|
||||
items_out = {
|
||||
str(iid): {
|
||||
"key": catalog[iid]["key"],
|
||||
"dname": catalog[iid]["dname"],
|
||||
"name_loc": catalog[iid].get("name_loc") or catalog[iid]["dname"],
|
||||
}
|
||||
for iid in sorted(used_item_ids)
|
||||
if iid in catalog
|
||||
}
|
||||
|
||||
pros_out = {
|
||||
str(aid): prof
|
||||
for aid, prof in sorted(pro_index.items(), key=lambda t: t[0])
|
||||
}
|
||||
|
||||
return {
|
||||
"meta": {
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
"source": "opendota",
|
||||
"attribution": "https://www.opendota.com",
|
||||
"match_source": match_source,
|
||||
"pro_count": len(pro_index),
|
||||
"hero_count": len(by_hero),
|
||||
"match_count": total_matches,
|
||||
"note_zh": (
|
||||
"职业选手样本：OpenDota 注册选手 + 联赛对局终局出装/加点；"
|
||||
"出装频率 = 该装备出现在样本终局栏位的比例；"
|
||||
"加点为完整升级顺序的最常见方案。"
|
||||
),
|
||||
},
|
||||
"pros": pros_out,
|
||||
"items": items_out,
|
||||
"by_hero": by_hero,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--out", type=Path, default=OUT)
|
||||
ap.add_argument(
|
||||
"--matches",
|
||||
type=Path,
|
||||
default=MATCHES_IN,
|
||||
help="Input hero_matches.json league rows (default: data/hero_matches.json)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--pro-matches",
|
||||
type=Path,
|
||||
default=PRO_MATCHES_IN,
|
||||
help="Merge pro player rows from this file (default: data/pro_matches.json)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--fetch",
|
||||
action="store_true",
|
||||
help="Fetch league matches from OpenDota instead of only reading --matches",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=12,
|
||||
help="League matches per hero when --fetch (default: 12)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--heroes",
|
||||
default="",
|
||||
help="Comma-separated hero keys (default: all when --fetch; else keys in matches file)",
|
||||
)
|
||||
ap.add_argument("--delay", type=float, default=0.35)
|
||||
args = ap.parse_args()
|
||||
|
||||
print("loading pro players ...", flush=True)
|
||||
pro_index = fetch_pro_index()
|
||||
print(f" {len(pro_index)} registered pros", flush=True)
|
||||
|
||||
catalog, _ = load_item_catalog()
|
||||
id_map = load_ability_id_map()
|
||||
heroes = hero_table()
|
||||
by_key = {h["key"]: h for h in heroes}
|
||||
hero_filter = [k.strip() for k in args.heroes.split(",") if k.strip()]
|
||||
|
||||
by_hero_rows: dict[str, list[dict]] = {}
|
||||
|
||||
if args.fetch:
|
||||
keys = hero_filter or sorted(by_key)
|
||||
limit = max(1, int(args.limit))
|
||||
for i, key in enumerate(keys, 1):
|
||||
hero = by_key.get(key)
|
||||
if not hero:
|
||||
print(f" skip unknown hero {key}", flush=True)
|
||||
continue
|
||||
hid = int(hero["id"])
|
||||
print(f"[{i}/{len(keys)}] fetch league {key} ...", flush=True)
|
||||
rows = fetch_league_rows_for_hero(
|
||||
hid,
|
||||
limit=limit,
|
||||
id_map=id_map,
|
||||
catalog=catalog,
|
||||
delay=args.delay,
|
||||
)
|
||||
if rows:
|
||||
by_hero_rows[key] = rows
|
||||
else:
|
||||
by_hero_rows = load_league_rows(args.matches)
|
||||
pro_path = args.pro_matches
|
||||
if pro_path and pro_path.is_file():
|
||||
pro_rows = load_pro_league_rows(pro_path)
|
||||
for key, rows in pro_rows.items():
|
||||
bucket = by_hero_rows.setdefault(key, [])
|
||||
seen = {int(r.get("match_id") or 0) for r in bucket if isinstance(r, dict)}
|
||||
for row in rows:
|
||||
try:
|
||||
mid = int(row.get("match_id") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if mid in seen:
|
||||
continue
|
||||
seen.add(mid)
|
||||
bucket.append(row)
|
||||
if hero_filter:
|
||||
by_hero_rows = {k: v for k, v in by_hero_rows.items() if k in hero_filter}
|
||||
if not by_hero_rows:
|
||||
raise SystemExit(
|
||||
f"No league/pro rows in {args.matches} or {args.pro_matches}. "
|
||||
"Run: python fetch_pro_matches.py or fetch_hero_matches.py --source league"
|
||||
)
|
||||
|
||||
sources = []
|
||||
if args.fetch:
|
||||
sources.append("fetch")
|
||||
else:
|
||||
sources.append(str(args.matches.name))
|
||||
if args.pro_matches and args.pro_matches.is_file():
|
||||
sources.append(str(args.pro_matches.name))
|
||||
match_source = "+".join(sources)
|
||||
payload = build_payload(
|
||||
by_hero_rows=by_hero_rows,
|
||||
pro_index=pro_index,
|
||||
catalog=catalog,
|
||||
match_source=match_source,
|
||||
)
|
||||
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
meta = payload["meta"]
|
||||
print(
|
||||
f"done heroes={meta['hero_count']} matches={meta['match_count']} → {args.out}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user