vosonandCursor
3ec8007077
Ship OpenDota counter-stats reordering for feared items, finalize SITE_VERSION/docs for rankings/streamers/trends/matches/mechanics and draft archetypes, and ignore regenerable Web data caches. Co-authored-by: Cursor <cursoragent@cursor.com>
435 lines
13 KiB
Python
435 lines
13 KiB
Python
"""Fetch recent league/tournament matches for OpenDota registered pros.
|
||
|
||
Pulls /proPlayers, then per player /players/{id}/matches (lobby practice +
|
||
tournament), enriches with /matches/{id} for final items + skill builds.
|
||
|
||
Output: data/pro_matches.json (Climperor web only; not used by recommend).
|
||
|
||
Usage:
|
||
python fetch_pro_matches.py --limit-pros 20 --limit 8
|
||
python fetch_pro_matches.py --players 1296625,117421467
|
||
python fetch_pro_matches.py --with-team --active-days 45
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import time
|
||
import urllib.error
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
|
||
from common import DATA
|
||
from fetch_hero_matches import (
|
||
collect_item_ids,
|
||
extract_player_row,
|
||
fetch_match,
|
||
load_ability_id_map,
|
||
)
|
||
from fetch_hero_items import load_item_catalog
|
||
from fetch_pro_builds import fetch_pro_index
|
||
from grid import hero_table
|
||
from http_utils import http_json
|
||
|
||
OPENDOTA = "https://api.opendota.com/api"
|
||
OUT = DATA / "pro_matches.json"
|
||
DEFAULT_LIMIT = 8
|
||
DEFAULT_LIMIT_PROS = 40
|
||
# OpenDota lobby_type: 1=practice, 2=tournament (pro/league biased).
|
||
LOBBY_LEAGUE = (1, 2)
|
||
|
||
|
||
def parse_pro_filter(raw: str, pro_index: dict[int, dict]) -> list[int]:
|
||
"""Comma-separated account ids or registered pro names."""
|
||
if not raw.strip():
|
||
return []
|
||
name_to_id: dict[str, int] = {}
|
||
for aid, prof in pro_index.items():
|
||
for key in ("name",):
|
||
val = prof.get(key)
|
||
if isinstance(val, str) and val.strip():
|
||
name_to_id[val.strip().lower()] = aid
|
||
out: list[int] = []
|
||
for part in raw.split(","):
|
||
token = part.strip()
|
||
if not token:
|
||
continue
|
||
if token.isdigit():
|
||
out.append(int(token))
|
||
continue
|
||
aid = name_to_id.get(token.lower())
|
||
if aid:
|
||
out.append(aid)
|
||
else:
|
||
print(f" warn: unknown pro {token!r}", flush=True)
|
||
return out
|
||
|
||
|
||
def filter_pros(
|
||
pro_index: dict[int, dict],
|
||
*,
|
||
with_team: bool,
|
||
active_days: int | None,
|
||
limit_pros: int,
|
||
player_ids: list[int],
|
||
) -> list[tuple[int, dict]]:
|
||
if player_ids:
|
||
rows: list[tuple[int, dict]] = []
|
||
for aid in player_ids:
|
||
prof = pro_index.get(aid)
|
||
if prof:
|
||
rows.append((aid, prof))
|
||
return rows
|
||
|
||
cutoff = None
|
||
if active_days is not None and active_days > 0:
|
||
cutoff = datetime.now(timezone.utc) - timedelta(days=active_days)
|
||
|
||
candidates: list[tuple[int, dict, float]] = []
|
||
for aid, prof in pro_index.items():
|
||
if with_team and not prof.get("team_tag") and not prof.get("team_name"):
|
||
continue
|
||
last = prof.get("last_match_time")
|
||
score = 0.0
|
||
if isinstance(last, str) and last.strip():
|
||
try:
|
||
ts = datetime.fromisoformat(last.replace("Z", "+00:00"))
|
||
if cutoff and ts < cutoff:
|
||
continue
|
||
score = ts.timestamp()
|
||
except ValueError:
|
||
if cutoff:
|
||
continue
|
||
candidates.append((aid, prof, score))
|
||
|
||
candidates.sort(key=lambda t: (-t[2], str(t[1].get("name") or ""), t[0]))
|
||
picked = candidates[: max(1, limit_pros)]
|
||
return [(aid, prof) for aid, prof, _ in picked]
|
||
|
||
|
||
def player_match_metas(
|
||
account_id: int,
|
||
limit: int,
|
||
*,
|
||
lobby_types: tuple[int, ...] = LOBBY_LEAGUE,
|
||
) -> list[dict]:
|
||
"""Recent match list rows for a pro (deduped, newest first)."""
|
||
per_lt = max(limit, limit // max(1, len(lobby_types)) + 2)
|
||
by_id: dict[int, dict] = {}
|
||
for lt in lobby_types:
|
||
url = f"{OPENDOTA}/players/{account_id}/matches?limit={per_lt}&lobby_type={lt}"
|
||
try:
|
||
raw = http_json(url)
|
||
except (
|
||
urllib.error.HTTPError,
|
||
urllib.error.URLError,
|
||
TimeoutError,
|
||
json.JSONDecodeError,
|
||
OSError,
|
||
):
|
||
continue
|
||
if not isinstance(raw, list):
|
||
continue
|
||
for row in raw:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
try:
|
||
mid = int(row.get("match_id") or 0)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if mid <= 0:
|
||
continue
|
||
prev = by_id.get(mid)
|
||
if prev is None:
|
||
by_id[mid] = row
|
||
continue
|
||
try:
|
||
st_new = int(row.get("start_time") or 0)
|
||
st_old = int(prev.get("start_time") or 0)
|
||
except (TypeError, ValueError):
|
||
st_new = st_old = 0
|
||
if st_new >= st_old:
|
||
by_id[mid] = row
|
||
|
||
ranked = sorted(
|
||
by_id.values(),
|
||
key=lambda r: (-int(r.get("start_time") or 0), -int(r.get("match_id") or 0)),
|
||
)
|
||
return ranked[:limit]
|
||
|
||
|
||
def fetch_player_matches(
|
||
account_id: int,
|
||
*,
|
||
limit: int,
|
||
id_map: dict[int, str],
|
||
catalog: dict[int, dict],
|
||
id_to_key: dict[int, str],
|
||
delay: float,
|
||
lobby_types: tuple[int, ...],
|
||
) -> list[dict]:
|
||
metas = player_match_metas(account_id, limit, lobby_types=lobby_types)
|
||
rows: list[dict] = []
|
||
for meta in metas:
|
||
try:
|
||
mid = int(meta.get("match_id") or 0)
|
||
hid = int(meta.get("hero_id") or 0)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if mid <= 0 or hid <= 0:
|
||
continue
|
||
detail = fetch_match(mid)
|
||
if delay > 0:
|
||
time.sleep(delay)
|
||
if not detail:
|
||
continue
|
||
slim = extract_player_row(
|
||
detail,
|
||
hid,
|
||
origin="pro",
|
||
id_map=id_map,
|
||
item_catalog=catalog,
|
||
list_meta=meta,
|
||
)
|
||
if not slim:
|
||
continue
|
||
row_aid = slim.get("account_id")
|
||
try:
|
||
row_aid_i = int(row_aid) if row_aid is not None else 0
|
||
except (TypeError, ValueError):
|
||
row_aid_i = 0
|
||
if row_aid_i and row_aid_i != account_id:
|
||
continue
|
||
slim["hero_id"] = hid
|
||
slim["hero_key"] = id_to_key.get(hid)
|
||
lt = meta.get("lobby_type")
|
||
try:
|
||
slim["lobby_type"] = int(lt) if lt is not None else None
|
||
except (TypeError, ValueError):
|
||
slim["lobby_type"] = None
|
||
rows.append(slim)
|
||
return rows
|
||
|
||
|
||
def build_indexes(
|
||
by_pro: dict[str, dict],
|
||
id_to_key: dict[int, str],
|
||
) -> dict[str, dict]:
|
||
by_hero: dict[str, list[dict]] = {}
|
||
for cell in by_pro.values():
|
||
for row in cell.get("matches") or []:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
key = row.get("hero_key")
|
||
if not key:
|
||
hid = row.get("hero_id")
|
||
try:
|
||
key = id_to_key.get(int(hid)) if hid is not None else None
|
||
except (TypeError, ValueError):
|
||
key = None
|
||
if not key:
|
||
continue
|
||
by_hero.setdefault(str(key), []).append(row)
|
||
out: dict[str, dict] = {}
|
||
for key, rows in by_hero.items():
|
||
seen: set[int] = set()
|
||
deduped: list[dict] = []
|
||
for row in sorted(
|
||
rows,
|
||
key=lambda r: (-int(r.get("start_time") or 0), -int(r.get("match_id") or 0)),
|
||
):
|
||
try:
|
||
mid = int(row.get("match_id") or 0)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if mid in seen:
|
||
continue
|
||
seen.add(mid)
|
||
deduped.append(row)
|
||
out[key] = {"matches": deduped}
|
||
return out
|
||
|
||
|
||
def write_out(
|
||
path: Path,
|
||
*,
|
||
by_pro: dict[str, dict],
|
||
by_hero: dict[str, dict],
|
||
pros_meta: dict[str, dict],
|
||
item_catalog: dict[int, dict],
|
||
limit: int,
|
||
limit_pros: int,
|
||
lobby_types: tuple[int, ...],
|
||
) -> None:
|
||
used = collect_item_ids(by_hero)
|
||
items_out = {
|
||
str(iid): {
|
||
"key": item_catalog[iid]["key"],
|
||
"dname": item_catalog[iid]["dname"],
|
||
"name_loc": item_catalog[iid].get("name_loc") or item_catalog[iid]["dname"],
|
||
}
|
||
for iid in sorted(used)
|
||
if iid in item_catalog
|
||
}
|
||
match_count = sum(
|
||
len(cell.get("matches") or [])
|
||
for cell in by_pro.values()
|
||
if isinstance(cell, dict)
|
||
)
|
||
payload = {
|
||
"meta": {
|
||
"source": "opendota",
|
||
"attribution": "https://www.opendota.com",
|
||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||
"limit_per_pro": limit,
|
||
"limit_pros": limit_pros,
|
||
"lobby_types": list(lobby_types),
|
||
"pro_count": len(by_pro),
|
||
"match_count": match_count,
|
||
"hero_count": len(by_hero),
|
||
"note_zh": (
|
||
"OpenDota 注册职业选手近期联赛/锦标赛对局；"
|
||
"lobby_type 1=训练/practice、2=tournament；"
|
||
"含终局出装、加点与联赛名（若有）。"
|
||
),
|
||
},
|
||
"items": items_out,
|
||
"pros": pros_meta,
|
||
"by_pro": by_pro,
|
||
"by_hero": by_hero,
|
||
}
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(
|
||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
|
||
|
||
def _log(msg: str) -> None:
|
||
try:
|
||
print(msg, flush=True)
|
||
except UnicodeEncodeError:
|
||
print(msg.encode("ascii", "backslashreplace").decode("ascii"), flush=True)
|
||
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--out", type=Path, default=OUT)
|
||
ap.add_argument(
|
||
"--limit",
|
||
type=int,
|
||
default=DEFAULT_LIMIT,
|
||
help=f"Matches per pro (default: {DEFAULT_LIMIT})",
|
||
)
|
||
ap.add_argument(
|
||
"--limit-pros",
|
||
type=int,
|
||
default=DEFAULT_LIMIT_PROS,
|
||
help=f"Max pros when --players omitted (default: {DEFAULT_LIMIT_PROS})",
|
||
)
|
||
ap.add_argument(
|
||
"--players",
|
||
default="",
|
||
help="Comma-separated account_id or registered pro name (overrides --limit-pros)",
|
||
)
|
||
ap.add_argument(
|
||
"--with-team",
|
||
action="store_true",
|
||
help="Only pros with a team_tag/team_name when picking from /proPlayers",
|
||
)
|
||
ap.add_argument(
|
||
"--active-days",
|
||
type=int,
|
||
default=60,
|
||
help="Skip pros with no last_match_time within N days (0=disable; default: 60)",
|
||
)
|
||
ap.add_argument("--delay", type=float, default=0.35)
|
||
ap.add_argument(
|
||
"--include-pubs",
|
||
action="store_true",
|
||
help="Also include ranked pub lobby_type=7 (high-MMR scrims)",
|
||
)
|
||
args = ap.parse_args()
|
||
|
||
limit = max(1, int(args.limit))
|
||
limit_pros = max(1, int(args.limit_pros))
|
||
active_days = int(args.active_days) if args.active_days > 0 else None
|
||
lobby_types: tuple[int, ...] = LOBBY_LEAGUE
|
||
if args.include_pubs:
|
||
lobby_types = LOBBY_LEAGUE + (7,)
|
||
|
||
_log("loading pro players ...")
|
||
pro_index = fetch_pro_index()
|
||
_log(f" {len(pro_index)} registered pros")
|
||
|
||
player_ids = parse_pro_filter(args.players, pro_index)
|
||
picked = filter_pros(
|
||
pro_index,
|
||
with_team=args.with_team,
|
||
active_days=active_days,
|
||
limit_pros=limit_pros,
|
||
player_ids=player_ids,
|
||
)
|
||
if not picked:
|
||
raise SystemExit("No pros matched filters")
|
||
|
||
heroes = hero_table()
|
||
id_to_key = {int(h["id"]): h["key"] for h in heroes}
|
||
catalog, _ = load_item_catalog()
|
||
id_map = load_ability_id_map()
|
||
|
||
by_pro: dict[str, dict] = {}
|
||
pros_meta: dict[str, dict] = {}
|
||
total_matches = 0
|
||
|
||
for i, (aid, prof) in enumerate(picked, 1):
|
||
label = prof.get("name") or prof.get("team_tag") or str(aid)
|
||
_log(f"[{i}/{len(picked)}] {label} ({aid}) ...")
|
||
matches = fetch_player_matches(
|
||
aid,
|
||
limit=limit,
|
||
id_map=id_map,
|
||
catalog=catalog,
|
||
id_to_key=id_to_key,
|
||
delay=args.delay,
|
||
lobby_types=lobby_types,
|
||
)
|
||
sid = str(aid)
|
||
by_pro[sid] = {
|
||
"account_id": aid,
|
||
"name": prof.get("name"),
|
||
"team_tag": prof.get("team_tag"),
|
||
"team_name": prof.get("team_name"),
|
||
"country_code": prof.get("country_code"),
|
||
"match_count": len(matches),
|
||
"matches": matches,
|
||
}
|
||
pros_meta[sid] = {
|
||
"account_id": aid,
|
||
"name": prof.get("name"),
|
||
"team_tag": prof.get("team_tag"),
|
||
"team_name": prof.get("team_name"),
|
||
"country_code": prof.get("country_code"),
|
||
}
|
||
total_matches += len(matches)
|
||
_log(f" {len(matches)} matches")
|
||
|
||
by_hero = build_indexes(by_pro, id_to_key)
|
||
write_out(
|
||
args.out,
|
||
by_pro=by_pro,
|
||
by_hero=by_hero,
|
||
pros_meta=pros_meta,
|
||
item_catalog=catalog,
|
||
limit=limit,
|
||
limit_pros=len(picked),
|
||
lobby_types=lobby_types,
|
||
)
|
||
_log(
|
||
f"done pros={len(by_pro)} matches={total_matches} heroes={len(by_hero)} → {args.out}"
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|