From 0f7e7b68dd79600aa6d32660ddce147264cc6f69 Mon Sep 17 00:00:00 2001 From: voson Date: Thu, 30 Jul 2026 22:59:34 +0800 Subject: [PATCH] Rotate OpenDota pro-match refresh to stay under rate limits. Daily now refreshes the 15 oldest watchlist pros, keeps prior cache for the rest, fails fast on consecutive 429s, and still writes a partial result. Co-authored-by: Cursor --- AGENTS.md | 4 +- CHANGELOG.md | 10 + web/fetch_pro_matches.py | 276 ++++++++++++++++++++-- web/refresh_web.py | 8 +- web/tests/test_pro_matches_incremental.py | 144 +++++++++++ 5 files changed, 420 insertions(+), 22 deletions(-) create mode 100644 web/tests/test_pro_matches_incremental.py diff --git a/AGENTS.md b/AGENTS.md index 3a555cc..293e95e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,7 +88,7 @@ climperor/ | `web/fetch_hero_items.py` | 拉取 OpenDota 热门装备 → `web/data/hero_items.json` + `web/assets/item_icons/` | | `web/fetch_hero_stats.py` | 拉取 OpenDota 各段位场次/胜场 → `web/data/hero_stats.json`（**仅上分帝 Web**；勿写入 relations/heroes，勿进 recommend） | | `web/fetch_hero_matches.py` | 拉取同英雄近期比赛 + 终局出装/加点 → `web/data/hero_matches.json`（`--source league\|public\|both`；合并后保留最近 N 场胜局，默认 10；天梯需传奇及以上；公开列表过滤 bot/Turbo、仅 ranked lobby；`--public-region china` 优先国服；`--workers` 详情并发 + 凑满即停；可选 `OPENDOTA_API_KEY`；增量补缺失/不足 N 场；`--enrich-item-times` 补购买时间；**仅上分帝 Web**；勿进 recommend） | -| `web/fetch_pro_matches.py` | 按明星名单拉近期联赛/锦标赛对局 → `web/data/pro_matches.json`（默认读 `web/data/pro_player_watchlist.json`；`--include-pubs` 另拉天梯 lobby 7；每种 lobby 各保留 `--limit` 场；`--players` 覆盖整文件；`--all-pros` 从 `/proPlayers` 盲抽；含终局出装/加点；进 `refresh_web` daily 且带 `--include-pubs`；**仅上分帝 Web「比赛」**；勿进 recommend） | +| `web/fetch_pro_matches.py` | 按明星名单拉近期联赛/锦标赛对局 → `web/data/pro_matches.json`（默认读 `web/data/pro_player_watchlist.json`；`--include-pubs` 另拉天梯 lobby 7；每种 lobby 各保留 `--limit` 场；`--refresh-limit N` 按 `fetched_at` 只刷新最陈旧 N 人并保留其余；连续 429 熔断后仍写盘；可选 `OPENDOTA_API_KEY`；`--players` 覆盖整文件且不做轮换；`--all-pros` 从 `/proPlayers` 盲抽；含终局出装/加点；进 `refresh_web` daily 且带 `--include-pubs --refresh-limit 15`；**仅上分帝 Web「比赛」**；勿进 recommend） | | `web/fetch_leaderboards.py` | 拉取 Valve Immortal 四区榜 Top100 → `web/data/leaderboards.json`（**仅上分帝 Web「排行」**；无 MMR/account_id；勿进 recommend） | | `web/fetch_streamers.py` | 从抖音 / 斗鱼主页补全 `web/data/streamers.json` 的昵称/签名/关注粉丝获赞(播放)/头像（手工名单 + 直播间/主页 URL；抖音支持 `v.douyin.com` 短链；斗鱼优先 `v.douyu.com/author/`（兼容 `author-video`）的 `$DATA`，仅房间号时从直播间 HTML 解析 `up_id` 再拉作者页，失败才回退 `betard`；失败保留旧值；**不**探测开播（由 `fetch_streamer_live.py` 负责）；**仅上分帝 Web「主播」**；勿进 recommend；进 `refresh_web` daily） | | `web/fetch_streamer_live.py` | 探测主播真实在播状态回写 `web/data/streamers.json` 的 `is_live`/`live_probed_at`：抖音解析直播间 SSR 页 `roomStore.roomInfo.room.status`（2 在播 / 4 下播；预热 cookie + ~1s 间隔；web_rid 校验），B 站走 `Room/get_info`（`live_status==1` 在播，轮播算下播），斗鱼走 `betard/{room_id}`（`show_status==1` 在播，`videoLoop==1` 轮播算下播）；探测失败清为 `is_live:false` 并去掉 `live_probed_at`（与 `/api/live-status` 一致，不沿用旧直播中）、始终 exit 0；`--ids a,b` 限范围、`--dry-run` 只打印；仅 Web；进 `refresh_web` daily（角标以访问触发的 live API 为准，daily 仅作 data.json 兜底） | @@ -187,7 +187,7 @@ python web/fetch_hero_items.py # OpenDota 热门装备缓存（上分帝 python web/fetch_hero_stats.py # OpenDota 各段位胜率/场次（上分帝 Web 走势兜底） python web/fetch_stratz_meta.py # STRATZ 周胜率/分路/对位 Top（走势/对位/Meta；需 token） python web/fetch_hero_matches.py # 同英雄近期比赛出装/加点（上分帝 Web；可 --heroes antimage；--workers 6；--enrich-item-times） -python web/fetch_pro_matches.py # 明星选手近期联赛对局（默认 watchlist；--include-pubs 含天梯；可 --players Ame,898754153） +python web/fetch_pro_matches.py # 明星选手近期联赛对局（默认 watchlist；--include-pubs 含天梯；--refresh-limit 15 轮换；可 --players Ame,898754153） python web/fetch_leaderboards.py # Valve Immortal 四区 Top100（排行页） python web/fetch_streamers.py # 抖音主播主页补全（主播页；手工名单；亦由 daily 定时软失败刷新） python web/fetch_streamer_live.py # 探测真实在播状态回写 is_live（主播页角标；daily；--dry-run 只打印） diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7b978..24dc36e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,20 @@ - PC 本人锁定英雄后，按阵容推荐装备：核心装（`hero_items` 相对热度）+ 敌方应对装（定性规则），overlay 以图标条展示；结果写入 `recommendations.items`。 +### Changed + +- 上分帝 Web 每日刷新明星比赛改为按 `fetched_at` 轮换（默认每次 15 人），遇 OpenDota 连续 429 熔断并保留旧缓存，避免整档 daily 被限流拖死。 + ### Removed - PC 选将 overlay 不再在顶栏头像下显示定位图标；仍保留网格「克/搭/补」与阵容分析条。 +## [0.6.9] - 2026-07-30 + +### Fixed + +- 恢复 OSS 英雄横版头像（曾被 96×96 匹配模板误覆盖）；静态导出/上传不再回退到 `pc/templates/cdn`，缺图或非宽图会直接失败；头像 URL 缓存戳改为 `v=wide2`。 + ## [0.6.8] - 2026-07-30 ### Added diff --git a/web/fetch_pro_matches.py b/web/fetch_pro_matches.py index 45278e0..57d5d9d 100644 --- a/web/fetch_pro_matches.py +++ b/web/fetch_pro_matches.py @@ -7,10 +7,16 @@ Per player: /players/{id}/matches per lobby_type (practice/tournament, and ranked with --include-pubs), up to --limit newest each; then /matches/{id} for final items + skill builds. League rows do not crowd out ranked pubs. +Daily refresh uses --refresh-limit to rotate the oldest / missing pros so the +OpenDota anonymous quota is not burned on a full 90-player crawl every night. +Optional OPENDOTA_API_KEY raises rate limits. Consecutive 429s trip a circuit +breaker: remaining pros keep prior cache and the script still writes. + Output: web/data/pro_matches.json (Climperor web only; not used by recommend). Usage: python web/fetch_pro_matches.py + python web/fetch_pro_matches.py --include-pubs --refresh-limit 15 python web/fetch_pro_matches.py --include-pubs --limit 8 python web/fetch_pro_matches.py --players 898754153,Ame python web/fetch_pro_matches.py --all-pros --limit-pros 20 --with-team @@ -36,8 +42,8 @@ from shared.paths import DATA from fetch_hero_matches import ( collect_item_ids, extract_player_row, - fetch_match, load_ability_id_map, + opendota_url, ) from fetch_hero_items import load_item_catalog from fetch_pro_builds import fetch_pro_index @@ -47,10 +53,48 @@ OUT = DATA / "pro_matches.json" WATCHLIST = DATA / "pro_player_watchlist.json" DEFAULT_LIMIT = 8 DEFAULT_LIMIT_PROS = 40 +DEFAULT_REFRESH_LIMIT = 0 # 0 = refresh all selected pros +DEFAULT_429_STREAK = 3 # OpenDota lobby_type: 1=practice, 2=tournament (pro/league biased). LOBBY_LEAGUE = (1, 2) +class RateLimitTripped(Exception): + """OpenDota returned enough consecutive 429s to abort the remaining batch.""" + + +class OpenDotaClient: + """Fail-fast OpenDota JSON client with consecutive-429 circuit breaker.""" + + def __init__(self, *, consecutive_limit: int = DEFAULT_429_STREAK) -> None: + self.consecutive_limit = max(1, int(consecutive_limit)) + self.consecutive_429 = 0 + self.tripped = False + + def json(self, path: str) -> dict | list: + if self.tripped: + raise RateLimitTripped("opendota rate-limit circuit open") + url = opendota_url(path) + try: + # No 5/10/20/40s retry chain — daily must finish within budget. + data = http_json(url, retries=0) + except urllib.error.HTTPError as e: + if e.code == 429: + self.consecutive_429 += 1 + _log( + f"HTTP 429 {url} — streak " + f"{self.consecutive_429}/{self.consecutive_limit}" + ) + if self.consecutive_429 >= self.consecutive_limit: + self.tripped = True + raise RateLimitTripped( + f"opendota 429 x{self.consecutive_429}" + ) from e + raise + self.consecutive_429 = 0 + return data + + def _log(msg: str) -> None: try: print(msg, flush=True) @@ -113,6 +157,58 @@ def load_watchlist(path: Path) -> list[dict]: return out +def load_existing(path: Path) -> dict: + """Load prior pro_matches.json; empty dict when missing/unreadable.""" + if not path.is_file(): + return {} + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return raw if isinstance(raw, dict) else {} + + +def parse_ts(value: object) -> datetime | None: + if not isinstance(value, str) or not value.strip(): + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def cell_fetched_at(cell: dict | None) -> datetime | None: + if not isinstance(cell, dict): + return None + return parse_ts(cell.get("fetched_at")) + + +def select_refresh_batch( + picked: list[tuple[int, dict]], + existing_by_pro: dict[str, dict], + refresh_limit: int, +) -> tuple[list[tuple[int, dict]], list[tuple[int, dict]]]: + """Split watchlist into (to_refresh, to_retain) by staleness. + + Missing / unscored cells sort oldest. ``refresh_limit <= 0`` refreshes all. + """ + if refresh_limit <= 0 or refresh_limit >= len(picked): + return list(picked), [] + + ranked: list[tuple[float, int, tuple[int, dict]]] = [] + for idx, item in enumerate(picked): + aid, _prof = item + ts = cell_fetched_at(existing_by_pro.get(str(aid))) + # Missing timestamp => oldest (refresh first). + score = ts.timestamp() if ts is not None else float("-inf") + ranked.append((score, idx, item)) + ranked.sort(key=lambda row: (row[0], row[1])) + to_refresh = [item for _score, _idx, item in ranked[:refresh_limit]] + refresh_ids = {aid for aid, _ in to_refresh} + to_retain = [item for item in picked if item[0] not in refresh_ids] + return to_refresh, to_retain + + def parse_pro_filter(raw: str, pro_index: dict[int, dict]) -> list[int]: """Comma-separated account ids or registered pro names.""" if not raw.strip(): @@ -219,6 +315,7 @@ def player_match_metas( account_id: int, limit: int, *, + client: OpenDotaClient, lobby_types: tuple[int, ...] = LOBBY_LEAGUE, ) -> list[dict]: """Recent match list rows for a pro (deduped, newest first). @@ -229,9 +326,11 @@ def player_match_metas( """ by_id: dict[int, dict] = {} for lt in lobby_types: - url = f"{OPENDOTA}/players/{account_id}/matches?limit={limit}&lobby_type={lt}" + path = f"/players/{account_id}/matches?limit={limit}&lobby_type={lt}" try: - raw = http_json(url) + raw = client.json(path) + except RateLimitTripped: + raise except ( urllib.error.HTTPError, urllib.error.URLError, @@ -269,9 +368,26 @@ def player_match_metas( ) +def fetch_match_detail(client: OpenDotaClient, match_id: int) -> dict | None: + try: + raw = client.json(f"/matches/{match_id}") + except RateLimitTripped: + raise + except ( + urllib.error.HTTPError, + urllib.error.URLError, + TimeoutError, + json.JSONDecodeError, + OSError, + ): + return None + return raw if isinstance(raw, dict) else None + + def fetch_player_matches( account_id: int, *, + client: OpenDotaClient, limit: int, id_map: dict[int, str], catalog: dict[int, dict], @@ -279,7 +395,9 @@ def fetch_player_matches( delay: float, lobby_types: tuple[int, ...], ) -> list[dict]: - metas = player_match_metas(account_id, limit, lobby_types=lobby_types) + metas = player_match_metas( + account_id, limit, client=client, lobby_types=lobby_types + ) rows: list[dict] = [] for meta in metas: try: @@ -295,7 +413,7 @@ def fetch_player_matches( except (TypeError, ValueError): lt_i = None row_origin = "public" if lt_i == 7 else "pro" - detail = fetch_match(mid) + detail = fetch_match_detail(client, mid) if delay > 0: time.sleep(delay) if not detail: @@ -363,6 +481,32 @@ def build_indexes( return out +def retain_cell( + aid: int, + prof: dict, + existing: dict | None, +) -> dict: + """Keep prior matches for a pro not refreshed this round.""" + if isinstance(existing, dict) and isinstance(existing.get("matches"), list): + cell = dict(existing) + cell["account_id"] = aid + for key in ("name", "team_tag", "team_name", "country_code"): + if prof.get(key) and not cell.get(key): + cell[key] = prof.get(key) + cell["match_count"] = len(cell.get("matches") or []) + return cell + return { + "account_id": aid, + "name": prof.get("name"), + "team_tag": prof.get("team_tag"), + "team_name": prof.get("team_name"), + "country_code": prof.get("country_code"), + "match_count": 0, + "matches": [], + "fetched_at": None, + } + + def write_out( path: Path, *, @@ -374,6 +518,10 @@ def write_out( limit_pros: int, lobby_types: tuple[int, ...], player_source: str, + refreshed_count: int, + retained_count: int, + refresh_limit: int, + rate_limited: bool, ) -> None: used = collect_item_ids(by_hero) items_out = { @@ -402,10 +550,15 @@ def write_out( "pro_count": len(by_pro), "match_count": match_count, "hero_count": len(by_hero), + "refresh_limit": refresh_limit, + "refreshed_count": refreshed_count, + "retained_count": retained_count, + "rate_limited": bool(rate_limited), "note_zh": ( "OpenDota 明星选手近期联赛/锦标赛对局（可选含天梯 lobby_type=7）；" "默认名单见 web/data/pro_player_watchlist.json；" "每种 lobby 各保留最近 limit 场，避免联赛挤掉天梯；" + "daily 按 fetched_at 轮换最陈旧选手（--refresh-limit）；" "lobby_type 1=训练/practice、2=tournament、7=ranked；" "含终局出装、加点与联赛名（若有）。" ), @@ -431,7 +584,7 @@ def main() -> None: "--limit", type=int, default=DEFAULT_LIMIT, - help=f"Matches per pro (default: {DEFAULT_LIMIT})", + help=f"Matches per lobby per pro (default: {DEFAULT_LIMIT})", ) ap.add_argument( "--limit-pros", @@ -439,6 +592,16 @@ def main() -> None: default=DEFAULT_LIMIT_PROS, help=f"Max pros when --all-pros (default: {DEFAULT_LIMIT_PROS})", ) + ap.add_argument( + "--refresh-limit", + type=int, + default=DEFAULT_REFRESH_LIMIT, + help=( + "Max pros to refresh this run by oldest fetched_at " + f"(0=all; default: {DEFAULT_REFRESH_LIMIT}). " + "Ignored when --players is set." + ), + ) ap.add_argument( "--players", default="", @@ -466,15 +629,25 @@ def main() -> None: action="store_true", help="Also include ranked pub lobby_type=7 (high-MMR scrims)", ) + ap.add_argument( + "--429-streak", + dest="streak_429", + type=int, + default=DEFAULT_429_STREAK, + help=f"Consecutive 429s before aborting remaining pros (default: {DEFAULT_429_STREAK})", + ) args = ap.parse_args() limit = max(1, int(args.limit)) limit_pros = max(1, int(args.limit_pros)) + refresh_limit = max(0, int(args.refresh_limit)) active_days = int(args.active_days) if args.active_days > 0 else None lobby_types: tuple[int, ...] = LOBBY_LEAGUE if args.include_pubs: lobby_types = LOBBY_LEAGUE + (7,) + client = OpenDotaClient(consecutive_limit=int(args.streak_429)) + _log("loading pro players ...") pro_index = fetch_pro_index() _log(f" {len(pro_index)} registered pros") @@ -508,6 +681,25 @@ def main() -> None: if not picked: raise SystemExit("No pros matched filters") + existing = load_existing(args.out) + existing_by_pro = existing.get("by_pro") if isinstance(existing.get("by_pro"), dict) else {} + assert isinstance(existing_by_pro, dict) + + # Explicit --players means refresh those fully (no rotation). + if player_ids: + to_refresh, to_retain = list(picked), [] + effective_refresh_limit = 0 + else: + to_refresh, to_retain = select_refresh_batch( + picked, existing_by_pro, refresh_limit + ) + effective_refresh_limit = refresh_limit + + _log( + f" refresh batch={len(to_refresh)} retain={len(to_retain)} " + f"refresh_limit={effective_refresh_limit or 'all'}" + ) + heroes = hero_table() id_to_key = {int(h["id"]): h["key"] for h in heroes} catalog, _ = load_item_catalog() @@ -516,20 +708,57 @@ def main() -> None: by_pro: dict[str, dict] = {} pros_meta: dict[str, dict] = {} total_matches = 0 + refreshed_count = 0 + rate_limited = False + now_iso = datetime.now(timezone.utc).isoformat() - for i, (aid, prof) in enumerate(picked, 1): - label = prof.get("name") or prof.get("team_tag") or str(aid) - _log(f"[{i}/{len(picked)}] {label} ({aid}) ...") - matches = fetch_player_matches( - aid, - limit=limit, - id_map=id_map, - catalog=catalog, - id_to_key=id_to_key, - delay=args.delay, - lobby_types=lobby_types, - ) + for aid, prof in to_retain: sid = str(aid) + cell = retain_cell(aid, prof, existing_by_pro.get(sid)) + by_pro[sid] = cell + pros_meta[sid] = { + "account_id": aid, + "name": cell.get("name") or prof.get("name"), + "team_tag": cell.get("team_tag") or prof.get("team_tag"), + "team_name": cell.get("team_name") or prof.get("team_name"), + "country_code": cell.get("country_code") or prof.get("country_code"), + } + total_matches += len(cell.get("matches") or []) + + for i, (aid, prof) in enumerate(to_refresh, 1): + label = prof.get("name") or prof.get("team_tag") or str(aid) + sid = str(aid) + _log(f"[{i}/{len(to_refresh)}] {label} ({aid}) ...") + try: + matches = fetch_player_matches( + aid, + client=client, + limit=limit, + id_map=id_map, + catalog=catalog, + id_to_key=id_to_key, + delay=args.delay, + lobby_types=lobby_types, + ) + except RateLimitTripped as exc: + rate_limited = True + _log(f" rate-limited; keeping prior cache for remaining ({exc})") + # Keep old (or empty) for this pro and every leftover refresh target. + remaining = to_refresh[i - 1 :] + for raid, rprof in remaining: + rsid = str(raid) + cell = retain_cell(raid, rprof, existing_by_pro.get(rsid)) + by_pro[rsid] = cell + pros_meta[rsid] = { + "account_id": raid, + "name": cell.get("name") or rprof.get("name"), + "team_tag": cell.get("team_tag") or rprof.get("team_tag"), + "team_name": cell.get("team_name") or rprof.get("team_name"), + "country_code": cell.get("country_code") or rprof.get("country_code"), + } + total_matches += len(cell.get("matches") or []) + break + by_pro[sid] = { "account_id": aid, "name": prof.get("name"), @@ -538,6 +767,7 @@ def main() -> None: "country_code": prof.get("country_code"), "match_count": len(matches), "matches": matches, + "fetched_at": now_iso, } pros_meta[sid] = { "account_id": aid, @@ -546,9 +776,11 @@ def main() -> None: "team_name": prof.get("team_name"), "country_code": prof.get("country_code"), } + refreshed_count += 1 total_matches += len(matches) _log(f" {len(matches)} matches") + retained_count = len(by_pro) - refreshed_count by_hero = build_indexes(by_pro, id_to_key) write_out( args.out, @@ -560,9 +792,15 @@ def main() -> None: limit_pros=len(picked), lobby_types=lobby_types, player_source=player_source, + refreshed_count=refreshed_count, + retained_count=max(0, retained_count), + refresh_limit=effective_refresh_limit, + rate_limited=rate_limited, ) _log( - f"done pros={len(by_pro)} matches={total_matches} heroes={len(by_hero)} → {args.out}" + f"done pros={len(by_pro)} refreshed={refreshed_count} " + f"retained={max(0, retained_count)} matches={total_matches} " + f"heroes={len(by_hero)} rate_limited={rate_limited} → {args.out}" ) diff --git a/web/refresh_web.py b/web/refresh_web.py index bf65265..9c5b376 100644 --- a/web/refresh_web.py +++ b/web/refresh_web.py @@ -275,7 +275,13 @@ def run_daily(*, dry_run: bool = False) -> bool: run_script("fetch_hero_stats.py", dry_run=dry_run) run_script("fetch_leaderboards.py", dry_run=dry_run) run_script("fetch_hero_matches.py", "--source", "league", dry_run=dry_run) - run_script("fetch_pro_matches.py", "--include-pubs", dry_run=dry_run) + run_script( + "fetch_pro_matches.py", + "--include-pubs", + "--refresh-limit", + "15", + dry_run=dry_run, + ) # Soft-fail Douyin enrichment (script itself exits 0; keep previous values on miss). run_script("fetch_streamers.py", dry_run=dry_run, soft_fail=True) # Soft-fail live probe (exits 0; probe failures clear is_live / live_probed_at). diff --git a/web/tests/test_pro_matches_incremental.py b/web/tests/test_pro_matches_incremental.py new file mode 100644 index 0000000..0bfbe00 --- /dev/null +++ b/web/tests/test_pro_matches_incremental.py @@ -0,0 +1,144 @@ +"""Unit tests for pro_matches incremental refresh + 429 circuit breaker.""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +import urllib.error +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "web")) + +from web.fetch_pro_matches import ( # noqa: E402 + OpenDotaClient, + RateLimitTripped, + load_existing, + retain_cell, + select_refresh_batch, + write_out, +) + + +class SelectRefreshBatchTests(unittest.TestCase): + def test_oldest_and_missing_first(self) -> None: + picked = [ + (1, {"name": "A"}), + (2, {"name": "B"}), + (3, {"name": "C"}), + (4, {"name": "D"}), + ] + existing = { + "1": {"fetched_at": "2026-07-28T00:00:00+00:00", "matches": []}, + "2": {"fetched_at": "2026-07-20T00:00:00+00:00", "matches": []}, + # 3 missing => oldest + "4": {"fetched_at": "2026-07-29T00:00:00+00:00", "matches": []}, + } + refresh, retain = select_refresh_batch(picked, existing, 2) + self.assertEqual([aid for aid, _ in refresh], [3, 2]) + self.assertEqual([aid for aid, _ in retain], [1, 4]) + + def test_refresh_limit_zero_means_all(self) -> None: + picked = [(1, {}), (2, {})] + refresh, retain = select_refresh_batch(picked, {}, 0) + self.assertEqual(len(refresh), 2) + self.assertEqual(retain, []) + + def test_retain_keeps_prior_matches(self) -> None: + existing = { + "account_id": 9, + "name": "Old", + "matches": [{"match_id": 1}], + "fetched_at": "2026-07-01T00:00:00+00:00", + } + cell = retain_cell(9, {"name": "New"}, existing) + self.assertEqual(cell["matches"], [{"match_id": 1}]) + self.assertEqual(cell["fetched_at"], "2026-07-01T00:00:00+00:00") + self.assertEqual(cell["match_count"], 1) + + +class RateLimitClientTests(unittest.TestCase): + def test_trips_after_consecutive_429(self) -> None: + client = OpenDotaClient(consecutive_limit=2) + + def boom(_url: str, **_kwargs): + raise urllib.error.HTTPError( + "https://api.opendota.com/api/x", 429, "Too Many", hdrs=None, fp=None + ) + + with patch("web.fetch_pro_matches.http_json", side_effect=boom): + with self.assertRaises(urllib.error.HTTPError): + client.json("/players/1/matches") + with self.assertRaises(RateLimitTripped): + client.json("/players/1/matches") + self.assertTrue(client.tripped) + + def test_success_resets_streak(self) -> None: + client = OpenDotaClient(consecutive_limit=3) + calls = {"n": 0} + + def flaky(url: str, **_kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise urllib.error.HTTPError(url, 429, "Too Many", hdrs=None, fp=None) + return [] + + with patch("web.fetch_pro_matches.http_json", side_effect=flaky): + with self.assertRaises(urllib.error.HTTPError): + client.json("/players/1/matches") + self.assertEqual(client.consecutive_429, 1) + self.assertEqual(client.json("/players/2/matches"), []) + self.assertEqual(client.consecutive_429, 0) + + +class WriteOutMergeTests(unittest.TestCase): + def test_write_out_records_refresh_meta(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "pro_matches.json" + by_pro = { + "1": { + "account_id": 1, + "name": "A", + "matches": [], + "match_count": 0, + "fetched_at": datetime.now(timezone.utc).isoformat(), + }, + "2": { + "account_id": 2, + "name": "B", + "matches": [{"match_id": 9, "hero_key": "axe"}], + "match_count": 1, + "fetched_at": "2026-07-01T00:00:00+00:00", + }, + } + write_out( + path, + by_pro=by_pro, + by_hero={"axe": {"matches": [{"match_id": 9}]}}, + pros_meta={"1": {"account_id": 1}, "2": {"account_id": 2}}, + item_catalog={}, + limit=8, + limit_pros=2, + lobby_types=(1, 2, 7), + player_source="watchlist:x.json", + refreshed_count=1, + retained_count=1, + refresh_limit=15, + rate_limited=True, + ) + payload = json.loads(path.read_text(encoding="utf-8")) + meta = payload["meta"] + self.assertEqual(meta["refreshed_count"], 1) + self.assertEqual(meta["retained_count"], 1) + self.assertEqual(meta["refresh_limit"], 15) + self.assertTrue(meta["rate_limited"]) + self.assertEqual(load_existing(path)["by_pro"]["2"]["match_count"], 1) + + +if __name__ == "__main__": + unittest.main()