"""Orchestrate Climperor web data refresh tiers, then optional OSS + deploy. Tiers (see AGENTS.md / Gitea Actions workflows): daily — OpenDota stats, leaderboards, matches, pro matches, streamers, streamer live probe; patch check (live badge still trails reality by up to a day — truly real-time would need a dedicated higher-frequency workflow, e.g. 15-min) weekly — STRATZ meta, hero items, items_meta, item_fears patch — patch list check; on has_new fetch details + version-linked scripts all — weekly then daily (patch check included in daily/patch) Usage: python refresh_web.py --tier patch --skip-deploy --skip-oss python refresh_web.py --tier daily --skip-deploy python refresh_web.py --tier weekly python refresh_web.py --tier all --dry-run """ from __future__ import annotations import argparse import hashlib import json import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent # Paths whose content change should trigger deploy / OSS. DATA_WATCH = [ ROOT / "data" / "hero_stats.json", ROOT / "data" / "leaderboards.json", ROOT / "data" / "hero_matches.json", ROOT / "data" / "pro_matches.json", ROOT / "data" / "streamers.json", ROOT / "data" / "stratz_hero_meta.json", ROOT / "data" / "stratz_matchup_tops.json", ROOT / "data" / "hero_items.json", ROOT / "data" / "items_meta.json", ROOT / "data" / "hero_item_fears.json", ROOT / "data" / "patches.json", ROOT / "data" / "hero_abilities.json", ROOT / "data" / "item_shop.json", ] ASSET_DIRS = [ ROOT / "assets" / "item_icons", ROOT / "assets" / "ability_icons", ROOT / "assets" / "item_cat_icons", ROOT / "assets" / "hero_portraits", ROOT / "assets" / "streamer_avatars", ROOT / "assets" / "streamer_videos", ] def _file_digest(path: Path) -> str | None: if not path.is_file(): return None h = hashlib.sha256() with path.open("rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def _dir_digest(path: Path) -> str | None: if not path.is_dir(): return None h = hashlib.sha256() for p in sorted(path.rglob("*")): if not p.is_file(): continue rel = p.relative_to(path).as_posix().encode() h.update(rel) h.update(b"\0") digest = _file_digest(p) if digest: h.update(digest.encode()) return h.hexdigest() def snapshot() -> dict[str, str | None]: out: dict[str, str | None] = {} for p in DATA_WATCH: out[str(p.relative_to(ROOT))] = _file_digest(p) for d in ASSET_DIRS: out[str(d.relative_to(ROOT)) + "/"] = _dir_digest(d) return out def diff_snapshots(before: dict[str, str | None], after: dict[str, str | None]) -> tuple[bool, bool]: """Return (data_changed, assets_changed).""" data_changed = False assets_changed = False keys = set(before) | set(after) for k in keys: if before.get(k) == after.get(k): continue if k.endswith("/"): assets_changed = True else: data_changed = True return data_changed, assets_changed def run_script(script: str, *args: str, dry_run: bool = False, soft_fail: bool = False) -> bool: cmd = [sys.executable, str(ROOT / script), *args] print(f"+ {' '.join(cmd)}", flush=True) if dry_run: return True proc = subprocess.run(cmd, cwd=str(ROOT), check=False) if proc.returncode == 0: return True if soft_fail: print( f"soft-fail: {script} exited {proc.returncode}; continuing", flush=True, ) return False raise subprocess.CalledProcessError(proc.returncode, cmd) def patch_check() -> dict: """Always hits the network (read-only list); safe under --dry-run.""" cmd = [sys.executable, str(ROOT / "fetch_patches.py"), "--check"] print(f"+ {' '.join(cmd)}", flush=True) proc = subprocess.run( cmd, cwd=str(ROOT), check=True, capture_output=True, text=True, encoding="utf-8", ) line = (proc.stdout or "").strip().splitlines()[-1] if (proc.stdout or "").strip() else "" if not line: raise RuntimeError("fetch_patches.py --check produced empty stdout") result = json.loads(line) print(json.dumps(result, ensure_ascii=False), flush=True) return result def run_patch_linked(*, dry_run: bool = False) -> None: run_script("fetch_patches.py", dry_run=dry_run) run_script("fetch_hero_abilities.py", "--icons", dry_run=dry_run) run_script("fetch_item_shop.py", dry_run=dry_run) run_script("fetch_items_meta.py", dry_run=dry_run) run_script("item_fears.py", dry_run=dry_run) # Soft: patch workflow may lack STRATZ token; weekly still does the hard refresh. run_script( "fetch_stratz_meta.py", "--matchups-only", dry_run=dry_run, soft_fail=True, ) def run_daily(*, dry_run: bool = False) -> bool: """Return True if patch-linked fetch ran.""" run_script("fetch_hero_stats.py", dry_run=dry_run) run_script("fetch_leaderboards.py", dry_run=dry_run) run_script("fetch_hero_matches.py", "--source", "league", dry_run=dry_run) run_script("fetch_pro_matches.py", dry_run=dry_run) # Soft-fail Douyin enrichment (script itself exits 0; keep previous values on miss). run_script("fetch_streamers.py", dry_run=dry_run) # Soft-fail live probe (exits 0; probe failures keep previous is_live). run_script("fetch_streamer_live.py", dry_run=dry_run) check = patch_check() if check.get("has_new"): print( f"new patches detected: {check.get('new_versions')}; running version-linked fetch", flush=True, ) run_patch_linked(dry_run=dry_run) return True print("no new patches", flush=True) return False def run_weekly(*, dry_run: bool = False) -> None: # Default full matchup refresh (not --resume-matchups); stale cells kept on failure. run_script("fetch_stratz_meta.py", dry_run=dry_run) run_script("fetch_hero_items.py", dry_run=dry_run) run_script("fetch_items_meta.py", dry_run=dry_run) run_script("fetch_item_counter_stats.py", "--soft-fail", dry_run=dry_run) run_script("item_fears.py", dry_run=dry_run) def run_patch_tier(*, dry_run: bool = False) -> bool: """Return True if version-linked fetch ran (or would run under dry-run).""" check = patch_check() if not check.get("has_new"): print("no new patches; skip detail fetch", flush=True) return False print( f"new patches detected: {check.get('new_versions')}; running version-linked fetch", flush=True, ) run_patch_linked(dry_run=dry_run) return True def main() -> None: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument( "--tier", choices=("daily", "weekly", "patch", "all"), required=True, help="which refresh tier to run", ) ap.add_argument("--dry-run", action="store_true", help="print commands only") ap.add_argument("--skip-deploy", action="store_true", help="do not call deploy_relations.py") ap.add_argument("--skip-oss", action="store_true", help="do not upload static assets to OSS") ap.add_argument( "--force-deploy", action="store_true", help="deploy even when snapshot digests are unchanged", ) args = ap.parse_args() before = snapshot() if not args.dry_run else {} did_work = False if args.tier in ("weekly", "all"): run_weekly(dry_run=args.dry_run) did_work = True if args.tier in ("daily", "all"): if run_daily(dry_run=args.dry_run): did_work = True else: did_work = True # daily fetch scripts still ran if args.tier == "patch": if run_patch_tier(dry_run=args.dry_run): did_work = True if args.dry_run: print("dry-run complete (no deploy/oss)", flush=True) return after = snapshot() data_changed, assets_changed = diff_snapshots(before, after) print( f"changes: data={data_changed} assets={assets_changed} " f"force_deploy={args.force_deploy} did_work={did_work}", flush=True, ) if assets_changed and not args.skip_oss: run_script("_oss_static_assets.py", "upload") elif assets_changed and args.skip_oss: print("assets changed but --skip-oss set; skipping OSS upload", flush=True) should_deploy = args.force_deploy or data_changed or assets_changed deployed = False if should_deploy and not args.skip_deploy: run_script("deploy_relations.py") deployed = True elif should_deploy and args.skip_deploy: print("deploy needed but --skip-deploy set; skipping", flush=True) else: print("no data/asset changes; skip deploy", flush=True) summary = { "ok": True, "tier": args.tier, "data_changed": data_changed, "assets_changed": assets_changed, "deployed": deployed, "force_deploy": args.force_deploy, } print("REFRESH_SUMMARY " + json.dumps(summary, ensure_ascii=False), flush=True) if __name__ == "__main__": main()