vosonandCursor
3ec8007077
Ship OpenDota counter-stats reordering for feared items, finalize SITE_VERSION/docs for rankings/streamers/trends/matches/mechanics and draft archetypes, and ignore regenerable Web data caches. Co-authored-by: Cursor <cursoragent@cursor.com>
255 lines
8.3 KiB
Python
255 lines
8.3 KiB
Python
"""Orchestrate Climperor web data refresh tiers, then optional OSS + deploy.
|
|
|
|
Tiers (see AGENTS.md / Gitea Actions workflows):
|
|
daily — OpenDota stats, leaderboards, matches, pro matches, streamers;
|
|
patch check
|
|
weekly — STRATZ meta, hero items, items_meta, item_fears
|
|
patch — patch list check; on has_new fetch details + version-linked scripts
|
|
all — weekly then daily (patch check included in daily/patch)
|
|
|
|
Usage:
|
|
python refresh_web.py --tier patch --skip-deploy --skip-oss
|
|
python refresh_web.py --tier daily --skip-deploy
|
|
python refresh_web.py --tier weekly
|
|
python refresh_web.py --tier all --dry-run
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
|
|
# Paths whose content change should trigger deploy / OSS.
|
|
DATA_WATCH = [
|
|
ROOT / "data" / "hero_stats.json",
|
|
ROOT / "data" / "leaderboards.json",
|
|
ROOT / "data" / "hero_matches.json",
|
|
ROOT / "data" / "pro_matches.json",
|
|
ROOT / "data" / "streamers.json",
|
|
ROOT / "data" / "stratz_hero_meta.json",
|
|
ROOT / "data" / "stratz_matchup_tops.json",
|
|
ROOT / "data" / "hero_items.json",
|
|
ROOT / "data" / "items_meta.json",
|
|
ROOT / "data" / "hero_item_fears.json",
|
|
ROOT / "data" / "patches.json",
|
|
ROOT / "data" / "hero_abilities.json",
|
|
ROOT / "data" / "item_shop.json",
|
|
]
|
|
ASSET_DIRS = [
|
|
ROOT / "assets" / "item_icons",
|
|
ROOT / "assets" / "ability_icons",
|
|
ROOT / "assets" / "item_cat_icons",
|
|
ROOT / "assets" / "hero_portraits",
|
|
ROOT / "assets" / "streamer_avatars",
|
|
ROOT / "assets" / "streamer_videos",
|
|
]
|
|
|
|
|
|
def _file_digest(path: Path) -> str | None:
|
|
if not path.is_file():
|
|
return None
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as f:
|
|
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def _dir_digest(path: Path) -> str | None:
|
|
if not path.is_dir():
|
|
return None
|
|
h = hashlib.sha256()
|
|
for p in sorted(path.rglob("*")):
|
|
if not p.is_file():
|
|
continue
|
|
rel = p.relative_to(path).as_posix().encode()
|
|
h.update(rel)
|
|
h.update(b"\0")
|
|
digest = _file_digest(p)
|
|
if digest:
|
|
h.update(digest.encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def snapshot() -> dict[str, str | None]:
|
|
out: dict[str, str | None] = {}
|
|
for p in DATA_WATCH:
|
|
out[str(p.relative_to(ROOT))] = _file_digest(p)
|
|
for d in ASSET_DIRS:
|
|
out[str(d.relative_to(ROOT)) + "/"] = _dir_digest(d)
|
|
return out
|
|
|
|
|
|
def diff_snapshots(before: dict[str, str | None], after: dict[str, str | None]) -> tuple[bool, bool]:
|
|
"""Return (data_changed, assets_changed)."""
|
|
data_changed = False
|
|
assets_changed = False
|
|
keys = set(before) | set(after)
|
|
for k in keys:
|
|
if before.get(k) == after.get(k):
|
|
continue
|
|
if k.endswith("/"):
|
|
assets_changed = True
|
|
else:
|
|
data_changed = True
|
|
return data_changed, assets_changed
|
|
|
|
|
|
def run_script(script: str, *args: str, dry_run: bool = False) -> None:
|
|
cmd = [sys.executable, str(ROOT / script), *args]
|
|
print(f"+ {' '.join(cmd)}", flush=True)
|
|
if dry_run:
|
|
return
|
|
subprocess.run(cmd, cwd=str(ROOT), check=True)
|
|
|
|
|
|
def patch_check() -> dict:
|
|
"""Always hits the network (read-only list); safe under --dry-run."""
|
|
cmd = [sys.executable, str(ROOT / "fetch_patches.py"), "--check"]
|
|
print(f"+ {' '.join(cmd)}", flush=True)
|
|
proc = subprocess.run(
|
|
cmd,
|
|
cwd=str(ROOT),
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
)
|
|
line = (proc.stdout or "").strip().splitlines()[-1] if (proc.stdout or "").strip() else ""
|
|
if not line:
|
|
raise RuntimeError("fetch_patches.py --check produced empty stdout")
|
|
result = json.loads(line)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
return result
|
|
|
|
|
|
def run_patch_linked(*, dry_run: bool = False) -> None:
|
|
run_script("fetch_patches.py", dry_run=dry_run)
|
|
run_script("fetch_hero_abilities.py", "--icons", dry_run=dry_run)
|
|
run_script("fetch_item_shop.py", dry_run=dry_run)
|
|
run_script("fetch_items_meta.py", dry_run=dry_run)
|
|
run_script("item_fears.py", dry_run=dry_run)
|
|
|
|
|
|
def run_daily(*, dry_run: bool = False) -> bool:
|
|
"""Return True if patch-linked fetch ran."""
|
|
run_script("fetch_hero_stats.py", dry_run=dry_run)
|
|
run_script("fetch_leaderboards.py", dry_run=dry_run)
|
|
run_script("fetch_hero_matches.py", "--source", "league", dry_run=dry_run)
|
|
run_script("fetch_pro_matches.py", dry_run=dry_run)
|
|
# Soft-fail Douyin enrichment (script itself exits 0; keep previous values on miss).
|
|
run_script("fetch_streamers.py", dry_run=dry_run)
|
|
check = patch_check()
|
|
if check.get("has_new"):
|
|
print(
|
|
f"new patches detected: {check.get('new_versions')}; running version-linked fetch",
|
|
flush=True,
|
|
)
|
|
run_patch_linked(dry_run=dry_run)
|
|
return True
|
|
print("no new patches", flush=True)
|
|
return False
|
|
|
|
|
|
def run_weekly(*, dry_run: bool = False) -> None:
|
|
run_script("fetch_stratz_meta.py", dry_run=dry_run)
|
|
run_script("fetch_hero_items.py", dry_run=dry_run)
|
|
run_script("fetch_items_meta.py", dry_run=dry_run)
|
|
run_script("fetch_item_counter_stats.py", "--soft-fail", dry_run=dry_run)
|
|
run_script("item_fears.py", dry_run=dry_run)
|
|
|
|
|
|
def run_patch_tier(*, dry_run: bool = False) -> bool:
|
|
"""Return True if version-linked fetch ran (or would run under dry-run)."""
|
|
check = patch_check()
|
|
if not check.get("has_new"):
|
|
print("no new patches; skip detail fetch", flush=True)
|
|
return False
|
|
print(
|
|
f"new patches detected: {check.get('new_versions')}; running version-linked fetch",
|
|
flush=True,
|
|
)
|
|
run_patch_linked(dry_run=dry_run)
|
|
return True
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument(
|
|
"--tier",
|
|
choices=("daily", "weekly", "patch", "all"),
|
|
required=True,
|
|
help="which refresh tier to run",
|
|
)
|
|
ap.add_argument("--dry-run", action="store_true", help="print commands only")
|
|
ap.add_argument("--skip-deploy", action="store_true", help="do not call deploy_relations.py")
|
|
ap.add_argument("--skip-oss", action="store_true", help="do not upload static assets to OSS")
|
|
ap.add_argument(
|
|
"--force-deploy",
|
|
action="store_true",
|
|
help="deploy even when snapshot digests are unchanged",
|
|
)
|
|
args = ap.parse_args()
|
|
|
|
before = snapshot() if not args.dry_run else {}
|
|
did_work = False
|
|
|
|
if args.tier in ("weekly", "all"):
|
|
run_weekly(dry_run=args.dry_run)
|
|
did_work = True
|
|
if args.tier in ("daily", "all"):
|
|
if run_daily(dry_run=args.dry_run):
|
|
did_work = True
|
|
else:
|
|
did_work = True # daily fetch scripts still ran
|
|
if args.tier == "patch":
|
|
if run_patch_tier(dry_run=args.dry_run):
|
|
did_work = True
|
|
|
|
if args.dry_run:
|
|
print("dry-run complete (no deploy/oss)", flush=True)
|
|
return
|
|
|
|
after = snapshot()
|
|
data_changed, assets_changed = diff_snapshots(before, after)
|
|
print(
|
|
f"changes: data={data_changed} assets={assets_changed} "
|
|
f"force_deploy={args.force_deploy} did_work={did_work}",
|
|
flush=True,
|
|
)
|
|
|
|
if assets_changed and not args.skip_oss:
|
|
run_script("_oss_static_assets.py", "upload")
|
|
elif assets_changed and args.skip_oss:
|
|
print("assets changed but --skip-oss set; skipping OSS upload", flush=True)
|
|
|
|
should_deploy = args.force_deploy or data_changed or assets_changed
|
|
deployed = False
|
|
if should_deploy and not args.skip_deploy:
|
|
run_script("deploy_relations.py")
|
|
deployed = True
|
|
elif should_deploy and args.skip_deploy:
|
|
print("deploy needed but --skip-deploy set; skipping", flush=True)
|
|
else:
|
|
print("no data/asset changes; skip deploy", flush=True)
|
|
|
|
summary = {
|
|
"ok": True,
|
|
"tier": args.tier,
|
|
"data_changed": data_changed,
|
|
"assets_changed": assets_changed,
|
|
"deployed": deployed,
|
|
"force_deploy": args.force_deploy,
|
|
}
|
|
print("REFRESH_SUMMARY " + json.dumps(summary, ensure_ascii=False), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|