climperor/web/refresh_web.py

"""Orchestrate Climperor web data refresh tiers, then optional OSS + deploy.

Tiers (see AGENTS.md / Gitea Actions workflows):
  daily  — OpenDota stats, leaderboards, matches, pro matches, streamers,
           streamer live probe; patch check
           (live badge is served by /api/live-status on visit; daily probe is
           only a data.json fallback until the edge API returns)
  weekly — STRATZ meta, hero items, items_meta, item_counter_stats, item_fears
  patch  — patch list check; on has_new fetch details + version-linked scripts
  all    — weekly then daily (patch check included in daily/patch)

Usage:
  python refresh_web.py --tier patch --skip-deploy --skip-oss
  python refresh_web.py --tier daily --skip-deploy
  python refresh_web.py --tier weekly
  python refresh_web.py --tier all --dry-run
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parent
REPO_ROOT = ROOT.parent
sys.path.insert(0, str(REPO_ROOT))

from shared.http_utils import http_json, write_json_atomic
from shared.paths import HEROES_JSON, RELATIONS_JSON, WEB_FRONTEND

OPENDOTA_PROBE_URL = "https://api.opendota.com/api/heroStats"
# Must be non-empty before any Pages deploy — empty shells overwrite production.
CRITICAL_BY_HERO_FILES = (
    ROOT / "data" / "hero_stats.json",
    ROOT / "data" / "stratz_hero_meta.json",
    ROOT / "data" / "stratz_matchup_tops.json",
)

# Paths whose content change should trigger deploy / OSS.
DATA_WATCH = [
    ROOT / "data" / "hero_stats.json",
    ROOT / "data" / "leaderboards.json",
    ROOT / "data" / "hero_matches.json",
    ROOT / "data" / "pro_matches.json",
    ROOT / "data" / "streamers.json",
    ROOT / "data" / "stratz_hero_meta.json",
    ROOT / "data" / "stratz_matchup_tops.json",
    ROOT / "data" / "hero_items.json",
    ROOT / "data" / "items_meta.json",
    ROOT / "data" / "hero_item_fears.json",
    ROOT / "data" / "patches.json",
    ROOT / "data" / "hero_abilities.json",
    ROOT / "data" / "item_shop.json",
    ROOT / "data" / "item_counter_stats.json",
]
# Static site inputs that are not rewritten by fetch tiers but still ship in
# data.json / dist (frontend + qualitative relations + hero table / grid).
INPUT_WATCH = [
    RELATIONS_JSON,
    HEROES_JSON,
    ROOT / "data" / "hero_grid_order.json",
    ROOT / "data" / "patch_summaries.json",
    WEB_FRONTEND / "index.html",
    WEB_FRONTEND / "app.js",
    WEB_FRONTEND / "style.css",
    WEB_FRONTEND / "router.js",
    WEB_FRONTEND / "mobile-gate.js",
    WEB_FRONTEND / "config.js",
    WEB_FRONTEND / "_headers",
    WEB_FRONTEND / "_redirects",
    WEB_FRONTEND / "robots.txt",
]
FRONTEND_FUNCTIONS_DIR = WEB_FRONTEND / "functions"
ASSET_DIRS = [
    ROOT / "assets" / "item_icons",
    ROOT / "assets" / "ability_icons",
    ROOT / "assets" / "item_cat_icons",
    ROOT / "assets" / "hero_portraits",
    ROOT / "assets" / "streamer_avatars",
    ROOT / "assets" / "streamer_videos",
]
SUMMARY_PATH = ROOT / ".refresh" / "summary.json"
VOLATILE_JSON_KEYS = frozenset(
    {
        "fetched_at",
        "generated_at",
        "last_attempt_at",
        "live_probed_at",
        "profile_fetched_at",
        "updated_at",
        # Live badge is owned by /api/live-status; flipping is_live alone must
        # not trigger a full static redeploy.
        "is_live",
    }
)
RUN_RESULTS: list[dict] = []
PATCH_RESULT: dict = {}


def _watch_key(path: Path) -> str:
    """Stable snapshot key relative to the repo root when possible."""
    resolved = path.resolve()
    try:
        return resolved.relative_to(REPO_ROOT.resolve()).as_posix()
    except ValueError:
        return resolved.as_posix()


def _file_digest(path: Path) -> str | None:
    if not path.is_file():
        return None
    h = hashlib.sha256()
    with path.open("rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def _without_volatile_fields(value):
    if isinstance(value, dict):
        return {
            key: _without_volatile_fields(child)
            for key, child in value.items()
            if key not in VOLATILE_JSON_KEYS
        }
    if isinstance(value, list):
        return [_without_volatile_fields(child) for child in value]
    return value


def _semantic_file_digest(path: Path) -> str | None:
    """Hash JSON business content while ignoring refresh-only timestamps."""
    if not path.is_file():
        return None
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, UnicodeDecodeError, json.JSONDecodeError):
        return _file_digest(path)
    canonical = json.dumps(
        _without_volatile_fields(payload),
        ensure_ascii=False,
        sort_keys=True,
        separators=(",", ":"),
    ).encode("utf-8")
    return hashlib.sha256(canonical).hexdigest()


def _dir_digest(path: Path) -> str | None:
    if not path.is_dir():
        return None
    h = hashlib.sha256()
    for p in sorted(path.rglob("*")):
        if not p.is_file():
            continue
        rel = p.relative_to(path).as_posix().encode()
        h.update(rel)
        h.update(b"\0")
        digest = _file_digest(p)
        if digest:
            h.update(digest.encode())
    return h.hexdigest()


def snapshot() -> dict[str, str | None]:
    out: dict[str, str | None] = {}
    for p in DATA_WATCH:
        out[_watch_key(p)] = _semantic_file_digest(p)
    for p in INPUT_WATCH:
        # Frontend/source files use byte digests; JSON inputs use semantic hash.
        if p.suffix.lower() == ".json":
            out[_watch_key(p)] = _semantic_file_digest(p)
        else:
            out[_watch_key(p)] = _file_digest(p)
    out[_watch_key(FRONTEND_FUNCTIONS_DIR) + "/"] = _dir_digest(FRONTEND_FUNCTIONS_DIR)
    for d in ASSET_DIRS:
        out[_watch_key(d) + "/"] = _dir_digest(d)
    return out


def diff_snapshots(before: dict[str, str | None], after: dict[str, str | None]) -> tuple[bool, bool]:
    """Return (data_changed, assets_changed).

    ``data_changed`` covers generated JSON, frontend/source inputs, and Pages
    Functions. Only ``web/assets/...`` directory digests count as assets
    (OSS upload).
    """
    data_changed = False
    assets_changed = False
    asset_keys = {_watch_key(d) + "/" for d in ASSET_DIRS}
    keys = set(before) | set(after)
    for k in keys:
        if before.get(k) == after.get(k):
            continue
        if k in asset_keys:
            assets_changed = True
        else:
            data_changed = True
    return data_changed, assets_changed

def run_script(script: str, *args: str, dry_run: bool = False, soft_fail: bool = False) -> bool:
    cmd = [sys.executable, str(ROOT / script), *args]
    print(f"+ {' '.join(cmd)}", flush=True)
    started = time.monotonic()
    if dry_run:
        RUN_RESULTS.append({"step": script, "ok": True, "dry_run": True, "duration_s": 0.0})
        return True
    proc = subprocess.run(cmd, cwd=str(ROOT), check=False)
    result = {
        "step": script,
        "args": list(args),
        "ok": proc.returncode == 0,
        "soft_fail": soft_fail,
        "returncode": proc.returncode,
        "duration_s": round(time.monotonic() - started, 3),
    }
    RUN_RESULTS.append(result)
    if proc.returncode == 0:
        return True
    if soft_fail:
        print(
            f"soft-fail: {script} exited {proc.returncode}; continuing",
            flush=True,
        )
        return False
    raise subprocess.CalledProcessError(proc.returncode, cmd)


def patch_check() -> dict:
    """Always hits the network (read-only list); safe under --dry-run."""
    cmd = [sys.executable, str(ROOT / "fetch_patches.py"), "--check"]
    print(f"+ {' '.join(cmd)}", flush=True)
    started = time.monotonic()
    proc = subprocess.run(
        cmd,
        cwd=str(ROOT),
        check=True,
        capture_output=True,
        text=True,
        encoding="utf-8",
    )
    line = (proc.stdout or "").strip().splitlines()[-1] if (proc.stdout or "").strip() else ""
    if not line:
        raise RuntimeError("fetch_patches.py --check produced empty stdout")
    result = json.loads(line)
    PATCH_RESULT.clear()
    PATCH_RESULT.update(result)
    RUN_RESULTS.append(
        {
            "step": "fetch_patches.py --check",
            "ok": True,
            "duration_s": round(time.monotonic() - started, 3),
            "has_new": bool(result.get("has_new")),
            "new_versions": list(result.get("new_versions") or []),
        }
    )
    print(json.dumps(result, ensure_ascii=False), flush=True)
    return result


def run_patch_linked(*, dry_run: bool = False) -> None:
    run_script("fetch_patches.py", dry_run=dry_run)
    run_script("fetch_hero_abilities.py", "--icons", dry_run=dry_run)
    run_script("fetch_item_shop.py", dry_run=dry_run)
    run_script("fetch_items_meta.py", dry_run=dry_run)
    run_script("item_fears.py", dry_run=dry_run)
    # Soft: patch workflow may lack STRATZ token; weekly still does the hard refresh.
    run_script(
        "fetch_stratz_meta.py",
        "--matchups-only",
        dry_run=dry_run,
        soft_fail=True,
    )


def probe_opendota_available() -> bool:
    """Single fail-fast GET; False means skip OpenDota daily fetches this run."""
    try:
        http_json(OPENDOTA_PROBE_URL, timeout=15, retries=0)
    except Exception as exc:
        print(
            f"OpenDota probe failed ({type(exc).__name__}: {exc}); "
            "skipping hero_stats / hero_matches / pro_matches this run",
            flush=True,
        )
        RUN_RESULTS.append(
            {
                "step": "opendota_probe",
                "ok": False,
                "skipped_opendota": True,
                "error": f"{type(exc).__name__}: {exc}",
            }
        )
        return False
    print("OpenDota probe ok", flush=True)
    RUN_RESULTS.append({"step": "opendota_probe", "ok": True})
    return True


def assert_critical_data_ready() -> None:
    """Refuse deploy when restored/generated caches would publish empty shells."""
    problems: list[str] = []
    for path in CRITICAL_BY_HERO_FILES:
        rel = path.name
        if not path.is_file():
            problems.append(f"{rel}: missing")
            continue
        try:
            raw = json.loads(path.read_text(encoding="utf-8"))
        except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
            problems.append(f"{rel}: unreadable ({exc})")
            continue
        by_hero = raw.get("by_hero") if isinstance(raw, dict) else None
        if not isinstance(by_hero, dict) or not by_hero:
            problems.append(f"{rel}: empty by_hero")
    if problems:
        raise RuntimeError(
            "refusing deploy: critical web data incomplete ("
            + "; ".join(problems)
            + ")"
        )


def run_daily(*, dry_run: bool = False) -> bool:
    """Return True if patch-linked fetch ran."""
    # Probe once: if OpenDota is 429/down, keep restored cache and do not burn quota.
    opendota_ok = True if dry_run else probe_opendota_available()
    if opendota_ok:
        # Soft-fail OpenDota quota burns: keep restored cache and continue the tier.
        run_script("fetch_hero_stats.py", dry_run=dry_run, soft_fail=True)
        run_script(
            "fetch_hero_matches.py",
            "--source",
            "league",
            dry_run=dry_run,
            soft_fail=True,
        )
        run_script(
            "fetch_pro_matches.py",
            "--include-pubs",
            "--refresh-limit",
            "15",
            dry_run=dry_run,
            soft_fail=True,
        )
    else:
        print(
            "skipped OpenDota daily fetches (probe failed); keeping prior cache",
            flush=True,
        )
    run_script("fetch_leaderboards.py", dry_run=dry_run)
    # Soft-fail Douyin enrichment (script itself exits 0; keep previous values on miss).
    run_script("fetch_streamers.py", dry_run=dry_run, soft_fail=True)
    # Soft-fail live probe (exits 0; probe failures clear is_live / live_probed_at).
    run_script("fetch_streamer_live.py", dry_run=dry_run, soft_fail=True)
    check = patch_check()
    if check.get("has_new"):
        print(
            f"new patches detected: {check.get('new_versions')}; running version-linked fetch",
            flush=True,
        )
        run_patch_linked(dry_run=dry_run)
        return True
    print("no new patches", flush=True)
    return False


def run_weekly(*, dry_run: bool = False) -> None:
    # Default full matchup refresh (not --resume-matchups); stale cells kept on failure.
    run_script("fetch_stratz_meta.py", dry_run=dry_run)
    run_script("fetch_hero_items.py", "--force", dry_run=dry_run)
    run_script("fetch_items_meta.py", dry_run=dry_run)
    run_script("fetch_item_counter_stats.py", "--soft-fail", dry_run=dry_run)
    run_script("item_fears.py", dry_run=dry_run)


def run_patch_tier(*, dry_run: bool = False) -> bool:
    """Return True if version-linked fetch ran (or would run under dry-run)."""
    check = patch_check()
    if not check.get("has_new"):
        print("no new patches; skip detail fetch", flush=True)
        return False
    print(
        f"new patches detected: {check.get('new_versions')}; running version-linked fetch",
        flush=True,
    )
    run_patch_linked(dry_run=dry_run)
    return True


def _stale_metrics() -> dict:
    metrics = {
        "stratz_meta_stale": False,
        "stratz_matchup_stale": 0,
        "streamer_profile_missing": 0,
        "streamer_live_probe_missing": 0,
    }
    meta_path = ROOT / "data" / "stratz_hero_meta.json"
    matchup_path = ROOT / "data" / "stratz_matchup_tops.json"
    try:
        meta = json.loads(meta_path.read_text(encoding="utf-8"))
        metrics["stratz_meta_stale"] = bool(meta.get("stale"))
        metrics["stratz_meta_stale_brackets"] = list(meta.get("stale_brackets") or [])
    except (OSError, json.JSONDecodeError):
        metrics["stratz_meta_missing"] = True
    try:
        matchup = json.loads(matchup_path.read_text(encoding="utf-8"))
        metrics["stratz_matchup_stale"] = sum(
            1
            for cell in (matchup.get("by_hero") or {}).values()
            if isinstance(cell, dict) and cell.get("stale")
        )
    except (OSError, json.JSONDecodeError):
        metrics["stratz_matchup_missing"] = True
    try:
        streamers = json.loads(
            (ROOT / "data" / "streamers.json").read_text(encoding="utf-8")
        )
        rows = streamers.get("streamers") or []
        metrics["streamer_profile_missing"] = sum(
            1
            for row in rows
            if isinstance(row, dict)
            and row.get("profile_url")
            and not row.get("profile_fetched_at")
        )
        metrics["streamer_live_probe_missing"] = sum(
            1
            for row in rows
            if isinstance(row, dict)
            and row.get("live_url")
            and not row.get("live_probed_at")
        )
    except (OSError, json.JSONDecodeError):
        metrics["streamers_missing"] = True
    metrics["item_counter_missing"] = not (
        ROOT / "data" / "item_counter_stats.json"
    ).is_file()
    return metrics


def _finalize_summary(summary: dict) -> dict:
    summary["steps"] = list(RUN_RESULTS)
    summary["patch_check"] = dict(PATCH_RESULT)
    summary["health"] = _stale_metrics()
    summary["completed_at"] = datetime.now(timezone.utc).isoformat()
    return summary


def _write_summary(summary: dict) -> None:
    _finalize_summary(summary)
    write_json_atomic(SUMMARY_PATH, summary)
    print("REFRESH_SUMMARY " + json.dumps(summary, ensure_ascii=False), flush=True)


def validate_publish_options(
    *,
    assets_changed: bool,
    skip_oss: bool,
    skip_deploy: bool,
) -> None:
    if assets_changed and skip_oss and not skip_deploy:
        raise RuntimeError(
            "assets changed but --skip-oss would publish missing OSS assets; "
            "also set --skip-deploy for a local-only refresh"
        )


def acquire_refresh_lock(tier: str):
    """Serialize refresh/deploy work on a self-hosted runner.

    Patch checks are deferred instead of waiting behind a full refresh; the
    six-hour schedule will retry them without exporting a stale restored cache.
    """
    lock_path = Path(
        os.environ.get("CLIMPEROR_REFRESH_LOCK")
        or (Path.home() / ".climperor-web-refresh.lock")
    )
    lock_path.parent.mkdir(parents=True, exist_ok=True)
    handle = lock_path.open("a+b")
    handle.seek(0)
    if handle.tell() == 0 and lock_path.stat().st_size == 0:
        handle.write(b"\0")
        handle.flush()
    handle.seek(0)
    try:
        if os.name == "nt":
            import msvcrt

            mode = msvcrt.LK_NBLCK if tier == "patch" else msvcrt.LK_LOCK
            msvcrt.locking(handle.fileno(), mode, 1)
        else:
            import fcntl

            flags = fcntl.LOCK_EX | (fcntl.LOCK_NB if tier == "patch" else 0)
            fcntl.flock(handle.fileno(), flags)
    except (BlockingIOError, OSError):
        handle.close()
        if tier == "patch":
            return None
        raise
    return handle


def release_refresh_lock(handle) -> None:
    try:
        if os.name == "nt":
            import msvcrt

            handle.seek(0)
            msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
        else:
            import fcntl

            fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
    finally:
        handle.close()


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument(
        "--tier",
        choices=("daily", "weekly", "patch", "all"),
        required=True,
        help="which refresh tier to run",
    )
    ap.add_argument("--dry-run", action="store_true", help="print commands only")
    ap.add_argument("--skip-deploy", action="store_true", help="do not call deploy_relations.py")
    ap.add_argument("--skip-oss", action="store_true", help="do not upload static assets to OSS")
    ap.add_argument(
        "--force-deploy",
        action="store_true",
        help="deploy even when snapshot digests are unchanged",
    )
    args = ap.parse_args()

    RUN_RESULTS.clear()
    PATCH_RESULT.clear()
    started = time.monotonic()
    summary = {
        "ok": False,
        "tier": args.tier,
        "run_id": os.environ.get("REFRESH_RUN_ID") or "",
        "started_at": datetime.now(timezone.utc).isoformat(),
        "data_changed": False,
        "assets_changed": False,
        "changed_paths": [],
        "deployed": False,
        "force_deploy": args.force_deploy,
    }
    lock_handle = acquire_refresh_lock(args.tier)
    if lock_handle is None:
        summary.update(
            {
                "ok": True,
                "skipped": True,
                "skip_reason": "full refresh already running; next patch schedule will retry",
                "duration_s": round(time.monotonic() - started, 3),
            }
        )
        _write_summary(summary)
        return
    try:
        cache_enabled = (
            os.environ.get("REFRESH_CACHE_ENABLED") == "1" and not args.dry_run
        )
        if cache_enabled:
            run_script("refresh_cache.py", "restore")
        before = snapshot() if not args.dry_run else {}
        if args.tier in ("weekly", "all"):
            run_weekly(dry_run=args.dry_run)
        if args.tier in ("daily", "all"):
            run_daily(dry_run=args.dry_run)
        if args.tier == "patch":
            run_patch_tier(dry_run=args.dry_run)

        if args.dry_run:
            summary["ok"] = True
            summary["dry_run"] = True
            print(
                "dry-run complete (no cache restore/save, deploy, or oss)",
                flush=True,
            )
            return

        after = snapshot()
        data_changed, assets_changed = diff_snapshots(before, after)
        changed_paths = sorted(
            key for key in set(before) | set(after) if before.get(key) != after.get(key)
        )
        summary.update(
            {
                "data_changed": data_changed,
                "assets_changed": assets_changed,
                "changed_paths": changed_paths,
            }
        )
        print(
            f"changes: data={data_changed} assets={assets_changed} "
            f"force_deploy={args.force_deploy}",
            flush=True,
        )

        validate_publish_options(
            assets_changed=assets_changed,
            skip_oss=args.skip_oss,
            skip_deploy=args.skip_deploy,
        )
        if assets_changed and not args.skip_oss:
            run_script("_oss_static_assets.py", "upload")
        elif assets_changed:
            print("assets changed but --skip-oss set; skipping OSS upload", flush=True)

        should_deploy = args.force_deploy or data_changed or assets_changed
        if should_deploy and not args.skip_deploy:
            assert_critical_data_ready()
            run_script("deploy_relations.py")
            summary["deployed"] = True
        elif should_deploy:
            print("deploy needed but --skip-deploy set; skipping", flush=True)
        else:
            print("no business data/asset changes; skip deploy", flush=True)
        if cache_enabled:
            run_script("refresh_cache.py", "save")
        summary["ok"] = True
    except BaseException as exc:
        summary["error"] = f"{type(exc).__name__}: {exc}"
        raise
    finally:
        summary["duration_s"] = round(time.monotonic() - started, 3)
        if args.dry_run:
            # Print for humans; do not persist dry-run artifacts.
            _finalize_summary(summary)
            print(
                "REFRESH_SUMMARY " + json.dumps(summary, ensure_ascii=False),
                flush=True,
            )
        else:
            _write_summary(summary)
        release_refresh_lock(lock_handle)


if __name__ == "__main__":
    main()