climperor/web/refresh_web.py

"""Orchestrate Climperor web data refresh tiers, then optional OSS + deploy.

Tiers (see AGENTS.md / Gitea Actions workflows):
  daily  — OpenDota stats, leaderboards, matches, pro matches, streamers,
           streamer live probe; patch check
           (live badge still trails reality by up to a day — truly real-time
           would need a dedicated higher-frequency workflow, e.g. 15-min)
  weekly — STRATZ meta, hero items, items_meta, item_fears
  patch  — patch list check; on has_new fetch details + version-linked scripts
  all    — weekly then daily (patch check included in daily/patch)

Usage:
  python refresh_web.py --tier patch --skip-deploy --skip-oss
  python refresh_web.py --tier daily --skip-deploy
  python refresh_web.py --tier weekly
  python refresh_web.py --tier all --dry-run
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT.parent))

from shared.http_utils import write_json_atomic

# Paths whose content change should trigger deploy / OSS.
DATA_WATCH = [
    ROOT / "data" / "hero_stats.json",
    ROOT / "data" / "leaderboards.json",
    ROOT / "data" / "hero_matches.json",
    ROOT / "data" / "pro_matches.json",
    ROOT / "data" / "streamers.json",
    ROOT / "data" / "stratz_hero_meta.json",
    ROOT / "data" / "stratz_matchup_tops.json",
    ROOT / "data" / "hero_items.json",
    ROOT / "data" / "items_meta.json",
    ROOT / "data" / "hero_item_fears.json",
    ROOT / "data" / "patches.json",
    ROOT / "data" / "hero_abilities.json",
    ROOT / "data" / "item_shop.json",
    ROOT / "data" / "item_counter_stats.json",
]
ASSET_DIRS = [
    ROOT / "assets" / "item_icons",
    ROOT / "assets" / "ability_icons",
    ROOT / "assets" / "item_cat_icons",
    ROOT / "assets" / "hero_portraits",
    ROOT / "assets" / "streamer_avatars",
    ROOT / "assets" / "streamer_videos",
]
SUMMARY_PATH = ROOT / ".refresh" / "summary.json"
VOLATILE_JSON_KEYS = frozenset(
    {
        "fetched_at",
        "generated_at",
        "last_attempt_at",
        "live_probed_at",
        "profile_fetched_at",
        "updated_at",
    }
)
RUN_RESULTS: list[dict] = []
PATCH_RESULT: dict = {}


def _file_digest(path: Path) -> str | None:
    if not path.is_file():
        return None
    h = hashlib.sha256()
    with path.open("rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def _without_volatile_fields(value):
    if isinstance(value, dict):
        return {
            key: _without_volatile_fields(child)
            for key, child in value.items()
            if key not in VOLATILE_JSON_KEYS
        }
    if isinstance(value, list):
        return [_without_volatile_fields(child) for child in value]
    return value


def _semantic_file_digest(path: Path) -> str | None:
    """Hash JSON business content while ignoring refresh-only timestamps."""
    if not path.is_file():
        return None
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, UnicodeDecodeError, json.JSONDecodeError):
        return _file_digest(path)
    canonical = json.dumps(
        _without_volatile_fields(payload),
        ensure_ascii=False,
        sort_keys=True,
        separators=(",", ":"),
    ).encode("utf-8")
    return hashlib.sha256(canonical).hexdigest()


def _dir_digest(path: Path) -> str | None:
    if not path.is_dir():
        return None
    h = hashlib.sha256()
    for p in sorted(path.rglob("*")):
        if not p.is_file():
            continue
        rel = p.relative_to(path).as_posix().encode()
        h.update(rel)
        h.update(b"\0")
        digest = _file_digest(p)
        if digest:
            h.update(digest.encode())
    return h.hexdigest()


def snapshot() -> dict[str, str | None]:
    out: dict[str, str | None] = {}
    for p in DATA_WATCH:
        out[str(p.relative_to(ROOT))] = _semantic_file_digest(p)
    for d in ASSET_DIRS:
        out[str(d.relative_to(ROOT)) + "/"] = _dir_digest(d)
    return out


def diff_snapshots(before: dict[str, str | None], after: dict[str, str | None]) -> tuple[bool, bool]:
    """Return (data_changed, assets_changed)."""
    data_changed = False
    assets_changed = False
    keys = set(before) | set(after)
    for k in keys:
        if before.get(k) == after.get(k):
            continue
        if k.endswith("/"):
            assets_changed = True
        else:
            data_changed = True
    return data_changed, assets_changed


def run_script(script: str, *args: str, dry_run: bool = False, soft_fail: bool = False) -> bool:
    cmd = [sys.executable, str(ROOT / script), *args]
    print(f"+ {' '.join(cmd)}", flush=True)
    started = time.monotonic()
    if dry_run:
        RUN_RESULTS.append({"step": script, "ok": True, "dry_run": True, "duration_s": 0.0})
        return True
    proc = subprocess.run(cmd, cwd=str(ROOT), check=False)
    result = {
        "step": script,
        "args": list(args),
        "ok": proc.returncode == 0,
        "soft_fail": soft_fail,
        "returncode": proc.returncode,
        "duration_s": round(time.monotonic() - started, 3),
    }
    RUN_RESULTS.append(result)
    if proc.returncode == 0:
        return True
    if soft_fail:
        print(
            f"soft-fail: {script} exited {proc.returncode}; continuing",
            flush=True,
        )
        return False
    raise subprocess.CalledProcessError(proc.returncode, cmd)


def patch_check() -> dict:
    """Always hits the network (read-only list); safe under --dry-run."""
    cmd = [sys.executable, str(ROOT / "fetch_patches.py"), "--check"]
    print(f"+ {' '.join(cmd)}", flush=True)
    started = time.monotonic()
    proc = subprocess.run(
        cmd,
        cwd=str(ROOT),
        check=True,
        capture_output=True,
        text=True,
        encoding="utf-8",
    )
    line = (proc.stdout or "").strip().splitlines()[-1] if (proc.stdout or "").strip() else ""
    if not line:
        raise RuntimeError("fetch_patches.py --check produced empty stdout")
    result = json.loads(line)
    PATCH_RESULT.clear()
    PATCH_RESULT.update(result)
    RUN_RESULTS.append(
        {
            "step": "fetch_patches.py --check",
            "ok": True,
            "duration_s": round(time.monotonic() - started, 3),
            "has_new": bool(result.get("has_new")),
            "new_versions": list(result.get("new_versions") or []),
        }
    )
    print(json.dumps(result, ensure_ascii=False), flush=True)
    return result


def run_patch_linked(*, dry_run: bool = False) -> None:
    run_script("fetch_patches.py", dry_run=dry_run)
    run_script("fetch_hero_abilities.py", "--icons", dry_run=dry_run)
    run_script("fetch_item_shop.py", dry_run=dry_run)
    run_script("fetch_items_meta.py", dry_run=dry_run)
    run_script("item_fears.py", dry_run=dry_run)
    # Soft: patch workflow may lack STRATZ token; weekly still does the hard refresh.
    run_script(
        "fetch_stratz_meta.py",
        "--matchups-only",
        dry_run=dry_run,
        soft_fail=True,
    )


def run_daily(*, dry_run: bool = False) -> bool:
    """Return True if patch-linked fetch ran."""
    run_script("fetch_hero_stats.py", dry_run=dry_run)
    run_script("fetch_leaderboards.py", dry_run=dry_run)
    run_script("fetch_hero_matches.py", "--source", "league", dry_run=dry_run)
    run_script("fetch_pro_matches.py", "--include-pubs", dry_run=dry_run)
    # Soft-fail Douyin enrichment (script itself exits 0; keep previous values on miss).
    run_script("fetch_streamers.py", dry_run=dry_run)
    # Soft-fail live probe (exits 0; probe failures keep previous is_live).
    run_script("fetch_streamer_live.py", dry_run=dry_run)
    check = patch_check()
    if check.get("has_new"):
        print(
            f"new patches detected: {check.get('new_versions')}; running version-linked fetch",
            flush=True,
        )
        run_patch_linked(dry_run=dry_run)
        return True
    print("no new patches", flush=True)
    return False


def run_weekly(*, dry_run: bool = False) -> None:
    # Default full matchup refresh (not --resume-matchups); stale cells kept on failure.
    run_script("fetch_stratz_meta.py", dry_run=dry_run)
    run_script("fetch_hero_items.py", "--force", dry_run=dry_run)
    run_script("fetch_items_meta.py", dry_run=dry_run)
    run_script("fetch_item_counter_stats.py", "--soft-fail", dry_run=dry_run)
    run_script("item_fears.py", dry_run=dry_run)


def run_patch_tier(*, dry_run: bool = False) -> bool:
    """Return True if version-linked fetch ran (or would run under dry-run)."""
    check = patch_check()
    if not check.get("has_new"):
        print("no new patches; skip detail fetch", flush=True)
        return False
    print(
        f"new patches detected: {check.get('new_versions')}; running version-linked fetch",
        flush=True,
    )
    run_patch_linked(dry_run=dry_run)
    return True


def _stale_metrics() -> dict:
    metrics = {
        "stratz_meta_stale": False,
        "stratz_matchup_stale": 0,
        "streamer_profile_missing": 0,
        "streamer_live_probe_missing": 0,
    }
    meta_path = ROOT / "data" / "stratz_hero_meta.json"
    matchup_path = ROOT / "data" / "stratz_matchup_tops.json"
    try:
        meta = json.loads(meta_path.read_text(encoding="utf-8"))
        metrics["stratz_meta_stale"] = bool(meta.get("stale"))
        metrics["stratz_meta_stale_brackets"] = list(meta.get("stale_brackets") or [])
    except (OSError, json.JSONDecodeError):
        metrics["stratz_meta_missing"] = True
    try:
        matchup = json.loads(matchup_path.read_text(encoding="utf-8"))
        metrics["stratz_matchup_stale"] = sum(
            1
            for cell in (matchup.get("by_hero") or {}).values()
            if isinstance(cell, dict) and cell.get("stale")
        )
    except (OSError, json.JSONDecodeError):
        metrics["stratz_matchup_missing"] = True
    try:
        streamers = json.loads(
            (ROOT / "data" / "streamers.json").read_text(encoding="utf-8")
        )
        rows = streamers.get("streamers") or []
        metrics["streamer_profile_missing"] = sum(
            1
            for row in rows
            if isinstance(row, dict)
            and row.get("profile_url")
            and not row.get("profile_fetched_at")
        )
        metrics["streamer_live_probe_missing"] = sum(
            1
            for row in rows
            if isinstance(row, dict)
            and row.get("live_url")
            and not row.get("live_probed_at")
        )
    except (OSError, json.JSONDecodeError):
        metrics["streamers_missing"] = True
    metrics["item_counter_missing"] = not (
        ROOT / "data" / "item_counter_stats.json"
    ).is_file()
    return metrics


def _write_summary(summary: dict) -> None:
    summary["steps"] = list(RUN_RESULTS)
    summary["patch_check"] = dict(PATCH_RESULT)
    summary["health"] = _stale_metrics()
    summary["completed_at"] = datetime.now(timezone.utc).isoformat()
    write_json_atomic(SUMMARY_PATH, summary)
    print("REFRESH_SUMMARY " + json.dumps(summary, ensure_ascii=False), flush=True)


def validate_publish_options(
    *,
    assets_changed: bool,
    skip_oss: bool,
    skip_deploy: bool,
) -> None:
    if assets_changed and skip_oss and not skip_deploy:
        raise RuntimeError(
            "assets changed but --skip-oss would publish missing OSS assets; "
            "also set --skip-deploy for a local-only refresh"
        )


def acquire_refresh_lock(tier: str):
    """Serialize refresh/deploy work on a self-hosted runner.

    Patch checks are deferred instead of waiting behind a full refresh; the
    six-hour schedule will retry them without exporting a stale restored cache.
    """
    lock_path = Path(
        os.environ.get("CLIMPEROR_REFRESH_LOCK")
        or (Path.home() / ".climperor-web-refresh.lock")
    )
    lock_path.parent.mkdir(parents=True, exist_ok=True)
    handle = lock_path.open("a+b")
    handle.seek(0)
    if handle.tell() == 0 and lock_path.stat().st_size == 0:
        handle.write(b"\0")
        handle.flush()
    handle.seek(0)
    try:
        if os.name == "nt":
            import msvcrt

            mode = msvcrt.LK_NBLCK if tier == "patch" else msvcrt.LK_LOCK
            msvcrt.locking(handle.fileno(), mode, 1)
        else:
            import fcntl

            flags = fcntl.LOCK_EX | (fcntl.LOCK_NB if tier == "patch" else 0)
            fcntl.flock(handle.fileno(), flags)
    except (BlockingIOError, OSError):
        handle.close()
        if tier == "patch":
            return None
        raise
    return handle


def release_refresh_lock(handle) -> None:
    try:
        if os.name == "nt":
            import msvcrt

            handle.seek(0)
            msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
        else:
            import fcntl

            fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
    finally:
        handle.close()


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument(
        "--tier",
        choices=("daily", "weekly", "patch", "all"),
        required=True,
        help="which refresh tier to run",
    )
    ap.add_argument("--dry-run", action="store_true", help="print commands only")
    ap.add_argument("--skip-deploy", action="store_true", help="do not call deploy_relations.py")
    ap.add_argument("--skip-oss", action="store_true", help="do not upload static assets to OSS")
    ap.add_argument(
        "--force-deploy",
        action="store_true",
        help="deploy even when snapshot digests are unchanged",
    )
    args = ap.parse_args()

    RUN_RESULTS.clear()
    PATCH_RESULT.clear()
    started = time.monotonic()
    summary = {
        "ok": False,
        "tier": args.tier,
        "run_id": os.environ.get("REFRESH_RUN_ID") or "",
        "started_at": datetime.now(timezone.utc).isoformat(),
        "data_changed": False,
        "assets_changed": False,
        "changed_paths": [],
        "deployed": False,
        "force_deploy": args.force_deploy,
    }
    lock_handle = acquire_refresh_lock(args.tier)
    if lock_handle is None:
        summary.update(
            {
                "ok": True,
                "skipped": True,
                "skip_reason": "full refresh already running; next patch schedule will retry",
                "duration_s": round(time.monotonic() - started, 3),
            }
        )
        _write_summary(summary)
        return
    try:
        cache_enabled = os.environ.get("REFRESH_CACHE_ENABLED") == "1"
        if cache_enabled:
            run_script("refresh_cache.py", "restore")
        before = snapshot() if not args.dry_run else {}
        if args.tier in ("weekly", "all"):
            run_weekly(dry_run=args.dry_run)
        if args.tier in ("daily", "all"):
            run_daily(dry_run=args.dry_run)
        if args.tier == "patch":
            run_patch_tier(dry_run=args.dry_run)

        if args.dry_run:
            summary["ok"] = True
            summary["dry_run"] = True
            print("dry-run complete (no deploy/oss)", flush=True)
            return

        after = snapshot()
        data_changed, assets_changed = diff_snapshots(before, after)
        changed_paths = sorted(
            key for key in set(before) | set(after) if before.get(key) != after.get(key)
        )
        summary.update(
            {
                "data_changed": data_changed,
                "assets_changed": assets_changed,
                "changed_paths": changed_paths,
            }
        )
        print(
            f"changes: data={data_changed} assets={assets_changed} "
            f"force_deploy={args.force_deploy}",
            flush=True,
        )

        validate_publish_options(
            assets_changed=assets_changed,
            skip_oss=args.skip_oss,
            skip_deploy=args.skip_deploy,
        )
        if assets_changed and not args.skip_oss:
            run_script("_oss_static_assets.py", "upload")
        elif assets_changed:
            print("assets changed but deploy is disabled; skipping OSS upload", flush=True)

        should_deploy = args.force_deploy or data_changed or assets_changed
        if should_deploy and not args.skip_deploy:
            run_script("deploy_relations.py")
            summary["deployed"] = True
        elif should_deploy:
            print("deploy needed but --skip-deploy set; skipping", flush=True)
        else:
            print("no business data/asset changes; skip deploy", flush=True)
        if cache_enabled:
            run_script("refresh_cache.py", "save")
        summary["ok"] = True
    except BaseException as exc:
        summary["error"] = f"{type(exc).__name__}: {exc}"
        raise
    finally:
        summary["duration_s"] = round(time.monotonic() - started, 3)
        _write_summary(summary)
        release_refresh_lock(lock_handle)


if __name__ == "__main__":
    main()