climperor/http_utils.py

"""Shared HTTP helpers: fetch JSON/bytes, download icons, load Valve datafeeds.

Used by all fetch_*.py scripts and serve_relations.py so HTTP logic, User-Agent,
timeout, and retry conventions live in exactly one place.
"""

from __future__ import annotations

import json
import time
import urllib.error
import urllib.request
from pathlib import Path

UA = "climperor"
DEFAULT_TIMEOUT = 60
# OpenDota and similar APIs occasionally 429; back off before failing CI/refresh.
DEFAULT_RETRIES = 4
DEFAULT_RETRY_BACKOFF = 5.0

HEROES_URL = "https://www.dota2.com/datafeed/herolist?language={lang}"
ITEMLIST_URL = "https://www.dota2.com/datafeed/itemlist?language={lang}"


def _http_open(url: str, *, timeout: int, retries: int, backoff: float):
    """urlopen with retries on 429 / 5xx / transient network errors."""
    last_err: BaseException | None = None
    for attempt in range(retries + 1):
        req = urllib.request.Request(url, headers={"User-Agent": UA})
        try:
            return urllib.request.urlopen(req, timeout=timeout)
        except urllib.error.HTTPError as e:
            last_err = e
            if e.code not in (429, 500, 502, 503, 504) or attempt >= retries:
                raise
            sleep_s = backoff * (2**attempt)
            retry_after = e.headers.get("Retry-After") if e.headers else None
            if retry_after:
                try:
                    sleep_s = max(sleep_s, float(retry_after))
                except ValueError:
                    pass
            print(
                f"HTTP {e.code} {url} — retry {attempt + 1}/{retries} in {sleep_s:.0f}s",
                flush=True,
            )
            time.sleep(sleep_s)
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            last_err = e
            if attempt >= retries:
                raise
            sleep_s = backoff * (2**attempt)
            print(
                f"HTTP error {e} {url} — retry {attempt + 1}/{retries} in {sleep_s:.0f}s",
                flush=True,
            )
            time.sleep(sleep_s)
    assert last_err is not None
    raise last_err


def http_json(
    url: str,
    *,
    timeout: int = DEFAULT_TIMEOUT,
    retries: int = DEFAULT_RETRIES,
    backoff: float = DEFAULT_RETRY_BACKOFF,
) -> dict | list:
    with _http_open(url, timeout=timeout, retries=retries, backoff=backoff) as resp:
        return json.loads(resp.read().decode())


def http_bytes(
    url: str,
    *,
    timeout: int = DEFAULT_TIMEOUT,
    retries: int = DEFAULT_RETRIES,
    backoff: float = DEFAULT_RETRY_BACKOFF,
) -> bytes:
    with _http_open(url, timeout=timeout, retries=retries, backoff=backoff) as resp:
        return resp.read()


def download_icons(
    keys,
    url_template: str,
    dest_dir: Path,
    *,
    force: bool = False,
    delay: float = 0.0,
    min_size: int = 32,
    skip_keys: frozenset[str] | set[str] | None = None,
) -> tuple[int, int, int]:
    """Download PNG icons from a CDN. Returns (saved, skipped_existing, fail).

    keys       — iterable of template substitution values (hero/item/ability keys).
    url_template — e.g. "https://cdn.../abilities/{key}.png".
    dest_dir   — target directory (created if missing).
    force      — re-download even if the file exists.
    delay      — seconds to sleep between requests (rate limiting).
    min_size   — files smaller than this are treated as empty and re-downloaded.
    skip_keys  — keys to ignore entirely (e.g. bundled icons that 404 on CDN).
    """
    dest_dir.mkdir(parents=True, exist_ok=True)
    skip = set(skip_keys or ())
    saved = skipped = fail = 0
    for key in sorted(keys):
        if key in skip or "/" in key or "\\" in key or ".." in key:
            continue
        dest = dest_dir / f"{key}.png"
        if dest.is_file() and dest.stat().st_size >= min_size and not force:
            skipped += 1
            continue
        try:
            data = http_bytes(url_template.format(key=key))
            if not data or len(data) < min_size:
                raise ValueError("empty icon")
            dest.write_bytes(data)
            saved += 1
            print(f"  icon saved {key}.png ({len(data)} bytes)", flush=True)
        except (urllib.error.URLError, TimeoutError, ValueError, OSError) as e:
            print(f"  icon {key}: {e}", flush=True)
            fail += 1
        if delay > 0:
            time.sleep(delay)
    return saved, skipped, fail


def load_itemlist(lang: str = "schinese") -> dict[int, dict[str, str]]:
    """Valve datafeed item list: item id -> {name_loc, name}."""
    raw = http_json(ITEMLIST_URL.format(lang=lang))
    rows = (((raw or {}).get("result") or {}).get("data") or {}).get("itemabilities") or []
    out: dict[int, dict[str, str]] = {}
    for row in rows:
        if not isinstance(row, dict):
            continue
        iid = row.get("id")
        if iid is None:
            continue
        out[int(iid)] = {
            "name_loc": (row.get("name_loc") or "").strip(),
            "name": (row.get("name") or "").strip(),
        }
    return out


def load_itemlist_zh() -> dict[int, str]:
    """Convenience: item id -> Chinese localized name."""
    return {iid: v["name_loc"] for iid, v in load_itemlist().items() if v["name_loc"]}


def fetch_hero_list(lang: str = "schinese") -> list[dict]:
    """Hero list from Valve's datafeed (id, key, name_loc, primary_attr)."""
    data = http_json(HEROES_URL.format(lang=lang))
    heroes = data.get("result", {}).get("data", {}).get("heroes") or data.get("heroes")
    if not heroes:
        raise SystemExit("hero list came back empty")
    return heroes


def fetch_hero_keys() -> list[str]:
    """Hero keys (e.g. antimage, earthshaker) from the English datafeed."""
    heroes = fetch_hero_list(lang="english")
    keys = []
    for h in heroes:
        name = h.get("name") or ""
        keys.append(name.removeprefix("npc_dota_hero_"))
    return keys