"""Fetch Douyin / Douyu profile fields into data/streamers.json.

Manual seed fields (id / platform / live_url / profile_url / heroes / tagline)
are preserved. Profile enrichment (nickname, signature, counts, avatar) is
best-effort:

- Douyin: HTML RENDER_DATA + text fallback
- Douyu: ``v.douyu.com/author/<hash>`` (or ``author-video/<hash>``) page
  ``window.$DATA`` (fans / following / plays / avatar). Room-only rows
  resolve ``up_id`` from the live-room HTML then fetch the author page;
  if that fails, fall back to ``betard`` (nickname / avatar / bio, no fans).

Failures keep the previous values.

Preview only — do not merge into relations/heroes or recommend.
Part of refresh_web ``daily`` (soft-fail: never aborts the tier).

Usage:
  python fetch_streamers.py
  python fetch_streamers.py --ids xiaowang k9
  python fetch_streamers.py --out data/streamers.json
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import argparse
import http.cookiejar
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from typing import Any

from shared.http_utils import http_json, write_json_atomic
from shared.paths import DATA, ROOT, STREAMER_AVATARS

OUT = DATA / "streamers.json"
AVATAR_DIR = STREAMER_AVATARS
BROWSER_UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/120.0.0.0 Safari/537.36"
)
RENDER_DATA_RE = re.compile(
    r'<script[^>]+id=["\']RENDER_DATA["\'][^>]*>([^<]+)</script>',
    re.IGNORECASE,
)
HTML_COUNTS_RE = re.compile(
    r"关注\s*([\d.]+万?亿?)\s*粉丝\s*([\d.]+万?亿?)\s*获赞\s*([\d.]+万?亿?)",
    re.DOTALL,
)
HTML_UNIQUE_RE = re.compile(r"抖音号[：:]\s*([A-Za-z0-9_.-]+)")
HTML_TITLE_RE = re.compile(r"<title>\s*([^<]+?)\s*的抖音", re.IGNORECASE)
HTML_AVATAR_RE = re.compile(
    r'<img[^>]+alt="[^"]*头像"[^>]+src="([^"]+)"|'
    r'<img[^>]+src="([^"]+)"[^>]+alt="[^"]*头像"',
    re.IGNORECASE,
)
SEC_UID_RE = re.compile(r"/user/(MS4wLjABAAAA[A-Za-z0-9_-]+)")
DOUYU_AUTHOR_HASH_RE = re.compile(
    r"(?:v\.)?douyu\.com/author(?:-video)?/([A-Za-z0-9]+)", re.IGNORECASE
)
DOUYU_ROOM_RE = re.compile(
    r"(?:www\.)?douyu\.com/(\d+)(?:/|$|\?)", re.IGNORECASE
)
DOUYU_DATA_RE = re.compile(r"window\.\$DATA=(\{.*?\}),\$", re.DOTALL)
DOUYU_BARE_KEY_RE = re.compile(r"([{\s,])([A-Za-z_][A-Za-z0-9_]*)\s*:")
# Room HTML embeds up_id in plain JSON and/or JSON-escaped script strings.
DOUYU_UP_ID_RE = re.compile(r'\\?"up_id\\?"\s*:\s*\\?"([A-Za-z0-9]+)\\?"')
DOUYU_BETARD_URL = "https://www.douyu.com/betard/{room_id}"
# Fields fetch may overwrite; manual seed keys are never removed.
PROFILE_KEYS = (
    "nickname",
    "unique_id",
    "signature",
    "following_count",
    "follower_count",
    "total_favorited",
    "avatar",
    "profile_fetched_at",
)


def _now_iso() -> str:
    return datetime.now(timezone.utc).isoformat()


def _load(path: Path) -> dict:
    if not path.is_file():
        return {
            "fetched_at": None,
            "source": "manual+douyin",
            "platform_meta": {
                "douyin": {
                    "label_zh": "抖音",
                    "icon": "ui-icon/platform_douyin.png",
                }
            },
            "streamers": [],
        }
    raw = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(raw, dict):
        raise SystemExit(f"{path}: expected object")
    return raw


def _save(path: Path, payload: dict) -> None:
    write_json_atomic(path, payload)


def _opener() -> urllib.request.OpenerDirector:
    jar = http.cookiejar.CookieJar()
    return urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))


def _get(
    opener: urllib.request.OpenerDirector,
    url: str,
    *,
    timeout: int = 30,
    referer: str = "https://www.douyin.com/",
) -> tuple[str, str]:
    """Return (final_url, html)."""
    req = urllib.request.Request(
        url,
        headers={
            "User-Agent": BROWSER_UA,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Referer": referer,
        },
    )
    with opener.open(req, timeout=timeout) as resp:
        final = resp.geturl()
        html = resp.read().decode("utf-8", "replace")
        return final, html


def parse_cn_count(raw: str) -> int | None:
    s = (raw or "").strip().replace(",", "").replace("\n", "")
    if not s:
        return None
    try:
        if s.endswith("亿"):
            return int(float(s[:-1]) * 100_000_000)
        if s.endswith("万"):
            return int(float(s[:-1]) * 10_000)
        return int(float(s))
    except ValueError:
        return None


def _parse_render_data(html: str) -> dict | None:
    m = RENDER_DATA_RE.search(html)
    if not m:
        return None
    try:
        decoded = urllib.parse.unquote(m.group(1))
        data = json.loads(decoded)
    except (ValueError, json.JSONDecodeError):
        return None
    return data if isinstance(data, dict) else None


def _walk_user_nodes(obj: Any, found: list[dict]) -> None:
    if isinstance(obj, dict):
        if "nickname" in obj and (
            "follower_count" in obj
            or "followerCount" in obj
            or "mplatform_followers_count" in obj
        ):
            found.append(obj)
        for v in obj.values():
            _walk_user_nodes(v, found)
    elif isinstance(obj, list):
        for v in obj:
            _walk_user_nodes(v, found)


def _first_int(*vals: Any) -> int | None:
    for v in vals:
        if isinstance(v, bool):
            continue
        if isinstance(v, int):
            return v
        if isinstance(v, float):
            return int(v)
        if isinstance(v, str):
            n = parse_cn_count(v)
            if n is not None:
                return n
    return None


def _first_str(*vals: Any) -> str | None:
    for v in vals:
        if isinstance(v, str) and v.strip():
            return v.strip()
    return None


def _avatar_url(user: dict) -> str | None:
    for key in (
        "avatar_larger",
        "avatar_medium",
        "avatar_thumb",
        "avatarUrl",
        "avatar_url",
    ):
        cell = user.get(key)
        if isinstance(cell, str) and cell.startswith("http"):
            return cell
        if isinstance(cell, dict):
            urls = cell.get("url_list") or cell.get("urlList") or []
            if isinstance(urls, list):
                for u in urls:
                    if isinstance(u, str) and u.startswith("http"):
                        return u
    return None


def extract_profile_from_render(render: dict) -> dict | None:
    candidates: list[dict] = []
    _walk_user_nodes(render, candidates)
    if not candidates:
        return None

    def score(u: dict) -> int:
        n = _first_int(
            u.get("follower_count"),
            u.get("followerCount"),
            u.get("mplatform_followers_count"),
        )
        return n if n is not None else -1

    user = max(candidates, key=score)
    return {
        "nickname": _first_str(user.get("nickname"), user.get("nickName")),
        "unique_id": _first_str(
            user.get("unique_id"),
            user.get("uniqueId"),
            user.get("short_id"),
            user.get("shortId"),
        ),
        "signature": _first_str(user.get("signature"), user.get("desc")),
        "following_count": _first_int(
            user.get("following_count"), user.get("followingCount")
        ),
        "follower_count": _first_int(
            user.get("follower_count"),
            user.get("followerCount"),
            user.get("mplatform_followers_count"),
        ),
        "total_favorited": _first_int(
            user.get("total_favorited"),
            user.get("totalFavorited"),
            user.get("favoriting_count"),
        ),
        "avatar_url": _avatar_url(user),
    }


def extract_profile_from_html(html: str) -> dict:
    """Best-effort parse when RENDER_DATA user blob is missing/blocked."""
    out: dict[str, Any] = {}
    m = HTML_COUNTS_RE.search(html)
    if m:
        out["following_count"] = parse_cn_count(m.group(1))
        out["follower_count"] = parse_cn_count(m.group(2))
        out["total_favorited"] = parse_cn_count(m.group(3))
    uid = HTML_UNIQUE_RE.search(html)
    if uid:
        out["unique_id"] = uid.group(1)
    title = HTML_TITLE_RE.search(html)
    if title:
        out["nickname"] = title.group(1).strip()
    av = HTML_AVATAR_RE.search(html)
    if av:
        url = av.group(1) or av.group(2)
        if url and url.startswith("http"):
            out["avatar_url"] = url.replace("&amp;", "&")
    return out


def merge_profile_dicts(primary: dict | None, fallback: dict) -> dict:
    out = dict(fallback)
    if primary:
        for k, v in primary.items():
            if v is None or v == "":
                continue
            out[k] = v
    return out


def canonicalize_douyin_user_url(url: str) -> str:
    """Map share / short-link destinations to www.douyin.com/user/<sec_uid>."""
    m = SEC_UID_RE.search(url)
    if m:
        return f"https://www.douyin.com/user/{m.group(1)}"
    return url


def resolve_profile_url(
    opener: urllib.request.OpenerDirector, profile_url: str
) -> tuple[str, str | None]:
    """Follow redirects; return (canonical_url, html_or_None if not yet fetched)."""
    url = profile_url.strip()
    host = urllib.parse.urlparse(url).netloc.lower()
    if "v.douyin.com" in host or "iesdouyin.com" in host:
        final, html = _get(opener, url)
        canon = canonicalize_douyin_user_url(final)
        if canon != final and "douyin.com/user/" in canon:
            # Re-fetch canonical profile for RENDER_DATA when possible.
            try:
                _, html2 = _get(opener, canon)
                return canon, html2
            except (urllib.error.URLError, TimeoutError, OSError):
                return canon, html
        return canon, html
    return url, None


def download_avatar(url: str, dest: Path, *, referer: str | None = None) -> bool:
    dest.parent.mkdir(parents=True, exist_ok=True)
    # Prefer a larger CDN variant when the URL embeds a size token.
    candidates = [url]
    if "/100x100/" in url:
        candidates.insert(0, url.replace("/100x100/", "/720x720/"))
    if "300x300" in url:
        candidates.insert(0, url.replace("300x300", "720x720"))
    if "_avatar_middle." in url:
        candidates.insert(0, url.replace("_avatar_middle.", "_avatar_big."))
    if "_middle.jpg" in url:
        candidates.insert(0, url.replace("_middle.jpg", "_big.jpg"))
    host = urllib.parse.urlparse(url).netloc.lower()
    if referer is None:
        if "douyu" in host:
            referer = "https://www.douyu.com/"
        else:
            referer = "https://www.douyin.com/"
    for candidate in candidates:
        req = urllib.request.Request(
            candidate,
            headers={
                "User-Agent": BROWSER_UA,
                "Referer": referer,
            },
        )
        try:
            with urllib.request.urlopen(req, timeout=30) as resp:
                data = resp.read()
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            print(f"  avatar download failed: {e}", flush=True)
            continue
        if not data or len(data) < 64:
            continue
        dest.write_bytes(data)
        return True
    print("  avatar download empty", flush=True)
    return False


def parse_douyu_dollar_data(blob: str) -> dict:
    """Parse Douyu ``window.$DATA={...}`` JS object (bare keys) into a dict."""
    quoted = DOUYU_BARE_KEY_RE.sub(r'\1"\2":', blob.strip())
    data = json.loads(quoted)
    if not isinstance(data, dict):
        raise ValueError("douyu $DATA is not an object")
    return data


def extract_profile_from_douyu_data(data: dict) -> dict:
    """Map Douyu author ``$DATA`` fields onto streamer profile keys."""
    out: dict[str, Any] = {}
    nick = _first_str(data.get("name"), data.get("nickname"))
    if nick:
        out["nickname"] = nick
    room_id = _first_str(data.get("roomId"), data.get("room_id"))
    if room_id:
        out["unique_id"] = room_id
        out["live_url"] = f"https://www.douyu.com/{room_id}"
    up_id = _first_str(data.get("upId"), data.get("up_id"))
    if up_id:
        # Canonical homepage (author space); author-video is an alias.
        out["profile_url"] = f"https://v.douyu.com/author/{up_id}"
    # Author bio only — do not fall back to live room title (would wipe
    # hand-seeded / betard signatures on every refresh).
    contents = _first_str(data.get("contents"), data.get("ownerAuthContents"))
    if contents:
        out["signature"] = contents
    out["following_count"] = _first_int(data.get("upFollowNum"), data.get("up_follow_num"))
    out["follower_count"] = _first_int(data.get("subscribeNum"), data.get("subscribe_num"))
    # Video play count — shown as「播放」on Douyu cards.
    out["total_favorited"] = _first_int(data.get("playCount"), data.get("play_count"))
    avatar = _first_str(data.get("avatar"))
    if avatar:
        out["avatar_url"] = avatar.replace(r"\/", "/")
    return out


def fetch_douyu_author_profile(
    opener: urllib.request.OpenerDirector, profile_url: str
) -> dict:
    """Fetch Douyu author / author-video page and parse ``window.$DATA``."""
    m = DOUYU_AUTHOR_HASH_RE.search(profile_url)
    if not m:
        raise ValueError(f"not a douyu author url: {profile_url!r}")
    hash_id = m.group(1)
    # Prefer /author/; fall back to /author-video/ if $DATA is missing.
    errors: list[str] = []
    for path in (f"author/{hash_id}", f"author-video/{hash_id}"):
        page_url = f"https://v.douyu.com/{path}"
        try:
            _, html = _get(opener, page_url, referer="https://v.douyu.com/")
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            errors.append(f"{path}: {e}")
            continue
        data_m = DOUYU_DATA_RE.search(html)
        if not data_m:
            errors.append(f"{path}: no window.$DATA")
            continue
        profile = extract_profile_from_douyu_data(
            parse_douyu_dollar_data(data_m.group(1))
        )
        if not any(
            profile.get(k) is not None
            for k in ("nickname", "follower_count", "avatar_url", "unique_id")
        ):
            errors.append(f"{path}: parsed empty")
            continue
        return profile
    raise ValueError("; ".join(errors) or "douyu author page failed")


def extract_douyu_up_id_from_room_html(html: str) -> str | None:
    """Pull author ``up_id`` hash embedded in the live-room page HTML."""
    m = DOUYU_UP_ID_RE.search(html or "")
    return m.group(1) if m else None


def resolve_douyu_author_url_from_room(
    opener: urllib.request.OpenerDirector, room_id: str
) -> str:
    """Fetch ``www.douyu.com/<rid>`` and build the author homepage URL."""
    page_url = f"https://www.douyu.com/{room_id}"
    _, html = _get(opener, page_url, referer="https://www.douyu.com/")
    up_id = extract_douyu_up_id_from_room_html(html)
    if not up_id:
        raise ValueError(f"douyu room {room_id}: no up_id in HTML")
    return f"https://v.douyu.com/author/{up_id}"


def fetch_douyu_room_profile(room_id: str) -> dict:
    """Fallback enrichment from live-room ``betard`` (no fan counts)."""
    payload = http_json(DOUYU_BETARD_URL.format(room_id=room_id), timeout=20)
    if not isinstance(payload, dict):
        raise ValueError("douyu betard returned non-object")
    room = payload.get("room")
    if not isinstance(room, dict):
        raise ValueError("douyu betard returned no room")
    out: dict[str, Any] = {
        "unique_id": str(room_id),
        "live_url": f"https://www.douyu.com/{room_id}",
    }
    nick = _first_str(room.get("nickname"), room.get("owner_name"))
    if nick:
        out["nickname"] = nick
    details = _first_str(room.get("show_details"), room.get("room_name"))
    if details:
        out["signature"] = details
    avatar = room.get("avatar")
    avatar_url = None
    if isinstance(avatar, dict):
        avatar_url = _first_str(avatar.get("big"), avatar.get("middle"), avatar.get("small"))
    elif isinstance(avatar, str):
        avatar_url = avatar
    if not avatar_url:
        avatar_url = _first_str(room.get("owner_avatar"), room.get("avatar_mid"))
    if avatar_url:
        out["avatar_url"] = avatar_url
    if not out.get("nickname") and not out.get("avatar_url"):
        raise ValueError("douyu betard parsed empty")
    return out


def fetch_douyu_profile(
    opener: urllib.request.OpenerDirector, row: dict
) -> dict:
    """Enrich a Douyu streamer from author URL, room→up_id, else betard."""
    profile_url = str(row.get("profile_url") or "").strip()
    live_url = str(row.get("live_url") or "").strip()
    if DOUYU_AUTHOR_HASH_RE.search(profile_url):
        return fetch_douyu_author_profile(opener, profile_url)
    room_id = None
    for candidate in (profile_url, live_url):
        m = DOUYU_ROOM_RE.search(candidate)
        if m:
            room_id = m.group(1)
            break
    if not room_id:
        raise ValueError("douyu row needs author profile_url or room live_url")
    try:
        author_url = resolve_douyu_author_url_from_room(opener, room_id)
        return fetch_douyu_author_profile(opener, author_url)
    except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e:
        print(f"  douyu room→author failed ({e}); betard fallback", flush=True)
        return fetch_douyu_room_profile(room_id)


def fetch_douyin_profile(
    opener: urllib.request.OpenerDirector, profile_url: str
) -> dict:
    try:
        _get(opener, "https://www.douyin.com/")
    except (urllib.error.URLError, TimeoutError, OSError) as e:
        print(f"  douyin homepage warm-up failed: {e}", flush=True)

    canon, html = resolve_profile_url(opener, profile_url)
    if html is None:
        _, html = _get(opener, canon)

    primary = None
    render = _parse_render_data(html)
    if render is not None:
        primary = extract_profile_from_render(render)
    fallback = extract_profile_from_html(html)
    profile = merge_profile_dicts(primary, fallback)
    if not any(
        profile.get(k) is not None
        for k in (
            "nickname",
            "follower_count",
            "following_count",
            "total_favorited",
            "unique_id",
            "avatar_url",
        )
    ):
        raise ValueError("no profile fields parsed (blocked or layout changed)")
    return profile


def merge_profile(row: dict, profile: dict, *, streamer_id: str) -> None:
    for key in (
        "nickname",
        "unique_id",
        "signature",
        "following_count",
        "follower_count",
        "total_favorited",
    ):
        val = profile.get(key)
        if val is None or val == "":
            continue
        row[key] = val
    # Fill missing live/profile URLs from platform enrichment; never wipe seeds.
    for key in ("live_url", "profile_url"):
        val = profile.get(key)
        if isinstance(val, str) and val and not str(row.get(key) or "").strip():
            row[key] = val
    # Prefer canonical Douyu author homepage when enrichment found one.
    prof = profile.get("profile_url")
    if isinstance(prof, str) and DOUYU_AUTHOR_HASH_RE.search(prof):
        row["profile_url"] = prof
    avatar_url = profile.get("avatar_url")
    if isinstance(avatar_url, str) and avatar_url:
        dest = AVATAR_DIR / f"{streamer_id}.jpg"
        if download_avatar(avatar_url, dest):
            row["avatar"] = f"streamer_avatars/{streamer_id}.jpg"
            print(f"  avatar saved {dest.relative_to(ROOT)}", flush=True)
    row["profile_fetched_at"] = _now_iso()


def enrich_streamers(
    payload: dict, *, ids: set[str] | None = None
) -> tuple[int, int, int]:
    rows = payload.get("streamers")
    if not isinstance(rows, list):
        raise SystemExit("streamers.json: missing streamers array")
    opener = _opener()
    ok = skip = fail = 0
    for row in rows:
        if not isinstance(row, dict):
            continue
        sid = str(row.get("id") or "").strip()
        if not sid:
            continue
        if ids is not None and sid not in ids:
            continue
        platform = str(row.get("platform") or "").strip().lower()
        if platform not in ("douyin", "douyu"):
            print(
                f"skip {sid}: platform={platform!r} "
                f"(supported: douyin, douyu)",
                flush=True,
            )
            skip += 1
            continue
        print(f"fetching {sid} ({platform}) ...", flush=True)
        try:
            if platform == "douyin":
                profile_url = str(row.get("profile_url") or "").strip()
                if not profile_url:
                    print(f"skip {sid}: missing profile_url", flush=True)
                    skip += 1
                    continue
                profile = fetch_douyin_profile(opener, profile_url)
            else:
                profile = fetch_douyu_profile(opener, row)
            merge_profile(row, profile, streamer_id=sid)
            nick = row.get("nickname") or "?"
            print(
                f"  ok {nick} followers={row.get('follower_count')} "
                f"likes={row.get('total_favorited')}",
                flush=True,
            )
            ok += 1
        except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e:
            print(f"  FAIL {sid}: {e} (keeping previous values)", flush=True)
            fail += 1
        time.sleep(0.8)
    if ok > 0:
        payload["fetched_at"] = _now_iso()
    payload["source"] = payload.get("source") or "manual+douyin+douyu"
    meta = payload.get("platform_meta")
    if not isinstance(meta, dict):
        meta = {}
    meta.setdefault(
        "douyin",
        {"label_zh": "抖音", "icon": "ui-icon/platform_douyin.png"},
    )
    meta.setdefault(
        "douyu",
        {"label_zh": "斗鱼", "icon": "ui-icon/platform_douyu.png"},
    )
    payload["platform_meta"] = meta
    _ = PROFILE_KEYS
    return ok, skip, fail


def main() -> int:
    ap = argparse.ArgumentParser(
        description="Enrich streamers.json from Douyin / Douyu profiles"
    )
    ap.add_argument("--out", type=Path, default=OUT)
    ap.add_argument(
        "--ids",
        nargs="+",
        default=None,
        help="only refresh these streamer ids",
    )
    args = ap.parse_args()
    payload = _load(args.out)
    id_set = set(args.ids) if args.ids else None
    ok, skip, fail = enrich_streamers(payload, ids=id_set)
    _save(args.out, payload)
    print(f"wrote {args.out} ok={ok} skip={skip} fail={fail}", flush=True)
    # Soft-fail for CI/refresh_web: always exit 0 after writing (keep old values).
    return 0


if __name__ == "__main__":
    raise SystemExit(main())