v0.5.114: add Douyu streamers with fan enrichment and live probe.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-30 02:42:47 +08:00
parent fdd926e9bb
commit 38f46ad2ea
17 changed files with 477 additions and 38 deletions
@@ -1,16 +1,23 @@
-"""Fetch Douyin profile fields into data/streamers.json.
+"""Fetch Douyin / Douyu profile fields into data/streamers.json.

 Manual seed fields (id / platform / live_url / profile_url / heroes / tagline)
 are preserved. Profile enrichment (nickname, signature, counts, avatar) is
-best-effort via Douyin HTML RENDER_DATA + text fallback; failures keep the
-previous values.
+best-effort:
+
+- Douyin: HTML RENDER_DATA + text fallback
+- Douyu: ``v.douyu.com/author/<hash>`` (or ``author-video/<hash>``) page
+  ``window.$DATA`` (fans / following / plays / avatar). Room-only rows
+  resolve ``up_id`` from the live-room HTML then fetch the author page;
+  if that fails, fall back to ``betard`` (nickname / avatar / bio, no fans).
+
+Failures keep the previous values.

 Preview only — do not merge into relations/heroes or recommend.
 Part of refresh_web ``daily`` (soft-fail: never aborts the tier).

 Usage:
  python fetch_streamers.py
-  python fetch_streamers.py --ids xiaowang
+  python fetch_streamers.py --ids xiaowang k9
  python fetch_streamers.py --out data/streamers.json
 """

@@ -32,8 +39,8 @@ import urllib.request
 from datetime import datetime, timezone
 from typing import Any

-from shared.http_utils import write_json_atomic
-from shared.paths import DATA, STREAMER_AVATARS
+from shared.http_utils import http_json, write_json_atomic
+from shared.paths import DATA, ROOT, STREAMER_AVATARS

 OUT = DATA / "streamers.json"
 AVATAR_DIR = STREAMER_AVATARS
@@ -58,6 +65,17 @@ HTML_AVATAR_RE = re.compile(
    re.IGNORECASE,
 )
 SEC_UID_RE = re.compile(r"/user/(MS4wLjABAAAA[A-Za-z0-9_-]+)")
+DOUYU_AUTHOR_HASH_RE = re.compile(
+    r"(?:v\.)?douyu\.com/author(?:-video)?/([A-Za-z0-9]+)", re.IGNORECASE
+)
+DOUYU_ROOM_RE = re.compile(
+    r"(?:www\.)?douyu\.com/(\d+)(?:/|$|\?)", re.IGNORECASE
+)
+DOUYU_DATA_RE = re.compile(r"window\.\$DATA=(\{.*?\}),\$", re.DOTALL)
+DOUYU_BARE_KEY_RE = re.compile(r"([{\s,])([A-Za-z_][A-Za-z0-9_]*)\s*:")
+# Room HTML embeds up_id in plain JSON and/or JSON-escaped script strings.
+DOUYU_UP_ID_RE = re.compile(r'\\?"up_id\\?"\s*:\s*\\?"([A-Za-z0-9]+)\\?"')
+DOUYU_BETARD_URL = "https://www.douyu.com/betard/{room_id}"
 # Fields fetch may overwrite; manual seed keys are never removed.
 PROFILE_KEYS = (
    "nickname",
@@ -104,7 +122,11 @@ def _opener() -> urllib.request.OpenerDirector:


 def _get(
-    opener: urllib.request.OpenerDirector, url: str, *, timeout: int = 30
+    opener: urllib.request.OpenerDirector,
+    url: str,
+    *,
+    timeout: int = 30,
+    referer: str = "https://www.douyin.com/",
 ) -> tuple[str, str]:
    """Return (final_url, html)."""
    req = urllib.request.Request(
@@ -113,7 +135,7 @@ def _get(
            "User-Agent": BROWSER_UA,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-            "Referer": "https://www.douyin.com/",
+            "Referer": referer,
        },
    )
    with opener.open(req, timeout=timeout) as resp:
@@ -306,7 +328,7 @@ def resolve_profile_url(
    return url, None


-def download_avatar(url: str, dest: Path) -> bool:
+def download_avatar(url: str, dest: Path, *, referer: str | None = None) -> bool:
    dest.parent.mkdir(parents=True, exist_ok=True)
    # Prefer a larger CDN variant when the URL embeds a size token.
    candidates = [url]
@@ -314,12 +336,22 @@ def download_avatar(url: str, dest: Path) -> bool:
        candidates.insert(0, url.replace("/100x100/", "/720x720/"))
    if "300x300" in url:
        candidates.insert(0, url.replace("300x300", "720x720"))
+    if "_avatar_middle." in url:
+        candidates.insert(0, url.replace("_avatar_middle.", "_avatar_big."))
+    if "_middle.jpg" in url:
+        candidates.insert(0, url.replace("_middle.jpg", "_big.jpg"))
+    host = urllib.parse.urlparse(url).netloc.lower()
+    if referer is None:
+        if "douyu" in host:
+            referer = "https://www.douyu.com/"
+        else:
+            referer = "https://www.douyin.com/"
    for candidate in candidates:
        req = urllib.request.Request(
            candidate,
            headers={
                "User-Agent": BROWSER_UA,
-                "Referer": "https://www.douyin.com/",
+                "Referer": referer,
            },
        )
        try:
@@ -336,6 +368,153 @@ def download_avatar(url: str, dest: Path) -> bool:
    return False


+def parse_douyu_dollar_data(blob: str) -> dict:
+    """Parse Douyu ``window.$DATA={...}`` JS object (bare keys) into a dict."""
+    quoted = DOUYU_BARE_KEY_RE.sub(r'\1"\2":', blob.strip())
+    data = json.loads(quoted)
+    if not isinstance(data, dict):
+        raise ValueError("douyu $DATA is not an object")
+    return data
+
+
+def extract_profile_from_douyu_data(data: dict) -> dict:
+    """Map Douyu author ``$DATA`` fields onto streamer profile keys."""
+    out: dict[str, Any] = {}
+    nick = _first_str(data.get("name"), data.get("nickname"))
+    if nick:
+        out["nickname"] = nick
+    room_id = _first_str(data.get("roomId"), data.get("room_id"))
+    if room_id:
+        out["unique_id"] = room_id
+        out["live_url"] = f"https://www.douyu.com/{room_id}"
+    up_id = _first_str(data.get("upId"), data.get("up_id"))
+    if up_id:
+        # Canonical homepage (author space); author-video is an alias.
+        out["profile_url"] = f"https://v.douyu.com/author/{up_id}"
+    # Author bio only — do not fall back to live room title (would wipe
+    # hand-seeded / betard signatures on every refresh).
+    contents = _first_str(data.get("contents"), data.get("ownerAuthContents"))
+    if contents:
+        out["signature"] = contents
+    out["following_count"] = _first_int(data.get("upFollowNum"), data.get("up_follow_num"))
+    out["follower_count"] = _first_int(data.get("subscribeNum"), data.get("subscribe_num"))
+    # Video play count — shown as「播放」on Douyu cards.
+    out["total_favorited"] = _first_int(data.get("playCount"), data.get("play_count"))
+    avatar = _first_str(data.get("avatar"))
+    if avatar:
+        out["avatar_url"] = avatar.replace(r"\/", "/")
+    return out
+
+
+def fetch_douyu_author_profile(
+    opener: urllib.request.OpenerDirector, profile_url: str
+) -> dict:
+    """Fetch Douyu author / author-video page and parse ``window.$DATA``."""
+    m = DOUYU_AUTHOR_HASH_RE.search(profile_url)
+    if not m:
+        raise ValueError(f"not a douyu author url: {profile_url!r}")
+    hash_id = m.group(1)
+    # Prefer /author/; fall back to /author-video/ if $DATA is missing.
+    errors: list[str] = []
+    for path in (f"author/{hash_id}", f"author-video/{hash_id}"):
+        page_url = f"https://v.douyu.com/{path}"
+        try:
+            _, html = _get(opener, page_url, referer="https://v.douyu.com/")
+        except (urllib.error.URLError, TimeoutError, OSError) as e:
+            errors.append(f"{path}: {e}")
+            continue
+        data_m = DOUYU_DATA_RE.search(html)
+        if not data_m:
+            errors.append(f"{path}: no window.$DATA")
+            continue
+        profile = extract_profile_from_douyu_data(
+            parse_douyu_dollar_data(data_m.group(1))
+        )
+        if not any(
+            profile.get(k) is not None
+            for k in ("nickname", "follower_count", "avatar_url", "unique_id")
+        ):
+            errors.append(f"{path}: parsed empty")
+            continue
+        return profile
+    raise ValueError("; ".join(errors) or "douyu author page failed")
+
+
+def extract_douyu_up_id_from_room_html(html: str) -> str | None:
+    """Pull author ``up_id`` hash embedded in the live-room page HTML."""
+    m = DOUYU_UP_ID_RE.search(html or "")
+    return m.group(1) if m else None
+
+
+def resolve_douyu_author_url_from_room(
+    opener: urllib.request.OpenerDirector, room_id: str
+) -> str:
+    """Fetch ``www.douyu.com/<rid>`` and build the author homepage URL."""
+    page_url = f"https://www.douyu.com/{room_id}"
+    _, html = _get(opener, page_url, referer="https://www.douyu.com/")
+    up_id = extract_douyu_up_id_from_room_html(html)
+    if not up_id:
+        raise ValueError(f"douyu room {room_id}: no up_id in HTML")
+    return f"https://v.douyu.com/author/{up_id}"
+
+
+def fetch_douyu_room_profile(room_id: str) -> dict:
+    """Fallback enrichment from live-room ``betard`` (no fan counts)."""
+    payload = http_json(DOUYU_BETARD_URL.format(room_id=room_id), timeout=20)
+    if not isinstance(payload, dict):
+        raise ValueError("douyu betard returned non-object")
+    room = payload.get("room")
+    if not isinstance(room, dict):
+        raise ValueError("douyu betard returned no room")
+    out: dict[str, Any] = {
+        "unique_id": str(room_id),
+        "live_url": f"https://www.douyu.com/{room_id}",
+    }
+    nick = _first_str(room.get("nickname"), room.get("owner_name"))
+    if nick:
+        out["nickname"] = nick
+    details = _first_str(room.get("show_details"), room.get("room_name"))
+    if details:
+        out["signature"] = details
+    avatar = room.get("avatar")
+    avatar_url = None
+    if isinstance(avatar, dict):
+        avatar_url = _first_str(avatar.get("big"), avatar.get("middle"), avatar.get("small"))
+    elif isinstance(avatar, str):
+        avatar_url = avatar
+    if not avatar_url:
+        avatar_url = _first_str(room.get("owner_avatar"), room.get("avatar_mid"))
+    if avatar_url:
+        out["avatar_url"] = avatar_url
+    if not out.get("nickname") and not out.get("avatar_url"):
+        raise ValueError("douyu betard parsed empty")
+    return out
+
+
+def fetch_douyu_profile(
+    opener: urllib.request.OpenerDirector, row: dict
+) -> dict:
+    """Enrich a Douyu streamer from author URL, room→up_id, else betard."""
+    profile_url = str(row.get("profile_url") or "").strip()
+    live_url = str(row.get("live_url") or "").strip()
+    if DOUYU_AUTHOR_HASH_RE.search(profile_url):
+        return fetch_douyu_author_profile(opener, profile_url)
+    room_id = None
+    for candidate in (profile_url, live_url):
+        m = DOUYU_ROOM_RE.search(candidate)
+        if m:
+            room_id = m.group(1)
+            break
+    if not room_id:
+        raise ValueError("douyu row needs author profile_url or room live_url")
+    try:
+        author_url = resolve_douyu_author_url_from_room(opener, room_id)
+        return fetch_douyu_author_profile(opener, author_url)
+    except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e:
+        print(f"  douyu room→author failed ({e}); betard fallback", flush=True)
+        return fetch_douyu_room_profile(room_id)
+
+
 def fetch_douyin_profile(
    opener: urllib.request.OpenerDirector, profile_url: str
 ) -> dict:
@@ -382,6 +561,15 @@ def merge_profile(row: dict, profile: dict, *, streamer_id: str) -> None:
        if val is None or val == "":
            continue
        row[key] = val
+    # Fill missing live/profile URLs from platform enrichment; never wipe seeds.
+    for key in ("live_url", "profile_url"):
+        val = profile.get(key)
+        if isinstance(val, str) and val and not str(row.get(key) or "").strip():
+            row[key] = val
+    # Prefer canonical Douyu author homepage when enrichment found one.
+    prof = profile.get("profile_url")
+    if isinstance(prof, str) and DOUYU_AUTHOR_HASH_RE.search(prof):
+        row["profile_url"] = prof
    avatar_url = profile.get("avatar_url")
    if isinstance(avatar_url, str) and avatar_url:
        dest = AVATAR_DIR / f"{streamer_id}.jpg"
@@ -408,18 +596,25 @@ def enrich_streamers(
        if ids is not None and sid not in ids:
            continue
        platform = str(row.get("platform") or "").strip().lower()
-        if platform != "douyin":
-            print(f"skip {sid}: platform={platform!r} (only douyin supported)", flush=True)
+        if platform not in ("douyin", "douyu"):
+            print(
+                f"skip {sid}: platform={platform!r} "
+                f"(supported: douyin, douyu)",
+                flush=True,
+            )
            skip += 1
            continue
-        profile_url = str(row.get("profile_url") or "").strip()
-        if not profile_url:
-            print(f"skip {sid}: missing profile_url", flush=True)
-            skip += 1
-            continue
-        print(f"fetching {sid} ...", flush=True)
+        print(f"fetching {sid} ({platform}) ...", flush=True)
        try:
-            profile = fetch_douyin_profile(opener, profile_url)
+            if platform == "douyin":
+                profile_url = str(row.get("profile_url") or "").strip()
+                if not profile_url:
+                    print(f"skip {sid}: missing profile_url", flush=True)
+                    skip += 1
+                    continue
+                profile = fetch_douyin_profile(opener, profile_url)
+            else:
+                profile = fetch_douyu_profile(opener, row)
            merge_profile(row, profile, streamer_id=sid)
            nick = row.get("nickname") or "?"
            print(
@@ -434,7 +629,7 @@ def enrich_streamers(
        time.sleep(0.8)
    if ok > 0:
        payload["fetched_at"] = _now_iso()
-    payload["source"] = payload.get("source") or "manual+douyin"
+    payload["source"] = payload.get("source") or "manual+douyin+douyu"
    meta = payload.get("platform_meta")
    if not isinstance(meta, dict):
        meta = {}
@@ -442,13 +637,19 @@ def enrich_streamers(
        "douyin",
        {"label_zh": "抖音", "icon": "ui-icon/platform_douyin.png"},
    )
+    meta.setdefault(
+        "douyu",
+        {"label_zh": "斗鱼", "icon": "ui-icon/platform_douyu.png"},
+    )
    payload["platform_meta"] = meta
    _ = PROFILE_KEYS
    return ok, skip, fail


 def main() -> int:
-    ap = argparse.ArgumentParser(description="Enrich streamers.json from Douyin profiles")
+    ap = argparse.ArgumentParser(
+        description="Enrich streamers.json from Douyin / Douyu profiles"
+    )
    ap.add_argument("--out", type=Path, default=OUT)
    ap.add_argument(
        "--ids",