vosonandCursor
3ec8007077
Ship OpenDota counter-stats reordering for feared items, finalize SITE_VERSION/docs for rankings/streamers/trends/matches/mechanics and draft archetypes, and ignore regenerable Web data caches. Co-authored-by: Cursor <cursoragent@cursor.com>
470 lines
14 KiB
Python
470 lines
14 KiB
Python
"""Fetch Douyin profile fields into data/streamers.json.
|
||
|
||
Manual seed fields (id / platform / live_url / profile_url / heroes / tagline)
|
||
are preserved. Profile enrichment (nickname, signature, counts, avatar) is
|
||
best-effort via Douyin HTML RENDER_DATA + text fallback; failures keep the
|
||
previous values.
|
||
|
||
Preview only — do not merge into relations/heroes or recommend.
|
||
Part of refresh_web ``daily`` (soft-fail: never aborts the tier).
|
||
|
||
Usage:
|
||
python fetch_streamers.py
|
||
python fetch_streamers.py --ids xiaowang
|
||
python fetch_streamers.py --out data/streamers.json
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import http.cookiejar
|
||
import json
|
||
import re
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from common import DATA, ROOT
|
||
|
||
OUT = DATA / "streamers.json"
|
||
AVATAR_DIR = ROOT / "assets" / "streamer_avatars"
|
||
BROWSER_UA = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/120.0.0.0 Safari/537.36"
|
||
)
|
||
RENDER_DATA_RE = re.compile(
|
||
r'<script[^>]+id=["\']RENDER_DATA["\'][^>]*>([^<]+)</script>',
|
||
re.IGNORECASE,
|
||
)
|
||
HTML_COUNTS_RE = re.compile(
|
||
r"关注\s*([\d.]+万?亿?)\s*粉丝\s*([\d.]+万?亿?)\s*获赞\s*([\d.]+万?亿?)",
|
||
re.DOTALL,
|
||
)
|
||
HTML_UNIQUE_RE = re.compile(r"抖音号[：:]\s*([A-Za-z0-9_.-]+)")
|
||
HTML_TITLE_RE = re.compile(r"<title>\s*([^<]+?)\s*的抖音", re.IGNORECASE)
|
||
HTML_AVATAR_RE = re.compile(
|
||
r'<img[^>]+alt="[^"]*头像"[^>]+src="([^"]+)"|'
|
||
r'<img[^>]+src="([^"]+)"[^>]+alt="[^"]*头像"',
|
||
re.IGNORECASE,
|
||
)
|
||
SEC_UID_RE = re.compile(r"/user/(MS4wLjABAAAA[A-Za-z0-9_-]+)")
|
||
# Fields fetch may overwrite; manual seed keys are never removed.
|
||
PROFILE_KEYS = (
|
||
"nickname",
|
||
"unique_id",
|
||
"signature",
|
||
"following_count",
|
||
"follower_count",
|
||
"total_favorited",
|
||
"avatar",
|
||
"profile_fetched_at",
|
||
)
|
||
|
||
|
||
def _now_iso() -> str:
|
||
return datetime.now(timezone.utc).isoformat()
|
||
|
||
|
||
def _load(path: Path) -> dict:
|
||
if not path.is_file():
|
||
return {
|
||
"fetched_at": None,
|
||
"source": "manual+douyin",
|
||
"platform_meta": {
|
||
"douyin": {
|
||
"label_zh": "抖音",
|
||
"icon": "ui-icon/platform_douyin.png",
|
||
}
|
||
},
|
||
"streamers": [],
|
||
}
|
||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||
if not isinstance(raw, dict):
|
||
raise SystemExit(f"{path}: expected object")
|
||
return raw
|
||
|
||
|
||
def _save(path: Path, payload: dict) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(
|
||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
|
||
|
||
def _opener() -> urllib.request.OpenerDirector:
|
||
jar = http.cookiejar.CookieJar()
|
||
return urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
|
||
|
||
|
||
def _get(
|
||
opener: urllib.request.OpenerDirector, url: str, *, timeout: int = 30
|
||
) -> tuple[str, str]:
|
||
"""Return (final_url, html)."""
|
||
req = urllib.request.Request(
|
||
url,
|
||
headers={
|
||
"User-Agent": BROWSER_UA,
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
"Referer": "https://www.douyin.com/",
|
||
},
|
||
)
|
||
with opener.open(req, timeout=timeout) as resp:
|
||
final = resp.geturl()
|
||
html = resp.read().decode("utf-8", "replace")
|
||
return final, html
|
||
|
||
|
||
def parse_cn_count(raw: str) -> int | None:
|
||
s = (raw or "").strip().replace(",", "").replace("\n", "")
|
||
if not s:
|
||
return None
|
||
try:
|
||
if s.endswith("亿"):
|
||
return int(float(s[:-1]) * 100_000_000)
|
||
if s.endswith("万"):
|
||
return int(float(s[:-1]) * 10_000)
|
||
return int(float(s))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _parse_render_data(html: str) -> dict | None:
|
||
m = RENDER_DATA_RE.search(html)
|
||
if not m:
|
||
return None
|
||
try:
|
||
decoded = urllib.parse.unquote(m.group(1))
|
||
data = json.loads(decoded)
|
||
except (ValueError, json.JSONDecodeError):
|
||
return None
|
||
return data if isinstance(data, dict) else None
|
||
|
||
|
||
def _walk_user_nodes(obj: Any, found: list[dict]) -> None:
|
||
if isinstance(obj, dict):
|
||
if "nickname" in obj and (
|
||
"follower_count" in obj
|
||
or "followerCount" in obj
|
||
or "mplatform_followers_count" in obj
|
||
):
|
||
found.append(obj)
|
||
for v in obj.values():
|
||
_walk_user_nodes(v, found)
|
||
elif isinstance(obj, list):
|
||
for v in obj:
|
||
_walk_user_nodes(v, found)
|
||
|
||
|
||
def _first_int(*vals: Any) -> int | None:
|
||
for v in vals:
|
||
if isinstance(v, bool):
|
||
continue
|
||
if isinstance(v, int):
|
||
return v
|
||
if isinstance(v, float):
|
||
return int(v)
|
||
if isinstance(v, str):
|
||
n = parse_cn_count(v)
|
||
if n is not None:
|
||
return n
|
||
return None
|
||
|
||
|
||
def _first_str(*vals: Any) -> str | None:
|
||
for v in vals:
|
||
if isinstance(v, str) and v.strip():
|
||
return v.strip()
|
||
return None
|
||
|
||
|
||
def _avatar_url(user: dict) -> str | None:
|
||
for key in (
|
||
"avatar_larger",
|
||
"avatar_medium",
|
||
"avatar_thumb",
|
||
"avatarUrl",
|
||
"avatar_url",
|
||
):
|
||
cell = user.get(key)
|
||
if isinstance(cell, str) and cell.startswith("http"):
|
||
return cell
|
||
if isinstance(cell, dict):
|
||
urls = cell.get("url_list") or cell.get("urlList") or []
|
||
if isinstance(urls, list):
|
||
for u in urls:
|
||
if isinstance(u, str) and u.startswith("http"):
|
||
return u
|
||
return None
|
||
|
||
|
||
def extract_profile_from_render(render: dict) -> dict | None:
|
||
candidates: list[dict] = []
|
||
_walk_user_nodes(render, candidates)
|
||
if not candidates:
|
||
return None
|
||
|
||
def score(u: dict) -> int:
|
||
n = _first_int(
|
||
u.get("follower_count"),
|
||
u.get("followerCount"),
|
||
u.get("mplatform_followers_count"),
|
||
)
|
||
return n if n is not None else -1
|
||
|
||
user = max(candidates, key=score)
|
||
return {
|
||
"nickname": _first_str(user.get("nickname"), user.get("nickName")),
|
||
"unique_id": _first_str(
|
||
user.get("unique_id"),
|
||
user.get("uniqueId"),
|
||
user.get("short_id"),
|
||
user.get("shortId"),
|
||
),
|
||
"signature": _first_str(user.get("signature"), user.get("desc")),
|
||
"following_count": _first_int(
|
||
user.get("following_count"), user.get("followingCount")
|
||
),
|
||
"follower_count": _first_int(
|
||
user.get("follower_count"),
|
||
user.get("followerCount"),
|
||
user.get("mplatform_followers_count"),
|
||
),
|
||
"total_favorited": _first_int(
|
||
user.get("total_favorited"),
|
||
user.get("totalFavorited"),
|
||
user.get("favoriting_count"),
|
||
),
|
||
"avatar_url": _avatar_url(user),
|
||
}
|
||
|
||
|
||
def extract_profile_from_html(html: str) -> dict:
|
||
"""Best-effort parse when RENDER_DATA user blob is missing/blocked."""
|
||
out: dict[str, Any] = {}
|
||
m = HTML_COUNTS_RE.search(html)
|
||
if m:
|
||
out["following_count"] = parse_cn_count(m.group(1))
|
||
out["follower_count"] = parse_cn_count(m.group(2))
|
||
out["total_favorited"] = parse_cn_count(m.group(3))
|
||
uid = HTML_UNIQUE_RE.search(html)
|
||
if uid:
|
||
out["unique_id"] = uid.group(1)
|
||
title = HTML_TITLE_RE.search(html)
|
||
if title:
|
||
out["nickname"] = title.group(1).strip()
|
||
av = HTML_AVATAR_RE.search(html)
|
||
if av:
|
||
url = av.group(1) or av.group(2)
|
||
if url and url.startswith("http"):
|
||
out["avatar_url"] = url.replace("&", "&")
|
||
return out
|
||
|
||
|
||
def merge_profile_dicts(primary: dict | None, fallback: dict) -> dict:
|
||
out = dict(fallback)
|
||
if primary:
|
||
for k, v in primary.items():
|
||
if v is None or v == "":
|
||
continue
|
||
out[k] = v
|
||
return out
|
||
|
||
|
||
def canonicalize_douyin_user_url(url: str) -> str:
|
||
"""Map share / short-link destinations to www.douyin.com/user/<sec_uid>."""
|
||
m = SEC_UID_RE.search(url)
|
||
if m:
|
||
return f"https://www.douyin.com/user/{m.group(1)}"
|
||
return url
|
||
|
||
|
||
def resolve_profile_url(
|
||
opener: urllib.request.OpenerDirector, profile_url: str
|
||
) -> tuple[str, str | None]:
|
||
"""Follow redirects; return (canonical_url, html_or_None if not yet fetched)."""
|
||
url = profile_url.strip()
|
||
host = urllib.parse.urlparse(url).netloc.lower()
|
||
if "v.douyin.com" in host or "iesdouyin.com" in host:
|
||
final, html = _get(opener, url)
|
||
canon = canonicalize_douyin_user_url(final)
|
||
if canon != final and "douyin.com/user/" in canon:
|
||
# Re-fetch canonical profile for RENDER_DATA when possible.
|
||
try:
|
||
_, html2 = _get(opener, canon)
|
||
return canon, html2
|
||
except (urllib.error.URLError, TimeoutError, OSError):
|
||
return canon, html
|
||
return canon, html
|
||
return url, None
|
||
|
||
|
||
def download_avatar(url: str, dest: Path) -> bool:
|
||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||
# Prefer a larger CDN variant when the URL embeds a size token.
|
||
candidates = [url]
|
||
if "/100x100/" in url:
|
||
candidates.insert(0, url.replace("/100x100/", "/720x720/"))
|
||
if "300x300" in url:
|
||
candidates.insert(0, url.replace("300x300", "720x720"))
|
||
for candidate in candidates:
|
||
req = urllib.request.Request(
|
||
candidate,
|
||
headers={
|
||
"User-Agent": BROWSER_UA,
|
||
"Referer": "https://www.douyin.com/",
|
||
},
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
data = resp.read()
|
||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||
print(f" avatar download failed: {e}", flush=True)
|
||
continue
|
||
if not data or len(data) < 64:
|
||
continue
|
||
dest.write_bytes(data)
|
||
return True
|
||
print(" avatar download empty", flush=True)
|
||
return False
|
||
|
||
|
||
def fetch_douyin_profile(
|
||
opener: urllib.request.OpenerDirector, profile_url: str
|
||
) -> dict:
|
||
try:
|
||
_get(opener, "https://www.douyin.com/")
|
||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||
print(f" douyin homepage warm-up failed: {e}", flush=True)
|
||
|
||
canon, html = resolve_profile_url(opener, profile_url)
|
||
if html is None:
|
||
_, html = _get(opener, canon)
|
||
|
||
primary = None
|
||
render = _parse_render_data(html)
|
||
if render is not None:
|
||
primary = extract_profile_from_render(render)
|
||
fallback = extract_profile_from_html(html)
|
||
profile = merge_profile_dicts(primary, fallback)
|
||
if not any(
|
||
profile.get(k) is not None
|
||
for k in (
|
||
"nickname",
|
||
"follower_count",
|
||
"following_count",
|
||
"total_favorited",
|
||
"unique_id",
|
||
"avatar_url",
|
||
)
|
||
):
|
||
raise ValueError("no profile fields parsed (blocked or layout changed)")
|
||
return profile
|
||
|
||
|
||
def merge_profile(row: dict, profile: dict, *, streamer_id: str) -> None:
|
||
for key in (
|
||
"nickname",
|
||
"unique_id",
|
||
"signature",
|
||
"following_count",
|
||
"follower_count",
|
||
"total_favorited",
|
||
):
|
||
val = profile.get(key)
|
||
if val is None or val == "":
|
||
continue
|
||
row[key] = val
|
||
avatar_url = profile.get("avatar_url")
|
||
if isinstance(avatar_url, str) and avatar_url:
|
||
dest = AVATAR_DIR / f"{streamer_id}.jpg"
|
||
if download_avatar(avatar_url, dest):
|
||
row["avatar"] = f"streamer_avatars/{streamer_id}.jpg"
|
||
print(f" avatar saved {dest.relative_to(ROOT)}", flush=True)
|
||
row["profile_fetched_at"] = _now_iso()
|
||
|
||
|
||
def enrich_streamers(
|
||
payload: dict, *, ids: set[str] | None = None
|
||
) -> tuple[int, int, int]:
|
||
rows = payload.get("streamers")
|
||
if not isinstance(rows, list):
|
||
raise SystemExit("streamers.json: missing streamers array")
|
||
opener = _opener()
|
||
ok = skip = fail = 0
|
||
for row in rows:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
sid = str(row.get("id") or "").strip()
|
||
if not sid:
|
||
continue
|
||
if ids is not None and sid not in ids:
|
||
continue
|
||
platform = str(row.get("platform") or "").strip().lower()
|
||
if platform != "douyin":
|
||
print(f"skip {sid}: platform={platform!r} (only douyin supported)", flush=True)
|
||
skip += 1
|
||
continue
|
||
profile_url = str(row.get("profile_url") or "").strip()
|
||
if not profile_url:
|
||
print(f"skip {sid}: missing profile_url", flush=True)
|
||
skip += 1
|
||
continue
|
||
print(f"fetching {sid} ...", flush=True)
|
||
try:
|
||
profile = fetch_douyin_profile(opener, profile_url)
|
||
merge_profile(row, profile, streamer_id=sid)
|
||
nick = row.get("nickname") or "?"
|
||
print(
|
||
f" ok {nick} followers={row.get('follower_count')} "
|
||
f"likes={row.get('total_favorited')}",
|
||
flush=True,
|
||
)
|
||
ok += 1
|
||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e:
|
||
print(f" FAIL {sid}: {e} (keeping previous values)", flush=True)
|
||
fail += 1
|
||
time.sleep(0.8)
|
||
if ok > 0:
|
||
payload["fetched_at"] = _now_iso()
|
||
payload["source"] = payload.get("source") or "manual+douyin"
|
||
meta = payload.get("platform_meta")
|
||
if not isinstance(meta, dict):
|
||
meta = {}
|
||
meta.setdefault(
|
||
"douyin",
|
||
{"label_zh": "抖音", "icon": "ui-icon/platform_douyin.png"},
|
||
)
|
||
payload["platform_meta"] = meta
|
||
_ = PROFILE_KEYS
|
||
return ok, skip, fail
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser(description="Enrich streamers.json from Douyin profiles")
|
||
ap.add_argument("--out", type=Path, default=OUT)
|
||
ap.add_argument(
|
||
"--ids",
|
||
nargs="+",
|
||
default=None,
|
||
help="only refresh these streamer ids",
|
||
)
|
||
args = ap.parse_args()
|
||
payload = _load(args.out)
|
||
id_set = set(args.ids) if args.ids else None
|
||
ok, skip, fail = enrich_streamers(payload, ids=id_set)
|
||
_save(args.out, payload)
|
||
print(f"wrote {args.out} ok={ok} skip={skip} fail={fail}", flush=True)
|
||
# Soft-fail for CI/refresh_web: always exit 0 after writing (keep old values).
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|