climperor/web/frontend/functions/api/live-status.js

/**
 * Pages Function: GET /api/live-status
 *
 * Visit-triggered live-status probing with request coalescing via the edge
 * Cache API (no KV, no wrangler config). The first visitor after the 5-minute
 * freshness window triggers a re-probe of every streamer with `live_url`;
 * concurrent visitors within the window share the cached JSON. Concurrent cold
 * starts in the same isolate also share one in-flight probe promise.
 *
 * Probe logic is a JS port of fetch_streamer_live.py:
 * - Bilibili: api.live.bilibili.com Room/get_info; data.live_status === 1 is
 *   live (0 offline, 2 replay counts as offline).
 * - Douyin: warm up cookies (www.douyin.com + live.douyin.com), then GET
 *   live.douyin.com/<web_rid> with a browser UA and parse the escaped JSON in
 *   the SSR pace chunks: roomStore.roomInfo.room.status (2 live / 4 offline);
 *   the embedded web_rid must match the requested one.
 *
 * Soft-fail everywhere: douyin blocks from datacenter IPs are expected. When a
 * single probe fails, the streamer carries over the last known is_live from
 * data.json / the previous (stale) cache entry with `stale: true`; when every
 * probe fails the stale cache entry is served wholesale (X-Live-Cache:
 * stale-override), or an empty payload when nothing was ever cached
 * (X-Live-Cache: error). The handler never throws a 500 for probe failures.
 *
 * Named exports double as the local test surface; the Pages runtime only
 * routes onRequest* handlers.
 */

const CACHE_KEY = "https://live-status.internal/v1";
const FRESH_TTL_S = 300;
const FRESH_TTL_MS = FRESH_TTL_S * 1000;
// Store longer than the 5-min freshness window so expired-for-serve entries
// remain readable as carry-over material; freshness is governed by probed_at.
const CACHE_STORE_MAX_AGE_S = 6 * 60 * 60;
const CLIENT_MAX_AGE_S = 60;
const PROBE_TIMEOUT_MS = 8000;
const BATCH_SIZE = 3;
const DOUYIN_SPACING_MS = 800;

const BROWSER_UA =
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
  "AppleWebKit/537.36 (KHTML, like Gecko) " +
  "Chrome/120.0.0.0 Safari/537.36";

const DOUYIN_HOME = "https://www.douyin.com/";
const DOUYIN_LIVE_HOME = "https://live.douyin.com/";
const BILIBILI_INFO_URL =
  "https://api.live.bilibili.com/room/v1/Room/get_info?room_id=";

// Escaped JSON inside the SSR pace chunks: \"roomStore\":{\"roomInfo\":{\"room\":{
const DOUYIN_ROOMSTORE_RE = /\\"roomStore\\":\s*\{\\"roomInfo\\":\s*\{\\"room\\":\s*\{/;
const DOUYIN_STATUS_RE = /\\"status\\":\s*(\d)/;
const DOUYIN_WEBRID_RE = /\\"web_rid\\":\s*\\"(\d+)\\"/;

/** Isolate-local coalescing for concurrent cold starts (same Worker isolate). */
let inFlightProbe = null;

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function fetchWithTimeout(url, init = {}) {
  const ctrl = new AbortController();
  const timer = setTimeout(() => ctrl.abort(), PROBE_TIMEOUT_MS);
  return fetch(url, { ...init, signal: ctrl.signal }).finally(() =>
    clearTimeout(timer)
  );
}

/** Set-Cookie reader portable across Workers (getAll) and Node/undici. */
function setCookiesOf(res) {
  const h = res && res.headers;
  if (!h) return [];
  if (typeof h.getSetCookie === "function") return h.getSetCookie() || [];
  if (typeof h.getAll === "function") {
    try {
      return h.getAll("Set-Cookie") || [];
    } catch {
      return [];
    }
  }
  return [];
}

function collectCookies(res, jar) {
  for (const sc of setCookiesOf(res)) {
    const pair = String(sc).split(";")[0];
    const eq = pair.indexOf("=");
    if (eq > 0) jar.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
  }
}

function cookieHeader(jar) {
  return [...jar.entries()].map(([k, v]) => `${k}=${v}`).join("; ");
}

function douyinHeaders(referer, jar) {
  const headers = {
    "User-Agent": BROWSER_UA,
    Accept: "*/*",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    Referer: referer,
  };
  const cookie = jar && jar.size ? cookieHeader(jar) : "";
  if (cookie) headers.Cookie = cookie;
  return headers;
}

/** Seed cookies once so subsequent room-page requests are not blocked. */
export async function warmDouyinCookies(jar = new Map()) {
  for (const url of [DOUYIN_HOME, DOUYIN_LIVE_HOME]) {
    try {
      const res = await fetchWithTimeout(url, {
        headers: douyinHeaders(DOUYIN_HOME, jar),
      });
      collectCookies(res, jar);
      await res.arrayBuffer(); // drain the body
    } catch {
      // warm-up is best-effort; the room probe below is the real check
    }
    await sleep(DOUYIN_SPACING_MS);
  }
  return jar;
}

/** Parse roomStore status from the SSR live room page (2 live / 4 offline). */
export async function probeDouyinRoom(rid, jar) {
  const res = await fetchWithTimeout(DOUYIN_LIVE_HOME + rid, {
    headers: douyinHeaders(DOUYIN_LIVE_HOME, jar),
  });
  collectCookies(res, jar);
  const html = await res.text();
  if (!html) throw new Error("empty room page");
  const store = DOUYIN_ROOMSTORE_RE.exec(html);
  if (!store) throw new Error("no roomStore in page (blocked or layout changed)");
  // The room object opens with id_str/status; a short window is enough.
  const end = store.index + store[0].length;
  const win = html.slice(end, end + 3000);
  const statusM = DOUYIN_STATUS_RE.exec(win);
  if (!statusM) throw new Error("roomStore has no status field");
  const embedded = DOUYIN_WEBRID_RE.exec(html);
  if (!embedded || embedded[1] !== rid) {
    throw new Error("page resolved to a different room (stale web_rid?)");
  }
  const status = parseInt(statusM[1], 10);
  if (status === 2) return true;
  if (status === 4) return false;
  throw new Error(`unexpected room status ${status}`);
}

/** live_status: 0 offline, 1 live, 2 replay (replay counts as offline). */
export async function probeBilibiliRoom(roomId) {
  const res = await fetchWithTimeout(BILIBILI_INFO_URL + roomId, {
    headers: { "User-Agent": BROWSER_UA, Accept: "application/json" },
  });
  const payload = await res.json();
  if (!payload || payload.code !== 0) {
    throw new Error(`bilibili api error: code=${payload && payload.code}`);
  }
  const data = payload.data;
  if (!data || typeof data !== "object") {
    throw new Error("bilibili api returned no data");
  }
  return data.live_status === 1;
}

/** First path segment of the live room URL (douyin web_rid / bilibili room id). */
export function roomRefFromUrl(liveUrl) {
  let path = "";
  try {
    path = new URL(String(liveUrl).trim()).pathname;
  } catch {
    return null;
  }
  const seg = path.replace(/^\/+|\/+$/g, "").split("/")[0];
  return seg || null;
}

/** Probe backend follows the live room host (may differ from profile platform). */
export function livePlatformFromUrl(liveUrl, fallback = "") {
  try {
    const host = new URL(String(liveUrl).trim()).hostname.toLowerCase();
    if (host.includes("bilibili.com")) return "bilibili";
    if (host.includes("douyin.com")) return "douyin";
  } catch {
    /* keep fallback */
  }
  return String(fallback || "")
    .trim()
    .toLowerCase();
}

/**
 * Extract the streamer row list from a data.json payload.
 * Production nests under `streamers.streamers`; also accept a bare array.
 */
export function streamerRowsFromPayload(payload) {
  if (!payload || typeof payload !== "object") return [];
  if (Array.isArray(payload.streamers)) return payload.streamers;
  const nested = payload.streamers;
  if (nested && typeof nested === "object" && Array.isArray(nested.streamers)) {
    return nested.streamers;
  }
  return [];
}

/** Extract probe targets (id/platform/room ref) from a data.json payload. */
export function targetsFromPayload(payload) {
  const rows = streamerRowsFromPayload(payload);
  const targets = [];
  for (const row of rows) {
    if (!row || typeof row !== "object") continue;
    const sid = String(row.id || "").trim();
    const liveUrl = String(row.live_url || "").trim();
    if (!sid || !liveUrl) continue;
    const fallback = String(row.platform || "").trim().toLowerCase();
    const platform = livePlatformFromUrl(liveUrl, fallback);
    const ref = roomRefFromUrl(liveUrl);
    if (!ref) continue;
    targets.push({ id: sid, platform, ref });
  }
  return targets;
}

/** Seed carry-over map from data.json is_live (daily snapshot). */
export function seedLiveFromPayload(payload) {
  const out = {};
  for (const row of streamerRowsFromPayload(payload)) {
    if (!row || typeof row !== "object") continue;
    const sid = String(row.id || "").trim();
    if (!sid || typeof row.is_live !== "boolean") continue;
    out[sid] = { is_live: row.is_live };
  }
  return out;
}

async function probeAll(targets) {
  const jar = targets.some((t) => t.platform === "douyin")
    ? await warmDouyinCookies()
    : new Map();
  const results = new Map(); // id -> { is_live } | { error }
  for (let i = 0; i < targets.length; i += BATCH_SIZE) {
    const batch = targets.slice(i, i + BATCH_SIZE);
    await Promise.all(
      batch.map(async (t) => {
        try {
          let isLive;
          if (t.platform === "douyin") isLive = await probeDouyinRoom(t.ref, jar);
          else if (t.platform === "bilibili") isLive = await probeBilibiliRoom(t.ref);
          else throw new Error(`unsupported platform ${t.platform}`);
          results.set(t.id, { is_live: isLive });
        } catch (err) {
          results.set(t.id, { error: String((err && err.message) || err) });
        }
      })
    );
    // Douyin rate-limits aggressively; keep spacing between its requests.
    if (i + BATCH_SIZE < targets.length && jar.size) await sleep(DOUYIN_SPACING_MS);
  }
  return results;
}

function jsonResponse(body, cacheState, extraHeaders = {}) {
  return new Response(JSON.stringify(body), {
    status: 200,
    headers: {
      "Content-Type": "application/json; charset=utf-8",
      "Cache-Control": `public, max-age=${CLIENT_MAX_AGE_S}`,
      "X-Live-Cache": cacheState,
      ...extraHeaders,
    },
  });
}

async function readCachedPayload() {
  try {
    const cached = await caches.default.match(CACHE_KEY);
    if (!cached) return null;
    const data = await cached.json();
    return data && typeof data === "object" ? data : null;
  } catch {
    return null;
  }
}

function isFresh(data) {
  const ts = Date.parse(data && data.probed_at);
  return Number.isFinite(ts) && Date.now() - ts < FRESH_TTL_MS;
}

function emptyPayload() {
  return { probed_at: new Date().toISOString(), ttl: FRESH_TTL_S, streamers: {} };
}

async function putCache(body) {
  const cached = new Response(JSON.stringify(body), {
    headers: {
      "Content-Type": "application/json; charset=utf-8",
      "Cache-Control": `max-age=${CACHE_STORE_MAX_AGE_S}`,
    },
  });
  await caches.default.put(CACHE_KEY, cached);
}

/**
 * Load targets + build a fresh probe payload (or return null when there is
 * nothing to probe / total failure with no seed).
 * Returns { body, cacheState } where cacheState is miss | stale-override | error.
 */
async function probeFresh(requestUrl, cachedData) {
  let dataPayload = null;
  let targets = [];
  try {
    const dataUrl = new URL("/data.json", requestUrl);
    const res = await fetchWithTimeout(dataUrl.toString(), {
      headers: { Accept: "application/json" },
    });
    if (res.ok) {
      dataPayload = await res.json();
      targets = targetsFromPayload(dataPayload);
    }
  } catch {
    // data.json unreachable: fall through to stale/empty below
  }

  if (!targets.length) {
    if (cachedData) return { body: cachedData, cacheState: "stale-override" };
    return { body: emptyPayload(), cacheState: "error" };
  }

  const seeded = seedLiveFromPayload(dataPayload);
  const cachedStreamers =
    cachedData && cachedData.streamers && typeof cachedData.streamers === "object"
      ? cachedData.streamers
      : {};
  // Prefer previous edge cache over the daily snapshot when both exist.
  const staleStreamers = { ...seeded, ...cachedStreamers };
  const probed = await probeAll(targets);

  const streamers = {};
  let freshCount = 0;
  for (const t of targets) {
    const r = probed.get(t.id);
    if (r && typeof r.is_live === "boolean") {
      streamers[t.id] = { is_live: r.is_live };
      freshCount += 1;
      continue;
    }
    // Per-streamer soft-fail: carry over the last known state, marked stale.
    const prev = staleStreamers[t.id];
    if (prev && typeof prev.is_live === "boolean") {
      streamers[t.id] = { is_live: prev.is_live, stale: true };
    }
  }

  if (freshCount === 0) {
    // Total probe failure (e.g. douyin blocking this colo): serve the stale
    // snapshot if one exists, otherwise an explicitly empty payload.
    if (cachedData) return { body: cachedData, cacheState: "stale-override" };
    if (Object.keys(seeded).length) {
      const body = {
        probed_at: new Date().toISOString(),
        ttl: FRESH_TTL_S,
        streamers: Object.fromEntries(
          Object.entries(seeded).map(([id, cell]) => [
            id,
            { is_live: cell.is_live, stale: true },
          ])
        ),
      };
      return { body, cacheState: "stale-override" };
    }
    return { body: emptyPayload(), cacheState: "error" };
  }

  const body = { probed_at: new Date().toISOString(), ttl: FRESH_TTL_S, streamers };
  return { body, cacheState: "miss" };
}

async function handle(context) {
  const { request } = context;

  const cachedData = await readCachedPayload();
  if (cachedData && isFresh(cachedData)) {
    return jsonResponse(cachedData, "hit");
  }

  // Coalesce concurrent cold starts in this isolate onto one probe run.
  if (!inFlightProbe) {
    inFlightProbe = probeFresh(request.url, cachedData).finally(() => {
      inFlightProbe = null;
    });
  }
  const { body, cacheState } = await inFlightProbe;

  if (cacheState === "miss") {
    context.waitUntil(putCache(body));
  }
  return jsonResponse(body, cacheState);
}

export async function onRequestGet(context) {
  try {
    return await handle(context);
  } catch {
    // Never 500 because of probing: last-resort empty payload.
    return jsonResponse(emptyPayload(), "error");
  }
}