climperor/web/_cf_human_home.py

"""Human homepage opens: path=/ AND known browser UA (exclude bots/unknown)."""

from __future__ import annotations

import json
import os
import urllib.request
from datetime import date, timedelta

HOST = "dota2.refining.dev"
ZONE = "refining.dev"

HUMAN_BROWSERS = [
    "Chrome",
    "Firefox",
    "Safari",
    "Edge",
    "Opera",
    "MobileSafari",
    "ChromeMobile",
    "ChromeMobileWebview",
    "FirefoxMobile",
    "SamsungInternet",
    "EdgeMobile",
]


def headers() -> dict[str, str]:
    email = os.environ["KEYZOO_ASSET_META_USERNAME"]
    key = os.environ["KEYZOO_ASSET_SECRET_GLOBAL_API_KEY"]
    return {
        "X-Auth-Email": email,
        "X-Auth-Key": key,
        "Content-Type": "application/json",
    }


def gql(query: str, variables: dict) -> dict:
    req = urllib.request.Request(
        "https://api.cloudflare.com/client/v4/graphql",
        data=json.dumps({"query": query, "variables": variables}).encode(),
        method="POST",
        headers=headers(),
    )
    with urllib.request.urlopen(req) as r:
        return json.loads(r.read().decode())


def main() -> None:
    zid = json.loads(
        urllib.request.urlopen(
            urllib.request.Request(
                f"https://api.cloudflare.com/client/v4/zones?name={ZONE}",
                headers=headers(),
            )
        ).read()
    )["result"][0]["id"]

    q = """
    query($zoneTag:string!,$day:Date!,$host:string!){
      viewer{zones(filter:{zoneTag:$zoneTag}){
        homeByUA:httpRequestsAdaptiveGroups(
          limit:30
          filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/"}
          orderBy:[count_DESC]
        ){dimensions{userAgentBrowser clientCountryName} count sum{visits}}
        rumByUA:httpRequestsAdaptiveGroups(
          limit:20
          filter:{
            date:$day
            clientRequestHTTPHost:$host
            clientRequestPath:"/cdn-cgi/rum"
          }
          orderBy:[count_DESC]
        ){dimensions{userAgentBrowser} count sum{visits}}
        homeByCountry:httpRequestsAdaptiveGroups(
          limit:15
          filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/"}
          orderBy:[count_DESC]
        ){dimensions{clientCountryName} count sum{visits}}
      }}
    }
    """

    print(f"{HOST} homepage (/) by UA  |  RUM by UA\n")
    tot_human_home_req = tot_human_home_vis = 0
    tot_rum = 0
    tot_bot_home_vis = 0

    for i in range(7, -1, -1):
        day = date.today() - timedelta(days=i)
        data = gql(q, {"zoneTag": zid, "day": day.isoformat(), "host": HOST})
        if data.get("errors"):
            print(day, data["errors"][0]["message"][:120])
            continue
        z = data["data"]["viewer"]["zones"][0]
        home = z["homeByUA"] or []
        rum = z["rumByUA"] or []
        if not home and not rum:
            continue

        human_req = human_vis = bot_vis = rum_c = 0
        print(f"=== {day} ===")
        print("  homepage / by UA:")
        for r in home:
            ua = (r["dimensions"] or {}).get("userAgentBrowser")
            c = r["count"] or 0
            v = (r.get("sum") or {}).get("visits") or 0
            tag = "HUMAN" if ua in HUMAN_BROWSERS else "bot/other"
            print(f"    {c:>4} req  {v:>4} vis  [{tag}] {ua}")
            if ua in HUMAN_BROWSERS:
                human_req += c
                human_vis += v
            else:
                bot_vis += v
        print("  RUM by UA:")
        for r in rum:
            ua = (r["dimensions"] or {}).get("userAgentBrowser")
            c = r["count"] or 0
            rum_c += c
            print(f"    {c:>4} beacons  {ua}")
        print("  homepage / by country:")
        for r in z["homeByCountry"] or []:
            cc = (r["dimensions"] or {}).get("clientCountryName")
            v = (r.get("sum") or {}).get("visits") or 0
            print(f"    {r['count']:>4} req  {v:>4} vis  {cc}")
        print(
            f"  day human home: {human_req} req / {human_vis} visits | "
            f"bot-other home visits: {bot_vis} | rum: {rum_c}"
        )
        tot_human_home_req += human_req
        tot_human_home_vis += human_vis
        tot_bot_home_vis += bot_vis
        tot_rum += rum_c

    print("\n======== SUMMARY (8 days) ========")
    print(f"  Human browser homepage requests:  {tot_human_home_req}")
    print(f"  Human browser homepage visits:    {tot_human_home_vis}")
    print(f"  Non-human homepage visits:        {tot_bot_home_vis}")
    print(f"  RUM beacons (JS pageviews):       {tot_rum}")
    print()
    print("  Verdict (exclude crawlers):")
    print(f"    ~{tot_human_home_vis} real homepage visits (known browser UA + /)")
    print(f"    ~{tot_rum} RUM pageviews (browser executed JS)")
    # If RUM > human home visits, same session may fire multiple beacons
    # or SPA navigations; take human home visits as unique-ish sessions,
    # RUM as pageview count.
    lo = min(tot_human_home_vis, tot_rum) if tot_rum else tot_human_home_vis
    hi = max(tot_human_home_vis, tot_rum) if tot_rum else tot_human_home_vis
    print(f"    Best estimate: ~{lo}–{hi} real human opens")


if __name__ == "__main__":
    main()