vosonandCursor
96a9312194
Track reusable CF analytics utilities and the commercial prospects document while keeping local probe/sample artifacts untracked. Co-authored-by: Cursor <cursoragent@cursor.com>
387 lines
12 KiB
Python
387 lines
12 KiB
Python
"""Estimate human traffic for dota2.refining.dev (exclude obvious bots).
|
|
|
|
Free plan has no BotScore; we use:
|
|
- HTML document hits (/) vs asset/bot paths
|
|
- RUM beacons (/cdn-cgi/rum) as JS-executing browsers
|
|
- UA browser family when available
|
|
- Country + path heuristics
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import date, timedelta
|
|
|
|
HOST = "dota2.refining.dev"
|
|
ZONE_NAME = "refining.dev"
|
|
|
|
|
|
def headers() -> dict[str, str]:
|
|
email = os.environ.get("CLOUDFLARE_EMAIL") or os.environ.get(
|
|
"KEYZOO_ASSET_META_USERNAME"
|
|
)
|
|
key = os.environ.get("CLOUDFLARE_API_KEY") or os.environ.get(
|
|
"KEYZOO_ASSET_SECRET_GLOBAL_API_KEY"
|
|
)
|
|
if not email or not key:
|
|
raise SystemExit("missing credentials in env")
|
|
return {
|
|
"X-Auth-Email": email,
|
|
"X-Auth-Key": key,
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
|
|
def api(path: str) -> dict:
|
|
req = urllib.request.Request(
|
|
"https://api.cloudflare.com/client/v4" + path,
|
|
method="GET",
|
|
headers=headers(),
|
|
)
|
|
with urllib.request.urlopen(req) as r:
|
|
return json.loads(r.read().decode())
|
|
|
|
|
|
def gql(query: str, variables: dict) -> dict:
|
|
req = urllib.request.Request(
|
|
"https://api.cloudflare.com/client/v4/graphql",
|
|
data=json.dumps({"query": query, "variables": variables}).encode(),
|
|
method="POST",
|
|
headers=headers(),
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req) as r:
|
|
return json.loads(r.read().decode())
|
|
except urllib.error.HTTPError as e:
|
|
return {"errors": [{"message": e.read().decode()[:1500]}]}
|
|
|
|
|
|
def try_query(label: str, query: str, variables: dict) -> dict | None:
|
|
data = gql(query, variables)
|
|
if data.get("errors"):
|
|
msg = data["errors"][0].get("message", "")[:200]
|
|
print(f" [{label}] unavailable: {msg}")
|
|
return None
|
|
return data
|
|
|
|
|
|
def main() -> None:
|
|
zid = api(f"/zones?name={ZONE_NAME}")["result"][0]["id"]
|
|
days = [date.today() - timedelta(days=i) for i in range(7, -1, -1)]
|
|
|
|
print(f"\n{HOST} human-traffic estimate (Free plan, no BotScore)\n")
|
|
|
|
# --- probe schema fields once ---
|
|
day0 = days[-2] if len(days) > 1 else days[-1] # prefer yesterday
|
|
probes = [
|
|
(
|
|
"UA browser",
|
|
"""
|
|
query($zoneTag:string!,$day:Date!,$host:string!){
|
|
viewer{zones(filter:{zoneTag:$zoneTag}){
|
|
g:httpRequestsAdaptiveGroups(
|
|
limit:20
|
|
filter:{date:$day,clientRequestHTTPHost:$host}
|
|
orderBy:[count_DESC]
|
|
){dimensions{userAgentBrowser} count sum{visits}}
|
|
}}
|
|
}
|
|
""",
|
|
),
|
|
(
|
|
"device type",
|
|
"""
|
|
query($zoneTag:string!,$day:Date!,$host:string!){
|
|
viewer{zones(filter:{zoneTag:$zoneTag}){
|
|
g:httpRequestsAdaptiveGroups(
|
|
limit:10
|
|
filter:{date:$day,clientRequestHTTPHost:$host}
|
|
orderBy:[count_DESC]
|
|
){dimensions{clientDeviceType} count sum{visits}}
|
|
}}
|
|
}
|
|
""",
|
|
),
|
|
(
|
|
"botScore",
|
|
"""
|
|
query($zoneTag:string!,$day:Date!,$host:string!){
|
|
viewer{zones(filter:{zoneTag:$zoneTag}){
|
|
g:httpRequestsAdaptiveGroups(
|
|
limit:10
|
|
filter:{date:$day,clientRequestHTTPHost:$host,botScore_geq:30}
|
|
orderBy:[count_DESC]
|
|
){count sum{visits} dimensions{botScore}}
|
|
}}
|
|
}
|
|
""",
|
|
),
|
|
(
|
|
"content type",
|
|
"""
|
|
query($zoneTag:string!,$day:Date!,$host:string!){
|
|
viewer{zones(filter:{zoneTag:$zoneTag}){
|
|
g:httpRequestsAdaptiveGroups(
|
|
limit:15
|
|
filter:{date:$day,clientRequestHTTPHost:$host}
|
|
orderBy:[count_DESC]
|
|
){dimensions{edgeResponseContentTypeName} count sum{visits}}
|
|
}}
|
|
}
|
|
""",
|
|
),
|
|
]
|
|
print(f"Schema probes on {day0}:")
|
|
available = {}
|
|
for label, q in probes:
|
|
data = try_query(
|
|
label,
|
|
q,
|
|
{"zoneTag": zid, "day": day0.isoformat(), "host": HOST},
|
|
)
|
|
if data:
|
|
available[label] = data["data"]["viewer"]["zones"][0]["g"]
|
|
print(f" [{label}] OK")
|
|
|
|
for label, rows in available.items():
|
|
print(f"\n--- {label} ({day0}) ---")
|
|
for r in rows[:12]:
|
|
dims = r.get("dimensions") or {}
|
|
key = next(iter(dims.values()), "?") if dims else "?"
|
|
v = (r.get("sum") or {}).get("visits") or 0
|
|
print(f" {r['count']:>6} req {v:>5} vis {key}")
|
|
|
|
# --- per-day: total / homepage / rum / likely-bot paths ---
|
|
day_q = """
|
|
query($zoneTag:string!,$day:Date!,$host:string!){
|
|
viewer{zones(filter:{zoneTag:$zoneTag}){
|
|
all:httpRequestsAdaptiveGroups(
|
|
limit:1
|
|
filter:{date:$day,clientRequestHTTPHost:$host}
|
|
){count sum{visits}}
|
|
home:httpRequestsAdaptiveGroups(
|
|
limit:1
|
|
filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/"}
|
|
){count sum{visits}}
|
|
rum:httpRequestsAdaptiveGroups(
|
|
limit:1
|
|
filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/cdn-cgi/rum"}
|
|
){count sum{visits}}
|
|
html:httpRequestsAdaptiveGroups(
|
|
limit:1
|
|
filter:{
|
|
date:$day
|
|
clientRequestHTTPHost:$host
|
|
edgeResponseContentTypeName:"text/html"
|
|
}
|
|
){count sum{visits}}
|
|
byPath:httpRequestsAdaptiveGroups(
|
|
limit:40
|
|
filter:{date:$day,clientRequestHTTPHost:$host}
|
|
orderBy:[count_DESC]
|
|
){dimensions{clientRequestPath} count sum{visits}}
|
|
byUA:httpRequestsAdaptiveGroups(
|
|
limit:25
|
|
filter:{date:$day,clientRequestHTTPHost:$host}
|
|
orderBy:[count_DESC]
|
|
){dimensions{userAgentBrowser} count sum{visits}}
|
|
}}
|
|
}
|
|
"""
|
|
|
|
BOT_PATH_MARKERS = (
|
|
"wp-",
|
|
"wordpress",
|
|
"xmlrpc",
|
|
"wlwmanifest",
|
|
".env",
|
|
"phpmyadmin",
|
|
"robots.txt",
|
|
"sitemap",
|
|
"favicon.ico", # often bots; keep separate
|
|
)
|
|
HUMAN_UA = {
|
|
"Chrome",
|
|
"Firefox",
|
|
"Safari",
|
|
"Edge",
|
|
"Opera",
|
|
"Samsung Internet",
|
|
"Mobile Safari",
|
|
"Chrome Mobile",
|
|
"Firefox Mobile",
|
|
"Edg",
|
|
"Chromium",
|
|
}
|
|
BOT_UA = {
|
|
"Bot",
|
|
"Spider",
|
|
"Crawler",
|
|
"curl",
|
|
"Go-http-client",
|
|
"python",
|
|
"Python",
|
|
"Scrapy",
|
|
"Headless",
|
|
"Empty",
|
|
"Unknown",
|
|
"",
|
|
None,
|
|
}
|
|
|
|
print(f"\n{'date':<12} {'all_req':>8} {'all_vis':>8} {'home_vis':>8} {'rum':>6} {'html_vis':>8}")
|
|
totals = {
|
|
"all_req": 0,
|
|
"all_vis": 0,
|
|
"home_vis": 0,
|
|
"rum": 0,
|
|
"html_vis": 0,
|
|
"human_ua_vis": 0,
|
|
"bot_path_req": 0,
|
|
}
|
|
best_day = None
|
|
best_z = None
|
|
best_home = 0
|
|
|
|
for day in days:
|
|
data = try_query(
|
|
day.isoformat(),
|
|
day_q,
|
|
{"zoneTag": zid, "day": day.isoformat(), "host": HOST},
|
|
)
|
|
if not data:
|
|
# retry without html / UA if fields fail
|
|
day_q_min = """
|
|
query($zoneTag:string!,$day:Date!,$host:string!){
|
|
viewer{zones(filter:{zoneTag:$zoneTag}){
|
|
all:httpRequestsAdaptiveGroups(
|
|
limit:1 filter:{date:$day,clientRequestHTTPHost:$host}
|
|
){count sum{visits}}
|
|
home:httpRequestsAdaptiveGroups(
|
|
limit:1
|
|
filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/"}
|
|
){count sum{visits}}
|
|
rum:httpRequestsAdaptiveGroups(
|
|
limit:1
|
|
filter:{
|
|
date:$day
|
|
clientRequestHTTPHost:$host
|
|
clientRequestPath:"/cdn-cgi/rum"
|
|
}
|
|
){count sum{visits}}
|
|
byPath:httpRequestsAdaptiveGroups(
|
|
limit:40
|
|
filter:{date:$day,clientRequestHTTPHost:$host}
|
|
orderBy:[count_DESC]
|
|
){dimensions{clientRequestPath} count sum{visits}}
|
|
}}
|
|
}
|
|
"""
|
|
data = try_query(
|
|
day.isoformat() + "-min",
|
|
day_q_min,
|
|
{"zoneTag": zid, "day": day.isoformat(), "host": HOST},
|
|
)
|
|
if not data:
|
|
continue
|
|
z = data["data"]["viewer"]["zones"][0]
|
|
|
|
def one(name: str) -> tuple[int, int]:
|
|
rows = z.get(name) or []
|
|
if not rows:
|
|
return 0, 0
|
|
r = rows[0]
|
|
return r.get("count") or 0, (r.get("sum") or {}).get("visits") or 0
|
|
|
|
all_c, all_v = one("all")
|
|
home_c, home_v = one("home")
|
|
rum_c, _ = one("rum")
|
|
html_c, html_v = one("html") if "html" in z else (0, 0)
|
|
|
|
print(
|
|
f"{day.isoformat():<12} {all_c:>8} {all_v:>8} {home_v:>8} {rum_c:>6} {html_v:>8}"
|
|
)
|
|
totals["all_req"] += all_c
|
|
totals["all_vis"] += all_v
|
|
totals["home_vis"] += home_v
|
|
totals["rum"] += rum_c
|
|
totals["html_vis"] += html_v
|
|
|
|
# bot-looking paths
|
|
for r in z.get("byPath") or []:
|
|
p = (r["dimensions"] or {}).get("clientRequestPath") or ""
|
|
if any(m in p.lower() for m in BOT_PATH_MARKERS):
|
|
totals["bot_path_req"] += r["count"] or 0
|
|
|
|
# UA sum visits for human browsers
|
|
for r in z.get("byUA") or []:
|
|
ua = (r["dimensions"] or {}).get("userAgentBrowser")
|
|
v = (r.get("sum") or {}).get("visits") or 0
|
|
if ua in HUMAN_UA or (
|
|
ua
|
|
and not any(
|
|
str(b).lower() in str(ua).lower()
|
|
for b in BOT_UA
|
|
if b
|
|
)
|
|
and ua
|
|
not in ("", None)
|
|
):
|
|
# count known browsers only
|
|
if ua in HUMAN_UA or (
|
|
isinstance(ua, str)
|
|
and any(
|
|
x in ua
|
|
for x in (
|
|
"Chrome",
|
|
"Firefox",
|
|
"Safari",
|
|
"Edge",
|
|
"Opera",
|
|
)
|
|
)
|
|
):
|
|
totals["human_ua_vis"] += v
|
|
|
|
if home_v >= best_home:
|
|
best_home = home_v
|
|
best_day = day
|
|
best_z = z
|
|
|
|
print("\n--- 7-day totals ---")
|
|
print(f" all edge requests: {totals['all_req']}")
|
|
print(f" all edge visits: {totals['all_vis']}")
|
|
print(f" homepage visits (/): {totals['home_vis']}")
|
|
print(f" RUM beacons: {totals['rum']} (JS ran = real browser)")
|
|
print(f" HTML content visits: {totals['html_vis']}")
|
|
print(f" obvious bot-path req: {totals['bot_path_req']}")
|
|
|
|
# Best estimate narrative
|
|
rum = totals["rum"]
|
|
home = totals["home_vis"]
|
|
# Conservative human sessions: min(home visits, rum) .. max, with rum as best
|
|
print("\n=== Real-human estimate ===")
|
|
print(f" Best proxy (RUM / JS pageviews): ~{rum}")
|
|
print(f" Homepage edge visits (/): ~{home}")
|
|
print(
|
|
f" Suggested range: ~{min(rum, home) if rum and home else max(rum, home)}"
|
|
f" .. ~{max(rum, home)} unique-ish real opens"
|
|
)
|
|
print(
|
|
" (RUM fires only when a real browser runs JS; scrapers hitting wp-*/assets do not.)"
|
|
)
|
|
|
|
if best_z and best_day:
|
|
print(f"\n--- UA browsers on {best_day} (if available) ---")
|
|
for r in (best_z.get("byUA") or [])[:15]:
|
|
ua = (r["dimensions"] or {}).get("userAgentBrowser")
|
|
v = (r.get("sum") or {}).get("visits") or 0
|
|
print(f" {r['count']:>6} req {v:>5} vis {ua!r}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|