Add Cloudflare traffic helper scripts and commercialization notes.
Track reusable CF analytics utilities and the commercial prospects document while keeping local probe/sample artifacts untracked. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
voson
Cursor
@@ -0,0 +1,386 @@
|
||||
"""Estimate human traffic for dota2.refining.dev (exclude obvious bots).
|
||||
|
||||
Free plan has no BotScore; we use:
|
||||
- HTML document hits (/) vs asset/bot paths
|
||||
- RUM beacons (/cdn-cgi/rum) as JS-executing browsers
|
||||
- UA browser family when available
|
||||
- Country + path heuristics
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import date, timedelta
|
||||
|
||||
HOST = "dota2.refining.dev"
|
||||
ZONE_NAME = "refining.dev"
|
||||
|
||||
|
||||
def headers() -> dict[str, str]:
|
||||
email = os.environ.get("CLOUDFLARE_EMAIL") or os.environ.get(
|
||||
"KEYZOO_ASSET_META_USERNAME"
|
||||
)
|
||||
key = os.environ.get("CLOUDFLARE_API_KEY") or os.environ.get(
|
||||
"KEYZOO_ASSET_SECRET_GLOBAL_API_KEY"
|
||||
)
|
||||
if not email or not key:
|
||||
raise SystemExit("missing credentials in env")
|
||||
return {
|
||||
"X-Auth-Email": email,
|
||||
"X-Auth-Key": key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def api(path: str) -> dict:
|
||||
req = urllib.request.Request(
|
||||
"https://api.cloudflare.com/client/v4" + path,
|
||||
method="GET",
|
||||
headers=headers(),
|
||||
)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read().decode())
|
||||
|
||||
|
||||
def gql(query: str, variables: dict) -> dict:
|
||||
req = urllib.request.Request(
|
||||
"https://api.cloudflare.com/client/v4/graphql",
|
||||
data=json.dumps({"query": query, "variables": variables}).encode(),
|
||||
method="POST",
|
||||
headers=headers(),
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read().decode())
|
||||
except urllib.error.HTTPError as e:
|
||||
return {"errors": [{"message": e.read().decode()[:1500]}]}
|
||||
|
||||
|
||||
def try_query(label: str, query: str, variables: dict) -> dict | None:
|
||||
data = gql(query, variables)
|
||||
if data.get("errors"):
|
||||
msg = data["errors"][0].get("message", "")[:200]
|
||||
print(f" [{label}] unavailable: {msg}")
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def main() -> None:
|
||||
zid = api(f"/zones?name={ZONE_NAME}")["result"][0]["id"]
|
||||
days = [date.today() - timedelta(days=i) for i in range(7, -1, -1)]
|
||||
|
||||
print(f"\n{HOST} human-traffic estimate (Free plan, no BotScore)\n")
|
||||
|
||||
# --- probe schema fields once ---
|
||||
day0 = days[-2] if len(days) > 1 else days[-1] # prefer yesterday
|
||||
probes = [
|
||||
(
|
||||
"UA browser",
|
||||
"""
|
||||
query($zoneTag:string!,$day:Date!,$host:string!){
|
||||
viewer{zones(filter:{zoneTag:$zoneTag}){
|
||||
g:httpRequestsAdaptiveGroups(
|
||||
limit:20
|
||||
filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
orderBy:[count_DESC]
|
||||
){dimensions{userAgentBrowser} count sum{visits}}
|
||||
}}
|
||||
}
|
||||
""",
|
||||
),
|
||||
(
|
||||
"device type",
|
||||
"""
|
||||
query($zoneTag:string!,$day:Date!,$host:string!){
|
||||
viewer{zones(filter:{zoneTag:$zoneTag}){
|
||||
g:httpRequestsAdaptiveGroups(
|
||||
limit:10
|
||||
filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
orderBy:[count_DESC]
|
||||
){dimensions{clientDeviceType} count sum{visits}}
|
||||
}}
|
||||
}
|
||||
""",
|
||||
),
|
||||
(
|
||||
"botScore",
|
||||
"""
|
||||
query($zoneTag:string!,$day:Date!,$host:string!){
|
||||
viewer{zones(filter:{zoneTag:$zoneTag}){
|
||||
g:httpRequestsAdaptiveGroups(
|
||||
limit:10
|
||||
filter:{date:$day,clientRequestHTTPHost:$host,botScore_geq:30}
|
||||
orderBy:[count_DESC]
|
||||
){count sum{visits} dimensions{botScore}}
|
||||
}}
|
||||
}
|
||||
""",
|
||||
),
|
||||
(
|
||||
"content type",
|
||||
"""
|
||||
query($zoneTag:string!,$day:Date!,$host:string!){
|
||||
viewer{zones(filter:{zoneTag:$zoneTag}){
|
||||
g:httpRequestsAdaptiveGroups(
|
||||
limit:15
|
||||
filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
orderBy:[count_DESC]
|
||||
){dimensions{edgeResponseContentTypeName} count sum{visits}}
|
||||
}}
|
||||
}
|
||||
""",
|
||||
),
|
||||
]
|
||||
print(f"Schema probes on {day0}:")
|
||||
available = {}
|
||||
for label, q in probes:
|
||||
data = try_query(
|
||||
label,
|
||||
q,
|
||||
{"zoneTag": zid, "day": day0.isoformat(), "host": HOST},
|
||||
)
|
||||
if data:
|
||||
available[label] = data["data"]["viewer"]["zones"][0]["g"]
|
||||
print(f" [{label}] OK")
|
||||
|
||||
for label, rows in available.items():
|
||||
print(f"\n--- {label} ({day0}) ---")
|
||||
for r in rows[:12]:
|
||||
dims = r.get("dimensions") or {}
|
||||
key = next(iter(dims.values()), "?") if dims else "?"
|
||||
v = (r.get("sum") or {}).get("visits") or 0
|
||||
print(f" {r['count']:>6} req {v:>5} vis {key}")
|
||||
|
||||
# --- per-day: total / homepage / rum / likely-bot paths ---
|
||||
day_q = """
|
||||
query($zoneTag:string!,$day:Date!,$host:string!){
|
||||
viewer{zones(filter:{zoneTag:$zoneTag}){
|
||||
all:httpRequestsAdaptiveGroups(
|
||||
limit:1
|
||||
filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
){count sum{visits}}
|
||||
home:httpRequestsAdaptiveGroups(
|
||||
limit:1
|
||||
filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/"}
|
||||
){count sum{visits}}
|
||||
rum:httpRequestsAdaptiveGroups(
|
||||
limit:1
|
||||
filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/cdn-cgi/rum"}
|
||||
){count sum{visits}}
|
||||
html:httpRequestsAdaptiveGroups(
|
||||
limit:1
|
||||
filter:{
|
||||
date:$day
|
||||
clientRequestHTTPHost:$host
|
||||
edgeResponseContentTypeName:"text/html"
|
||||
}
|
||||
){count sum{visits}}
|
||||
byPath:httpRequestsAdaptiveGroups(
|
||||
limit:40
|
||||
filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
orderBy:[count_DESC]
|
||||
){dimensions{clientRequestPath} count sum{visits}}
|
||||
byUA:httpRequestsAdaptiveGroups(
|
||||
limit:25
|
||||
filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
orderBy:[count_DESC]
|
||||
){dimensions{userAgentBrowser} count sum{visits}}
|
||||
}}
|
||||
}
|
||||
"""
|
||||
|
||||
BOT_PATH_MARKERS = (
|
||||
"wp-",
|
||||
"wordpress",
|
||||
"xmlrpc",
|
||||
"wlwmanifest",
|
||||
".env",
|
||||
"phpmyadmin",
|
||||
"robots.txt",
|
||||
"sitemap",
|
||||
"favicon.ico", # often bots; keep separate
|
||||
)
|
||||
HUMAN_UA = {
|
||||
"Chrome",
|
||||
"Firefox",
|
||||
"Safari",
|
||||
"Edge",
|
||||
"Opera",
|
||||
"Samsung Internet",
|
||||
"Mobile Safari",
|
||||
"Chrome Mobile",
|
||||
"Firefox Mobile",
|
||||
"Edg",
|
||||
"Chromium",
|
||||
}
|
||||
BOT_UA = {
|
||||
"Bot",
|
||||
"Spider",
|
||||
"Crawler",
|
||||
"curl",
|
||||
"Go-http-client",
|
||||
"python",
|
||||
"Python",
|
||||
"Scrapy",
|
||||
"Headless",
|
||||
"Empty",
|
||||
"Unknown",
|
||||
"",
|
||||
None,
|
||||
}
|
||||
|
||||
print(f"\n{'date':<12} {'all_req':>8} {'all_vis':>8} {'home_vis':>8} {'rum':>6} {'html_vis':>8}")
|
||||
totals = {
|
||||
"all_req": 0,
|
||||
"all_vis": 0,
|
||||
"home_vis": 0,
|
||||
"rum": 0,
|
||||
"html_vis": 0,
|
||||
"human_ua_vis": 0,
|
||||
"bot_path_req": 0,
|
||||
}
|
||||
best_day = None
|
||||
best_z = None
|
||||
best_home = 0
|
||||
|
||||
for day in days:
|
||||
data = try_query(
|
||||
day.isoformat(),
|
||||
day_q,
|
||||
{"zoneTag": zid, "day": day.isoformat(), "host": HOST},
|
||||
)
|
||||
if not data:
|
||||
# retry without html / UA if fields fail
|
||||
day_q_min = """
|
||||
query($zoneTag:string!,$day:Date!,$host:string!){
|
||||
viewer{zones(filter:{zoneTag:$zoneTag}){
|
||||
all:httpRequestsAdaptiveGroups(
|
||||
limit:1 filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
){count sum{visits}}
|
||||
home:httpRequestsAdaptiveGroups(
|
||||
limit:1
|
||||
filter:{date:$day,clientRequestHTTPHost:$host,clientRequestPath:"/"}
|
||||
){count sum{visits}}
|
||||
rum:httpRequestsAdaptiveGroups(
|
||||
limit:1
|
||||
filter:{
|
||||
date:$day
|
||||
clientRequestHTTPHost:$host
|
||||
clientRequestPath:"/cdn-cgi/rum"
|
||||
}
|
||||
){count sum{visits}}
|
||||
byPath:httpRequestsAdaptiveGroups(
|
||||
limit:40
|
||||
filter:{date:$day,clientRequestHTTPHost:$host}
|
||||
orderBy:[count_DESC]
|
||||
){dimensions{clientRequestPath} count sum{visits}}
|
||||
}}
|
||||
}
|
||||
"""
|
||||
data = try_query(
|
||||
day.isoformat() + "-min",
|
||||
day_q_min,
|
||||
{"zoneTag": zid, "day": day.isoformat(), "host": HOST},
|
||||
)
|
||||
if not data:
|
||||
continue
|
||||
z = data["data"]["viewer"]["zones"][0]
|
||||
|
||||
def one(name: str) -> tuple[int, int]:
|
||||
rows = z.get(name) or []
|
||||
if not rows:
|
||||
return 0, 0
|
||||
r = rows[0]
|
||||
return r.get("count") or 0, (r.get("sum") or {}).get("visits") or 0
|
||||
|
||||
all_c, all_v = one("all")
|
||||
home_c, home_v = one("home")
|
||||
rum_c, _ = one("rum")
|
||||
html_c, html_v = one("html") if "html" in z else (0, 0)
|
||||
|
||||
print(
|
||||
f"{day.isoformat():<12} {all_c:>8} {all_v:>8} {home_v:>8} {rum_c:>6} {html_v:>8}"
|
||||
)
|
||||
totals["all_req"] += all_c
|
||||
totals["all_vis"] += all_v
|
||||
totals["home_vis"] += home_v
|
||||
totals["rum"] += rum_c
|
||||
totals["html_vis"] += html_v
|
||||
|
||||
# bot-looking paths
|
||||
for r in z.get("byPath") or []:
|
||||
p = (r["dimensions"] or {}).get("clientRequestPath") or ""
|
||||
if any(m in p.lower() for m in BOT_PATH_MARKERS):
|
||||
totals["bot_path_req"] += r["count"] or 0
|
||||
|
||||
# UA sum visits for human browsers
|
||||
for r in z.get("byUA") or []:
|
||||
ua = (r["dimensions"] or {}).get("userAgentBrowser")
|
||||
v = (r.get("sum") or {}).get("visits") or 0
|
||||
if ua in HUMAN_UA or (
|
||||
ua
|
||||
and not any(
|
||||
str(b).lower() in str(ua).lower()
|
||||
for b in BOT_UA
|
||||
if b
|
||||
)
|
||||
and ua
|
||||
not in ("", None)
|
||||
):
|
||||
# count known browsers only
|
||||
if ua in HUMAN_UA or (
|
||||
isinstance(ua, str)
|
||||
and any(
|
||||
x in ua
|
||||
for x in (
|
||||
"Chrome",
|
||||
"Firefox",
|
||||
"Safari",
|
||||
"Edge",
|
||||
"Opera",
|
||||
)
|
||||
)
|
||||
):
|
||||
totals["human_ua_vis"] += v
|
||||
|
||||
if home_v >= best_home:
|
||||
best_home = home_v
|
||||
best_day = day
|
||||
best_z = z
|
||||
|
||||
print("\n--- 7-day totals ---")
|
||||
print(f" all edge requests: {totals['all_req']}")
|
||||
print(f" all edge visits: {totals['all_vis']}")
|
||||
print(f" homepage visits (/): {totals['home_vis']}")
|
||||
print(f" RUM beacons: {totals['rum']} (JS ran = real browser)")
|
||||
print(f" HTML content visits: {totals['html_vis']}")
|
||||
print(f" obvious bot-path req: {totals['bot_path_req']}")
|
||||
|
||||
# Best estimate narrative
|
||||
rum = totals["rum"]
|
||||
home = totals["home_vis"]
|
||||
# Conservative human sessions: min(home visits, rum) .. max, with rum as best
|
||||
print("\n=== Real-human estimate ===")
|
||||
print(f" Best proxy (RUM / JS pageviews): ~{rum}")
|
||||
print(f" Homepage edge visits (/): ~{home}")
|
||||
print(
|
||||
f" Suggested range: ~{min(rum, home) if rum and home else max(rum, home)}"
|
||||
f" .. ~{max(rum, home)} unique-ish real opens"
|
||||
)
|
||||
print(
|
||||
" (RUM fires only when a real browser runs JS; scrapers hitting wp-*/assets do not.)"
|
||||
)
|
||||
|
||||
if best_z and best_day:
|
||||
print(f"\n--- UA browsers on {best_day} (if available) ---")
|
||||
for r in (best_z.get("byUA") or [])[:15]:
|
||||
ua = (r["dimensions"] or {}).get("userAgentBrowser")
|
||||
v = (r.get("sum") or {}).get("visits") or 0
|
||||
print(f" {r['count']:>6} req {v:>5} vis {ua!r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user