Reorganize repository into pc web shared monorepo
Separate the local recognition, web publishing, and shared data paths while preserving direct script execution and existing site content. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
voson
Cursor
@@ -0,0 +1,345 @@
|
||||
"""Build data/item_shop.json from the official CN shop layout.
|
||||
|
||||
Primary source: https://www.dota2.com.cn/itemscategory/json (basic / upgrade columns).
|
||||
Names / cost / components: OpenDota items.json + Valve itemlist (schinese).
|
||||
Icons → assets/item_icons/; category icons → assets/item_cat_icons/.
|
||||
|
||||
Usage:
|
||||
python fetch_item_shop.py
|
||||
python fetch_item_shop.py --skip-icons
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from shared.http_utils import download_icons, http_bytes, http_json, load_itemlist_zh
|
||||
from shared.paths import DATA, ITEM_CAT_ICONS, ITEM_ICONS
|
||||
|
||||
ITEMS_URL = (
|
||||
"https://raw.githubusercontent.com/odota/dotaconstants/master/build/items.json"
|
||||
)
|
||||
CN_CATEGORY_URL = "https://www.dota2.com.cn/itemscategory/json"
|
||||
ICON_URL = (
|
||||
"https://cdn.cloudflare.steamstatic.com/apps/dota2/images/dota_react/items/{key}.png"
|
||||
)
|
||||
CAT_ICON_BASE = "https://www.dota2.com.cn/items/images/"
|
||||
OUT = DATA / "item_shop.json"
|
||||
|
||||
# Official column label → category icon filename (from items_new.js).
|
||||
CAT_ICON_BY_LABEL = {
|
||||
"消耗品": "itemcat_consumables.png",
|
||||
"属性": "itemcat_attributes.png",
|
||||
"装备": "itemcat_armaments.png",
|
||||
"其它": "itemcat_arcane.png",
|
||||
"其他": "itemcat_arcane.png",
|
||||
"神秘商店": "itemcat_secret.png",
|
||||
"配件": "itemcat_common.png",
|
||||
"辅助": "itemcat_support.png",
|
||||
"法器": "itemcat_caster.png",
|
||||
"防具": "itemcat_armor.png",
|
||||
"兵刃": "itemcat_weapons.png",
|
||||
"宝物": "itemcat_artifacts.png",
|
||||
"军备": "itemcat_artifacts.png",
|
||||
}
|
||||
|
||||
# Stable section ids for UI / lookups.
|
||||
SECTION_ID_BY_LABEL = {
|
||||
"消耗品": "consumables",
|
||||
"属性": "attributes",
|
||||
"装备": "equipment",
|
||||
"其它": "misc",
|
||||
"其他": "misc",
|
||||
"神秘商店": "secretshop",
|
||||
"配件": "basics",
|
||||
"辅助": "support",
|
||||
"法器": "magics",
|
||||
"防具": "defense",
|
||||
"兵刃": "weapons",
|
||||
"宝物": "artifacts",
|
||||
"军备": "artifacts",
|
||||
}
|
||||
|
||||
|
||||
def slug_section(label: str, fallback: str) -> str:
|
||||
sid = SECTION_ID_BY_LABEL.get(label)
|
||||
if sid:
|
||||
return sid
|
||||
s = re.sub(r"[^a-z0-9]+", "_", fallback.lower()).strip("_")
|
||||
return s or "section"
|
||||
|
||||
|
||||
def make_item_row(key: str, row: dict, zh: dict[int, str], *, is_recipe: bool = False) -> dict:
|
||||
iid = row.get("id")
|
||||
try:
|
||||
cost = int(row.get("cost") or 0)
|
||||
except (TypeError, ValueError):
|
||||
cost = 0
|
||||
name_loc = zh.get(int(iid)) if iid is not None else None
|
||||
if not name_loc:
|
||||
name_loc = "卷轴" if is_recipe else str(row.get("dname") or key)
|
||||
return {
|
||||
"key": key,
|
||||
"id": int(iid) if iid is not None else None,
|
||||
"dname": str(row.get("dname") or key),
|
||||
"name_loc": name_loc,
|
||||
"cost": cost,
|
||||
"created": bool(row.get("created")),
|
||||
"qual": row.get("qual"),
|
||||
"is_recipe": is_recipe,
|
||||
}
|
||||
|
||||
|
||||
def download_shop_icons(icon_keys: set[str], *, force: bool = False) -> tuple[int, int, int]:
|
||||
"""Download item icons (recipe handled separately; recipe_* keys skipped)."""
|
||||
# recipe.png is the shared icon for all recipe_* items.
|
||||
recipe_dest = ITEM_ICONS / "recipe.png"
|
||||
recipe_fail = 0
|
||||
if force or not recipe_dest.is_file():
|
||||
try:
|
||||
recipe_dest.write_bytes(http_bytes(ICON_URL.format(key="recipe")))
|
||||
except (urllib.error.URLError, TimeoutError, ValueError, OSError) as e:
|
||||
print(f" icon fail recipe: {e}", flush=True)
|
||||
recipe_fail = 1
|
||||
real_keys = {k for k in icon_keys if not k.startswith("recipe_")}
|
||||
saved, skipped, fail = download_icons(
|
||||
real_keys, ICON_URL, ITEM_ICONS, force=force, delay=0.02
|
||||
)
|
||||
return saved, skipped, fail + recipe_fail
|
||||
|
||||
|
||||
def parse_cn_sections(raw_list: list, kind: str) -> list[dict]:
|
||||
"""Convert CN basic/upgrade arrays into sections; reverse items like the site JS."""
|
||||
sections: list[dict] = []
|
||||
for i, row in enumerate(raw_list or []):
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
label = str(row.get("name") or "").strip()
|
||||
if not label:
|
||||
continue
|
||||
keys: list[str] = []
|
||||
for entry in row.get("items") or []:
|
||||
if isinstance(entry, dict):
|
||||
name = entry.get("name")
|
||||
else:
|
||||
name = entry
|
||||
if name:
|
||||
keys.append(str(name))
|
||||
# Official site reverses each column for display.
|
||||
keys.reverse()
|
||||
sid = slug_section(label, f"{kind}_{i}")
|
||||
icon = CAT_ICON_BY_LABEL.get(label, "")
|
||||
sections.append(
|
||||
{
|
||||
"id": sid,
|
||||
"label": label,
|
||||
"icon": icon,
|
||||
"items": keys,
|
||||
}
|
||||
)
|
||||
return sections
|
||||
|
||||
|
||||
def attach_craft_graph(
|
||||
items_out: dict[str, dict],
|
||||
od: dict,
|
||||
catalog: dict[str, dict],
|
||||
zh: dict[int, str],
|
||||
shop_keys: set[str],
|
||||
) -> set[str]:
|
||||
builds_into: dict[str, list[str]] = {}
|
||||
for key, row in od.items():
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
key_s = str(key)
|
||||
if key_s.startswith("recipe_"):
|
||||
continue
|
||||
if row.get("tier") is not None:
|
||||
continue
|
||||
for raw in row.get("components") or []:
|
||||
if not raw:
|
||||
continue
|
||||
builds_into.setdefault(str(raw), []).append(key_s)
|
||||
|
||||
extra_keys: set[str] = set()
|
||||
for key, row in list(items_out.items()):
|
||||
od_row = od.get(key) if isinstance(od.get(key), dict) else {}
|
||||
comps = [str(c) for c in (od_row.get("components") or []) if c]
|
||||
# OpenDota often omits recipe_* from components; re-attach when present.
|
||||
has_recipe = any(c.startswith("recipe_") for c in comps)
|
||||
recipe_key = f"recipe_{key}"
|
||||
if not has_recipe and bool(od_row.get("created")):
|
||||
recipe_row = od.get(recipe_key)
|
||||
if isinstance(recipe_row, dict):
|
||||
try:
|
||||
rcost = int(recipe_row.get("cost") or 0)
|
||||
except (TypeError, ValueError):
|
||||
rcost = 0
|
||||
if rcost > 0:
|
||||
comps.append(recipe_key)
|
||||
else:
|
||||
# Fallback: gold gap between total and plain components.
|
||||
try:
|
||||
total = int(od_row.get("cost") or 0)
|
||||
except (TypeError, ValueError):
|
||||
total = 0
|
||||
part = 0
|
||||
ok = True
|
||||
for c in comps:
|
||||
crow = od.get(c) if isinstance(od.get(c), dict) else None
|
||||
if not crow:
|
||||
ok = False
|
||||
break
|
||||
try:
|
||||
part += int(crow.get("cost") or 0)
|
||||
except (TypeError, ValueError):
|
||||
ok = False
|
||||
break
|
||||
gap = total - part if ok else 0
|
||||
if gap > 0:
|
||||
comps.append(recipe_key)
|
||||
# Synthetic stub so UI can show cost even without odota recipe row.
|
||||
if recipe_key not in od:
|
||||
od[recipe_key] = {
|
||||
"id": None,
|
||||
"dname": f"{od_row.get('dname') or key} Recipe",
|
||||
"cost": gap,
|
||||
"components": None,
|
||||
"created": False,
|
||||
}
|
||||
row["components"] = comps
|
||||
ups = [u for u in builds_into.get(key, []) if u in shop_keys]
|
||||
ups.sort(key=lambda u: (items_out.get(u, {}).get("cost") or 0, u))
|
||||
row["builds_into"] = ups
|
||||
extra_keys.update(comps)
|
||||
|
||||
for key in sorted(extra_keys):
|
||||
if key in items_out:
|
||||
continue
|
||||
od_row = od.get(key)
|
||||
if not isinstance(od_row, dict):
|
||||
continue
|
||||
if key.startswith("recipe_"):
|
||||
items_out[key] = make_item_row(key, od_row, zh, is_recipe=True)
|
||||
elif key in catalog:
|
||||
stub = dict(catalog[key])
|
||||
stub.setdefault("components", [])
|
||||
stub.setdefault("builds_into", [])
|
||||
items_out[key] = stub
|
||||
else:
|
||||
stub = make_item_row(key, od_row, zh)
|
||||
stub["components"] = []
|
||||
stub["builds_into"] = [u for u in builds_into.get(key, []) if u in shop_keys]
|
||||
items_out[key] = stub
|
||||
return extra_keys
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--out", type=Path, default=OUT)
|
||||
ap.add_argument("--skip-icons", action="store_true")
|
||||
ap.add_argument("--force-icons", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("loading CN shop categories + OpenDota + Valve names...", flush=True)
|
||||
cn = http_json(CN_CATEGORY_URL)
|
||||
result = (cn or {}).get("result") or {}
|
||||
basic_sections = parse_cn_sections(result.get("basic") or [], "basic")
|
||||
up_sections = parse_cn_sections(result.get("upgrade") or [], "upgraded")
|
||||
if not basic_sections and not up_sections:
|
||||
raise SystemExit("CN itemscategory/json returned empty basic/upgrade")
|
||||
|
||||
od = http_json(ITEMS_URL)
|
||||
zh = load_itemlist_zh()
|
||||
catalog: dict[str, dict] = {}
|
||||
for key, row in od.items():
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
key_s = str(key)
|
||||
if key_s.startswith("recipe_"):
|
||||
continue
|
||||
catalog[key_s] = make_item_row(key_s, row, zh)
|
||||
|
||||
shop_keys: set[str] = set()
|
||||
items_out: dict[str, dict] = {}
|
||||
missing: list[str] = []
|
||||
cat_files: set[str] = set()
|
||||
|
||||
for kind, sections in (("basic", basic_sections), ("upgraded", up_sections)):
|
||||
for sec in sections:
|
||||
if sec.get("icon"):
|
||||
cat_files.add(sec["icon"])
|
||||
kept: list[str] = []
|
||||
for key in sec["items"]:
|
||||
if key not in catalog:
|
||||
missing.append(key)
|
||||
continue
|
||||
kept.append(key)
|
||||
shop_keys.add(key)
|
||||
if key not in items_out:
|
||||
row = dict(catalog[key])
|
||||
row["shop_kind"] = kind
|
||||
row["section"] = sec["id"]
|
||||
row["section_label"] = sec["label"]
|
||||
items_out[key] = row
|
||||
sec["items"] = kept
|
||||
|
||||
if missing:
|
||||
print(
|
||||
f" warn missing in OpenDota ({len(missing)}): "
|
||||
f"{', '.join(missing[:24])}{'…' if len(missing) > 24 else ''}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
craft_extra = attach_craft_graph(items_out, od, catalog, zh, shop_keys)
|
||||
icon_keys = set(shop_keys) | {k for k in craft_extra if not k.startswith("recipe_")}
|
||||
|
||||
payload = {
|
||||
"meta": {
|
||||
"source": "dota2.com.cn/itemscategory+opendota+valve",
|
||||
"layout": "cn_shop_columns",
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
"basic_count": sum(len(s["items"]) for s in basic_sections),
|
||||
"upgraded_count": sum(len(s["items"]) for s in up_sections),
|
||||
"craft_refs": len(craft_extra),
|
||||
},
|
||||
"basic": {"sections": basic_sections},
|
||||
"upgraded": {"sections": up_sections},
|
||||
"items": items_out,
|
||||
}
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
f"wrote {args.out}: basic={payload['meta']['basic_count']} "
|
||||
f"upgraded={payload['meta']['upgraded_count']} "
|
||||
f"cols={len(basic_sections)+len(up_sections)} "
|
||||
f"craft_refs={len(craft_extra)}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not args.skip_icons:
|
||||
print(f"downloading {len(icon_keys)} item icons (+ recipe)...", flush=True)
|
||||
saved, skipped, fail = download_shop_icons(icon_keys, force=args.force_icons)
|
||||
print(f" item icons saved={saved} skipped={skipped} fail={fail}", flush=True)
|
||||
print(f"downloading {len(cat_files)} category icons...", flush=True)
|
||||
c_saved, c_skipped, c_fail = download_icons(
|
||||
cat_files, CAT_ICON_BASE + "{key}", ITEM_CAT_ICONS, force=args.force_icons, delay=0.05
|
||||
)
|
||||
print(f" cat icons saved={c_saved} skipped={c_skipped} fail={c_fail}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user