"""Build data/item_shop.json from the official CN shop layout. Primary source: https://www.dota2.com.cn/itemscategory/json (basic / upgrade columns). Names / cost / components: OpenDota items.json + Valve itemlist (schinese). Icons → assets/item_icons/; category icons → assets/item_cat_icons/. Usage: python fetch_item_shop.py python fetch_item_shop.py --skip-icons """ from __future__ import annotations import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import argparse import json import re import time import urllib.error from datetime import datetime, timezone from shared.http_utils import download_icons, http_bytes, http_json, load_itemlist_zh from shared.paths import DATA, ITEM_CAT_ICONS, ITEM_ICONS ITEMS_URL = ( "https://raw.githubusercontent.com/odota/dotaconstants/master/build/items.json" ) CN_CATEGORY_URL = "https://www.dota2.com.cn/itemscategory/json" ICON_URL = ( "https://cdn.cloudflare.steamstatic.com/apps/dota2/images/dota_react/items/{key}.png" ) CAT_ICON_BASE = "https://www.dota2.com.cn/items/images/" OUT = DATA / "item_shop.json" # Official column label → category icon filename (from items_new.js). CAT_ICON_BY_LABEL = { "消耗品": "itemcat_consumables.png", "属性": "itemcat_attributes.png", "装备": "itemcat_armaments.png", "其它": "itemcat_arcane.png", "其他": "itemcat_arcane.png", "神秘商店": "itemcat_secret.png", "配件": "itemcat_common.png", "辅助": "itemcat_support.png", "法器": "itemcat_caster.png", "防具": "itemcat_armor.png", "兵刃": "itemcat_weapons.png", "宝物": "itemcat_artifacts.png", "军备": "itemcat_artifacts.png", } # Stable section ids for UI / lookups. SECTION_ID_BY_LABEL = { "消耗品": "consumables", "属性": "attributes", "装备": "equipment", "其它": "misc", "其他": "misc", "神秘商店": "secretshop", "配件": "basics", "辅助": "support", "法器": "magics", "防具": "defense", "兵刃": "weapons", "宝物": "artifacts", "军备": "artifacts", } def slug_section(label: str, fallback: str) -> str: sid = SECTION_ID_BY_LABEL.get(label) if sid: return sid s = re.sub(r"[^a-z0-9]+", "_", fallback.lower()).strip("_") return s or "section" def make_item_row(key: str, row: dict, zh: dict[int, str], *, is_recipe: bool = False) -> dict: iid = row.get("id") try: cost = int(row.get("cost") or 0) except (TypeError, ValueError): cost = 0 name_loc = zh.get(int(iid)) if iid is not None else None if not name_loc: name_loc = "卷轴" if is_recipe else str(row.get("dname") or key) return { "key": key, "id": int(iid) if iid is not None else None, "dname": str(row.get("dname") or key), "name_loc": name_loc, "cost": cost, "created": bool(row.get("created")), "qual": row.get("qual"), "is_recipe": is_recipe, } def download_shop_icons(icon_keys: set[str], *, force: bool = False) -> tuple[int, int, int]: """Download item icons (recipe handled separately; recipe_* keys skipped).""" # recipe.png is the shared icon for all recipe_* items. recipe_dest = ITEM_ICONS / "recipe.png" recipe_fail = 0 if force or not recipe_dest.is_file(): try: recipe_dest.write_bytes(http_bytes(ICON_URL.format(key="recipe"))) except (urllib.error.URLError, TimeoutError, ValueError, OSError) as e: print(f" icon fail recipe: {e}", flush=True) recipe_fail = 1 real_keys = {k for k in icon_keys if not k.startswith("recipe_")} saved, skipped, fail = download_icons( real_keys, ICON_URL, ITEM_ICONS, force=force, delay=0.02 ) return saved, skipped, fail + recipe_fail def parse_cn_sections(raw_list: list, kind: str) -> list[dict]: """Convert CN basic/upgrade arrays into sections; reverse items like the site JS.""" sections: list[dict] = [] for i, row in enumerate(raw_list or []): if not isinstance(row, dict): continue label = str(row.get("name") or "").strip() if not label: continue keys: list[str] = [] for entry in row.get("items") or []: if isinstance(entry, dict): name = entry.get("name") else: name = entry if name: keys.append(str(name)) # Official site reverses each column for display. keys.reverse() sid = slug_section(label, f"{kind}_{i}") icon = CAT_ICON_BY_LABEL.get(label, "") sections.append( { "id": sid, "label": label, "icon": icon, "items": keys, } ) return sections def attach_craft_graph( items_out: dict[str, dict], od: dict, catalog: dict[str, dict], zh: dict[int, str], shop_keys: set[str], ) -> set[str]: builds_into: dict[str, list[str]] = {} for key, row in od.items(): if not isinstance(row, dict): continue key_s = str(key) if key_s.startswith("recipe_"): continue if row.get("tier") is not None: continue for raw in row.get("components") or []: if not raw: continue builds_into.setdefault(str(raw), []).append(key_s) extra_keys: set[str] = set() for key, row in list(items_out.items()): od_row = od.get(key) if isinstance(od.get(key), dict) else {} comps = [str(c) for c in (od_row.get("components") or []) if c] # OpenDota often omits recipe_* from components; re-attach when present. has_recipe = any(c.startswith("recipe_") for c in comps) recipe_key = f"recipe_{key}" if not has_recipe and bool(od_row.get("created")): recipe_row = od.get(recipe_key) if isinstance(recipe_row, dict): try: rcost = int(recipe_row.get("cost") or 0) except (TypeError, ValueError): rcost = 0 if rcost > 0: comps.append(recipe_key) else: # Fallback: gold gap between total and plain components. try: total = int(od_row.get("cost") or 0) except (TypeError, ValueError): total = 0 part = 0 ok = True for c in comps: crow = od.get(c) if isinstance(od.get(c), dict) else None if not crow: ok = False break try: part += int(crow.get("cost") or 0) except (TypeError, ValueError): ok = False break gap = total - part if ok else 0 if gap > 0: comps.append(recipe_key) # Synthetic stub so UI can show cost even without odota recipe row. if recipe_key not in od: od[recipe_key] = { "id": None, "dname": f"{od_row.get('dname') or key} Recipe", "cost": gap, "components": None, "created": False, } row["components"] = comps ups = [u for u in builds_into.get(key, []) if u in shop_keys] ups.sort(key=lambda u: (items_out.get(u, {}).get("cost") or 0, u)) row["builds_into"] = ups extra_keys.update(comps) for key in sorted(extra_keys): if key in items_out: continue od_row = od.get(key) if not isinstance(od_row, dict): continue if key.startswith("recipe_"): items_out[key] = make_item_row(key, od_row, zh, is_recipe=True) elif key in catalog: stub = dict(catalog[key]) stub.setdefault("components", []) stub.setdefault("builds_into", []) items_out[key] = stub else: stub = make_item_row(key, od_row, zh) stub["components"] = [] stub["builds_into"] = [u for u in builds_into.get(key, []) if u in shop_keys] items_out[key] = stub return extra_keys def main() -> None: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--out", type=Path, default=OUT) ap.add_argument("--skip-icons", action="store_true") ap.add_argument("--force-icons", action="store_true") args = ap.parse_args() print("loading CN shop categories + OpenDota + Valve names...", flush=True) cn = http_json(CN_CATEGORY_URL) result = (cn or {}).get("result") or {} basic_sections = parse_cn_sections(result.get("basic") or [], "basic") up_sections = parse_cn_sections(result.get("upgrade") or [], "upgraded") if not basic_sections and not up_sections: raise SystemExit("CN itemscategory/json returned empty basic/upgrade") od = http_json(ITEMS_URL) zh = load_itemlist_zh() catalog: dict[str, dict] = {} for key, row in od.items(): if not isinstance(row, dict): continue key_s = str(key) if key_s.startswith("recipe_"): continue catalog[key_s] = make_item_row(key_s, row, zh) shop_keys: set[str] = set() items_out: dict[str, dict] = {} missing: list[str] = [] cat_files: set[str] = set() for kind, sections in (("basic", basic_sections), ("upgraded", up_sections)): for sec in sections: if sec.get("icon"): cat_files.add(sec["icon"]) kept: list[str] = [] for key in sec["items"]: if key not in catalog: missing.append(key) continue kept.append(key) shop_keys.add(key) if key not in items_out: row = dict(catalog[key]) row["shop_kind"] = kind row["section"] = sec["id"] row["section_label"] = sec["label"] items_out[key] = row sec["items"] = kept if missing: print( f" warn missing in OpenDota ({len(missing)}): " f"{', '.join(missing[:24])}{'…' if len(missing) > 24 else ''}", flush=True, ) craft_extra = attach_craft_graph(items_out, od, catalog, zh, shop_keys) icon_keys = set(shop_keys) | {k for k in craft_extra if not k.startswith("recipe_")} payload = { "meta": { "source": "dota2.com.cn/itemscategory+opendota+valve", "layout": "cn_shop_columns", "fetched_at": datetime.now(timezone.utc).isoformat(), "basic_count": sum(len(s["items"]) for s in basic_sections), "upgraded_count": sum(len(s["items"]) for s in up_sections), "craft_refs": len(craft_extra), }, "basic": {"sections": basic_sections}, "upgraded": {"sections": up_sections}, "items": items_out, } args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text( json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) print( f"wrote {args.out}: basic={payload['meta']['basic_count']} " f"upgraded={payload['meta']['upgraded_count']} " f"cols={len(basic_sections)+len(up_sections)} " f"craft_refs={len(craft_extra)}", flush=True, ) if not args.skip_icons: print(f"downloading {len(icon_keys)} item icons (+ recipe)...", flush=True) saved, skipped, fail = download_shop_icons(icon_keys, force=args.force_icons) print(f" item icons saved={saved} skipped={skipped} fail={fail}", flush=True) print(f"downloading {len(cat_files)} category icons...", flush=True) c_saved, c_skipped, c_fail = download_icons( cat_files, CAT_ICON_BASE + "{key}", ITEM_CAT_ICONS, force=args.force_icons, delay=0.05 ) print(f" cat icons saved={c_saved} skipped={c_skipped} fail={c_fail}", flush=True) if __name__ == "__main__": main()