climperor/web/fetch_item_shop.py

"""Build data/item_shop.json from the official CN shop layout.

Primary source: https://www.dota2.com.cn/itemscategory/json (basic / upgrade columns).
Names / cost / components: OpenDota items.json + Valve itemlist (schinese).
Icons → assets/item_icons/; category icons → assets/item_cat_icons/.

Usage:
  python fetch_item_shop.py
  python fetch_item_shop.py --skip-icons
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import argparse
import json
import re
import time
import urllib.error
from datetime import datetime, timezone

from shared.http_utils import (
    download_icons,
    http_bytes,
    http_json,
    load_itemlist_zh,
    write_json_atomic,
)
from shared.paths import DATA, ITEM_CAT_ICONS, ITEM_ICONS

ITEMS_URL = (
    "https://raw.githubusercontent.com/odota/dotaconstants/master/build/items.json"
)
CN_CATEGORY_URL = "https://www.dota2.com.cn/itemscategory/json"
ICON_URL = (
    "https://cdn.cloudflare.steamstatic.com/apps/dota2/images/dota_react/items/{key}.png"
)
CAT_ICON_BASE = "https://www.dota2.com.cn/items/images/"
OUT = DATA / "item_shop.json"

# Official column label → category icon filename (from items_new.js).
CAT_ICON_BY_LABEL = {
    "消耗品": "itemcat_consumables.png",
    "属性": "itemcat_attributes.png",
    "装备": "itemcat_armaments.png",
    "其它": "itemcat_arcane.png",
    "其他": "itemcat_arcane.png",
    "神秘商店": "itemcat_secret.png",
    "配件": "itemcat_common.png",
    "辅助": "itemcat_support.png",
    "法器": "itemcat_caster.png",
    "防具": "itemcat_armor.png",
    "兵刃": "itemcat_weapons.png",
    "宝物": "itemcat_artifacts.png",
    "军备": "itemcat_artifacts.png",
}

# Stable section ids for UI / lookups.
SECTION_ID_BY_LABEL = {
    "消耗品": "consumables",
    "属性": "attributes",
    "装备": "equipment",
    "其它": "misc",
    "其他": "misc",
    "神秘商店": "secretshop",
    "配件": "basics",
    "辅助": "support",
    "法器": "magics",
    "防具": "defense",
    "兵刃": "weapons",
    "宝物": "artifacts",
    "军备": "artifacts",
}


def slug_section(label: str, fallback: str) -> str:
    sid = SECTION_ID_BY_LABEL.get(label)
    if sid:
        return sid
    s = re.sub(r"[^a-z0-9]+", "_", fallback.lower()).strip("_")
    return s or "section"


def make_item_row(key: str, row: dict, zh: dict[int, str], *, is_recipe: bool = False) -> dict:
    iid = row.get("id")
    try:
        cost = int(row.get("cost") or 0)
    except (TypeError, ValueError):
        cost = 0
    name_loc = zh.get(int(iid)) if iid is not None else None
    if not name_loc:
        name_loc = "卷轴" if is_recipe else str(row.get("dname") or key)
    return {
        "key": key,
        "id": int(iid) if iid is not None else None,
        "dname": str(row.get("dname") or key),
        "name_loc": name_loc,
        "cost": cost,
        "created": bool(row.get("created")),
        "qual": row.get("qual"),
        "is_recipe": is_recipe,
    }


def download_shop_icons(icon_keys: set[str], *, force: bool = False) -> tuple[int, int, int]:
    """Download item icons (recipe handled separately; recipe_* keys skipped)."""
    # recipe.png is the shared icon for all recipe_* items.
    recipe_dest = ITEM_ICONS / "recipe.png"
    recipe_fail = 0
    if force or not recipe_dest.is_file():
        try:
            recipe_dest.write_bytes(http_bytes(ICON_URL.format(key="recipe")))
        except (urllib.error.URLError, TimeoutError, ValueError, OSError) as e:
            print(f"  icon fail recipe: {e}", flush=True)
            recipe_fail = 1
    real_keys = {k for k in icon_keys if not k.startswith("recipe_")}
    saved, skipped, fail = download_icons(
        real_keys, ICON_URL, ITEM_ICONS, force=force, delay=0.02
    )
    return saved, skipped, fail + recipe_fail


def parse_cn_sections(raw_list: list, kind: str) -> list[dict]:
    """Convert CN basic/upgrade arrays into sections; reverse items like the site JS."""
    sections: list[dict] = []
    for i, row in enumerate(raw_list or []):
        if not isinstance(row, dict):
            continue
        label = str(row.get("name") or "").strip()
        if not label:
            continue
        keys: list[str] = []
        for entry in row.get("items") or []:
            if isinstance(entry, dict):
                name = entry.get("name")
            else:
                name = entry
            if name:
                keys.append(str(name))
        # Official site reverses each column for display.
        keys.reverse()
        sid = slug_section(label, f"{kind}_{i}")
        icon = CAT_ICON_BY_LABEL.get(label, "")
        sections.append(
            {
                "id": sid,
                "label": label,
                "icon": icon,
                "items": keys,
            }
        )
    return sections


def attach_craft_graph(
    items_out: dict[str, dict],
    od: dict,
    catalog: dict[str, dict],
    zh: dict[int, str],
    shop_keys: set[str],
) -> set[str]:
    builds_into: dict[str, list[str]] = {}
    for key, row in od.items():
        if not isinstance(row, dict):
            continue
        key_s = str(key)
        if key_s.startswith("recipe_"):
            continue
        if row.get("tier") is not None:
            continue
        for raw in row.get("components") or []:
            if not raw:
                continue
            builds_into.setdefault(str(raw), []).append(key_s)

    extra_keys: set[str] = set()
    for key, row in list(items_out.items()):
        od_row = od.get(key) if isinstance(od.get(key), dict) else {}
        comps = [str(c) for c in (od_row.get("components") or []) if c]
        # OpenDota often omits recipe_* from components; re-attach when present.
        has_recipe = any(c.startswith("recipe_") for c in comps)
        recipe_key = f"recipe_{key}"
        if not has_recipe and bool(od_row.get("created")):
            recipe_row = od.get(recipe_key)
            if isinstance(recipe_row, dict):
                try:
                    rcost = int(recipe_row.get("cost") or 0)
                except (TypeError, ValueError):
                    rcost = 0
                if rcost > 0:
                    comps.append(recipe_key)
            else:
                # Fallback: gold gap between total and plain components.
                try:
                    total = int(od_row.get("cost") or 0)
                except (TypeError, ValueError):
                    total = 0
                part = 0
                ok = True
                for c in comps:
                    crow = od.get(c) if isinstance(od.get(c), dict) else None
                    if not crow:
                        ok = False
                        break
                    try:
                        part += int(crow.get("cost") or 0)
                    except (TypeError, ValueError):
                        ok = False
                        break
                gap = total - part if ok else 0
                if gap > 0:
                    comps.append(recipe_key)
                    # Synthetic stub so UI can show cost even without odota recipe row.
                    if recipe_key not in od:
                        od[recipe_key] = {
                            "id": None,
                            "dname": f"{od_row.get('dname') or key} Recipe",
                            "cost": gap,
                            "components": None,
                            "created": False,
                        }
        row["components"] = comps
        ups = [u for u in builds_into.get(key, []) if u in shop_keys]
        ups.sort(key=lambda u: (items_out.get(u, {}).get("cost") or 0, u))
        row["builds_into"] = ups
        extra_keys.update(comps)

    for key in sorted(extra_keys):
        if key in items_out:
            continue
        od_row = od.get(key)
        if not isinstance(od_row, dict):
            continue
        if key.startswith("recipe_"):
            items_out[key] = make_item_row(key, od_row, zh, is_recipe=True)
        elif key in catalog:
            stub = dict(catalog[key])
            stub.setdefault("components", [])
            stub.setdefault("builds_into", [])
            items_out[key] = stub
        else:
            stub = make_item_row(key, od_row, zh)
            stub["components"] = []
            stub["builds_into"] = [u for u in builds_into.get(key, []) if u in shop_keys]
            items_out[key] = stub
    return extra_keys


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--out", type=Path, default=OUT)
    ap.add_argument("--skip-icons", action="store_true")
    ap.add_argument("--force-icons", action="store_true")
    args = ap.parse_args()

    print("loading CN shop categories + OpenDota + Valve names...", flush=True)
    cn = http_json(CN_CATEGORY_URL)
    result = (cn or {}).get("result") or {}
    basic_sections = parse_cn_sections(result.get("basic") or [], "basic")
    up_sections = parse_cn_sections(result.get("upgrade") or [], "upgraded")
    if not basic_sections and not up_sections:
        raise SystemExit("CN itemscategory/json returned empty basic/upgrade")

    od = http_json(ITEMS_URL)
    zh = load_itemlist_zh()
    catalog: dict[str, dict] = {}
    for key, row in od.items():
        if not isinstance(row, dict):
            continue
        key_s = str(key)
        if key_s.startswith("recipe_"):
            continue
        catalog[key_s] = make_item_row(key_s, row, zh)

    shop_keys: set[str] = set()
    items_out: dict[str, dict] = {}
    missing: list[str] = []
    cat_files: set[str] = set()

    for kind, sections in (("basic", basic_sections), ("upgraded", up_sections)):
        for sec in sections:
            if sec.get("icon"):
                cat_files.add(sec["icon"])
            kept: list[str] = []
            for key in sec["items"]:
                if key not in catalog:
                    missing.append(key)
                    continue
                kept.append(key)
                shop_keys.add(key)
                if key not in items_out:
                    row = dict(catalog[key])
                    row["shop_kind"] = kind
                    row["section"] = sec["id"]
                    row["section_label"] = sec["label"]
                    items_out[key] = row
            sec["items"] = kept

    if missing:
        print(
            f"  warn missing in OpenDota ({len(missing)}): "
            f"{', '.join(missing[:24])}{'…' if len(missing) > 24 else ''}",
            flush=True,
        )

    craft_extra = attach_craft_graph(items_out, od, catalog, zh, shop_keys)
    icon_keys = set(shop_keys) | {k for k in craft_extra if not k.startswith("recipe_")}

    payload = {
        "meta": {
            "source": "dota2.com.cn/itemscategory+opendota+valve",
            "layout": "cn_shop_columns",
            "fetched_at": datetime.now(timezone.utc).isoformat(),
            "basic_count": sum(len(s["items"]) for s in basic_sections),
            "upgraded_count": sum(len(s["items"]) for s in up_sections),
            "craft_refs": len(craft_extra),
        },
        "basic": {"sections": basic_sections},
        "upgraded": {"sections": up_sections},
        "items": items_out,
    }
    write_json_atomic(args.out, payload)
    print(
        f"wrote {args.out}: basic={payload['meta']['basic_count']} "
        f"upgraded={payload['meta']['upgraded_count']} "
        f"cols={len(basic_sections)+len(up_sections)} "
        f"craft_refs={len(craft_extra)}",
        flush=True,
    )

    if not args.skip_icons:
        print(f"downloading {len(icon_keys)} item icons (+ recipe)...", flush=True)
        saved, skipped, fail = download_shop_icons(icon_keys, force=args.force_icons)
        print(f"  item icons saved={saved} skipped={skipped} fail={fail}", flush=True)
        print(f"downloading {len(cat_files)} category icons...", flush=True)
        # CAT_ICON_BY_LABEL values already include ".png"; download_icons appends
        # another ".png", so pass bare stems + a full URL template.
        cat_keys = {
            name[:-4] if name.endswith(".png") else name for name in cat_files
        }
        c_saved, c_skipped, c_fail = download_icons(
            cat_keys,
            CAT_ICON_BASE + "{key}.png",
            ITEM_CAT_ICONS,
            force=args.force_icons,
            delay=0.05,
        )
        print(f"  cat icons saved={c_saved} skipped={c_skipped} fail={c_fail}", flush=True)


if __name__ == "__main__":
    main()