diff --git a/.gitignore b/.gitignore index 48e1c1d..25a2a33 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ venv/ Thumbs.db *.log +# OSS ability-video upload runtime (regenerable) +.oss_ability_videos_bucket +oss_upload.pid + # Runtime / debug artifacts (regenerable) preview/ results/ diff --git a/AGENTS.md b/AGENTS.md index 1efd091..4e27fa5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,6 +32,7 @@ Windows 上的 Dota 2 **天梯选将识别**工具:用 GSI 感知选将阶段 | `export_relations_site.py` | 导出关系预览为纯静态站点 → `dist/relations/`(data.json 快照 + 前端 + 图片;`SITE_VERSION` 常量与 `web/relations/config.js` 同步;`--ability-video-base` 写 `config.js` 指向 OSS 技能视频;`--with-videos` 可选本地拷贝视频,生产部署勿用) | | `deploy_relations.py` | 一键部署关系预览静态站点到 Cloudflare Pages(导出 + 资产预检 + `wrangler` 直传 + 绑域名;默认 `--ability-video-base` 指向 `climperor` OSS;凭据经 keyzoo 注入或 env) | | `_cf_status.py` | 只读查询 Cloudflare Pages 项目 / 部署 / 自定义域名状态(凭据经 keyzoo 注入) | +| `_oss_ability_videos.py` | 阿里云 OSS 桶 `climperor` 建桶 / CORS / 同步 `assets/ability_videos/` → `ability-video/`(凭据经 keyzoo `digitevents/voson-RAM` 注入);`_oss_fix_public.py` / `_oss_launch_upload.py` 为配套辅助 | | `fetch_hero_portraits.py` | 拉取官网横版头像 → `assets/hero_portraits/`(关系预览) | | `fetch_hero_items.py` | 拉取 OpenDota 热门装备 → `data/hero_items.json` + `assets/item_icons/` | | `fetch_item_shop.py` | 官网商店 11 列目录(dota2.com.cn/itemscategory)+ 合成图 → `data/item_shop.json` + 图标 | diff --git a/_oss_ability_videos.py b/_oss_ability_videos.py new file mode 100644 index 0000000..44e2df4 --- /dev/null +++ b/_oss_ability_videos.py @@ -0,0 +1,294 @@ +"""One-shot helper: create OSS bucket + sync ability videos (keyzoo inject). + +Env (from keyzoo asset_exec on digitevents/voson-RAM): + KEYZOO_ASSET_META_ACCESSKEY_ID + KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET + +Usage: + python _oss_ability_videos.py setup + python _oss_ability_videos.py upload [--force] + python _oss_ability_videos.py verify +""" + +from __future__ import annotations + +import argparse +import mimetypes +import os +import sys +import time +from pathlib import Path + +import oss2 +from oss2.models import BucketCors, CorsRule + +from common import ABILITY_VIDEOS, ROOT + +BUCKET_CANDIDATES = ("climperor", "climperor-videos", "climperor-ability-videos") +ENDPOINT = "https://oss-cn-shanghai.aliyuncs.com" +REGION = "cn-shanghai" +PREFIX = "ability-video/" +DEFAULT_BASE = "https://climperor.oss-cn-shanghai.aliyuncs.com" + +CORS_ORIGINS = [ + "https://dota2.refining.dev", + "https://climperor-relations.pages.dev", + "http://localhost:8080", + "http://localhost:8765", + "http://127.0.0.1:8080", + "http://127.0.0.1:8765", + "http://localhost:3000", + "http://127.0.0.1:3000", +] + + +def _creds() -> tuple[str, str]: + ak = os.environ.get("KEYZOO_ASSET_META_ACCESSKEY_ID") or os.environ.get( + "OSS_ACCESS_KEY_ID" + ) + sk = os.environ.get("KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET") or os.environ.get( + "OSS_ACCESS_KEY_SECRET" + ) + if not ak or not sk: + raise SystemExit( + "missing AccessKey: inject via keyzoo or set OSS_ACCESS_KEY_ID / " + "OSS_ACCESS_KEY_SECRET" + ) + return ak, sk + + +def _auth() -> oss2.Auth: + ak, sk = _creds() + return oss2.Auth(ak, sk) + + +def _list_bucket_names(auth: oss2.Auth) -> list[str]: + service = oss2.Service(auth, ENDPOINT) + return [b.name for b in oss2.BucketIterator(service)] + + +def _bucket(auth: oss2.Auth, name: str) -> oss2.Bucket: + return oss2.Bucket(auth, ENDPOINT, name) + + +def setup_bucket() -> str: + auth = _auth() + existing = _list_bucket_names(auth) + print(f"existing buckets: {', '.join(existing) or '(none)'}") + + chosen = None + for name in BUCKET_CANDIDATES: + if name in existing: + chosen = name + print(f"bucket_exists: {name}") + break + bucket = _bucket(auth, name) + try: + bucket.create_bucket(oss2.BUCKET_ACL_PUBLIC_READ) + chosen = name + print(f"created: {name}") + break + except oss2.exceptions.BucketAlreadyExists: + print(f"name_conflict (other account?): {name}") + continue + except oss2.exceptions.OssError as exc: + print(f"create_failed {name}: {exc.code} {exc.message[:160]}") + continue + + if not chosen: + raise SystemExit("could not create or reuse any candidate bucket name") + + bucket = _bucket(auth, chosen) + try: + bucket.put_bucket_acl(oss2.BUCKET_ACL_PUBLIC_READ) + print("acl: public-read") + except oss2.exceptions.OssError as exc: + print(f"acl_warn: {exc.code} {exc.message[:120]}") + + rule = CorsRule( + allowed_origins=CORS_ORIGINS, + allowed_methods=["GET", "HEAD"], + allowed_headers=["*"], + expose_headers=["ETag", "Content-Type", "Content-Length"], + max_age_seconds=86400, + ) + try: + bucket.put_bucket_cors(BucketCors([rule])) + print("cors: ok") + except oss2.exceptions.OssError as exc: + print(f"cors_warn: {exc.code} {exc.message[:200]}") + + info = bucket.get_bucket_info() + base = f"https://{chosen}.oss-{REGION}.aliyuncs.com" + print(f"bucket: {chosen}") + print(f"location: {info.location}") + print(f"acl: {info.acl.grant if info.acl else '?'}") + print(f"public_base: {base}") + # Persist chosen name for upload step. + state = ROOT / ".oss_ability_videos_bucket" + state.write_text(chosen, encoding="utf-8") + return chosen + + +def _resolve_bucket_name(explicit: str | None) -> str: + if explicit: + return explicit + state = ROOT / ".oss_ability_videos_bucket" + if state.is_file(): + return state.read_text(encoding="utf-8").strip() + return BUCKET_CANDIDATES[0] + + +def _iter_local_videos() -> list[Path]: + if not ABILITY_VIDEOS.is_dir(): + raise SystemExit(f"missing local videos: {ABILITY_VIDEOS}") + files = sorted( + p + for p in ABILITY_VIDEOS.rglob("*") + if p.is_file() and p.suffix.lower() in {".webm", ".mp4"} + ) + return files + + +def _object_key(local: Path) -> str: + rel = local.relative_to(ABILITY_VIDEOS).as_posix() + return PREFIX + rel + + +def upload( + bucket_name: str, + *, + force: bool = False, + max_files: int | None = None, + offset: int = 0, +) -> None: + auth = _auth() + bucket = _bucket(auth, bucket_name) + files = _iter_local_videos() + if offset or max_files is not None: + end = None if max_files is None else offset + max_files + files = files[offset:end] + total = len(files) + total_bytes = sum(f.stat().st_size for f in files) + print( + f"upload -> oss://{bucket_name}/{PREFIX} " + f"({total} files in this batch, {total_bytes / 1e9:.2f} GB; " + f"offset={offset})" + ) + + # Remote index for skip-if-same-size. + remote_sizes: dict[str, int] = {} + if not force: + print("listing remote objects ...") + for obj in oss2.ObjectIterator(bucket, prefix=PREFIX): + remote_sizes[obj.key] = int(obj.size) + print(f"remote objects under prefix: {len(remote_sizes)}") + + uploaded = skipped = failed = 0 + t0 = time.time() + for i, path in enumerate(files, 1): + key = _object_key(path) + size = path.stat().st_size + if not force and remote_sizes.get(key) == size: + skipped += 1 + if i % 50 == 0 or i == total: + print(f"[{i}/{total}] skip {key}") + continue + headers = {} + ctype, _ = mimetypes.guess_type(str(path)) + if path.suffix.lower() == ".webm": + ctype = "video/webm" + elif path.suffix.lower() == ".mp4": + ctype = "video/mp4" + if ctype: + headers["Content-Type"] = ctype + try: + # Resumable multipart for large files. + oss2.resumable_upload( + bucket, + key, + str(path), + headers=headers, + multipart_threshold=8 * 1024 * 1024, + part_size=8 * 1024 * 1024, + num_threads=4, + ) + uploaded += 1 + if i % 10 == 0 or i == total or uploaded <= 3: + elapsed = time.time() - t0 + print( + f"[{i}/{total}] ok {key} " + f"({size / 1e6:.1f} MB) uploaded={uploaded} skipped={skipped} " + f"elapsed={elapsed:.0f}s" + ) + except Exception as exc: # noqa: BLE001 — keep uploading rest + failed += 1 + print(f"[{i}/{total}] FAIL {key}: {type(exc).__name__}: {exc}") + + elapsed = time.time() - t0 + print( + f"done: uploaded={uploaded} skipped={skipped} failed={failed} " + f"elapsed={elapsed:.0f}s" + ) + if failed: + raise SystemExit(1) + + +def verify(bucket_name: str, samples: int = 5) -> None: + import urllib.request + + auth = _auth() + bucket = _bucket(auth, bucket_name) + base = f"https://{bucket_name}.oss-{REGION}.aliyuncs.com" + files = _iter_local_videos() + # Prefer a few heroes spread across the alphabet. + picks = [] + if files: + step = max(1, len(files) // samples) + picks = [files[i] for i in range(0, len(files), step)][:samples] + print(f"spot-check {len(picks)} URLs against {base}") + ok = 0 + for path in picks: + key = _object_key(path) + url = f"{base}/{key}" + try: + req = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(req, timeout=30) as resp: + code = resp.status + ctype = resp.headers.get("Content-Type", "") + clen = resp.headers.get("Content-Length", "") + print(f" {code} {ctype} len={clen} {url}") + if code == 200: + ok += 1 + except Exception as exc: # noqa: BLE001 + print(f" FAIL {url}: {exc}") + # Count remote. + n = sum(1 for _ in oss2.ObjectIterator(bucket, prefix=PREFIX)) + print(f"remote object count under {PREFIX}: {n}") + print(f"spot-check ok: {ok}/{len(picks)}") + if ok < len(picks): + raise SystemExit(1) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=("setup", "upload", "verify", "setup-upload")) + ap.add_argument("--bucket", default=None) + ap.add_argument("--force", action="store_true") + ap.add_argument("--max-files", type=int, default=None, help="upload at most N files") + ap.add_argument("--offset", type=int, default=0, help="skip first N local files") + args = ap.parse_args() + + if args.cmd in ("setup", "setup-upload"): + name = setup_bucket() + else: + name = _resolve_bucket_name(args.bucket) + + if args.cmd in ("upload", "setup-upload"): + upload(name, force=args.force, max_files=args.max_files, offset=args.offset) + if args.cmd == "verify": + verify(name) + + +if __name__ == "__main__": + main() diff --git a/_oss_fix_public.py b/_oss_fix_public.py new file mode 100644 index 0000000..5fba222 --- /dev/null +++ b/_oss_fix_public.py @@ -0,0 +1,99 @@ +"""Fix climperor OSS public-read access (Block Public Access + policy).""" + +from __future__ import annotations + +import json +import os +import sys + +import oss2 + +BUCKET = "climperor" +ENDPOINT = "https://oss-cn-shanghai.aliyuncs.com" + +POLICY = { + "Version": "1", + "Statement": [ + { + "Sid": "PublicReadAbilityVideos", + "Effect": "Allow", + "Principal": "*", + "Action": ["oss:GetObject", "oss:GetObjectAcl"], + "Resource": [f"acs:oss:*:*:{BUCKET}/ability-video/*"], + } + ], +} + + +def main() -> None: + ak = os.environ["KEYZOO_ASSET_META_ACCESSKEY_ID"] + sk = os.environ["KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET"] + auth = oss2.Auth(ak, sk) + b = oss2.Bucket(auth, ENDPOINT, BUCKET) + + # 1) Try disable Block Public Access (required on newer Aliyun accounts). + try: + # oss2 >= 2.18: put_bucket_public_access_block(block_public_access=False) + if hasattr(b, "put_bucket_public_access_block"): + b.put_bucket_public_access_block(False) + print("public_access_block: disabled via SDK") + else: + # Raw REST: PUT /?publicAccessBlock with XML + xml = ( + '' + "" + "false" + "" + ) + resp = b._do("PUT", "", params={"publicAccessBlock": ""}, data=xml) + print(f"public_access_block: raw PUT status={resp.status}") + except Exception as exc: # noqa: BLE001 + print(f"public_access_block_warn: {type(exc).__name__}: {exc}") + + # 2) Bucket ACL public-read (may be denied by account policy). + try: + b.put_bucket_acl(oss2.BUCKET_ACL_PUBLIC_READ) + print("acl: public-read set") + except Exception as exc: # noqa: BLE001 + print(f"acl_warn: {type(exc).__name__}: {exc}") + + # 3) Bucket policy for anonymous GetObject under ability-video/. + try: + b.put_bucket_policy(json.dumps(POLICY)) + print("policy: public GetObject on ability-video/*") + except Exception as exc: # noqa: BLE001 + print(f"policy_fail: {type(exc).__name__}: {exc}") + raise SystemExit(1) + + info = b.get_bucket_info() + print(f"acl_now: {info.acl.grant if info.acl else '?'}") + + # 4) Probe one known object if any exist. + import urllib.request + + sample = None + for obj in oss2.ObjectIterator(b, prefix="ability-video/", max_keys=1): + sample = obj.key + break + if not sample: + print("no objects yet to probe") + return + url = f"https://{BUCKET}.oss-cn-shanghai.aliyuncs.com/{sample}" + try: + req = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(req, timeout=20) as resp: + print(f"probe: {resp.status} {resp.headers.get('Content-Type')} {url}") + except Exception as exc: # noqa: BLE001 + print(f"probe_fail: {exc}") + # Try signed URL to confirm object exists + signed = b.sign_url("HEAD", sample, 60) + print(f"signed_head_url_len={len(signed)} (object exists check via SDK)") + try: + meta = b.head_object(sample) + print(f"sdk_head: ok content_type={meta.content_type} size={meta.content_length}") + except Exception as exc2: # noqa: BLE001 + print(f"sdk_head_fail: {exc2}") + + +if __name__ == "__main__": + main() diff --git a/_oss_launch_upload.py b/_oss_launch_upload.py new file mode 100644 index 0000000..6e3de4a --- /dev/null +++ b/_oss_launch_upload.py @@ -0,0 +1,45 @@ +"""Launch OSS upload as a detached process inheriting current env (keyzoo inject).""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +LOG = ROOT / "oss_upload.log" +ERR = ROOT / "oss_upload_err.log" +PID = ROOT / "oss_upload.pid" + + +def main() -> None: + if not os.environ.get("KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET") and not os.environ.get( + "OSS_ACCESS_KEY_SECRET" + ): + raise SystemExit("missing AccessKey secret in env") + # Clear previous logs. + for p in (LOG, ERR): + if p.exists(): + p.unlink() + creationflags = 0 + if sys.platform == "win32": + creationflags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS + out = open(LOG, "w", encoding="utf-8") + err = open(ERR, "w", encoding="utf-8") + proc = subprocess.Popen( + [sys.executable, str(ROOT / "_oss_ability_videos.py"), "upload"], + cwd=str(ROOT), + stdout=out, + stderr=err, + env=os.environ.copy(), + creationflags=creationflags, + close_fds=True, + ) + PID.write_text(str(proc.pid), encoding="utf-8") + print(f"started_pid={proc.pid}") + print(f"log={LOG}") + + +if __name__ == "__main__": + main()