Add OSS ability-video upload helpers and ignore runtime artifacts.

Commit _oss_*.py ops scripts alongside existing _cf_* helpers; gitignore bucket marker and upload PID. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-27 20:31:00 +08:00
parent 124bdbb7c7
commit 74deb17888
5 changed files with 443 additions and 0 deletions
@@ -8,6 +8,10 @@ venv/
 Thumbs.db
 *.log

+# OSS ability-video upload runtime (regenerable)
+.oss_ability_videos_bucket
+oss_upload.pid
+
 # Runtime / debug artifacts (regenerable)
 preview/
 results/
@@ -32,6 +32,7 @@ Windows 上的 Dota 2 **天梯选将识别**工具：用 GSI 感知选将阶段
 | `export_relations_site.py` | 导出关系预览为纯静态站点 → `dist/relations/`（data.json 快照 + 前端 + 图片；`SITE_VERSION` 常量与 `web/relations/config.js` 同步；`--ability-video-base` 写 `config.js` 指向 OSS 技能视频；`--with-videos` 可选本地拷贝视频，生产部署勿用） |
 | `deploy_relations.py` | 一键部署关系预览静态站点到 Cloudflare Pages（导出 + 资产预检 + `wrangler` 直传 + 绑域名；默认 `--ability-video-base` 指向 `climperor` OSS；凭据经 keyzoo 注入或 env） |
 | `_cf_status.py` | 只读查询 Cloudflare Pages 项目 / 部署 / 自定义域名状态（凭据经 keyzoo 注入） |
+| `_oss_ability_videos.py` | 阿里云 OSS 桶 `climperor` 建桶 / CORS / 同步 `assets/ability_videos/` → `ability-video/`（凭据经 keyzoo `digitevents/voson-RAM` 注入）；`_oss_fix_public.py` / `_oss_launch_upload.py` 为配套辅助 |
 | `fetch_hero_portraits.py` | 拉取官网横版头像 → `assets/hero_portraits/`（关系预览） |
 | `fetch_hero_items.py` | 拉取 OpenDota 热门装备 → `data/hero_items.json` + `assets/item_icons/` |
 | `fetch_item_shop.py` | 官网商店 11 列目录（dota2.com.cn/itemscategory）+ 合成图 → `data/item_shop.json` + 图标 |
@@ -0,0 +1,294 @@
+"""One-shot helper: create OSS bucket + sync ability videos (keyzoo inject).
+
+Env (from keyzoo asset_exec on digitevents/voson-RAM):
+  KEYZOO_ASSET_META_ACCESSKEY_ID
+  KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET
+
+Usage:
+  python _oss_ability_videos.py setup
+  python _oss_ability_videos.py upload [--force]
+  python _oss_ability_videos.py verify
+"""
+
+from __future__ import annotations
+
+import argparse
+import mimetypes
+import os
+import sys
+import time
+from pathlib import Path
+
+import oss2
+from oss2.models import BucketCors, CorsRule
+
+from common import ABILITY_VIDEOS, ROOT
+
+BUCKET_CANDIDATES = ("climperor", "climperor-videos", "climperor-ability-videos")
+ENDPOINT = "https://oss-cn-shanghai.aliyuncs.com"
+REGION = "cn-shanghai"
+PREFIX = "ability-video/"
+DEFAULT_BASE = "https://climperor.oss-cn-shanghai.aliyuncs.com"
+
+CORS_ORIGINS = [
+    "https://dota2.refining.dev",
+    "https://climperor-relations.pages.dev",
+    "http://localhost:8080",
+    "http://localhost:8765",
+    "http://127.0.0.1:8080",
+    "http://127.0.0.1:8765",
+    "http://localhost:3000",
+    "http://127.0.0.1:3000",
+]
+
+
+def _creds() -> tuple[str, str]:
+    ak = os.environ.get("KEYZOO_ASSET_META_ACCESSKEY_ID") or os.environ.get(
+        "OSS_ACCESS_KEY_ID"
+    )
+    sk = os.environ.get("KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET") or os.environ.get(
+        "OSS_ACCESS_KEY_SECRET"
+    )
+    if not ak or not sk:
+        raise SystemExit(
+            "missing AccessKey: inject via keyzoo or set OSS_ACCESS_KEY_ID / "
+            "OSS_ACCESS_KEY_SECRET"
+        )
+    return ak, sk
+
+
+def _auth() -> oss2.Auth:
+    ak, sk = _creds()
+    return oss2.Auth(ak, sk)
+
+
+def _list_bucket_names(auth: oss2.Auth) -> list[str]:
+    service = oss2.Service(auth, ENDPOINT)
+    return [b.name for b in oss2.BucketIterator(service)]
+
+
+def _bucket(auth: oss2.Auth, name: str) -> oss2.Bucket:
+    return oss2.Bucket(auth, ENDPOINT, name)
+
+
+def setup_bucket() -> str:
+    auth = _auth()
+    existing = _list_bucket_names(auth)
+    print(f"existing buckets: {', '.join(existing) or '(none)'}")
+
+    chosen = None
+    for name in BUCKET_CANDIDATES:
+        if name in existing:
+            chosen = name
+            print(f"bucket_exists: {name}")
+            break
+        bucket = _bucket(auth, name)
+        try:
+            bucket.create_bucket(oss2.BUCKET_ACL_PUBLIC_READ)
+            chosen = name
+            print(f"created: {name}")
+            break
+        except oss2.exceptions.BucketAlreadyExists:
+            print(f"name_conflict (other account?): {name}")
+            continue
+        except oss2.exceptions.OssError as exc:
+            print(f"create_failed {name}: {exc.code} {exc.message[:160]}")
+            continue
+
+    if not chosen:
+        raise SystemExit("could not create or reuse any candidate bucket name")
+
+    bucket = _bucket(auth, chosen)
+    try:
+        bucket.put_bucket_acl(oss2.BUCKET_ACL_PUBLIC_READ)
+        print("acl: public-read")
+    except oss2.exceptions.OssError as exc:
+        print(f"acl_warn: {exc.code} {exc.message[:120]}")
+
+    rule = CorsRule(
+        allowed_origins=CORS_ORIGINS,
+        allowed_methods=["GET", "HEAD"],
+        allowed_headers=["*"],
+        expose_headers=["ETag", "Content-Type", "Content-Length"],
+        max_age_seconds=86400,
+    )
+    try:
+        bucket.put_bucket_cors(BucketCors([rule]))
+        print("cors: ok")
+    except oss2.exceptions.OssError as exc:
+        print(f"cors_warn: {exc.code} {exc.message[:200]}")
+
+    info = bucket.get_bucket_info()
+    base = f"https://{chosen}.oss-{REGION}.aliyuncs.com"
+    print(f"bucket: {chosen}")
+    print(f"location: {info.location}")
+    print(f"acl: {info.acl.grant if info.acl else '?'}")
+    print(f"public_base: {base}")
+    # Persist chosen name for upload step.
+    state = ROOT / ".oss_ability_videos_bucket"
+    state.write_text(chosen, encoding="utf-8")
+    return chosen
+
+
+def _resolve_bucket_name(explicit: str | None) -> str:
+    if explicit:
+        return explicit
+    state = ROOT / ".oss_ability_videos_bucket"
+    if state.is_file():
+        return state.read_text(encoding="utf-8").strip()
+    return BUCKET_CANDIDATES[0]
+
+
+def _iter_local_videos() -> list[Path]:
+    if not ABILITY_VIDEOS.is_dir():
+        raise SystemExit(f"missing local videos: {ABILITY_VIDEOS}")
+    files = sorted(
+        p
+        for p in ABILITY_VIDEOS.rglob("*")
+        if p.is_file() and p.suffix.lower() in {".webm", ".mp4"}
+    )
+    return files
+
+
+def _object_key(local: Path) -> str:
+    rel = local.relative_to(ABILITY_VIDEOS).as_posix()
+    return PREFIX + rel
+
+
+def upload(
+    bucket_name: str,
+    *,
+    force: bool = False,
+    max_files: int | None = None,
+    offset: int = 0,
+) -> None:
+    auth = _auth()
+    bucket = _bucket(auth, bucket_name)
+    files = _iter_local_videos()
+    if offset or max_files is not None:
+        end = None if max_files is None else offset + max_files
+        files = files[offset:end]
+    total = len(files)
+    total_bytes = sum(f.stat().st_size for f in files)
+    print(
+        f"upload -> oss://{bucket_name}/{PREFIX} "
+        f"({total} files in this batch, {total_bytes / 1e9:.2f} GB; "
+        f"offset={offset})"
+    )
+
+    # Remote index for skip-if-same-size.
+    remote_sizes: dict[str, int] = {}
+    if not force:
+        print("listing remote objects ...")
+        for obj in oss2.ObjectIterator(bucket, prefix=PREFIX):
+            remote_sizes[obj.key] = int(obj.size)
+        print(f"remote objects under prefix: {len(remote_sizes)}")
+
+    uploaded = skipped = failed = 0
+    t0 = time.time()
+    for i, path in enumerate(files, 1):
+        key = _object_key(path)
+        size = path.stat().st_size
+        if not force and remote_sizes.get(key) == size:
+            skipped += 1
+            if i % 50 == 0 or i == total:
+                print(f"[{i}/{total}] skip {key}")
+            continue
+        headers = {}
+        ctype, _ = mimetypes.guess_type(str(path))
+        if path.suffix.lower() == ".webm":
+            ctype = "video/webm"
+        elif path.suffix.lower() == ".mp4":
+            ctype = "video/mp4"
+        if ctype:
+            headers["Content-Type"] = ctype
+        try:
+            # Resumable multipart for large files.
+            oss2.resumable_upload(
+                bucket,
+                key,
+                str(path),
+                headers=headers,
+                multipart_threshold=8 * 1024 * 1024,
+                part_size=8 * 1024 * 1024,
+                num_threads=4,
+            )
+            uploaded += 1
+            if i % 10 == 0 or i == total or uploaded <= 3:
+                elapsed = time.time() - t0
+                print(
+                    f"[{i}/{total}] ok {key} "
+                    f"({size / 1e6:.1f} MB) uploaded={uploaded} skipped={skipped} "
+                    f"elapsed={elapsed:.0f}s"
+                )
+        except Exception as exc:  # noqa: BLE001 — keep uploading rest
+            failed += 1
+            print(f"[{i}/{total}] FAIL {key}: {type(exc).__name__}: {exc}")
+
+    elapsed = time.time() - t0
+    print(
+        f"done: uploaded={uploaded} skipped={skipped} failed={failed} "
+        f"elapsed={elapsed:.0f}s"
+    )
+    if failed:
+        raise SystemExit(1)
+
+
+def verify(bucket_name: str, samples: int = 5) -> None:
+    import urllib.request
+
+    auth = _auth()
+    bucket = _bucket(auth, bucket_name)
+    base = f"https://{bucket_name}.oss-{REGION}.aliyuncs.com"
+    files = _iter_local_videos()
+    # Prefer a few heroes spread across the alphabet.
+    picks = []
+    if files:
+        step = max(1, len(files) // samples)
+        picks = [files[i] for i in range(0, len(files), step)][:samples]
+    print(f"spot-check {len(picks)} URLs against {base}")
+    ok = 0
+    for path in picks:
+        key = _object_key(path)
+        url = f"{base}/{key}"
+        try:
+            req = urllib.request.Request(url, method="HEAD")
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                code = resp.status
+                ctype = resp.headers.get("Content-Type", "")
+                clen = resp.headers.get("Content-Length", "")
+            print(f"  {code} {ctype} len={clen} {url}")
+            if code == 200:
+                ok += 1
+        except Exception as exc:  # noqa: BLE001
+            print(f"  FAIL {url}: {exc}")
+    # Count remote.
+    n = sum(1 for _ in oss2.ObjectIterator(bucket, prefix=PREFIX))
+    print(f"remote object count under {PREFIX}: {n}")
+    print(f"spot-check ok: {ok}/{len(picks)}")
+    if ok < len(picks):
+        raise SystemExit(1)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("cmd", choices=("setup", "upload", "verify", "setup-upload"))
+    ap.add_argument("--bucket", default=None)
+    ap.add_argument("--force", action="store_true")
+    ap.add_argument("--max-files", type=int, default=None, help="upload at most N files")
+    ap.add_argument("--offset", type=int, default=0, help="skip first N local files")
+    args = ap.parse_args()
+
+    if args.cmd in ("setup", "setup-upload"):
+        name = setup_bucket()
+    else:
+        name = _resolve_bucket_name(args.bucket)
+
+    if args.cmd in ("upload", "setup-upload"):
+        upload(name, force=args.force, max_files=args.max_files, offset=args.offset)
+    if args.cmd == "verify":
+        verify(name)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,99 @@
+"""Fix climperor OSS public-read access (Block Public Access + policy)."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+import oss2
+
+BUCKET = "climperor"
+ENDPOINT = "https://oss-cn-shanghai.aliyuncs.com"
+
+POLICY = {
+    "Version": "1",
+    "Statement": [
+        {
+            "Sid": "PublicReadAbilityVideos",
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": ["oss:GetObject", "oss:GetObjectAcl"],
+            "Resource": [f"acs:oss:*:*:{BUCKET}/ability-video/*"],
+        }
+    ],
+}
+
+
+def main() -> None:
+    ak = os.environ["KEYZOO_ASSET_META_ACCESSKEY_ID"]
+    sk = os.environ["KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET"]
+    auth = oss2.Auth(ak, sk)
+    b = oss2.Bucket(auth, ENDPOINT, BUCKET)
+
+    # 1) Try disable Block Public Access (required on newer Aliyun accounts).
+    try:
+        # oss2 >= 2.18: put_bucket_public_access_block(block_public_access=False)
+        if hasattr(b, "put_bucket_public_access_block"):
+            b.put_bucket_public_access_block(False)
+            print("public_access_block: disabled via SDK")
+        else:
+            # Raw REST: PUT /?publicAccessBlock with XML
+            xml = (
+                '<?xml version="1.0" encoding="UTF-8"?>'
+                "<PublicAccessBlockConfiguration>"
+                "<BlockPublicAccess>false</BlockPublicAccess>"
+                "</PublicAccessBlockConfiguration>"
+            )
+            resp = b._do("PUT", "", params={"publicAccessBlock": ""}, data=xml)
+            print(f"public_access_block: raw PUT status={resp.status}")
+    except Exception as exc:  # noqa: BLE001
+        print(f"public_access_block_warn: {type(exc).__name__}: {exc}")
+
+    # 2) Bucket ACL public-read (may be denied by account policy).
+    try:
+        b.put_bucket_acl(oss2.BUCKET_ACL_PUBLIC_READ)
+        print("acl: public-read set")
+    except Exception as exc:  # noqa: BLE001
+        print(f"acl_warn: {type(exc).__name__}: {exc}")
+
+    # 3) Bucket policy for anonymous GetObject under ability-video/.
+    try:
+        b.put_bucket_policy(json.dumps(POLICY))
+        print("policy: public GetObject on ability-video/*")
+    except Exception as exc:  # noqa: BLE001
+        print(f"policy_fail: {type(exc).__name__}: {exc}")
+        raise SystemExit(1)
+
+    info = b.get_bucket_info()
+    print(f"acl_now: {info.acl.grant if info.acl else '?'}")
+
+    # 4) Probe one known object if any exist.
+    import urllib.request
+
+    sample = None
+    for obj in oss2.ObjectIterator(b, prefix="ability-video/", max_keys=1):
+        sample = obj.key
+        break
+    if not sample:
+        print("no objects yet to probe")
+        return
+    url = f"https://{BUCKET}.oss-cn-shanghai.aliyuncs.com/{sample}"
+    try:
+        req = urllib.request.Request(url, method="HEAD")
+        with urllib.request.urlopen(req, timeout=20) as resp:
+            print(f"probe: {resp.status} {resp.headers.get('Content-Type')} {url}")
+    except Exception as exc:  # noqa: BLE001
+        print(f"probe_fail: {exc}")
+        # Try signed URL to confirm object exists
+        signed = b.sign_url("HEAD", sample, 60)
+        print(f"signed_head_url_len={len(signed)} (object exists check via SDK)")
+        try:
+            meta = b.head_object(sample)
+            print(f"sdk_head: ok content_type={meta.content_type} size={meta.content_length}")
+        except Exception as exc2:  # noqa: BLE001
+            print(f"sdk_head_fail: {exc2}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,45 @@
+"""Launch OSS upload as a detached process inheriting current env (keyzoo inject)."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+LOG = ROOT / "oss_upload.log"
+ERR = ROOT / "oss_upload_err.log"
+PID = ROOT / "oss_upload.pid"
+
+
+def main() -> None:
+    if not os.environ.get("KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET") and not os.environ.get(
+        "OSS_ACCESS_KEY_SECRET"
+    ):
+        raise SystemExit("missing AccessKey secret in env")
+    # Clear previous logs.
+    for p in (LOG, ERR):
+        if p.exists():
+            p.unlink()
+    creationflags = 0
+    if sys.platform == "win32":
+        creationflags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS
+    out = open(LOG, "w", encoding="utf-8")
+    err = open(ERR, "w", encoding="utf-8")
+    proc = subprocess.Popen(
+        [sys.executable, str(ROOT / "_oss_ability_videos.py"), "upload"],
+        cwd=str(ROOT),
+        stdout=out,
+        stderr=err,
+        env=os.environ.copy(),
+        creationflags=creationflags,
+        close_fds=True,
+    )
+    PID.write_text(str(proc.pid), encoding="utf-8")
+    print(f"started_pid={proc.pid}")
+    print(f"log={LOG}")
+
+
+if __name__ == "__main__":
+    main()