"""One-shot helper: create OSS bucket + sync ability videos (keyzoo inject). Env (from keyzoo asset_exec on digitevents/voson-RAM): KEYZOO_ASSET_META_ACCESSKEY_ID KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET Usage: python _oss_ability_videos.py setup python _oss_ability_videos.py upload [--force] python _oss_ability_videos.py verify """ from __future__ import annotations import argparse import mimetypes import os import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import oss2 from oss2.models import BucketCors, CorsRule from shared.paths import ABILITY_VIDEOS, ROOT BUCKET_CANDIDATES = ("climperor", "climperor-videos", "climperor-ability-videos") ENDPOINT = "https://oss-cn-shanghai.aliyuncs.com" REGION = "cn-shanghai" PREFIX = "ability-video/" DEFAULT_BASE = "https://climperor.oss-cn-shanghai.aliyuncs.com" CORS_ORIGINS = [ "https://dota2.refining.dev", "https://climperor-relations.pages.dev", "http://localhost:8080", "http://localhost:8765", "http://127.0.0.1:8080", "http://127.0.0.1:8765", "http://localhost:3000", "http://127.0.0.1:3000", ] def _creds() -> tuple[str, str]: ak = os.environ.get("KEYZOO_ASSET_META_ACCESSKEY_ID") or os.environ.get( "OSS_ACCESS_KEY_ID" ) sk = os.environ.get("KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET") or os.environ.get( "OSS_ACCESS_KEY_SECRET" ) if not ak or not sk: raise SystemExit( "missing AccessKey: inject via keyzoo or set OSS_ACCESS_KEY_ID / " "OSS_ACCESS_KEY_SECRET" ) return ak, sk def _auth() -> oss2.Auth: ak, sk = _creds() return oss2.Auth(ak, sk) def _list_bucket_names(auth: oss2.Auth) -> list[str]: service = oss2.Service(auth, ENDPOINT) return [b.name for b in oss2.BucketIterator(service)] def _bucket(auth: oss2.Auth, name: str) -> oss2.Bucket: return oss2.Bucket(auth, ENDPOINT, name) def setup_bucket() -> str: auth = _auth() existing = _list_bucket_names(auth) print(f"existing buckets: {', '.join(existing) or '(none)'}") chosen = None for name in BUCKET_CANDIDATES: if name in existing: chosen = name print(f"bucket_exists: {name}") break bucket = _bucket(auth, name) try: bucket.create_bucket(oss2.BUCKET_ACL_PUBLIC_READ) chosen = name print(f"created: {name}") break except oss2.exceptions.BucketAlreadyExists: print(f"name_conflict (other account?): {name}") continue except oss2.exceptions.OssError as exc: print(f"create_failed {name}: {exc.code} {exc.message[:160]}") continue if not chosen: raise SystemExit("could not create or reuse any candidate bucket name") bucket = _bucket(auth, chosen) try: bucket.put_bucket_acl(oss2.BUCKET_ACL_PUBLIC_READ) print("acl: public-read") except oss2.exceptions.OssError as exc: print(f"acl_warn: {exc.code} {exc.message[:120]}") rule = CorsRule( allowed_origins=CORS_ORIGINS, allowed_methods=["GET", "HEAD"], allowed_headers=["*"], expose_headers=["ETag", "Content-Type", "Content-Length"], max_age_seconds=86400, ) try: bucket.put_bucket_cors(BucketCors([rule])) print("cors: ok") except oss2.exceptions.OssError as exc: print(f"cors_warn: {exc.code} {exc.message[:200]}") info = bucket.get_bucket_info() base = f"https://{chosen}.oss-{REGION}.aliyuncs.com" print(f"bucket: {chosen}") print(f"location: {info.location}") print(f"acl: {info.acl.grant if info.acl else '?'}") print(f"public_base: {base}") # Persist chosen name for upload step. state = ROOT / ".oss_ability_videos_bucket" state.write_text(chosen, encoding="utf-8") return chosen def _resolve_bucket_name(explicit: str | None) -> str: if explicit: return explicit state = ROOT / ".oss_ability_videos_bucket" if state.is_file(): return state.read_text(encoding="utf-8").strip() return BUCKET_CANDIDATES[0] def _iter_local_videos() -> list[Path]: if not ABILITY_VIDEOS.is_dir(): raise SystemExit(f"missing local videos: {ABILITY_VIDEOS}") files = sorted( p for p in ABILITY_VIDEOS.rglob("*") if p.is_file() and p.suffix.lower() in {".webm", ".mp4"} ) return files def _object_key(local: Path) -> str: rel = local.relative_to(ABILITY_VIDEOS).as_posix() return PREFIX + rel def upload( bucket_name: str, *, force: bool = False, max_files: int | None = None, offset: int = 0, ) -> None: auth = _auth() bucket = _bucket(auth, bucket_name) files = _iter_local_videos() if offset or max_files is not None: end = None if max_files is None else offset + max_files files = files[offset:end] total = len(files) total_bytes = sum(f.stat().st_size for f in files) print( f"upload -> oss://{bucket_name}/{PREFIX} " f"({total} files in this batch, {total_bytes / 1e9:.2f} GB; " f"offset={offset})" ) # Remote index for skip-if-same-size. remote_sizes: dict[str, int] = {} if not force: print("listing remote objects ...") for obj in oss2.ObjectIterator(bucket, prefix=PREFIX): remote_sizes[obj.key] = int(obj.size) print(f"remote objects under prefix: {len(remote_sizes)}") uploaded = skipped = failed = 0 t0 = time.time() for i, path in enumerate(files, 1): key = _object_key(path) size = path.stat().st_size if not force and remote_sizes.get(key) == size: skipped += 1 if i % 50 == 0 or i == total: print(f"[{i}/{total}] skip {key}") continue headers = {} ctype, _ = mimetypes.guess_type(str(path)) if path.suffix.lower() == ".webm": ctype = "video/webm" elif path.suffix.lower() == ".mp4": ctype = "video/mp4" if ctype: headers["Content-Type"] = ctype try: # Resumable multipart for large files. oss2.resumable_upload( bucket, key, str(path), headers=headers, multipart_threshold=8 * 1024 * 1024, part_size=8 * 1024 * 1024, num_threads=4, ) uploaded += 1 if i % 10 == 0 or i == total or uploaded <= 3: elapsed = time.time() - t0 print( f"[{i}/{total}] ok {key} " f"({size / 1e6:.1f} MB) uploaded={uploaded} skipped={skipped} " f"elapsed={elapsed:.0f}s" ) except Exception as exc: # noqa: BLE001 — keep uploading rest failed += 1 print(f"[{i}/{total}] FAIL {key}: {type(exc).__name__}: {exc}") elapsed = time.time() - t0 print( f"done: uploaded={uploaded} skipped={skipped} failed={failed} " f"elapsed={elapsed:.0f}s" ) if failed: raise SystemExit(1) def verify(bucket_name: str, samples: int = 5) -> None: import urllib.request auth = _auth() bucket = _bucket(auth, bucket_name) base = f"https://{bucket_name}.oss-{REGION}.aliyuncs.com" files = _iter_local_videos() # Prefer a few heroes spread across the alphabet. picks = [] if files: step = max(1, len(files) // samples) picks = [files[i] for i in range(0, len(files), step)][:samples] print(f"spot-check {len(picks)} URLs against {base}") ok = 0 for path in picks: key = _object_key(path) url = f"{base}/{key}" try: req = urllib.request.Request(url, method="HEAD") with urllib.request.urlopen(req, timeout=30) as resp: code = resp.status ctype = resp.headers.get("Content-Type", "") clen = resp.headers.get("Content-Length", "") print(f" {code} {ctype} len={clen} {url}") if code == 200: ok += 1 except Exception as exc: # noqa: BLE001 print(f" FAIL {url}: {exc}") # Count remote. n = sum(1 for _ in oss2.ObjectIterator(bucket, prefix=PREFIX)) print(f"remote object count under {PREFIX}: {n}") print(f"spot-check ok: {ok}/{len(picks)}") if ok < len(picks): raise SystemExit(1) def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("cmd", choices=("setup", "upload", "verify", "setup-upload")) ap.add_argument("--bucket", default=None) ap.add_argument("--force", action="store_true") ap.add_argument("--max-files", type=int, default=None, help="upload at most N files") ap.add_argument("--offset", type=int, default=0, help="skip first N local files") args = ap.parse_args() if args.cmd in ("setup", "setup-upload"): name = setup_bucket() else: name = _resolve_bucket_name(args.bucket) if args.cmd in ("upload", "setup-upload"): upload(name, force=args.force, max_files=args.max_files, offset=args.offset) if args.cmd == "verify": verify(name) if __name__ == "__main__": main()