Add OSS ability-video upload helpers and ignore runtime artifacts.
Commit _oss_*.py ops scripts alongside existing _cf_* helpers; gitignore bucket marker and upload PID. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
voson
Cursor
@@ -0,0 +1,294 @@
|
||||
"""One-shot helper: create OSS bucket + sync ability videos (keyzoo inject).
|
||||
|
||||
Env (from keyzoo asset_exec on digitevents/voson-RAM):
|
||||
KEYZOO_ASSET_META_ACCESSKEY_ID
|
||||
KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET
|
||||
|
||||
Usage:
|
||||
python _oss_ability_videos.py setup
|
||||
python _oss_ability_videos.py upload [--force]
|
||||
python _oss_ability_videos.py verify
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import mimetypes
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import oss2
|
||||
from oss2.models import BucketCors, CorsRule
|
||||
|
||||
from common import ABILITY_VIDEOS, ROOT
|
||||
|
||||
BUCKET_CANDIDATES = ("climperor", "climperor-videos", "climperor-ability-videos")
|
||||
ENDPOINT = "https://oss-cn-shanghai.aliyuncs.com"
|
||||
REGION = "cn-shanghai"
|
||||
PREFIX = "ability-video/"
|
||||
DEFAULT_BASE = "https://climperor.oss-cn-shanghai.aliyuncs.com"
|
||||
|
||||
CORS_ORIGINS = [
|
||||
"https://dota2.refining.dev",
|
||||
"https://climperor-relations.pages.dev",
|
||||
"http://localhost:8080",
|
||||
"http://localhost:8765",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://127.0.0.1:8765",
|
||||
"http://localhost:3000",
|
||||
"http://127.0.0.1:3000",
|
||||
]
|
||||
|
||||
|
||||
def _creds() -> tuple[str, str]:
|
||||
ak = os.environ.get("KEYZOO_ASSET_META_ACCESSKEY_ID") or os.environ.get(
|
||||
"OSS_ACCESS_KEY_ID"
|
||||
)
|
||||
sk = os.environ.get("KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET") or os.environ.get(
|
||||
"OSS_ACCESS_KEY_SECRET"
|
||||
)
|
||||
if not ak or not sk:
|
||||
raise SystemExit(
|
||||
"missing AccessKey: inject via keyzoo or set OSS_ACCESS_KEY_ID / "
|
||||
"OSS_ACCESS_KEY_SECRET"
|
||||
)
|
||||
return ak, sk
|
||||
|
||||
|
||||
def _auth() -> oss2.Auth:
|
||||
ak, sk = _creds()
|
||||
return oss2.Auth(ak, sk)
|
||||
|
||||
|
||||
def _list_bucket_names(auth: oss2.Auth) -> list[str]:
|
||||
service = oss2.Service(auth, ENDPOINT)
|
||||
return [b.name for b in oss2.BucketIterator(service)]
|
||||
|
||||
|
||||
def _bucket(auth: oss2.Auth, name: str) -> oss2.Bucket:
|
||||
return oss2.Bucket(auth, ENDPOINT, name)
|
||||
|
||||
|
||||
def setup_bucket() -> str:
|
||||
auth = _auth()
|
||||
existing = _list_bucket_names(auth)
|
||||
print(f"existing buckets: {', '.join(existing) or '(none)'}")
|
||||
|
||||
chosen = None
|
||||
for name in BUCKET_CANDIDATES:
|
||||
if name in existing:
|
||||
chosen = name
|
||||
print(f"bucket_exists: {name}")
|
||||
break
|
||||
bucket = _bucket(auth, name)
|
||||
try:
|
||||
bucket.create_bucket(oss2.BUCKET_ACL_PUBLIC_READ)
|
||||
chosen = name
|
||||
print(f"created: {name}")
|
||||
break
|
||||
except oss2.exceptions.BucketAlreadyExists:
|
||||
print(f"name_conflict (other account?): {name}")
|
||||
continue
|
||||
except oss2.exceptions.OssError as exc:
|
||||
print(f"create_failed {name}: {exc.code} {exc.message[:160]}")
|
||||
continue
|
||||
|
||||
if not chosen:
|
||||
raise SystemExit("could not create or reuse any candidate bucket name")
|
||||
|
||||
bucket = _bucket(auth, chosen)
|
||||
try:
|
||||
bucket.put_bucket_acl(oss2.BUCKET_ACL_PUBLIC_READ)
|
||||
print("acl: public-read")
|
||||
except oss2.exceptions.OssError as exc:
|
||||
print(f"acl_warn: {exc.code} {exc.message[:120]}")
|
||||
|
||||
rule = CorsRule(
|
||||
allowed_origins=CORS_ORIGINS,
|
||||
allowed_methods=["GET", "HEAD"],
|
||||
allowed_headers=["*"],
|
||||
expose_headers=["ETag", "Content-Type", "Content-Length"],
|
||||
max_age_seconds=86400,
|
||||
)
|
||||
try:
|
||||
bucket.put_bucket_cors(BucketCors([rule]))
|
||||
print("cors: ok")
|
||||
except oss2.exceptions.OssError as exc:
|
||||
print(f"cors_warn: {exc.code} {exc.message[:200]}")
|
||||
|
||||
info = bucket.get_bucket_info()
|
||||
base = f"https://{chosen}.oss-{REGION}.aliyuncs.com"
|
||||
print(f"bucket: {chosen}")
|
||||
print(f"location: {info.location}")
|
||||
print(f"acl: {info.acl.grant if info.acl else '?'}")
|
||||
print(f"public_base: {base}")
|
||||
# Persist chosen name for upload step.
|
||||
state = ROOT / ".oss_ability_videos_bucket"
|
||||
state.write_text(chosen, encoding="utf-8")
|
||||
return chosen
|
||||
|
||||
|
||||
def _resolve_bucket_name(explicit: str | None) -> str:
|
||||
if explicit:
|
||||
return explicit
|
||||
state = ROOT / ".oss_ability_videos_bucket"
|
||||
if state.is_file():
|
||||
return state.read_text(encoding="utf-8").strip()
|
||||
return BUCKET_CANDIDATES[0]
|
||||
|
||||
|
||||
def _iter_local_videos() -> list[Path]:
|
||||
if not ABILITY_VIDEOS.is_dir():
|
||||
raise SystemExit(f"missing local videos: {ABILITY_VIDEOS}")
|
||||
files = sorted(
|
||||
p
|
||||
for p in ABILITY_VIDEOS.rglob("*")
|
||||
if p.is_file() and p.suffix.lower() in {".webm", ".mp4"}
|
||||
)
|
||||
return files
|
||||
|
||||
|
||||
def _object_key(local: Path) -> str:
|
||||
rel = local.relative_to(ABILITY_VIDEOS).as_posix()
|
||||
return PREFIX + rel
|
||||
|
||||
|
||||
def upload(
|
||||
bucket_name: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
max_files: int | None = None,
|
||||
offset: int = 0,
|
||||
) -> None:
|
||||
auth = _auth()
|
||||
bucket = _bucket(auth, bucket_name)
|
||||
files = _iter_local_videos()
|
||||
if offset or max_files is not None:
|
||||
end = None if max_files is None else offset + max_files
|
||||
files = files[offset:end]
|
||||
total = len(files)
|
||||
total_bytes = sum(f.stat().st_size for f in files)
|
||||
print(
|
||||
f"upload -> oss://{bucket_name}/{PREFIX} "
|
||||
f"({total} files in this batch, {total_bytes / 1e9:.2f} GB; "
|
||||
f"offset={offset})"
|
||||
)
|
||||
|
||||
# Remote index for skip-if-same-size.
|
||||
remote_sizes: dict[str, int] = {}
|
||||
if not force:
|
||||
print("listing remote objects ...")
|
||||
for obj in oss2.ObjectIterator(bucket, prefix=PREFIX):
|
||||
remote_sizes[obj.key] = int(obj.size)
|
||||
print(f"remote objects under prefix: {len(remote_sizes)}")
|
||||
|
||||
uploaded = skipped = failed = 0
|
||||
t0 = time.time()
|
||||
for i, path in enumerate(files, 1):
|
||||
key = _object_key(path)
|
||||
size = path.stat().st_size
|
||||
if not force and remote_sizes.get(key) == size:
|
||||
skipped += 1
|
||||
if i % 50 == 0 or i == total:
|
||||
print(f"[{i}/{total}] skip {key}")
|
||||
continue
|
||||
headers = {}
|
||||
ctype, _ = mimetypes.guess_type(str(path))
|
||||
if path.suffix.lower() == ".webm":
|
||||
ctype = "video/webm"
|
||||
elif path.suffix.lower() == ".mp4":
|
||||
ctype = "video/mp4"
|
||||
if ctype:
|
||||
headers["Content-Type"] = ctype
|
||||
try:
|
||||
# Resumable multipart for large files.
|
||||
oss2.resumable_upload(
|
||||
bucket,
|
||||
key,
|
||||
str(path),
|
||||
headers=headers,
|
||||
multipart_threshold=8 * 1024 * 1024,
|
||||
part_size=8 * 1024 * 1024,
|
||||
num_threads=4,
|
||||
)
|
||||
uploaded += 1
|
||||
if i % 10 == 0 or i == total or uploaded <= 3:
|
||||
elapsed = time.time() - t0
|
||||
print(
|
||||
f"[{i}/{total}] ok {key} "
|
||||
f"({size / 1e6:.1f} MB) uploaded={uploaded} skipped={skipped} "
|
||||
f"elapsed={elapsed:.0f}s"
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — keep uploading rest
|
||||
failed += 1
|
||||
print(f"[{i}/{total}] FAIL {key}: {type(exc).__name__}: {exc}")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
print(
|
||||
f"done: uploaded={uploaded} skipped={skipped} failed={failed} "
|
||||
f"elapsed={elapsed:.0f}s"
|
||||
)
|
||||
if failed:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def verify(bucket_name: str, samples: int = 5) -> None:
|
||||
import urllib.request
|
||||
|
||||
auth = _auth()
|
||||
bucket = _bucket(auth, bucket_name)
|
||||
base = f"https://{bucket_name}.oss-{REGION}.aliyuncs.com"
|
||||
files = _iter_local_videos()
|
||||
# Prefer a few heroes spread across the alphabet.
|
||||
picks = []
|
||||
if files:
|
||||
step = max(1, len(files) // samples)
|
||||
picks = [files[i] for i in range(0, len(files), step)][:samples]
|
||||
print(f"spot-check {len(picks)} URLs against {base}")
|
||||
ok = 0
|
||||
for path in picks:
|
||||
key = _object_key(path)
|
||||
url = f"{base}/{key}"
|
||||
try:
|
||||
req = urllib.request.Request(url, method="HEAD")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
code = resp.status
|
||||
ctype = resp.headers.get("Content-Type", "")
|
||||
clen = resp.headers.get("Content-Length", "")
|
||||
print(f" {code} {ctype} len={clen} {url}")
|
||||
if code == 200:
|
||||
ok += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" FAIL {url}: {exc}")
|
||||
# Count remote.
|
||||
n = sum(1 for _ in oss2.ObjectIterator(bucket, prefix=PREFIX))
|
||||
print(f"remote object count under {PREFIX}: {n}")
|
||||
print(f"spot-check ok: {ok}/{len(picks)}")
|
||||
if ok < len(picks):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("cmd", choices=("setup", "upload", "verify", "setup-upload"))
|
||||
ap.add_argument("--bucket", default=None)
|
||||
ap.add_argument("--force", action="store_true")
|
||||
ap.add_argument("--max-files", type=int, default=None, help="upload at most N files")
|
||||
ap.add_argument("--offset", type=int, default=0, help="skip first N local files")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.cmd in ("setup", "setup-upload"):
|
||||
name = setup_bucket()
|
||||
else:
|
||||
name = _resolve_bucket_name(args.bucket)
|
||||
|
||||
if args.cmd in ("upload", "setup-upload"):
|
||||
upload(name, force=args.force, max_files=args.max_files, offset=args.offset)
|
||||
if args.cmd == "verify":
|
||||
verify(name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user