vosonandCursor
9c5aa5b610
Separate the local recognition, web publishing, and shared data paths while preserving direct script execution and existing site content. Co-authored-by: Cursor <cursoragent@cursor.com>
297 lines
9.2 KiB
Python
297 lines
9.2 KiB
Python
"""One-shot helper: create OSS bucket + sync ability videos (keyzoo inject).
|
|
|
|
Env (from keyzoo asset_exec on digitevents/voson-RAM):
|
|
KEYZOO_ASSET_META_ACCESSKEY_ID
|
|
KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET
|
|
|
|
Usage:
|
|
python _oss_ability_videos.py setup
|
|
python _oss_ability_videos.py upload [--force]
|
|
python _oss_ability_videos.py verify
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import mimetypes
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import oss2
|
|
from oss2.models import BucketCors, CorsRule
|
|
|
|
from shared.paths import ABILITY_VIDEOS, ROOT
|
|
|
|
BUCKET_CANDIDATES = ("climperor", "climperor-videos", "climperor-ability-videos")
|
|
ENDPOINT = "https://oss-cn-shanghai.aliyuncs.com"
|
|
REGION = "cn-shanghai"
|
|
PREFIX = "ability-video/"
|
|
DEFAULT_BASE = "https://climperor.oss-cn-shanghai.aliyuncs.com"
|
|
|
|
CORS_ORIGINS = [
|
|
"https://dota2.refining.dev",
|
|
"https://climperor-relations.pages.dev",
|
|
"http://localhost:8080",
|
|
"http://localhost:8765",
|
|
"http://127.0.0.1:8080",
|
|
"http://127.0.0.1:8765",
|
|
"http://localhost:3000",
|
|
"http://127.0.0.1:3000",
|
|
]
|
|
|
|
|
|
def _creds() -> tuple[str, str]:
|
|
ak = os.environ.get("KEYZOO_ASSET_META_ACCESSKEY_ID") or os.environ.get(
|
|
"OSS_ACCESS_KEY_ID"
|
|
)
|
|
sk = os.environ.get("KEYZOO_ASSET_SECRET_ACCESSKEY_SECRET") or os.environ.get(
|
|
"OSS_ACCESS_KEY_SECRET"
|
|
)
|
|
if not ak or not sk:
|
|
raise SystemExit(
|
|
"missing AccessKey: inject via keyzoo or set OSS_ACCESS_KEY_ID / "
|
|
"OSS_ACCESS_KEY_SECRET"
|
|
)
|
|
return ak, sk
|
|
|
|
|
|
def _auth() -> oss2.Auth:
|
|
ak, sk = _creds()
|
|
return oss2.Auth(ak, sk)
|
|
|
|
|
|
def _list_bucket_names(auth: oss2.Auth) -> list[str]:
|
|
service = oss2.Service(auth, ENDPOINT)
|
|
return [b.name for b in oss2.BucketIterator(service)]
|
|
|
|
|
|
def _bucket(auth: oss2.Auth, name: str) -> oss2.Bucket:
|
|
return oss2.Bucket(auth, ENDPOINT, name)
|
|
|
|
|
|
def setup_bucket() -> str:
|
|
auth = _auth()
|
|
existing = _list_bucket_names(auth)
|
|
print(f"existing buckets: {', '.join(existing) or '(none)'}")
|
|
|
|
chosen = None
|
|
for name in BUCKET_CANDIDATES:
|
|
if name in existing:
|
|
chosen = name
|
|
print(f"bucket_exists: {name}")
|
|
break
|
|
bucket = _bucket(auth, name)
|
|
try:
|
|
bucket.create_bucket(oss2.BUCKET_ACL_PUBLIC_READ)
|
|
chosen = name
|
|
print(f"created: {name}")
|
|
break
|
|
except oss2.exceptions.BucketAlreadyExists:
|
|
print(f"name_conflict (other account?): {name}")
|
|
continue
|
|
except oss2.exceptions.OssError as exc:
|
|
print(f"create_failed {name}: {exc.code} {exc.message[:160]}")
|
|
continue
|
|
|
|
if not chosen:
|
|
raise SystemExit("could not create or reuse any candidate bucket name")
|
|
|
|
bucket = _bucket(auth, chosen)
|
|
try:
|
|
bucket.put_bucket_acl(oss2.BUCKET_ACL_PUBLIC_READ)
|
|
print("acl: public-read")
|
|
except oss2.exceptions.OssError as exc:
|
|
print(f"acl_warn: {exc.code} {exc.message[:120]}")
|
|
|
|
rule = CorsRule(
|
|
allowed_origins=CORS_ORIGINS,
|
|
allowed_methods=["GET", "HEAD"],
|
|
allowed_headers=["*"],
|
|
expose_headers=["ETag", "Content-Type", "Content-Length"],
|
|
max_age_seconds=86400,
|
|
)
|
|
try:
|
|
bucket.put_bucket_cors(BucketCors([rule]))
|
|
print("cors: ok")
|
|
except oss2.exceptions.OssError as exc:
|
|
print(f"cors_warn: {exc.code} {exc.message[:200]}")
|
|
|
|
info = bucket.get_bucket_info()
|
|
base = f"https://{chosen}.oss-{REGION}.aliyuncs.com"
|
|
print(f"bucket: {chosen}")
|
|
print(f"location: {info.location}")
|
|
print(f"acl: {info.acl.grant if info.acl else '?'}")
|
|
print(f"public_base: {base}")
|
|
# Persist chosen name for upload step.
|
|
state = ROOT / ".oss_ability_videos_bucket"
|
|
state.write_text(chosen, encoding="utf-8")
|
|
return chosen
|
|
|
|
|
|
def _resolve_bucket_name(explicit: str | None) -> str:
|
|
if explicit:
|
|
return explicit
|
|
state = ROOT / ".oss_ability_videos_bucket"
|
|
if state.is_file():
|
|
return state.read_text(encoding="utf-8").strip()
|
|
return BUCKET_CANDIDATES[0]
|
|
|
|
|
|
def _iter_local_videos() -> list[Path]:
|
|
if not ABILITY_VIDEOS.is_dir():
|
|
raise SystemExit(f"missing local videos: {ABILITY_VIDEOS}")
|
|
files = sorted(
|
|
p
|
|
for p in ABILITY_VIDEOS.rglob("*")
|
|
if p.is_file() and p.suffix.lower() in {".webm", ".mp4"}
|
|
)
|
|
return files
|
|
|
|
|
|
def _object_key(local: Path) -> str:
|
|
rel = local.relative_to(ABILITY_VIDEOS).as_posix()
|
|
return PREFIX + rel
|
|
|
|
|
|
def upload(
|
|
bucket_name: str,
|
|
*,
|
|
force: bool = False,
|
|
max_files: int | None = None,
|
|
offset: int = 0,
|
|
) -> None:
|
|
auth = _auth()
|
|
bucket = _bucket(auth, bucket_name)
|
|
files = _iter_local_videos()
|
|
if offset or max_files is not None:
|
|
end = None if max_files is None else offset + max_files
|
|
files = files[offset:end]
|
|
total = len(files)
|
|
total_bytes = sum(f.stat().st_size for f in files)
|
|
print(
|
|
f"upload -> oss://{bucket_name}/{PREFIX} "
|
|
f"({total} files in this batch, {total_bytes / 1e9:.2f} GB; "
|
|
f"offset={offset})"
|
|
)
|
|
|
|
# Remote index for skip-if-same-size.
|
|
remote_sizes: dict[str, int] = {}
|
|
if not force:
|
|
print("listing remote objects ...")
|
|
for obj in oss2.ObjectIterator(bucket, prefix=PREFIX):
|
|
remote_sizes[obj.key] = int(obj.size)
|
|
print(f"remote objects under prefix: {len(remote_sizes)}")
|
|
|
|
uploaded = skipped = failed = 0
|
|
t0 = time.time()
|
|
for i, path in enumerate(files, 1):
|
|
key = _object_key(path)
|
|
size = path.stat().st_size
|
|
if not force and remote_sizes.get(key) == size:
|
|
skipped += 1
|
|
if i % 50 == 0 or i == total:
|
|
print(f"[{i}/{total}] skip {key}")
|
|
continue
|
|
headers = {}
|
|
ctype, _ = mimetypes.guess_type(str(path))
|
|
if path.suffix.lower() == ".webm":
|
|
ctype = "video/webm"
|
|
elif path.suffix.lower() == ".mp4":
|
|
ctype = "video/mp4"
|
|
if ctype:
|
|
headers["Content-Type"] = ctype
|
|
try:
|
|
# Resumable multipart for large files.
|
|
oss2.resumable_upload(
|
|
bucket,
|
|
key,
|
|
str(path),
|
|
headers=headers,
|
|
multipart_threshold=8 * 1024 * 1024,
|
|
part_size=8 * 1024 * 1024,
|
|
num_threads=4,
|
|
)
|
|
uploaded += 1
|
|
if i % 10 == 0 or i == total or uploaded <= 3:
|
|
elapsed = time.time() - t0
|
|
print(
|
|
f"[{i}/{total}] ok {key} "
|
|
f"({size / 1e6:.1f} MB) uploaded={uploaded} skipped={skipped} "
|
|
f"elapsed={elapsed:.0f}s"
|
|
)
|
|
except Exception as exc: # noqa: BLE001 — keep uploading rest
|
|
failed += 1
|
|
print(f"[{i}/{total}] FAIL {key}: {type(exc).__name__}: {exc}")
|
|
|
|
elapsed = time.time() - t0
|
|
print(
|
|
f"done: uploaded={uploaded} skipped={skipped} failed={failed} "
|
|
f"elapsed={elapsed:.0f}s"
|
|
)
|
|
if failed:
|
|
raise SystemExit(1)
|
|
|
|
|
|
def verify(bucket_name: str, samples: int = 5) -> None:
|
|
import urllib.request
|
|
|
|
auth = _auth()
|
|
bucket = _bucket(auth, bucket_name)
|
|
base = f"https://{bucket_name}.oss-{REGION}.aliyuncs.com"
|
|
files = _iter_local_videos()
|
|
# Prefer a few heroes spread across the alphabet.
|
|
picks = []
|
|
if files:
|
|
step = max(1, len(files) // samples)
|
|
picks = [files[i] for i in range(0, len(files), step)][:samples]
|
|
print(f"spot-check {len(picks)} URLs against {base}")
|
|
ok = 0
|
|
for path in picks:
|
|
key = _object_key(path)
|
|
url = f"{base}/{key}"
|
|
try:
|
|
req = urllib.request.Request(url, method="HEAD")
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
code = resp.status
|
|
ctype = resp.headers.get("Content-Type", "")
|
|
clen = resp.headers.get("Content-Length", "")
|
|
print(f" {code} {ctype} len={clen} {url}")
|
|
if code == 200:
|
|
ok += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
print(f" FAIL {url}: {exc}")
|
|
# Count remote.
|
|
n = sum(1 for _ in oss2.ObjectIterator(bucket, prefix=PREFIX))
|
|
print(f"remote object count under {PREFIX}: {n}")
|
|
print(f"spot-check ok: {ok}/{len(picks)}")
|
|
if ok < len(picks):
|
|
raise SystemExit(1)
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("cmd", choices=("setup", "upload", "verify", "setup-upload"))
|
|
ap.add_argument("--bucket", default=None)
|
|
ap.add_argument("--force", action="store_true")
|
|
ap.add_argument("--max-files", type=int, default=None, help="upload at most N files")
|
|
ap.add_argument("--offset", type=int, default=0, help="skip first N local files")
|
|
args = ap.parse_args()
|
|
|
|
if args.cmd in ("setup", "setup-upload"):
|
|
name = setup_bucket()
|
|
else:
|
|
name = _resolve_bucket_name(args.bucket)
|
|
|
|
if args.cmd in ("upload", "setup-upload"):
|
|
upload(name, force=args.force, max_files=args.max_files, offset=args.offset)
|
|
if args.cmd == "verify":
|
|
verify(name)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|