vosonandCursor
a089660ca9
Remove build_library and runtime artifacts, ignore regenerable outputs, and add README/AGENTS/DESIGN/CHANGELOG for the simplified project. Co-authored-by: Cursor <cursoragent@cursor.com>
73 lines
2.3 KiB
Python
73 lines
2.3 KiB
Python
"""Score the recogniser against every labelled frame at once.
|
|
|
|
Labels live in samples/labels.json as {frame filename: 10 hero keys}, '?' for
|
|
slots nobody has identified yet. Those slots are skipped, not counted wrong.
|
|
|
|
Usage:
|
|
python evaluate.py
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
|
|
import cv2
|
|
|
|
from common import ROOT, load_config, load_template_library
|
|
from recognize import recognize_image
|
|
|
|
LABELS_PATH = ROOT / "samples" / "labels.json"
|
|
RAW_DIR = ROOT / "samples" / "raw"
|
|
|
|
|
|
def main() -> None:
|
|
if not LABELS_PATH.is_file():
|
|
sys.exit(f"missing {LABELS_PATH}")
|
|
frames = json.loads(LABELS_PATH.read_text(encoding="utf-8"))["frames"]
|
|
|
|
cfg = load_config()
|
|
if not cfg["slots"]:
|
|
sys.exit("config.json has no slots - run autocalibrate.py first")
|
|
library = load_template_library()
|
|
print(f"library: {len(library)} CDN templates")
|
|
|
|
graded = correct = skipped = 0
|
|
misses: list[str] = []
|
|
for name, truth in frames.items():
|
|
path = RAW_DIR / name
|
|
img = cv2.imread(str(path))
|
|
if img is None:
|
|
print(f" {name}: MISSING, skipped")
|
|
continue
|
|
result = recognize_image(img, cfg, library)
|
|
|
|
hits = frame_graded = 0
|
|
worst = 1.0
|
|
for slot, expected in zip(result["slots"], truth):
|
|
if expected == "?":
|
|
skipped += 1
|
|
continue
|
|
frame_graded += 1
|
|
worst = min(worst, slot["score"])
|
|
if slot["hero"] == expected:
|
|
hits += 1
|
|
else:
|
|
misses.append(
|
|
f" {name} slot {slot['slot']}: expected {expected}, "
|
|
f"got {slot['hero']} (raw {slot['raw_best']} "
|
|
f"score {slot['score']} margin {slot['margin']})"
|
|
)
|
|
graded += frame_graded
|
|
correct += hits
|
|
flag = " ranked" if result.get("ranked_overlay") else ""
|
|
print(f" {name}: {hits}/{frame_graded} lowest score {worst:.3f} {result['elapsed_ms']}ms{flag}")
|
|
|
|
if misses:
|
|
print("\nmisses:")
|
|
print("\n".join(misses))
|
|
pct = 100 * correct / graded if graded else 0
|
|
print(f"\ntotal: {correct}/{graded} ({pct:.1f}%){f', {skipped} unlabelled slots skipped' if skipped else ''}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|