climperor/evaluate.py

"""Score the recogniser against every labelled frame at once.

Labels live in samples/labels.json as {frame filename: 10 hero keys}, '?' for
slots nobody has identified yet. Those slots are skipped, not counted wrong.

Usage:
    python evaluate.py                # use the full template library
    python evaluate.py --cdn-only     # ignore templates/real, measure the fallback layer
"""

import json
import sys
from pathlib import Path

import cv2

from common import ROOT, load_config, load_template_library
from recognize import recognize_image

LABELS_PATH = ROOT / "samples" / "labels.json"
RAW_DIR = ROOT / "samples" / "raw"


def main() -> None:
    cdn_only = "--cdn-only" in sys.argv
    if not LABELS_PATH.is_file():
        sys.exit(f"missing {LABELS_PATH}")
    frames = json.loads(LABELS_PATH.read_text(encoding="utf-8"))["frames"]

    cfg = load_config()
    if not cfg["slots"]:
        sys.exit("config.json has no slots - run autocalibrate.py first")
    library = load_template_library()
    if cdn_only:
        library = [t for t in library if t[1] == "cdn"]
    print(f"library: {len(library)} templates{' (cdn only)' if cdn_only else ''}")

    graded = correct = skipped = 0
    misses: list[str] = []
    for name, truth in frames.items():
        path = RAW_DIR / name
        img = cv2.imread(str(path))
        if img is None:
            print(f"  {name}: MISSING, skipped")
            continue
        result = recognize_image(img, cfg, library)

        hits = frame_graded = 0
        worst = 1.0
        for slot, expected in zip(result["slots"], truth):
            if expected == "?":
                skipped += 1
                continue
            frame_graded += 1
            worst = min(worst, slot["score"])
            if slot["hero"] == expected:
                hits += 1
            else:
                misses.append(
                    f"    {name} slot {slot['slot']}: expected {expected}, "
                    f"got {slot['hero']} (raw {slot['raw_best']} "
                    f"score {slot['score']} margin {slot['margin']})"
                )
        graded += frame_graded
        correct += hits
        flag = " ranked" if result.get("ranked_overlay") else ""
        print(f"  {name}: {hits}/{frame_graded}  lowest score {worst:.3f}  {result['elapsed_ms']}ms{flag}")

    if misses:
        print("\nmisses:")
        print("\n".join(misses))
    pct = 100 * correct / graded if graded else 0
    print(f"\ntotal: {correct}/{graded} ({pct:.1f}%){f', {skipped} unlabelled slots skipped' if skipped else ''}")


if __name__ == "__main__":
    main()