From 76fa474b7d6a01a4d696a86919c1ed9af26ceb64 Mon Sep 17 00:00:00 2001
From: voson <voson.wang.s@aliyun.com>
Date: Thu, 30 Jul 2026 23:06:45 +0800
Subject: [PATCH] Fail fast on OpenDota 429 in hero_matches fetch.

Stop the long retry chain after three consecutive 429s and restore prior
hero cells so a burned IP no longer stalls the whole daily tier.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 web/fetch_hero_matches.py | 61 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 58 insertions(+), 3 deletions(-)

diff --git a/web/fetch_hero_matches.py b/web/fetch_hero_matches.py
index 68ed017..4cc2009 100644
--- a/web/fetch_hero_matches.py
+++ b/web/fetch_hero_matches.py
@@ -61,6 +61,34 @@ from shared.http_utils import download_icons, http_json, write_json_atomic
 from shared.paths import DATA, ITEM_ICONS
 
 OPENDOTA = "https://api.opendota.com/api"
+# Consecutive OpenDota 429s before aborting the remaining hero batch.
+_429_STREAK = 0
+_429_STREAK_LIMIT = 3
+
+
+class RateLimitTripped(Exception):
+    """Enough consecutive OpenDota 429s to stop the rest of this run."""
+
+
+def _opendota_json(url: str):
+    """Fail-fast OpenDota JSON (no 5/10/20/40s retry chain on 429)."""
+    global _429_STREAK
+    try:
+        data = http_json(url, retries=0)
+    except urllib.error.HTTPError as e:
+        if e.code == 429:
+            _429_STREAK += 1
+            print(
+                f"HTTP 429 {url} — streak {_429_STREAK}/{_429_STREAK_LIMIT}",
+                flush=True,
+            )
+            if _429_STREAK >= _429_STREAK_LIMIT:
+                raise RateLimitTripped(
+                    f"opendota 429 x{_429_STREAK}"
+                ) from e
+        raise
+    _429_STREAK = 0
+    return data
 ABILITY_IDS_URL = (
     "https://raw.githubusercontent.com/odota/dotaconstants/master/build/ability_ids.json"
 )
@@ -605,7 +633,9 @@ def opendota_url(path: str) -> str:
 
 def fetch_match(match_id: int) -> dict | None:
     try:
-        raw = http_json(opendota_url(f"/matches/{match_id}"))
+        raw = _opendota_json(opendota_url(f"/matches/{match_id}"))
+    except RateLimitTripped:
+        raise
     except (
         urllib.error.HTTPError,
         urllib.error.URLError,
@@ -620,7 +650,9 @@ def fetch_match(match_id: int) -> dict | None:
 def league_match_ids(hero_id: int, limit: int) -> list[dict]:
     """Return win list metas from /heroes/{id}/matches (up to candidate cap)."""
     try:
-        raw = http_json(opendota_url(f"/heroes/{hero_id}/matches"))
+        raw = _opendota_json(opendota_url(f"/heroes/{hero_id}/matches"))
+    except RateLimitTripped:
+        raise
     except (
         urllib.error.HTTPError,
         urllib.error.URLError,
@@ -700,7 +732,9 @@ def public_match_ids(
             sep = "&" if "?" in url else "?"
             url = f"{url}{sep}less_than_match_id={less_than}"
         try:
-            raw = http_json(url)
+            raw = _opendota_json(url)
+        except RateLimitTripped:
+            raise
         except (
             urllib.error.HTTPError,
             urllib.error.URLError,
@@ -1401,6 +1435,27 @@ def main() -> None:
                 public_region=public_region,
                 workers=workers,
             )
+        except RateLimitTripped as exc:
+            print(
+                f"  rate-limited; keeping prior cache for remaining ({exc})",
+                flush=True,
+            )
+            prior = existing.get("by_hero") if isinstance(existing, dict) else {}
+            if not isinstance(prior, dict):
+                prior = {}
+            for rest_key in pending[i - 1 :]:
+                prev = prior.get(rest_key)
+                if prev is not None:
+                    by_hero[rest_key] = prev
+            write_out(
+                args.out,
+                by_hero,
+                source=args.source,
+                limit=limit,
+                item_catalog=item_catalog,
+                public_region=public_region,
+            )
+            break
         except (
             urllib.error.HTTPError,
             urllib.error.URLError,