Auto-discover all benchmark:official leaderboards on the Hub

update_data.py

@@ -30,18 +30,96 @@ from huggingface_hub import HfApi
 
 SPACE_REPO = "davanstrien/benchmark-race"
 
-BENCHMARK_CONFIGS = [
-    {"dataset": "SWE-bench/SWE-bench_Verified", "key": "sweVerified", "name": "SWE-bench Verified", "gated": False},
-    {"dataset": "ScaleAI/SWE-bench_Pro", "key": "swePro", "name": "SWE-bench Pro", "gated": False},
-    {"dataset": "TIGER-Lab/MMLU-Pro", "key": "mmluPro", "name": "MMLU-Pro", "gated": False},
-    {"dataset": "Idavidrein/gpqa", "key": "gpqa", "name": "GPQA Diamond", "gated": True},
-    {"dataset": "cais/hle", "key": "hle", "name": "HLE", "gated": True},
-    {"dataset": "MathArena/aime_2026", "key": "aime2026", "name": "AIME 2026", "gated": False},
-    {"dataset": "MathArena/hmmt_feb_2026", "key": "hmmt2026", "name": "HMMT Feb 2026", "gated": False},
-    {"dataset": "allenai/olmOCR-bench", "key": "olmOcr", "name": "olmOCR-bench", "gated": False},
-    {"dataset": "harborframework/terminal-bench-2.0", "key": "terminalBench", "name": "Terminal-Bench 2.0", "gated": False},
-    {"dataset": "FutureMa/EvasionBench", "key": "evasionBench", "name": "EvasionBench", "gated": False},
-]
+# Benchmarks are auto-discovered from datasets tagged `benchmark:official` on
+# the Hub. The originals get keys preserved so the UI's hardcoded default
+# (`sweVerified` in index.html) keeps working; new benchmarks get
+# slugified keys and a name from cardData.pretty_name (or basename).
+OVERRIDES = {
+    "SWE-bench/SWE-bench_Verified":       ("sweVerified",   "SWE-bench Verified"),
+    "ScaleAI/SWE-bench_Pro":              ("swePro",        "SWE-bench Pro"),
+    "TIGER-Lab/MMLU-Pro":                 ("mmluPro",       "MMLU-Pro"),
+    "Idavidrein/gpqa":                    ("gpqa",          "GPQA Diamond"),
+    "cais/hle":                           ("hle",           "HLE"),
+    "MathArena/aime_2026":                ("aime2026",      "AIME 2026"),
+    "MathArena/hmmt_feb_2026":            ("hmmt2026",      "HMMT Feb 2026"),
+    "allenai/olmOCR-bench":               ("olmOcr",        "olmOCR-bench"),
+    "harborframework/terminal-bench-2.0": ("terminalBench", "Terminal-Bench 2.0"),
+    "FutureMa/EvasionBench":              ("evasionBench",  "EvasionBench"),
+}
+MIN_MODELS = 2
+
+
+def slugify(dataset_id: str) -> str:
+    base = dataset_id.split("/")[-1]
+    s = re.sub(r"[^a-zA-Z0-9]+", "_", base).strip("_")
+    return s or dataset_id.replace("/", "_")
+
+
+def discover_benchmarks(hf_token: str | None) -> list[dict]:
+    """Fetch every benchmark:official dataset with a usable leaderboard."""
+    print("Discovering official benchmarks...")
+    resp = httpx.get(
+        "https://huggingface.co/api/datasets",
+        params={"filter": "benchmark:official", "limit": 500},
+        timeout=30,
+    )
+    resp.raise_for_status()
+    datasets = resp.json()
+    print(f"  found {len(datasets)} datasets with benchmark:official tag")
+
+    configs = []
+    for d in datasets:
+        did = d["id"]
+        try:
+            info = httpx.get(f"https://huggingface.co/api/datasets/{did}", timeout=15).json()
+        except Exception as e:
+            print(f"  {did}: skipped (info fetch failed: {e})")
+            continue
+        gated = bool(info.get("gated"))
+        card = info.get("cardData") or {}
+        if did in OVERRIDES:
+            key, pretty = OVERRIDES[did]
+        else:
+            key = slugify(did)
+            pretty = card.get("pretty_name") or did.split("/")[-1]
+
+        headers = {"Authorization": f"Bearer {hf_token}"} if (gated and hf_token) else {}
+        if gated and not hf_token:
+            print(f"  {did}: skipped (gated, no token)")
+            continue
+        try:
+            lb = httpx.get(
+                f"https://huggingface.co/api/datasets/{did}/leaderboard",
+                headers=headers,
+                timeout=30,
+            )
+        except Exception as e:
+            print(f"  {did}: skipped (leaderboard fetch failed: {e})")
+            continue
+        if lb.status_code != 200:
+            print(f"  {did}: skipped (status {lb.status_code})")
+            continue
+        rows = lb.json()
+        if not isinstance(rows, list) or len(rows) < MIN_MODELS:
+            print(f"  {did}: skipped (only {len(rows) if isinstance(rows, list) else '?'} rows)")
+            continue
+
+        lower_is_better = False
+        for r in rows:
+            if isinstance(r, dict) and "lower_is_better" in r:
+                lower_is_better = bool(r["lower_is_better"])
+                break
+
+        configs.append({
+            "dataset": did,
+            "key": key,
+            "name": pretty,
+            "gated": gated,
+            "lower_is_better": lower_is_better,
+        })
+        print(f"  {did} -> {key} ({len(rows)} rows, lower_is_better={lower_is_better})")
+
+    return configs
 
 PALETTE = [
     "#6366f1", "#0d9488", "#d97706", "#e11d48", "#7c3aed",
@@ -73,13 +151,21 @@ def fetch_leaderboard(config: dict, hf_token: str | None) -> list[dict]:
         print(f"    error: {e}")
         return []
 
-    seen = {}
+    lower = config.get("lower_is_better", False)
+    seen: dict[str, float] = {}
     for entry in data:
+        if not isinstance(entry, dict):
+            continue
         model_id = entry.get("modelId")
         score = entry.get("value")
         if model_id and score is not None:
-            score = float(score)
-            if model_id not in seen or score > seen[model_id]:
+            try:
+                score = float(score)
+            except (TypeError, ValueError):
+                continue
+            if model_id not in seen:
+                seen[model_id] = score
+            elif (lower and score < seen[model_id]) or (not lower and score > seen[model_id]):
                 seen[model_id] = score
 
     print(f"    {len(seen)} models")
@@ -144,13 +230,21 @@ def main():
     hf_token = os.environ.get("HF_TOKEN")
     print("Generating data.json for bar chart race\n")
 
-    all_scores: dict[str, list[dict]] = {}
+    benchmark_configs = discover_benchmarks(hf_token)
+    print(f"\n{len(benchmark_configs)} usable benchmarks\n")
+
+    all_scores: dict[str, dict] = {}
     all_model_ids: set[str] = set()
 
-    for config in BENCHMARK_CONFIGS:
+    for config in benchmark_configs:
         rows = fetch_leaderboard(config, hf_token)
         if rows:
-            all_scores[config["key"]] = {"name": config["name"], "rows": rows}
+            all_scores[config["key"]] = {
+                "name": config["name"],
+                "dataset": config["dataset"],
+                "lower_is_better": config["lower_is_better"],
+                "rows": rows,
+            }
             all_model_ids.update(r["model_id"] for r in rows)
 
     print(f"\n{len(all_model_ids)} unique models across {len(all_scores)} benchmarks")
@@ -179,8 +273,13 @@ def main():
                 "score": round(row["score"], 2),
                 "date": model_dates[mid]["date"],
             })
-        if models:
-            benchmarks[key] = {"name": info["name"], "models": models}
+        if len(models) >= MIN_MODELS:
+            benchmarks[key] = {
+                "name": info["name"],
+                "dataset": info["dataset"],
+                "lower_is_better": info["lower_is_better"],
+                "models": models,
+            }
 
     print(f"\nFetching logos for {len(all_providers)} providers...")
     logos = fetch_all_logos(all_providers)