add map

app.py

@@ -36,12 +36,17 @@ TIMEOUT = 90
 SHOW_HINT_STRING = True  # Set to False to hide the hint string altogether
 HINT_STRING = "Once signed in, your votes will be recorded securely."
 
-# Load context length limits
-with open("context_window.json", "r") as file:
-    context_window = json.load(file)
+# Load model metadata
+model_metadata = pd.read_json("model_metadata.jsonl", lines=True)
 
-# Get list of available models
-available_models = list(context_window.keys())
+# Create a dictionary mapping model names to their context lengths
+model_context_window = model_metadata.set_index("model_name")["context_window"].to_dict()
+
+# Create a dictionary mapping model names to their links
+model_links = model_metadata.set_index("model_name")["link"].to_dict()
+
+# Get the list of available models
+available_models = model_metadata["model_name"].tolist()
 
 
 def fetch_github_content(url):
@@ -204,7 +209,7 @@ def truncate_prompt(model_alias, models, conversation_state):
     full_conversation = conversation_state[f"{model_alias}_chat"]
 
     # Get the context length for the model
-    context_length = context_window[models[model_alias]]
+    context_length = model_context_window[models[model_alias]]
 
     # Single loop to handle both FIFO removal and content truncation
     while len(json.dumps(full_conversation)) > context_length:
@@ -364,14 +369,14 @@ def load_content_from_hf(repo_name="SWE-Arena/model_votes"):
 
 def get_leaderboard_data(vote_entry=None, use_cache=True):
     year = str(datetime.now().year)
-    
+
     # Try to load cached leaderboard first
     if use_cache:
         try:
             cached_path = hf_hub_download(
                 repo_id="SWE-Arena/model_leaderboards",
                 filename=f"{year}.json",
-                repo_type="dataset"
+                repo_type="dataset",
             )
             with open(cached_path, "r") as f:
                 leaderboard_data = pd.read_json(f)
@@ -391,7 +396,7 @@ def get_leaderboard_data(vote_entry=None, use_cache=True):
                 return leaderboard_data
         except Exception as e:
             print(f"No cached leaderboard found for {year}, computing from votes...")
-    
+
     # Load feedback data from the Hugging Face repository
     vote_data = load_content_from_hf()
     vote_df = pd.DataFrame(vote_data)
@@ -511,14 +516,25 @@ def get_leaderboard_data(vote_entry=None, use_cache=True):
     )
 
     # Clean up potential inf/NaN values in the results
-    for result in [avr_result, bt_result, newman_result, eigen_result, elo_result, pagerank_result]:
-        result.scores = result.scores.replace([float('inf'), float('-inf')], float('nan'))
+    for result in [
+        avr_result,
+        bt_result,
+        newman_result,
+        eigen_result,
+        elo_result,
+        pagerank_result,
+    ]:
+        result.scores = result.scores.replace(
+            [float("inf"), float("-inf")], float("nan")
+        )
 
     # Calculate CEI results
     cei_result = {}
     for model in elo_result.scores.index:
         if model in model_stats and model_stats[model]["cei_max"] > 0:
-            cei_result[model] = round(model_stats[model]["cei_sum"] / model_stats[model]["cei_max"], 2)
+            cei_result[model] = round(
+                model_stats[model]["cei_sum"] / model_stats[model]["cei_max"], 2
+            )
         else:
             cei_result[model] = None
     cei_result = pd.Series(cei_result)
@@ -527,7 +543,9 @@ def get_leaderboard_data(vote_entry=None, use_cache=True):
     mcs_result = {}
     for model in elo_result.scores.index:
         if model in model_stats and model_stats[model]["self_matches"] > 0:
-            mcs_result[model] = round(model_stats[model]["self_draws"] / model_stats[model]["self_matches"], 2)
+            mcs_result[model] = round(
+                model_stats[model]["self_draws"] / model_stats[model]["self_matches"], 2
+            )
         else:
             mcs_result[model] = None
     mcs_result = pd.Series(mcs_result)
@@ -569,25 +587,33 @@ def get_leaderboard_data(vote_entry=None, use_cache=True):
         ["Rank"] + [col for col in leaderboard_data.columns if col != "Rank"]
     ]
 
+    # Make model names clickable with their corresponding links
+    leaderboard_data["Model"] = leaderboard_data["Model"].apply(
+        lambda model_name: f'<a href="{model_links[model_name]}" target="_blank">{model_name}</a>'
+    )
+
     # Save leaderboard data if this is a new vote
     if vote_entry is not None:
         try:
             # Convert DataFrame to JSON and save
-            json_content = leaderboard_data.to_json(orient='records', indent=4).encode('utf-8')
+            json_content = leaderboard_data.to_json(orient="records", indent=4).encode(
+                "utf-8"
+            )
             file_like_object = io.BytesIO(json_content)
-            
+
             upload_file(
                 path_or_fileobj=file_like_object,
                 path_in_repo=f"{year}.json",
                 repo_id="SWE-Arena/model_leaderboards",
                 repo_type="dataset",
-                use_auth_token=HfFolder.get_token()
+                use_auth_token=HfFolder.get_token(),
             )
         except Exception as e:
             print(f"Failed to save leaderboard cache: {e}")
-    
+
     return leaderboard_data
 
+
 # Function to enable or disable submit buttons based on textbox content
 def toggle_submit_button(text):
     if not text or text.strip() == "":
@@ -632,6 +658,7 @@ with gr.Blocks() as app:
                 "Newman Modularity Score",
                 "PageRank Score",
             ],
+            datatype=["number", "html", "number", "number", "number", "number", "number", "number", "number", "number"],
         )
         # Add a citation block in Markdown
         citation_component = gr.Markdown(
@@ -1242,7 +1269,9 @@ with gr.Blocks() as app:
             file_name = now.strftime("%Y%m%d_%H%M%S")
 
             # Save feedback back to the Hugging Face dataset
-            save_content_to_hf(vote_entry, "SWE-Arena/model_votes", folder_name, file_name)
+            save_content_to_hf(
+                vote_entry, "SWE-Arena/model_votes", folder_name, file_name
+            )
 
             conversation_state["right_chat"][0]["content"] = conversation_state[
                 "right_chat"
@@ -1253,7 +1282,10 @@ with gr.Blocks() as app:
 
             # Save conversations back to the Hugging Face dataset
             save_content_to_hf(
-                conversation_state, "SWE-Arena/model_conversations", folder_name, file_name
+                conversation_state,
+                "SWE-Arena/model_conversations",
+                folder_name,
+                file_name,
             )
 
             # Clear state

context_window.json

@@ -1,53 +0,0 @@
-{
-    "claude-3-haiku-20240307": 200000,
-    "claude-3-opus-20240229": 200000,
-    "claude-3-5-haiku-20241022": 200000,
-    "claude-3-5-sonnet-20241022": 200000,
-    "claude-3-7-sonnet-latest": 200000,
-    "claude-3-7-sonnet-20250219#thinking": 200000,
-    "claude-opus-4-20250514": 200000,
-    "claude-sonnet-4-20250514": 200000,
-    "claude-opus-4-1-20250805": 200000,
-    "doubao-1-5-pro-256k-250115": 256000,
-    "doubao-1-5-thinking-pro-250415": 256000,
-    "doubao-seed-1-6-250615": 256000,
-    "doubao-seed-1-6-thinking-250615": 256000,
-    "deepseek-chat": 64000,
-    "deepseek-r1": 64000,
-    "deepseek-v3.1": 1000000,
-    "deepseek-v3.1#thinking": 1000000,
-    "gemini-2.5-flash-preview-non-thinking": 1048576,
-    "gemini-2.5-flash-preview-thinking": 1048576,
-    "gemini-2.5-pro-preview-06-05": 1048576,
-    "gemma-3-27b-it": 128000,
-    "gpt-3.5-turbo": 16000,
-    "gpt-4-turbo": 128000,
-    "gpt-4o": 128000,
-    "gpt-4o-mini": 128000,
-    "gpt-4.1": 1000000,
-    "gpt-4.1-mini": 1000000,
-    "gpt-5": 400000,
-    "gpt-5-chat-latest": 400000,
-    "gpt-5-mini": 400000,
-    "gpt-5-nano": 400000,
-    "gpt-oss-120b": 128000,
-    "gpt-oss-20b": 128000,
-    "grok-3-fast-beta": 1000000,
-    "grok-3-beta": 1000000,
-    "grok-3-mini-fast-beta": 1000000,
-    "grok-3-mini-beta": 1000000,
-    "llama-3.1-405b": 128000,
-    "llama-3.3-70b": 128000,
-    "llama4-scout-instruct-basic": 10000000,
-    "llama4-maverick-instruct-basic": 10000000,
-    "mistral-large-latest": 131000,
-    "mistral-small-latest": 32000,
-    "o1": 128000,
-    "o1-mini": 128000,
-    "o3": 200000,
-    "o3-mini": 200000,
-    "o4-mini": 200000,
-    "qwen3-30b-a3b": 32768,
-    "qwen3-32b": 32768,
-    "qwen3-235b-a22b": 32768
-}

model_metadata.jsonl

@@ -0,0 +1,51 @@
+{"model_name": "claude-3-haiku-20240307", "context_window": 200000, "link": "https://www.anthropic.com/news/claude-3-haiku"}
+{"model_name": "claude-3-opus-20240229", "context_window": 200000, "link": "https://www.anthropic.com/news/claude-3-family"}
+{"model_name": "claude-3-5-haiku-20241022", "context_window": 200000, "link": "https://www.anthropic.com/claude/haiku"}
+{"model_name": "claude-3-5-sonnet-20241022", "context_window": 200000, "link": "https://www.anthropic.com/news/claude-3-5-sonnet"}
+{"model_name": "claude-3-7-sonnet-latest", "context_window": 200000, "link": "https://www.anthropic.com/claude/sonnet"}
+{"model_name": "claude-3-7-sonnet-20250219#thinking", "context_window": 200000, "link": "https://www.anthropic.com/claude/sonnet"}
+{"model_name": "claude-opus-4-20250514", "context_window": 200000, "link": "https://www.anthropic.com/news/claude-4"}
+{"model_name": "claude-sonnet-4-20250514", "context_window": 200000, "link": "https://www.anthropic.com/claude/sonnet"}
+{"model_name": "claude-opus-4-1-20250805", "context_window": 200000, "link": "https://www.anthropic.com/news/claude-opus-4-1"}
+{"model_name": "doubao-1-5-pro-256k-250115", "context_window": 256000, "link": "https://seed.bytedance.com"}
+{"model_name": "doubao-1-5-thinking-pro-250415", "context_window": 256000, "link": "https://seed.bytedance.com"}
+{"model_name": "doubao-seed-1-6-250615", "context_window": 256000, "link": "https://seed.bytedance.com"}
+{"model_name": "doubao-seed-1-6-thinking-250615", "context_window": 256000, "link": "https://seed.bytedance.com"}
+{"model_name": "deepseek-chat", "context_window": 64000, "link": "https://www.deepseek.com"}
+{"model_name": "deepseek-r1", "context_window": 64000, "link": "https://www.deepseek.com"}
+{"model_name": "deepseek-v3.1", "context_window": 1000000, "link": "https://www.deepseek.com"}
+{"model_name": "deepseek-v3.1#thinking", "context_window": 1000000, "link": "https://www.deepseek.com"}
+{"model_name": "gemini-2.5-flash-preview-non-thinking", "context_window": 1048576, "link": "https://deepmind.google/models/gemini/flash"}
+{"model_name": "gemini-2.5-flash-preview-thinking", "context_window": 1048576, "link": "https://deepmind.google/models/gemini/flash"}
+{"model_name": "gemini-2.5-pro-preview-06-05", "context_window": 1048576, "link": "https://deepmind.google/models/gemini/pro"}
+{"model_name": "gemma-3-27b-it", "context_window": 128000, "link": "https://ai.google.dev/gemma"}
+{"model_name": "gpt-3.5-turbo", "context_window": 16000, "link": "https://openai.com"}
+{"model_name": "gpt-4-turbo", "context_window": 128000, "link": "https://openai.com/index/gpt-4"}
+{"model_name": "gpt-4o", "context_window": 128000, "link": "https://openai.com"}
+{"model_name": "gpt-4o-mini", "context_window": 128000, "link": "https://openai.com"}
+{"model_name": "gpt-4.1", "context_window": 1000000, "link": "https://openai.com/index/gpt-4-1"}
+{"model_name": "gpt-4.1-mini", "context_window": 1000000, "link": "https://openai.com/index/gpt-4-1"}
+{"model_name": "gpt-5", "context_window": 400000, "link": "https://openai.com/gpt-5"}
+{"model_name": "gpt-5-chat-latest", "context_window": 400000, "link": "https://openai.com/gpt-5"}
+{"model_name": "gpt-5-mini", "context_window": 400000, "link": "https://openai.com/gpt-5"}
+{"model_name": "gpt-5-nano", "context_window": 400000, "link": "https://openai.com/gpt-5"}
+{"model_name": "gpt-oss-120b", "context_window": 128000, "link": "https://openai.com"}
+{"model_name": "gpt-oss-20b", "context_window": 128000, "link": "https://openai.com"}
+{"model_name": "grok-3-fast-beta", "context_window": 1000000, "link": "https://x.ai/news/grok-3"}
+{"model_name": "grok-3-beta", "context_window": 1000000, "link": "https://x.ai/news/grok-3"}
+{"model_name": "grok-3-mini-fast-beta", "context_window": 1000000, "link": "https://x.ai/news/grok-3"}
+{"model_name": "grok-3-mini-beta", "context_window": 1000000, "link": "https://x.ai/news/grok-3"}
+{"model_name": "llama-3.1-405b", "context_window": 128000, "link": "https://ai.meta.com/blog/meta-llama-3-1"}
+{"model_name": "llama-3.3-70b", "context_window": 128000, "link": "https://www.llama.com"}
+{"model_name": "llama4-scout-instruct-basic", "context_window": 10000000, "link": "https://www.llama.com/models/llama-4"}
+{"model_name": "llama4-maverick-instruct-basic", "context_window": 10000000, "link": "https://www.llama.com/models/llama-4"}
+{"model_name": "mistral-large-latest", "context_window": 131000, "link": "https://mistral.ai/news/mistral-large"}
+{"model_name": "mistral-small-latest", "context_window": 32000, "link": "https://mistral.ai/news/mistral-small-3-1"}
+{"model_name": "o1", "context_window": 128000, "link": "https://openai.com/o1"}
+{"model_name": "o1-mini", "context_window": 128000, "link": "https://openai.com/o1"}
+{"model_name": "o3", "context_window": 200000, "link": "https://openai.com/index/introducing-o3-and-o4-mini"}
+{"model_name": "o3-mini", "context_window": 200000, "link": "https://openai.com/index/introducing-o3-and-o4-mini"}
+{"model_name": "o4-mini", "context_window": 200000, "link": "https://openai.com/index/introducing-o3-and-o4-mini"}
+{"model_name": "qwen3-30b-a3b", "context_window": 32768, "link": "https://qwen-3.com"}
+{"model_name": "qwen3-32b", "context_window": 32768, "link": "https://qwen-3.com"}
+{"model_name": "qwen3-235b-a22b", "context_window": 32768, "link": "https://qwen-3.com"}