from html import escape
import random
import gradio as gr
import pandas as pd
import plotly.graph_objects as go
FILTER_COLUMNS = ["Platform", "Endpoint Owner", "Model", "Optimized"]
MAX_COMPARE_MODELS = 4
DEFAULT_COMPARE_PROMPTS = 3
MAX_COMPARE_PROMPTS = 8
ALL_LEADERBOARD_NOTE = """
> **π‘ Note:** Each efficiency metric and quality metric captures only one dimension of
> model capacity. Rankings may vary when considering other metrics. "Endpoint Owner"
> refers to the publisher of the evaluated endpoint and may differ from the original
> model creator. "Optimized" indicates that the endpoint uses an explicit optimization
> mode or serves a modified/accelerated variant.
"""
ONEIG_SCORES_CONTENT = """
### Reported OneIG scores
- **Anime Alignment** β alignment for anime and stylization prompts
- **Human Alignment** β alignment for portrait prompts
- **Object Alignment** β alignment for general-object prompts
The leaderboard's **OneIG Overall Score** is the mean of each model's available
category scores.
"""
ONEIG_RUN_DETAILS_CONTENT = """
### Reading the results
Each row records the evaluation date, generation time, price per image, and links to
the source evaluation runs when available. Use the filters in the leaderboard to
compare providers, models, and optimized variants.
"""
P_JUDGE_SCORES_CONTENT = """
### Reported P-Judge scores
- **P-Judge Overall** β automatic preference / quality score for generated images
Higher scores indicate stronger performance on the P-Judge evaluation.
"""
P_JUDGE_RUN_DETAILS_CONTENT = """
### Reading the results
Each row records the evaluation date, generation time, and price per image when
available. Rankings are ordered by **P-Judge Overall**.
"""
DATAPOINT_ELO_SCORES_CONTENT = """
### Reported Datapoint Elo scores
- **Datapoint Elo** β human-preference Elo rating from pairwise comparisons
Elo values move whenever new models are added and re-evaluated against the field,
so the Date column reflects the leaderboard snapshot date.
"""
DATAPOINT_ELO_RUN_DETAILS_CONTENT = """
### Reading the results
Each row records the snapshot date, generation time, and price per image when
available. Rankings are ordered by **Datapoint Elo**.
"""
RAPIDATA_ELO_SCORES_CONTENT = """
### Reported Rapidata Elo scores
- **Rapidata Elo** β Elo rating from the Rapidata evaluation suite
Higher Elo indicates stronger relative performance on that suite.
"""
RAPIDATA_ELO_RUN_DETAILS_CONTENT = """
### Reading the results
Each row records generation time and price per image when available. Rankings are
ordered by **Rapidata Elo**.
"""
BENCHMARK_AI_ELO_SCORES_CONTENT = """
### Reported Benchmark.ai Elo scores
- **Benchmark.ai Elo** β Elo rating from the Benchmark.ai leaderboard
This score is not from the Qwen Image Bench prompt suite; it is shown alongside for
cross-benchmark comparison.
"""
BENCHMARK_AI_ELO_RUN_DETAILS_CONTENT = """
### Reading the results
Each row records generation time and price per image when available. Rankings for
this metric alone are ordered by **Benchmark.ai Elo**.
"""
ABOUT_OVERVIEW_CONTENT = """
# About InferBench
InferBench compares **text-to-image models** on quality, preference, latency, and
price. Results are organized by prompt suite (benchmark), not by a single opaque
score.
## What you can do here
- **Home** β snapshot of each prompt suite and unique model count.
- **Benchmarks** β open a prompt suite to see its metric columns, graphs (including a
quality-vs-price Pareto frontier), and side-by-side sample generations.
- **About** β this page.
## Current prompt suites
### OneIG Alignment
The **alignment** slice of OneIG (not the full OneIG suite), across three categories:
- Anime / stylization
- Human / portrait
- General object
The table reports category **alignment scores**, **Datapoint Elo** columns (Anime /
Human / Object), median / min generation time, and price per image. Rankings use the
mean of each model's available category alignment scores (missing categories are
skipped for that model).
### Qwen Image Bench
A shared prompt suite with multiple evaluation tracks shown as columns:
- **P-Judge Overall** β automatic preference / quality score
- **Datapoint Elo** β human-preference Elo (default sort key)
- **Rapidata Elo** β Elo from the Rapidata evaluation on this suite
Plus latency and price metadata, and combined generations for visual comparison.
## How to read the numbers
Quality metrics from different suites are **not interchangeable** β a high OneIG
Alignment score is not the same quantity as a Datapoint Elo. Prefer comparing models
*within* a benchmark column, and use price / generation time when you care about
efficiency.
The Pareto plot highlights models that are not dominated on both **higher score** and
**lower price**.
"""
ABOUT_DETAILS_CONTENT = """
# Data & caveats
- Endpoint **price** and **generation time** come from the evaluation snapshots linked
in each table where available.
- Some models are missing individual metric columns; empty cells mean that track was
not run (or not reported) for that model.
- Elo ratings can shift when the comparison pool changes β treat them as relative
rankings for the snapshot, not absolute constants.
- Optimized / accelerated endpoints (when labeled) may differ from the base model
publisher's default serving stack.
Built by [Pruna AI](https://www.pruna.ai/). Contributions and new evaluation runs welcome.
"""
COMMUNITY_CONTENT = """
"""
CITATION_CONTENT = """
```bibtex
@misc{InferBench,
title={InferBench: A Leaderboard for Text-to-Image Models},
author={PrunaAI},
year={2026},
howpublished={\\url{https://huggingface.co/spaces/PrunaAI/InferBench}}
}
```
"""
def render_header():
gr.HTML(
"""
InferBench
Compare text-to-image models on quality, speed, and price
"""
)
def _top_models(data, score_column, n=3):
if score_column not in data.columns or "Model" not in data.columns:
return []
ranked = (
data[["Model", score_column]]
.dropna(subset=[score_column])
.loc[lambda df: ~df["Model"].astype(str).str.startswith("#")]
.sort_values(score_column, ascending=False)
.head(n)
)
return [
(str(row["Model"]), float(row[score_column]))
for _, row in ranked.iterrows()
]
def _home_highlights(benchmarks):
"""Quality leaders per suite β more relevant than cheapest/fastest outliers."""
highlights = []
unique_models = set()
for benchmark in benchmarks:
data = benchmark.get("data")
if data is None or "Model" not in getattr(data, "columns", []):
continue
active = data[~data["Model"].astype(str).str.startswith("#")]
unique_models.update(active["Model"].astype(str).tolist())
score_column = benchmark.get("overall_column")
score_columns = benchmark.get("score_columns") or []
if not score_column or score_column not in data.columns:
score_column = score_columns[0] if score_columns else None
top = _top_models(data, score_column, n=1) if score_column else []
if not top:
continue
model, score = top[0]
highlights.append(
{
"label": f"BEST {benchmark['title'].upper()}",
"model": model,
"detail": f"{_display_label(score_column)} Β· {_format_score(score)}",
}
)
if unique_models:
highlights.append(
{
"label": "MODELS SCORED",
"model": str(len(unique_models)),
"detail": "unique across prompt suites",
}
)
return highlights
def render_home(benchmarks):
highlights = _home_highlights(benchmarks)
gr.Markdown(
"""
InferBench is organized by **prompt suite**. There is no single global
βbest modelβ score β open a benchmark for full tables, graphs, and sample
comparisons.
"""
)
if highlights:
callout_bits = [
f"{escape(item['label'])}"
f"{escape(item['model'])}"
f"{escape(item['detail'])}
"
for item in highlights
]
gr.HTML(f'{"".join(callout_bits)}
')
gr.Markdown("### Benchmark snapshots")
with gr.Row(equal_height=True, elem_classes="benchmark-catalogue-row"):
for benchmark in benchmarks:
data = benchmark["data"]
score_column = benchmark.get("overall_column")
score_columns = benchmark.get("score_columns") or []
if not score_column or score_column not in data.columns:
score_column = score_columns[0] if score_columns else None
top = _top_models(data, score_column, n=3) if score_column else []
score_label = _display_label(score_column) if score_column else "Score"
rows_html = "".join(
f"{idx}"
f"{escape(model)}"
f"{_format_score(score)}"
for idx, (model, score) in enumerate(top, start=1)
) or "No scores yet."
with gr.Column(scale=1, min_width=280):
gr.HTML(
f"""
{escape(benchmark.get("emoji", "π"))}
{escape(benchmark["title"])}
{escape(benchmark.get("card_description", ""))}
Top 3 by {escape(score_label)}
{rows_html}
"""
)
def _format_leaderboard_cell(column, value):
if pd.isna(value) or value is None or value == "":
return "β"
label = str(column).lower()
if label == "rank":
return str(int(value))
if "price" in label:
return _format_price(value)
if "time" in label or "generation" in label:
try:
return f"{float(value):.2f}"
except (TypeError, ValueError):
return escape(str(value))
if label in {"model", "platform", "endpoint owner", "optimized"}:
return escape(str(value))
try:
number = float(value)
except (TypeError, ValueError):
return escape(str(value))
if abs(number) >= 100:
return f"{number:.1f}"
return f"{number:.4f}".rstrip("0").rstrip(".")
def _leaderboard_sort_value(column, value):
"""Raw value used by client-side column sorting."""
if pd.isna(value) or value is None or value == "":
return ""
label = str(column).lower()
if label in {"model", "platform", "endpoint owner", "optimized", "date"}:
return str(value).casefold()
try:
return f"{float(value):.10g}"
except (TypeError, ValueError):
return str(value).casefold()
def _leaderboard_sort_type(column):
label = str(column).lower()
if label in {"model", "platform", "endpoint owner", "optimized", "date"}:
return "text"
return "number"
def _leaderboard_html(data, columns, score_columns, overall_column):
leaderboard = _leaderboard_dataframe(
data, columns, score_columns, overall_column
)
if leaderboard.empty:
return (
'"
)
header_cells = []
for index, column in enumerate(leaderboard.columns):
sort_type = _leaderboard_sort_type(column)
header_cells.append(
f'{escape(str(column))} | '
)
body_rows = []
for _, row in leaderboard.iterrows():
cells = []
for column in leaderboard.columns:
css = "rank" if column == "Rank" else "metric-score"
sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)
if column == "Model":
css = "model-cell"
cells.append(
f''
f"{_format_leaderboard_cell(column, row[column])} | "
)
else:
cells.append(
f''
f"{_format_leaderboard_cell(column, row[column])} | "
)
body_rows.append(f"{''.join(cells)}
")
return f"""
"""
def render_leaderboard(
data,
columns,
note=None,
score_columns=None,
overall_column=None,
):
score_columns = list(score_columns or _infer_score_columns(columns))
overall_column = overall_column or _default_overall_column(score_columns)
platform_choices = _filter_choices(data, "Platform")
owner_choices = _filter_choices(data, "Endpoint Owner")
optimized_choices = _filter_choices(data, "Optimized")
if note:
gr.Markdown(note)
filter_inputs = []
with gr.Row(elem_classes="leaderboard-controls"):
search = gr.Textbox(
label="Search models",
placeholder="Search by model or provider",
scale=3,
)
filter_inputs.append(search)
platform = None
owner = None
optimized = None
if platform_choices:
platform = gr.Dropdown(
choices=platform_choices,
value=[],
label="Providers",
multiselect=True,
scale=1,
)
filter_inputs.append(platform)
if owner_choices:
owner = gr.Dropdown(
choices=owner_choices,
value=[],
label="Endpoint owners",
multiselect=True,
scale=1,
)
filter_inputs.append(owner)
if optimized_choices:
optimized = gr.Dropdown(
choices=optimized_choices,
value=[],
label="Optimized",
multiselect=True,
scale=1,
)
filter_inputs.append(optimized)
ranking = gr.HTML(
_leaderboard_html(data, columns, score_columns, overall_column),
elem_classes="ranking-table-host",
)
def update_ranking(
search_term,
platform_value=None,
owner_value=None,
optimized_value=None,
):
filtered_data = _filter_leaderboard(
data,
search_term,
platform_value or [],
owner_value or [],
optimized_value or [],
)
return _leaderboard_html(
filtered_data, columns, score_columns, overall_column
)
# Wire only the filters that actually exist for this table.
change_inputs = [search]
if platform is not None:
change_inputs.append(platform)
if owner is not None:
change_inputs.append(owner)
if optimized is not None:
change_inputs.append(optimized)
for component in filter_inputs:
component.change(
update_ranking,
inputs=change_inputs,
outputs=ranking,
)
def _infer_score_columns(columns):
return [column for column in columns if column.startswith("OneIG (")]
def _default_overall_column(score_columns):
if len(score_columns) == 1:
return score_columns[0]
return "OneIG Overall Score"
def _filter_choices(data, column):
if column not in data.columns:
return []
return sorted(data[column].dropna().astype(str).unique().tolist())
def _filter_leaderboard(data, search_term, platform, owner, optimized):
filtered = data.copy()
if search_term:
search_columns = [
column
for column in ["Model", "Platform", "Endpoint Owner"]
if column in filtered.columns
]
matches = pd.Series(False, index=filtered.index)
for column in search_columns:
matches |= filtered[column].astype(str).str.contains(
search_term, case=False, na=False
)
filtered = filtered[matches]
for column, values in [
("Platform", platform),
("Endpoint Owner", owner),
("Optimized", optimized),
]:
if values and column in filtered.columns:
filtered = filtered[filtered[column].astype(str).isin(values)]
return filtered
def _leaderboard_dataframe(data, columns, score_columns, overall_column):
# Honor the caller-provided column list so extra metrics (e.g. Elo) are not
# dropped just because they are not part of the ranking score_columns.
skip_columns = {"URL", "Rank"}
preferred_prefix = [
column
for column in ["Model", "Platform", "Endpoint Owner", "Optimized"]
if column in data.columns
]
preferred_suffix = [
column
for column in [
"Median Generation Time (s)",
"Min Generation Time (s)",
"Price / Image (USD)",
"Evaluation Date (UTC)",
"Date",
]
if column in data.columns
]
# Keep overall_column visible when the caller includes it (e.g. Datapoint Elo).
# Synthetic aggregates like OneIG Overall Score are simply omitted from `columns`.
middle = [
column
for column in columns
if column in data.columns
and column not in skip_columns
and column not in preferred_prefix
and column not in preferred_suffix
]
ordered_columns = []
seen = set()
for column in [*preferred_prefix, *middle, *preferred_suffix]:
if column not in seen:
seen.add(column)
ordered_columns.append(column)
leaderboard = data[ordered_columns].copy()
# Rank by overall when available, even if that column is not displayed.
if overall_column and overall_column in data.columns:
leaderboard = (
leaderboard.assign(_sort_key=data[overall_column])
.sort_values("_sort_key", ascending=False, na_position="last")
.drop(columns=["_sort_key"])
.reset_index(drop=True)
)
else:
leaderboard = leaderboard.reset_index(drop=True)
leaderboard.insert(0, "Rank", leaderboard.index + 1)
return leaderboard.rename(columns=_display_label)
def _display_label(column):
labels = {
"_overall_score": "Overall score",
"OneIG Overall Score": "Overall",
"OneIG (Anime Alignment)": "Anime",
"OneIG (Human Alignment)": "Human",
"OneIG (Object Alignment)": "Object",
"OneIG Anime Elo": "Anime Elo (Datapoint)",
"OneIG Human Elo": "Human Elo (Datapoint)",
"OneIG Object Elo": "Object Elo (Datapoint)",
"P-Judge Overall": "P-Judge",
"Datapoint Elo": "Datapoint Elo",
"Rapidata Elo": "Rapidata Elo",
"Benchmark.ai Elo": "Benchmark.ai Elo",
"Raw Win Rate": "Raw win rate",
"Median Generation Time (s)": "Median generation time",
"Min Generation Time (s)": "Min generation time",
"Price / Image (USD)": "Price per image",
"Evaluation Date (UTC)": "Date",
"Date": "Date",
}
return labels.get(column, column)
def _text_value(value):
return "β" if pd.isna(value) or value is None else escape(str(value))
def _format_score(value):
return "β" if pd.isna(value) or value is None else f"{float(value):.3f}"
def _format_price(value):
return "β" if pd.isna(value) or value is None else f"${float(value):.3f}"
def render_benchmark_detail(benchmark):
gr.Markdown(
f"""
# {benchmark["title"]}
{benchmark["intro"]}
"""
)
with gr.Tabs(elem_classes="subtabs", selected=0) as detail_tabs:
with gr.TabItem("Leaderboard"):
render_leaderboard(
benchmark["data"],
benchmark["columns"],
note=benchmark.get("note"),
score_columns=benchmark.get("score_columns"),
overall_column=benchmark.get("overall_column"),
)
with gr.TabItem("Graphs"):
render_benchmark_graphs(benchmark)
with gr.TabItem("Compare samples"):
render_compare_samples(benchmark)
return detail_tabs
def render_compare_samples(benchmark):
samples = benchmark.get("samples")
if not samples:
gr.Markdown(
"""
Sample comparison is not available for this benchmark yet.
When generations are linked, you will be able to pick models and browse
side-by-side outputs for the same prompts.
"""
)
return
models = samples["models"]
default_models = models[: min(2, len(models))]
gr.Markdown(
f"""
Pick up to {MAX_COMPARE_MODELS} models, then browse shared
prompts side by side. Images come from the public generation URLs for this
benchmark.
"""
)
with gr.Row(elem_classes="leaderboard-controls"):
model_picker = gr.Dropdown(
choices=models,
value=default_models,
multiselect=True,
max_choices=MAX_COMPARE_MODELS,
label="Models",
info=f"Select 1β{MAX_COMPARE_MODELS} models to compare",
scale=3,
)
prompt_count = gr.Slider(
minimum=1,
maximum=MAX_COMPARE_PROMPTS,
value=DEFAULT_COMPARE_PROMPTS,
step=1,
label="Prompts to show",
scale=1,
)
shuffle_button = gr.Button("Shuffle prompts", scale=1)
gallery = gr.HTML(
value=_build_compare_samples_html(
samples,
default_models,
DEFAULT_COMPARE_PROMPTS,
seed=0,
)
)
seed_state = gr.State(0)
def update_gallery(selected_models, num_prompts, seed):
return _build_compare_samples_html(
samples,
selected_models,
int(num_prompts),
seed=int(seed or 0),
)
def shuffle_gallery(selected_models, num_prompts, seed):
next_seed = int(seed or 0) + 1
return next_seed, _build_compare_samples_html(
samples,
selected_models,
int(num_prompts),
seed=next_seed,
)
model_picker.change(
update_gallery,
inputs=[model_picker, prompt_count, seed_state],
outputs=gallery,
)
prompt_count.change(
update_gallery,
inputs=[model_picker, prompt_count, seed_state],
outputs=gallery,
)
shuffle_button.click(
shuffle_gallery,
inputs=[model_picker, prompt_count, seed_state],
outputs=[seed_state, gallery],
)
def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
selected_models = [
model
for model in (selected_models or [])
if model in samples["images"]
][:MAX_COMPARE_MODELS]
if not selected_models:
return (
''
"Select at least one model to compare samples."
"
"
)
shared_prompt_ids = None
for model in selected_models:
model_prompt_ids = set(samples["images"][model])
shared_prompt_ids = (
model_prompt_ids
if shared_prompt_ids is None
else shared_prompt_ids & model_prompt_ids
)
shared_prompt_ids = sorted(shared_prompt_ids or [])
if not shared_prompt_ids:
return (
''
"No shared prompts found for the selected models."
"
"
)
rng = random.Random(seed)
prompt_pool = list(shared_prompt_ids)
rng.shuffle(prompt_pool)
chosen = prompt_pool[: max(1, min(int(num_prompts), len(prompt_pool)))]
columns = len(selected_models)
blocks = []
for index, prompt_id in enumerate(chosen, start=1):
prompt_text = escape(samples["prompts"].get(prompt_id, ""))
cells = []
for model in selected_models:
image_url = escape(samples["images"][model][prompt_id], quote=True)
cells.append(
f"""
{escape(model)}
"""
)
blocks.append(
f"""
Prompt {index}
{escape(prompt_id)}
{prompt_text}
{''.join(cells)}
"""
)
return "\n".join(blocks)
def render_benchmarks(benchmarks):
"""Catalogue cards + detail pages; back button returns to the catalogue."""
open_buttons = []
detail_entries = []
with gr.Column(visible=True, elem_classes="benchmark-catalogue") as catalogue:
gr.Markdown(
"""
# Benchmarks
Choose a prompt suite. Each one has a **Leaderboard** table, **Graphs**,
and **Compare samples**.
"""
)
card_rows = [benchmarks[i : i + 2] for i in range(0, len(benchmarks), 2)]
for row in card_rows:
with gr.Row(equal_height=True, elem_classes="benchmark-catalogue-row"):
for benchmark in row:
with gr.Column(scale=1, min_width=280):
with gr.Group(elem_classes="benchmark-card"):
gr.Markdown(
f"""
## {benchmark.get("emoji", "π")} {benchmark["title"]}
{benchmark["card_description"]}
"""
)
open_buttons.append(
(
benchmark["id"],
gr.Button("View benchmark β", variant="primary"),
)
)
for benchmark in benchmarks:
with gr.Column(visible=False) as detail:
back_button = gr.Button("β All benchmarks", size="sm")
render_benchmark_detail(benchmark)
detail_entries.append((benchmark["id"], detail, back_button))
nav_outputs = [catalogue, *[detail for _, detail, _ in detail_entries]]
def show_catalogue(_evt=None):
return (
gr.Column(visible=True),
*[gr.Column(visible=False) for _ in detail_entries],
)
def show_detail(selected_id):
return (
gr.Column(visible=False),
*[
gr.Column(visible=(benchmark_id == selected_id))
for benchmark_id, _, _ in detail_entries
],
)
for benchmark_id, button in open_buttons:
button.click(
lambda selected_id=benchmark_id: show_detail(selected_id),
outputs=nav_outputs,
)
for _, _, back_button in detail_entries:
back_button.click(show_catalogue, outputs=nav_outputs)
return show_catalogue, nav_outputs
def _pareto_frontier_mask(x_values, scores):
"""True for non-dominated points when maximizing score and minimizing x."""
n = len(x_values)
mask = [True] * n
for i in range(n):
for j in range(n):
if i == j:
continue
better_or_equal = x_values[j] <= x_values[i] and scores[j] >= scores[i]
strictly_better = x_values[j] < x_values[i] or scores[j] > scores[i]
if better_or_equal and strictly_better:
mask[i] = False
break
return mask
def _build_pareto_figure(
data,
score_column,
x_column,
x_title,
x_hover_prefix="",
x_hover_suffix="",
):
scatter = (
data[["Model", score_column, x_column]]
.dropna()
.copy()
.reset_index(drop=True)
)
if scatter.empty:
return None
x_values = scatter[x_column].astype(float).tolist()
scores = scatter[score_column].astype(float).tolist()
on_frontier = _pareto_frontier_mask(x_values, scores)
dominated = scatter.loc[[not flag for flag in on_frontier]]
frontier = scatter.loc[on_frontier].sort_values(x_column)
hover = (
"%{text}
"
f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
"
Score: %{y:.4f}"
)
fig = go.Figure()
if not dominated.empty:
fig.add_trace(
go.Scatter(
x=dominated[x_column],
y=dominated[score_column],
mode="markers",
name="Below frontier",
text=dominated["Model"],
hovertemplate=hover,
marker={
"size": 9,
"color": "#c4b5fd",
"opacity": 0.75,
"line": {"width": 0},
},
)
)
if not frontier.empty:
fig.add_trace(
go.Scatter(
x=frontier[x_column],
y=frontier[score_column],
mode="lines+markers",
name="On frontier",
text=frontier["Model"],
hovertemplate=hover,
line={"color": "#7c3aed", "width": 2.5},
marker={
"size": 12,
"color": "#db2777",
"line": {"width": 1.5, "color": "#7c3aed"},
},
)
)
score_label = _display_label(score_column)
fig.update_layout(
title=None,
xaxis_title=x_title,
yaxis_title=score_label,
autosize=True,
height=420,
margin={"l": 56, "r": 28, "t": 28, "b": 80},
legend={
"orientation": "h",
"yanchor": "top",
"y": -0.24,
"xanchor": "center",
"x": 0.5,
"bgcolor": "rgba(0,0,0,0)",
"font": {"color": "#e9d5ff", "size": 12},
},
# Dark-theme plot: soft purple panel + light text (readable, not a white flash).
plot_bgcolor="#1e1b4b",
paper_bgcolor="#17153b",
font={"color": "#e9d5ff", "size": 13},
)
axis_font = {"color": "#f3e8ff", "size": 13}
tick_font = {"color": "#ddd6fe", "size": 12}
fig.update_xaxes(
showgrid=True,
gridcolor="rgba(167, 139, 250, 0.28)",
zeroline=False,
title_font=axis_font,
tickfont=tick_font,
color="#e9d5ff",
)
fig.update_yaxes(
showgrid=True,
gridcolor="rgba(167, 139, 250, 0.28)",
zeroline=False,
title_font=axis_font,
tickfont=tick_font,
color="#e9d5ff",
)
return fig
def render_benchmark_graphs(benchmark):
data = benchmark["data"]
score_columns = [
column
for column in (benchmark.get("score_columns") or [])
if column in data.columns
]
overall_column = benchmark.get("overall_column")
if not score_columns and overall_column and overall_column in data.columns:
score_columns = [overall_column]
if not score_columns:
gr.Markdown("No score data is available yet.")
return
# Pareto every displayed quality metric vs price.
# Skip only synthetic aggregates (e.g. OneIG mean), not real sort keys like Datapoint Elo.
pareto_skip = {
"Model",
"Platform",
"Endpoint Owner",
"Optimized",
"URL",
"Rank",
"Median Generation Time (s)",
"Min Generation Time (s)",
"Price / Image (USD)",
"Evaluation Date (UTC)",
"Date",
"Raw Win Rate",
"OneIG Overall Score",
}
display_columns = benchmark.get("columns") or []
pareto_columns = []
for column in [*score_columns, *display_columns]:
if (
column in data.columns
and column not in pareto_skip
and column not in pareto_columns
and pd.api.types.is_numeric_dtype(data[column])
):
pareto_columns.append(column)
price_column = "Price / Image (USD)"
time_column = "Min Generation Time (s)"
price_figures = []
time_figures = []
for plot_column in pareto_columns:
if price_column in data.columns:
price_fig = _build_pareto_figure(
data,
plot_column,
x_column=price_column,
x_title="Price per image (USD)",
x_hover_prefix="$",
)
if price_fig is not None:
price_figures.append((plot_column, price_fig))
if time_column in data.columns:
time_fig = _build_pareto_figure(
data,
plot_column,
x_column=time_column,
x_title="Min generation time (s)",
x_hover_suffix="s",
)
if time_fig is not None:
time_figures.append((plot_column, time_fig))
if price_figures or time_figures:
gr.Markdown(
"### Pareto frontiers\n\n"
""
"Pink = on the frontier (lower cost or time at the same or better score). "
"Light purple = below the frontier."
""
)
with gr.Row(equal_height=False):
with gr.Column(scale=1, min_width=320):
gr.Markdown("#### Price vs score")
if not price_figures:
gr.Markdown("_No price data available._")
for plot_column, pareto_fig in price_figures:
gr.Markdown(f"**{_display_label(plot_column)}**")
gr.Plot(
value=pareto_fig,
show_label=False,
elem_classes="pareto-plot",
)
with gr.Column(scale=1, min_width=320):
gr.Markdown("#### Min generation time vs score")
if not time_figures:
gr.Markdown("_No min generation time data available._")
for plot_column, pareto_fig in time_figures:
gr.Markdown(f"**{_display_label(plot_column)}**")
gr.Plot(
value=pareto_fig,
show_label=False,
elem_classes="pareto-plot",
)
def render_about():
with gr.Row():
with gr.Column():
gr.Markdown(ABOUT_OVERVIEW_CONTENT)
with gr.Column():
gr.Markdown(ABOUT_DETAILS_CONTENT)
def render_footer():
gr.HTML(COMMUNITY_CONTENT)
with gr.Accordion("Citation", open=False):
gr.Markdown(CITATION_CONTENT)