Spaces:

PrunaAI
/

InferBench

Running

App Files Files Community

begumcig commited on 5 days ago

Commit

5fc63f1

1 Parent(s): 0e5bb7f

add: p-image-2-ideogram-benchmarking-results

Browse files

Files changed (9) hide show

app.py +864 -178
data/oneig_combined/generations.jsonl +3 -0
data/oneig_combined/prompts.jsonl +0 -0
data/oneig_model_price_and_median_generation_time.csv +40 -0
data/qwen_image_bench_combined/generations.jsonl +3 -0
data/qwen_image_bench_combined/prompts.jsonl +0 -0
data/qwen_image_bench_model_price_and_median_generation_time.csv +60 -0
requirements.txt +2 -1
ui.py +1123 -0

app.py CHANGED Viewed

@@ -1,8 +1,17 @@
 from pathlib import Path
 import gradio as gr
 import pandas as pd
-from gradio_leaderboard import Leaderboard
 custom_css = """
 .logo {
@@ -16,11 +25,470 @@ custom_css = """
 .text {
     font-size: 16px !important;
 }
-.tabs button {
-    font-size: 20px;
-}
 .subtabs button {
     font-size: 20px;
 }
 h1, h2 {
     margin: 0;
@@ -28,207 +496,425 @@ h1, h2 {
 }
 """
-# override method to avoid bugg
-Leaderboard.raise_error_if_incorrect_config = lambda self: None
-abs_path = Path(__file__).parent / "data"
-# Load the JSONL file into a pandas DataFrame using the json library
-df = pd.read_json(abs_path / "text_to_image.jsonl", lines=True)
-# Format URL column, handling None/empty URLs
-df["URL"] = df.apply(
-    lambda row: f'<a target="_blank" href="{row["URL"]}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">link</a>'
-    if pd.notna(row["URL"]) and row["URL"]
-    else "",
-    axis=1,
 )
-# Compute aggregate GenEval score if individual GenEval columns exist but "GenEval" doesn't
-if "GenEval" not in df.columns:
-    # Find all GenEval-related columns (e.g., "GenEval (Single Object) (VQA)")
-    geneval_cols = [col for col in df.columns if col.startswith("GenEval")]
-    if geneval_cols:
-        # Compute mean of all GenEval columns, ignoring NaN values
-        df["GenEval"] = df[geneval_cols].mean(axis=1, skipna=True)
-        # If all values are NaN, set to NaN
-        df["GenEval"] = df["GenEval"].where(df[geneval_cols].notna().any(axis=1), None)
-df = df[
-    [
-        "URL",
         "Platform",
-        "Owner",
         "Device",
         "Model",
-        "Optimization",
         "Median Inference Time",
         "Price per Image",
     ]
-    + [
-        col
-        for col in df.columns.tolist()
-        if col
-        not in [
-            "URL",
-            "Model",
-            "Median Inference Time",
-            "Price per Image",
-            "Platform",
-            "Owner",
-            "Device",
-            "Optimization",
-        ]
     ]
-]
-# Sort by GenEval if it exists, otherwise try other common metrics
-sort_column = None
-if "GenEval" in df.columns:
-    sort_column = "GenEval"
-elif "HPS (v2.1)" in df.columns:
-    sort_column = "HPS (v2.1)"
-elif "GenAI-Bench (VQA)" in df.columns:
-    sort_column = "GenAI-Bench (VQA)"
-elif len(df.columns) > 0:
-    # Sort by first numeric column if available
     numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
-    if numeric_cols:
-        sort_column = numeric_cols[0]
-if sort_column:
-    df = df.sort_values(by=sort_column, ascending=False, na_position="last")
-else:
-    # If no sort column found, just keep original order
-    pass
-# Format numeric columns to have at most 4 decimal places
-numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
-for col in numeric_cols:
-    df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
-with gr.Blocks("ParityError/Interstellar", fill_width=True, css=custom_css) as demo:
-    gr.HTML(
-        """
-            <div style="text-align: center;">
-                <img src="https://huggingface.co/datasets/PrunaAI/documentation-images/resolve/main/inferbench/logo2-cropped.png" style="width: 200px; height: auto; max-width: 100%; margin: 0 auto;">
-                <h1>🏋️ InferBench 🏋️</h1>
-                <h2>A cost/quality/speed Leaderboard for Inference Providers!</h2>
-            </div>
-            """
     )
-    with gr.Tabs():
-        with gr.TabItem("Text-to-Image Leaderboard [WIP]"):
-            Leaderboard(
-                value=df,
-                select_columns=df.columns.tolist(),
-                datatype=[
-                    "markdown",
-                    "markdown",
-                    "markdown",
-                    "markdown",
-                    "markdown",
-                    "markdown",
-                ]
-                + ["number"] * (len(df.columns.tolist()) - 6),
-                filter_columns=[
-                    "Platform",
-                    "Owner",
-                    "Device",
-                    "Model",
-                    "Optimization",
-                ],
-            )
-            gr.Markdown(
-                """
-                > **💡 Note:** Each efficiency metric and quality metric captures only one dimension of model capacity. Rankings may vary when considering other metrics. This leaderboard is a work in progress and will be updated regularly. For now, some metrics are not computed on the entire benchmark.
-                """
-            )
-        with gr.TabItem("About"):
-            with gr.Row():
-                with gr.Column():
-                    gr.Markdown(
-                        """
-                        # 📊 Text-to-Image Leaderboard
-                        This leaderboard compares the performance of different text-to-image providers.
-                        We started with a comprehensive benchmark comparing our very own FLUX-juiced with the “FLUX.1 [dev]” endpoints offered by:
-                        - Replicate: https://replicate.com/black-forest-labs/flux-dev
-                        - Fal: https://fal.ai/models/fal-ai/flux/dev
-                        - Fireworks AI: https://fireworks.ai/models/fireworks/flux-1-dev-fp8
-                        - Together AI: https://www.together.ai/models/flux-1-dev
-                        We also included the following non-FLUX providers:
-                        - AWS Nova Canvas: https://aws.amazon.com/ai/generative-ai/nova/creative/
-                        All of these inference providers offer implementations but they don’t always communicate about the optimisation methods used in the background, and most endpoint have different response times and performance measures.
-                        For comparison purposes we used the same generation set-up for all the providers.
-                        - 28 inference steps
-                        - 1024×1024 resolution
-                        - Guidance scale of 3.5
-                        - H100 GPU (80GB)—only reported by Replicate
-                        Although we did test with this specific Pruna configuration and hardware, the applied compression methods work with different config and hardware too!
-                        > We published a full blog post on [the creation of our FLUX-juiced endpoint](https://www.pruna.ai/blog/flux-juiced-the-fastest-image-generation-endpoint).
-                        """
-                    )
-                with gr.Column():
-                    gr.Markdown(
-                        """
-                        # 🧃 FLUX.1-dev (juiced)
-                        FLUX.1-dev (juiced) is our optimized version of FLUX.1-dev, delivering up to **2.6x faster inference** than the official Replicate API, **without sacrificing image quality**.
-                        Under the hood, it uses a custom combination of:
-                        - **Graph compilation** for optimized execution paths
-                        - **Inference-time caching** for repeated operations
-                        We won’t go deep into the internals here, but here’s the gist:
-                        > We combine compiler-level execution graph optimization with selective caching of heavy operations (like attention layers), allowing inference to skip redundant computations without any loss in fidelity.
-                        These techniques are generalized and plug-and-play via the **Pruna Pro** pipeline, and can be applied to nearly any diffusion-based image model—not just FLUX. For a free but still very juicy model you can use our open source solution.
-                        > 🧪 Try FLUX-juiced now → [replicate.com/prunaai/flux.1-juiced](https://replicate.com/prunaai/flux.1-juiced)
-                        ## Sample Images
-                        The prompts were randomly sampled from the [parti-prompts dataset](https://github.com/google-research/parti). The reported times represent the full duration of each API call.
-                        > **For samples, check out the [Pruna Notion page](https://pruna.notion.site/FLUX-1-dev-vs-Pruna-s-FLUX-juiced-1d270a039e5f80c6a2a3c00fc0d75ef0)**
-                        """
-                    )
-        with gr.Accordion("🌍 Join the Pruna AI community!", open=False):
-            gr.HTML(
-                """
-                    <a rel="nofollow" href="https://twitter.com/PrunaAI"><img alt="Twitter" src="https://img.shields.io/twitter/follow/PrunaAI?style=social"></a>
-                    <a rel="nofollow" href="https://github.com/PrunaAI/pruna"><img alt="GitHub" src="https://img.shields.io/github/stars/prunaai/pruna"></a>
-                    <a rel="nofollow" href="https://www.linkedin.com/company/93832878/admin/feed/posts/?feedType=following"><img alt="LinkedIn" src="https://img.shields.io/badge/LinkedIn-Connect-blue"></a>
-                    <a rel="nofollow" href="https://discord.com/invite/rskEr4BZJx"><img alt="Discord" src="https://img.shields.io/badge/Discord-Join%20Us-blue?style=social&amp;logo=discord"></a>
-                    <a rel="nofollow" href="https://www.reddit.com/r/PrunaAI/"><img alt="Reddit" src="https://img.shields.io/reddit/subreddit-subscribers/PrunaAI?style=social"></a>
-                """
-            )
-        with gr.Accordion("Citation", open=True):
-            gr.Markdown(
-                """
-                ```bibtex
-                @article{InferBench,
-                    title={InferBench: A Leaderboard for Inference Providers},
-                    author={PrunaAI},
-                    year={2025},
-                    howpublished={\\url{https://huggingface.co/spaces/PrunaAI/InferBench}}
-                }
-                ```
-                """
             )
 if __name__ == "__main__":

+import json
+from collections import defaultdict
 from pathlib import Path
 import gradio as gr
 import pandas as pd
+from ui import (
+    render_about,
+    render_benchmarks,
+    render_footer,
+    render_header,
+    render_home,
+)
 custom_css = """
 .logo {
 .text {
     font-size: 16px !important;
 }
+.tabs button,
+.tab-nav button,
 .subtabs button {
     font-size: 20px;
+    color: #581c87 !important;
+}
+.tabs button.selected,
+.tabs button[aria-selected="true"],
+.tab-nav button.selected,
+.tab-nav button[aria-selected="true"],
+.subtabs button.selected,
+.subtabs button[aria-selected="true"] {
+    color: #ffffff !important;
+    background: #7c3aed !important;
+    border-color: #6d28d9 !important;
+}
+.dark .tabs button,
+.dark .tab-nav button,
+.dark .subtabs button {
+    color: #e9d5ff !important;
+}
+.dark .tabs button.selected,
+.dark .tabs button[aria-selected="true"],
+.dark .tab-nav button.selected,
+.dark .tab-nav button[aria-selected="true"],
+.dark .subtabs button.selected,
+.dark .subtabs button[aria-selected="true"] {
+    color: #ffffff !important;
+    background: #6d28d9 !important;
+}
+.benchmark-catalogue-row {
+    width: 100%;
+    gap: 16px;
+}
+.benchmark-card {
+    width: 100%;
+    max-width: none;
+    min-height: 220px;
+    border: 1px solid rgba(124, 58, 237, 0.45);
+    border-left: 4px solid #7c3aed;
+    border-radius: 12px;
+    background: rgba(139, 92, 246, 0.14);
+    padding: 16px 18px;
+    box-shadow: 0 4px 12px rgba(76, 29, 149, 0.12);
+}
+.benchmark-card:hover {
+    border-color: #5b21b6;
+    box-shadow: 0 6px 18px rgba(76, 29, 149, 0.22);
+}
+.community-footer {
+    margin-top: 24px;
+    padding: 18px 20px;
+    border: 1px solid #c4b5fd;
+    border-radius: 12px;
+    background: rgba(139, 92, 246, 0.16);
+}
+.community-footer h3 {
+    margin: 0 0 10px;
+    color: #581c87;
+    font-size: 1.05rem;
+}
+.community-footer-links {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 10px 14px;
+    align-items: center;
+}
+.dark .community-footer {
+    border-color: #6d28d9;
+    background: rgba(76, 29, 149, 0.35);
+}
+.dark .community-footer h3 {
+    color: #f5f3ff;
+}
+.home-callouts {
+    display: grid;
+    grid-template-columns: repeat(3, minmax(0, 1fr));
+    gap: 12px;
+    margin: 8px 0 20px;
+}
+.home-callouts > div {
+    padding: 14px 16px;
+    border: 1px solid #a78bfa;
+    border-radius: 10px;
+    background: #f5f3ff;
+}
+.home-callouts span {
+    color: #5b21b6;
+    font-size: 11px;
+    font-weight: 700;
+    letter-spacing: 0.05em;
+}
+.home-callouts strong {
+    display: block;
+    margin-top: 6px;
+    color: #1e1b4b;
+    font-size: 18px;
+    word-break: break-word;
+}
+.home-callouts em {
+    display: block;
+    margin-top: 4px;
+    color: #4c1d95;
+    font-style: normal;
+    font-size: 13px;
+}
+.home-benchmark-card {
+    height: 100%;
+    padding: 18px 18px 16px;
+    border: 1px solid #a78bfa;
+    border-left: 4px solid #7c3aed;
+    border-radius: 12px;
+    background: #f5f3ff;
+    box-shadow: 0 4px 12px rgba(76, 29, 149, 0.12);
+}
+.home-benchmark-title {
+    color: #1e1b4b;
+    font-size: 1.35rem;
+    font-weight: 700;
+}
+.home-benchmark-blurb {
+    margin: 8px 0 14px;
+    color: #312e81;
+    font-size: 0.95rem;
+    line-height: 1.45;
+}
+.home-top-label {
+    color: #5b21b6;
+    font-size: 11px;
+    font-weight: 700;
+    letter-spacing: 0.05em;
+    text-transform: uppercase;
+}
+.home-top-list {
+    list-style: none;
+    margin: 8px 0 0;
+    padding: 0;
+}
+.home-top-list li {
+    display: grid;
+    grid-template-columns: 28px 1fr auto;
+    gap: 10px;
+    align-items: center;
+    padding: 8px 0;
+    border-bottom: 1px solid #ddd6fe;
+    color: #1e1b4b;
+}
+.home-rank {
+    width: 28px;
+    height: 28px;
+    border-radius: 999px;
+    background: #ede9fe;
+    color: #5b21b6;
+    font-weight: 700;
+    font-size: 13px;
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+}
+.home-model {
+    font-weight: 600;
+    word-break: break-word;
+    color: #1e1b4b;
+}
+.home-score {
+    font-variant-numeric: tabular-nums;
+    color: #4c1d95;
+    font-weight: 600;
+}
+.home-empty {
+    color: #4c1d95;
+}
+.dark .home-callouts > div,
+.dark .home-benchmark-card {
+    border-color: #7c3aed;
+    background: #1e1b4b;
+}
+.dark .home-callouts span,
+.dark .home-top-label,
+.dark .home-rank {
+    color: #c4b5fd;
+}
+.dark .home-rank {
+    background: #312e81;
+}
+.dark .home-callouts strong,
+.dark .home-benchmark-title,
+.dark .home-model,
+.dark .home-top-list li {
+    color: #faf5ff;
+}
+.dark .home-callouts em,
+.dark .home-benchmark-blurb,
+.dark .home-score,
+.dark .home-empty {
+    color: #ddd6fe;
+}
+.dark .home-top-list li {
+    border-bottom-color: #4c1d95;
+}
+@media (max-width: 700px) {
+    .home-callouts {
+        grid-template-columns: 1fr;
+    }
+}
+.pareto-plot {
+    width: 100%;
+}
+.pareto-plot .plotly {
+    width: 100% !important;
+}
+.pareto-plot .js-plotly-plot,
+.pareto-plot .plot-container {
+    width: 100% !important;
+}
+/* Hide the Plotly toolbar so it doesn't cover points in the top-right. */
+.pareto-plot .modebar {
+    display: none !important;
+}
+.leaderboard-summary {
+    display: grid;
+    grid-template-columns: repeat(4, minmax(0, 1fr));
+    gap: 12px;
+    margin: 8px 0 16px;
+}
+.leaderboard-summary > div {
+    padding: 14px 16px;
+    border: 1px solid #c4b5fd;
+    border-radius: 10px;
+    background: rgba(139, 92, 246, 0.14);
+}
+.leaderboard-summary span,
+.results-heading {
+    color: #7e22ce;
+    font-size: 11px;
+    font-weight: 700;
+    letter-spacing: 0.05em;
+}
+.leaderboard-summary strong {
+    display: block;
+    margin-top: 4px;
+    color: #581c87;
+    font-size: 22px;
+}
+.leaderboard-controls {
+    align-items: end;
+    margin-bottom: 8px;
+}
+.ranking-table-host {
+    width: 100%;
+}
+.ranking-table {
+    margin-top: 4px;
+    width: 100%;
+}
+.results-heading {
+    display: flex;
+    justify-content: space-between;
+    margin: 16px 0 8px;
+    text-transform: uppercase;
+}
+.ranking-table-scroll {
+    overflow-x: auto;
+    border: 1px solid #c4b5fd;
+    border-radius: 10px;
+    background: rgba(139, 92, 246, 0.12);
+}
+.ranking-table table {
+    width: 100%;
+    min-width: 800px;
+    border-collapse: collapse;
+    color: #581c87;
+    font-size: 14px;
+}
+.ranking-table th {
+    padding: 11px 14px;
+    border-bottom: 1px solid #c4b5fd;
+    color: #7e22ce;
+    font-size: 11px;
+    letter-spacing: 0.04em;
+    text-align: left;
+    text-transform: uppercase;
+}
+.ranking-table th.sortable-col {
+    cursor: pointer;
+    user-select: none;
+    white-space: nowrap;
+}
+.ranking-table th.sortable-col:hover {
+    color: #4c1d95;
+}
+.ranking-table th.sortable-col.sorted-asc::after,
+.ranking-table th.sortable-col.sorted-desc::after {
+    margin-left: 0.35em;
+    font-size: 0.85em;
+    opacity: 0.85;
+}
+.ranking-table th.sortable-col.sorted-asc::after {
+    content: "▲";
+}
+.ranking-table th.sortable-col.sorted-desc::after {
+    content: "▼";
+}
+.ranking-table td {
+    padding: 13px 14px;
+    border-bottom: 1px solid #e9d5ff;
+}
+.ranking-table tbody tr:last-child td {
+    border-bottom: 0;
+}
+.ranking-table tbody tr:hover {
+    background: #6d28d9;
+}
+.ranking-table tbody tr:hover td,
+.ranking-table tbody tr:hover .rank,
+.ranking-table tbody tr:hover .metric-score,
+.ranking-table tbody tr:hover .model-cell strong,
+.ranking-table tbody tr:hover .model-cell span {
+    color: #ffffff;
+}
+.ranking-table .rank {
+    color: #7c3aed;
+    font-weight: 700;
+}
+.ranking-table .model-cell strong,
+.ranking-table .model-cell span {
+    display: block;
+}
+.ranking-table .model-cell span {
+    margin-top: 3px;
+    color: #9333ea;
+    font-size: 12px;
+}
+.ranking-table .metric-score {
+    color: #4c1d95;
+    font-variant-numeric: tabular-nums;
+    font-weight: 700;
+}
+.ranking-table .empty-state {
+    color: #9333ea;
+    padding: 32px;
+    text-align: center;
+}
+@media (prefers-color-scheme: dark) {
+    .leaderboard-summary > div,
+    .ranking-table-scroll {
+        border-color: rgba(221, 214, 254, 0.45);
+    }
+    .leaderboard-summary span,
+    .results-heading,
+    .ranking-table th,
+    .ranking-table .model-cell span,
+    .ranking-table .empty-state {
+        color: #ddd6fe;
+    }
+    .leaderboard-summary strong,
+    .ranking-table table,
+    .ranking-table .rank,
+    .ranking-table .metric-score {
+        color: #f5f3ff;
+    }
+    .ranking-table td {
+        border-bottom-color: rgba(221, 214, 254, 0.2);
+    }
+}
+.dark .leaderboard-summary > div,
+.dark .ranking-table-scroll {
+    border-color: rgba(221, 214, 254, 0.45);
+}
+.dark .leaderboard-summary span,
+.dark .results-heading,
+.dark .ranking-table th,
+.dark .ranking-table .model-cell span,
+.dark .ranking-table .empty-state {
+    color: #ddd6fe;
+}
+.dark .leaderboard-summary strong,
+.dark .ranking-table table,
+.dark .ranking-table .rank,
+.dark .ranking-table .metric-score {
+    color: #f5f3ff;
+}
+.compare-samples-help {
+    color: #6b21a8;
+    margin-bottom: 8px;
+}
+.compare-prompt-block {
+    margin: 0 0 22px;
+    padding: 14px 16px;
+    border: 1px solid #c4b5fd;
+    border-radius: 12px;
+    background: rgba(139, 92, 246, 0.10);
+}
+.compare-prompt-meta {
+    display: flex;
+    justify-content: space-between;
+    gap: 12px;
+    margin-bottom: 8px;
+    color: #7e22ce;
+    font-size: 12px;
+    font-weight: 700;
+    letter-spacing: 0.04em;
+    text-transform: uppercase;
+}
+.compare-prompt-text {
+    margin: 0 0 14px;
+    color: #3b0764;
+    font-size: 15px;
+    line-height: 1.45;
+}
+.compare-row {
+    display: grid;
+    gap: 12px;
+}
+.compare-cell {
+    min-width: 0;
+}
+.compare-model-label {
+    margin-bottom: 6px;
+    color: #5b21b6;
+    font-size: 13px;
+    font-weight: 700;
+    word-break: break-word;
+}
+.compare-cell img {
+    display: block;
+    width: 100%;
+    aspect-ratio: 1 / 1;
+    object-fit: cover;
+    border-radius: 10px;
+    border: 1px solid #ddd6fe;
+    background: #f5f3ff;
+}
+.compare-empty {
+    padding: 18px;
+    border: 1px dashed #c4b5fd;
+    border-radius: 10px;
+    color: #6b21a8;
+    text-align: center;
+}
+.dark .compare-samples-help,
+.dark .compare-prompt-meta,
+.dark .compare-model-label,
+.dark .compare-empty {
+    color: #e9d5ff;
+}
+.dark .compare-prompt-block {
+    border-color: #6d28d9;
+    background: rgba(76, 29, 149, 0.35);
+}
+.dark .compare-prompt-text {
+    color: #f5f3ff;
+}
+.dark .compare-cell img {
+    border-color: #6d28d9;
+    background: #2e1065;
+}
+.dark .ranking-table td {
+    border-bottom-color: rgba(221, 214, 254, 0.2);
+}
+@media (max-width: 700px) {
+    .leaderboard-summary {
+        grid-template-columns: repeat(2, minmax(0, 1fr));
+    }
 }
 h1, h2 {
     margin: 0;
 }
 """
+theme = gr.Theme.from_hub("Werli/Purple-Crimson-Gradio-Theme").set(
+    body_background_fill="#c4b5fd",
+    body_background_fill_dark="#120B1B",
+    body_text_color="#3b0764",
+    body_text_color_dark="#ffffff",
+    block_background_fill="#ede9fe",
+    block_background_fill_dark="#2e1065",
+)
+space_root = Path(__file__).resolve().parent
+data_dir = space_root / "data"
+def _resolve_data_path(*candidates):
+    """Prefer Space-local data/, fall back to monorepo parent for local edits."""
+    for path in candidates:
+        if path.exists():
+            return path
+    return candidates[0]
+oneig_path = _resolve_data_path(
+    data_dir / "oneig_model_price_and_median_generation_time.csv",
+    space_root.parent / "oneig_model_price_and_median_generation_time.csv",
 )
+def load_sample_comparison_data(folder):
+    """Load prompts + generations for the Compare samples tab."""
+    folder = Path(folder)
+    prompts_path = folder / "prompts.jsonl"
+    generations_path = folder / "generations.jsonl"
+    if not prompts_path.exists() or not generations_path.exists():
+        return None
+    prompts = {}
+    with prompts_path.open() as handle:
+        for line in handle:
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            prompts[row["prompt_id"]] = row.get("text", "")
+    images = defaultdict(dict)
+    with generations_path.open() as handle:
+        for line in handle:
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            model_id = row["model_id"]
+            prompt_id = row["prompt_id"]
+            image_url = row.get("image")
+            if model_id and prompt_id and image_url:
+                images[model_id][prompt_id] = image_url
+    models = sorted(images)
+    if not models or not prompts:
+        return None
+    return {
+        "prompts": prompts,
+        "images": {model: dict(prompt_map) for model, prompt_map in images.items()},
+        "models": models,
+        "prompt_ids": sorted(prompts),
+    }
+def format_url_links(value):
+    if pd.isna(value) or not value:
+        return ""
+    links = [url.strip() for url in str(value).split(",") if url.strip()]
+    if not links:
+        return ""
+    return " ".join(
+        f'<a target="_blank" href="{url}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">run {idx}</a>'
+        for idx, url in enumerate(links, start=1)
+    )
+def load_oneig_dataframe(path):
+    df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
+    df = df.rename(
+        columns={
+            "Owner": "Endpoint Owner",
+            "Optimization": "Optimization Details",
+            "Anime Alignment Score": "OneIG (Anime Alignment)",
+            "Human Alignment Score": "OneIG (Human Alignment)",
+            "Object Alignment Score": "OneIG (Object Alignment)",
+            "OneIG (Anime and Stylization) (Alignment Score)": "OneIG (Anime Alignment)",
+            "OneIG (Portrait) (Alignment Score)": "OneIG (Human Alignment)",
+            "OneIG (General Object) (Alignment Score)": "OneIG (Object Alignment)",
+        }
+    )
+    if "URL" in df.columns:
+        df["URL"] = df["URL"].apply(format_url_links)
+    if "Optimized" in df.columns:
+        df["Optimized"] = df["Optimized"].map(
+            {True: "Yes", False: "No", "TRUE": "Yes", "FALSE": "No"}
+        ).fillna(df["Optimized"])
+    for column in [
+        "Price / Image (USD)",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "OneIG (Anime Alignment)",
+        "OneIG (Human Alignment)",
+        "OneIG (Object Alignment)",
+        "OneIG Anime Elo",
+        "OneIG Human Elo",
+        "OneIG Object Elo",
+        "P-Judge Overall",
+        "Rapidata Elo",
+    ]:
+        if column in df.columns:
+            df[column] = pd.to_numeric(df[column], errors="coerce")
+    preferred_columns = [
         "Platform",
+        "Endpoint Owner",
         "Device",
         "Model",
+        "Optimized",
+        "Optimization Details",
+        "OneIG (Anime Alignment)",
+        "OneIG (Human Alignment)",
+        "OneIG (Object Alignment)",
+        "OneIG Anime Elo",
+        "OneIG Human Elo",
+        "OneIG Object Elo",
         "Median Inference Time",
+        "Median Inference Time (s)",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "Median Total Duration (s)",
         "Price per Image",
+        "Price / Image (USD)",
+        "Evaluation Date (UTC)",
+        "URL",
     ]
+    present_preferred_columns = [col for col in preferred_columns if col in df.columns]
+    df = df[
+        present_preferred_columns
+        + [col for col in df.columns.tolist() if col not in present_preferred_columns]
     ]
+    if "OneIG (Human Alignment)" in df.columns:
+        df = df.sort_values(
+            by="OneIG (Human Alignment)", ascending=False, na_position="last"
+        )
     numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
+    for col in numeric_cols:
+        df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
+    return df
+def load_qwen_combined_dataframe(path):
+    """Load the combined Qwen Image Bench metrics table."""
+    df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
+    model_column = "Rapidata Model" if "Rapidata Model" in df.columns else "Model"
+    df = df.rename(columns={model_column: "Model"})
+    # Rows prefixed with '#' are commented-out variants in the source exports.
+    df = df[~df["Model"].astype(str).str.startswith("#")].copy()
+    df["Model"] = df["Model"].astype(str).str.strip()
+    for column in [
+        "Price / Image (USD)",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "P-Judge Overall",
+        "Rapidata Elo",
+        "Datapoint Elo",
+    ]:
+        if column in df.columns:
+            df[column] = pd.to_numeric(df[column], errors="coerce")
+    sort_column = next(
+        (
+            col
+            for col in [
+                "Datapoint Elo",
+                "P-Judge Overall",
+                "Rapidata Elo",
+            ]
+            if col in df.columns
+        ),
+        None,
     )
+    if sort_column:
+        df = df.sort_values(by=sort_column, ascending=False, na_position="last")
+    numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
+    for col in numeric_cols:
+        df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
+    return df.reset_index(drop=True)
+df = load_oneig_dataframe(oneig_path)
+oneig_metric_columns = [
+    col
+    for col in [
+        "OneIG (Anime Alignment)",
+        "OneIG (Human Alignment)",
+        "OneIG (Object Alignment)",
+    ]
+    if col in df.columns
+]
+oneig_df = df.copy()
+if oneig_metric_columns:
+    oneig_df["OneIG Overall Score"] = oneig_df[oneig_metric_columns].mean(
+        axis=1, skipna=True
+    )
+    oneig_df = oneig_df.sort_values(
+        by="OneIG Overall Score", ascending=False, na_position="last"
+    )
+oneig_display_columns = [
+    col
+    for col in [
+        "Platform",
+        "Endpoint Owner",
+        "Model",
+        "Optimized",
+        *oneig_metric_columns,
+        "OneIG Anime Elo",
+        "OneIG Human Elo",
+        "OneIG Object Elo",
+        "P-Judge Overall",
+        "Rapidata Elo",
+        "Evaluation Date (UTC)",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "Price / Image (USD)",
+        "URL",
+    ]
+    if col in oneig_df.columns
+]
+# Top-level Leaderboard tab uses the same OneIG table.
+display_columns = oneig_display_columns
+oneig_combined_dir = _resolve_data_path(
+    data_dir / "oneig_combined",
+    space_root.parent / "oneig_combined",
+)
+qwen_combined_dir = _resolve_data_path(
+    data_dir / "qwen_image_bench_combined",
+    space_root.parent / "qwen_image_bench_combined",
+)
+qwen_path = _resolve_data_path(
+    data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
+    space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
+)
+qwen_df = load_qwen_combined_dataframe(qwen_path)
+qwen_score_columns = [
+    col
+    for col in [
+        "P-Judge Overall",
+        "Datapoint Elo",
+        "Rapidata Elo",
+    ]
+    if col in qwen_df.columns
+]
+qwen_display_columns = [
+    col
+    for col in [
+        "Model",
+        *qwen_score_columns,
+        "Raw Win Rate",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "Price / Image (USD)",
+    ]
+    if col in qwen_df.columns
+]
+qwen_overall_column = (
+    "Datapoint Elo"
+    if "Datapoint Elo" in qwen_df.columns
+    else (qwen_score_columns[0] if qwen_score_columns else None)
+)
+oneig_samples = load_sample_comparison_data(oneig_combined_dir)
+qwen_samples = load_sample_comparison_data(qwen_combined_dir)
+# Dataset-first catalogue: each card is a prompt suite; metrics are leaderboard columns.
+benchmarks = [
+    {
+        "id": "oneig",
+        "title": "OneIG Alignment",
+        "emoji": "🎯",
+        "card_description": (
+            "Anime/stylization, portrait, and general-object alignment prompts — "
+            "alignment scores and Datapoint Elo (not the full OneIG suite), plus "
+            "side-by-side generations."
+        ),
+        "intro": (
+            "OneIG Alignment covers the alignment slice of OneIG (not every OneIG "
+            "dimension). The leaderboard shows category alignment scores and Datapoint "
+            "Elo columns; Compare samples uses the combined alignment generations."
+        ),
+        "data": oneig_df,
+        "columns": oneig_display_columns,
+        "score_columns": oneig_metric_columns,
+        "overall_column": "OneIG Overall Score",
+        "best_score_label": "BEST ONEIG ALIGNMENT",
+        "metrics_same_scale": True,
+        "note": (
+            "> Rankings are ordered by the mean of the available alignment category "
+            "scores. Missing category scores are not included in that model's mean."
+        ),
+        "samples": oneig_samples,
+    },
+    {
+        "id": "qwen_image_bench",
+        "title": "Qwen Image Bench",
+        "emoji": "🖼️",
+        "card_description": (
+            "Qwen image-bench prompts with P-Judge, Datapoint Elo, and Rapidata Elo "
+            "as metric columns, plus combined generations for comparison."
+        ),
+        "intro": (
+            "Qwen Image Bench is a shared prompt suite. The leaderboard joins every "
+            "available metric for this benchmark; Compare samples uses the combined "
+            "Qwen generations."
+        ),
+        "data": qwen_df,
+        "columns": qwen_display_columns,
+        "score_columns": qwen_score_columns,
+        "overall_column": qwen_overall_column,
+        "best_score_label": "BEST DATAPOINT ELO",
+        "metrics_same_scale": False,
+        "note": (
+            "> Models are ordered by Datapoint Elo when available. Other metric "
+            "columns come from the P-Judge and Rapidata evaluations on the same "
+            "prompt suite."
+        ),
+        "samples": qwen_samples,
+    },
+]
+custom_head = """
+<script>
+(() => {
+  if (window.__inferbenchLeaderboardSortBound) return;
+  window.__inferbenchLeaderboardSortBound = true;
+  document.addEventListener("click", (event) => {
+    const th = event.target.closest(".ranking-table th.sortable-col");
+    if (!th) return;
+    const table = th.closest("table");
+    const tbody = table && table.querySelector("tbody");
+    if (!tbody) return;
+    const col = Number(th.dataset.col);
+    const type = th.dataset.type || "text";
+    const ascending = !th.classList.contains("sorted-asc");
+    table.querySelectorAll("th.sortable-col").forEach((header) => {
+      header.classList.remove("sorted-asc", "sorted-desc");
+    });
+    th.classList.add(ascending ? "sorted-asc" : "sorted-desc");
+    const rows = Array.from(tbody.querySelectorAll("tr"));
+    const parseValue = (row) => {
+      const cell = row.children[col];
+      const raw = cell ? cell.getAttribute("data-sort-value") : "";
+      if (raw === null || raw === "") return null;
+      if (type === "number") {
+        const number = Number(raw);
+        return Number.isFinite(number) ? number : null;
+      }
+      return raw;
+    };
+    rows.sort((left, right) => {
+      const a = parseValue(left);
+      const b = parseValue(right);
+      if (a === null && b === null) return 0;
+      if (a === null) return 1;
+      if (b === null) return -1;
+      if (type === "number") {
+        return ascending ? a - b : b - a;
+      }
+      const cmp = String(a).localeCompare(String(b), undefined, {
+        numeric: true,
+        sensitivity: "base",
+      });
+      return ascending ? cmp : -cmp;
+    });
+    rows.forEach((row) => tbody.appendChild(row));
+  });
+})();
+</script>
+"""
+with gr.Blocks(
+    theme=theme, fill_width=True, css=custom_css, head=custom_head
+) as demo:
+    render_header()
+    with gr.Tabs():
+        with gr.TabItem("Home"):
+            render_home(benchmarks)
+        with gr.TabItem("Benchmarks") as benchmarks_tab:
+            reset_benchmarks, reset_benchmark_outputs = render_benchmarks(benchmarks)
+            benchmarks_tab.select(
+                reset_benchmarks,
+                outputs=reset_benchmark_outputs,
             )
+        with gr.TabItem("About"):
+            render_about()
+    render_footer()
 if __name__ == "__main__":

data/oneig_combined/generations.jsonl ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7630c12b0c629a2a9fe66a40be90da3db8175d6a542b6a3df3440698e5b01940
+size 15535531

data/oneig_combined/prompts.jsonl ADDED Viewed

The diff for this file is too large to render. See raw diff

data/oneig_model_price_and_median_generation_time.csv ADDED Viewed

	@@ -0,0 +1,40 @@

+Model,Price / Image (USD),Min Generation Time (s),Median Generation Time (s),P-Judge Overall,Rapidata Elo,OneIG Anime Elo,OneIG Human Elo,OneIG Object Elo,Anime Alignment Score,Human Alignment Score,Object Alignment Score
+flux_2_pro,0.1,6.451752478955314,9.642776968947146,54.67505979381443,1085.79,,1021.0,1039.0,,0.8434398864526329,0.9019077567491852
+bria_4_fibo,0.04,11.443314291071149,15.73027866682969,56.735806,1040.34,,,1003.0,,,0.86779827444468
+z_image,0.005,1.0492684720084071,1.4465381670743227,51.31612759856628,998.29,986.0,1026.0,991.0,0.8670047225660046,0.8099311841844237,0.8347049973782066
+juggernaut_base_flux,0.035,3.7550464863112816,4.198047187528573,50.01986354515051,1022.26,974.0,989.0,976.0,0.8457611438462199,0.7737633369864975,0.7907706396009133
+juggernaut_lightning_flux,0.006,0.8921283883197854,1.2383652809658088,49.52208461538461,853.78,931.0,960.0,959.0,0.8571674526848435,0.7957690022519136,0.819303783379011
+juggernaut_pro_flux,0.055,3.325652208412066,3.6792007085168734,50.376797993311015,992.19,972.0,1007.0,984.0,0.8490802701542259,0.7894352383959775,0.7921262047886114
+flux_1_1_pro,0.04,2.299360333631436,3.3512960482621565,51.83815123674916,1051.25,1003.0,1005.0,1001.0,0.8716153899908383,0.7846021555377409,0.831820704487019
+flux_1_1_pro_ultra,0.06,6.1039779025595635,8.003180750412866,51.57635594405592,1002.23,1012.0,986.0,1002.0,0.8872399024525827,0.7953850837498103,0.845667366322335
+flux_2_dev,0.025,3.8249364433577284,4.122797580435872,52.64911717171714,1099.69,1019.0,1002.0,1006.0,0.9200030923152858,0.8408774253506177,0.8818942110481547
+flux_2_flex,0.06,7.688491132847655,9.843284582952036,53.50560874125876,1134.04,1052.0,987.0,1009.0,0.9221061950079986,0.83704222320422,0.8993309905484712
+flux_2_max,0.07,15.186774821311701,25.48462438955903,54.56841249999998,1044.7,1076.0,1023.0,1042.0,0.9269662930757988,0.8477831233094394,0.8948951475407387
+flux_dev,0.025,1.3863036371185444,1.6603330307407305,49.75226476510064,908.99,945.0,962.0,955.0,0.8511931856081816,0.7750798964898262,0.798883650395521
+flux_krea,0.025,1.399834333686158,1.647412791440729,51.85929899328858,1007.32,997.0,990.0,979.0,0.8841535719999664,0.8097028670162356,0.8398559499867212
+flux_schnell,0.003,0.8232578075646112,1.044991439965088,49.1443472972973,915.39,892.0,917.0,936.0,0.8656683997302044,0.7814102991777,0.8287984457553442
+imagen_4,0.04,5.230271888663992,8.709110853960738,51.65737018867922,998.27,1023.0,990.0,997.0,0.894310223704374,0.8137015075340288,0.8518758104139295
+imagen_4_fast,0.02,2.946248896070756,4.254744468489662,50.40180337078652,841.12,,951.0,966.0,,0.7680119279393379,0.8228953629726682
+imagen_4_ultra,0.06,8.757600130187347,12.61268443800509,53.72492471482888,1025.6,1066.0,1006.0,1016.0,0.9250454128525762,0.8453951500548762,0.9016222652489544
+nano_banana_2,0.067,16.819758291666705,21.93129874998704,55.173166211604126,1157.47,1063.0,1050.0,1076.0,0.9253234542983964,0.8489246879018892,0.9043643320237978
+nano_banana_pro,0.15,14.906700378924143,20.077973207924515,55.10878327759201,1062.45,1053.0,1034.0,1057.0,0.9097154955052252,0.8465375480054144,0.9036434044908332
+gpt_image_1_5,0.136,29.07584411953576,37.961403958383016,53.76628423913046,1188.48,1101.0,1066.0,,0.9233708744477797,0.8522874700035479,
+hidream_i1_dev,0.0086,1.9823376253868144,2.192608500132337,50.74670167224078,1118.03,1009.0,1007.0,1012.0,0.8720575175843863,0.7897814812758762,0.8188972054038817
+hidream_i1_fast,0.0051,1.2573314025842894,1.6601914172060788,50.70580066889632,1081.48,998.0,991.0,994.0,0.8674483659597709,0.7840883510194656,0.82328148897844
+hidream_i1_full,0.014,5.691079417127185,5.840440698026214,49.225683919598,945.34,906.0,,938.0,0.8629144755944692,,0.8163581350743205
+p_image,0.005,1.024232491850853,1.0473625104641542,50.47380301003343,955.42,964.0,1011.0,982.0,0.844481587914448,0.7973493346957257,0.7888656304845173
+p_image_2_ideogram_high_1k,0.015,3.1836699013329053,4.883333333333334,54.77,934.81,1014.0,1015.0,1029.0,0.8629618560618941,0.8063002387189694,0.8547826116307153
+p_image_2_ideogram_high_2k,0.03,7.530538211332896,9.75,54.09,913.17,996.0,1028.0,1023.0,0.8803449901921264,0.8307037713318671,0.8615976872714485
+p_image_2_ideogram_low_1k,0.0075,1.8912416643330896,4.97,53.71,864.21,1024.0,1014.0,1040.0,0.8644190279319922,0.8050990548194494,0.8358910999561152
+p_image_2_ideogram_low_2k,0.016,4.135335521333521,6.163333333333334,53.31,908.63,990.0,1011.0,1004.0,0.8801673257513037,0.8248396465935166,0.8412710602520973
+p_image_2_ideogram_medium_1k,0.01,2.1058022906678766,3.0533333333333337,54.27,967.5,1020.0,1028.0,1053.0,0.8560997548632995,0.8094780142672943,0.847484535459381
+p_image_2_ideogram_medium_2k,0.02,6.427383309333284,9.16,53.27,933.86,988.0,1024.0,994.0,0.8687718766012108,0.820451078677085,0.8418239715865493
+p_image_2_ideogram_very_low_1k,0.003,1.9598290003329264,3.4333333333333336,53.19,896.97,995.0,1020.0,1028.0,0.8632302099560225,0.7934524734166761,0.8255385460772456
+p_image_2_ideogram_very_low_2k,0.006,2.9796522429999945,5.556666666666667,52.12,890.07,983.0,990.0,974.0,0.8751773251258547,0.8240413668933404,0.8418756147962801
+sdxl_lightning,0.0014,1.3275378193551053,1.8803678539115936,,866.88,904.0,876.0,941.0,0.7328890298342023,0.6689303836044653,0.7324141421177138
+wan_2_2_image,0.02,2.6036043955245987,2.830037875450216,49.8521512562814,1000.45,,998.0,995.0,,0.7878415998328734,0.821705707700177
+qwen_image,0.025,3.419045856843392,3.5670732414582744,51.27888993288592,1176.37,1008.0,1004.0,966.0,0.905957670968254,0.8277556042985756,0.870535385375026
+hunyuan_image_3,0.08,18.571135867697496,18.82112309767399,52.848213103448295,1065.81,1014.0,991.0,1013.0,0.8981869529944881,0.8246260239317298,0.8582881719127173
+qwen_image_fast,0.021,5.7156417078804225,6.883911125012673,51.37910050505055,1127.4,,999.0,978.0,,0.8236622305555534,0.8629531817993643
+seedream_3,0.018,5.998649687040597,7.340932093793526,53.91764270833332,1157.55,,1031.0,1034.0,,0.8268986114060047,0.879781957515387
+sdxl,0.0019,3.6809837920591235,4.272879093885422,46.38236185567008,766.4,,868.0,953.0,,0.6990518231924773,0.764374879014914

data/qwen_image_bench_combined/generations.jsonl ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:98eb8211f242274d3afea988aacad318b13642f98d3aef1a371c40491a032afd
+size 5207632

data/qwen_image_bench_combined/prompts.jsonl ADDED Viewed

The diff for this file is too large to render. See raw diff

data/qwen_image_bench_model_price_and_median_generation_time.csv ADDED Viewed

	@@ -0,0 +1,60 @@

+Model,Price / Image (USD),Median Generation Time (s),Min Generation Time (s),P-Judge Overall,Raw Win Rate,Rapidata Elo,Datapoint Elo,Benchmark.ai Elo
+reve_2_1,N/A,N/A,28.3,,,,,1173.2
+ideogram_4_0_quality,N/A,N/A,66.6,,,,,1131.0
+gpt_image_2,0.21,77.8,77.8,59.02083099999998,,1172.58,1116,1124.5
+nano_banana_2_0,N/A,N/A,N/A,56.362956,59.4%,1071.79,1067,1056.3
+gpt_image_1_5,0.135,38.0,38.0,57.864434999999986,64.5%,1102.09,1064,910.3
+hidream_i1_dev,0.0086,2.823569217998738,2.06,49.17682099999999,51.7%,999.17,984,
+gpt_image_1,0.167,38.8,38.8,54.975741000000006,,1095.39,987,
+imagen_4_0_ultra,0.06,11.4,11.4,53.385603999999965,59.7%,1075.22,1023,
+flux_2_flex,0.06,10.856533817990567,8.07,53.9245676767677,57.8%,1054.34,1018,921.3
+qwen_image,0.025,4.8,4.8,51.561746,51.5%,1073.72,1005,975.6
+seedream_5_0,N/A,N/A,N/A,55.715153,,1070.35,1012,
+nano_banana_pro,0.134,17.2,17.2,56.452189898989914,58.0%,1029.51,1045,1102.4
+hidream_i1_fast,0.0051,9.920469530501578,1.40,48.89298600000002,50.1%,1024.67,984,
+imagen_4_fast,0.02,3.7531301500021073,2.71,50.155055208333344,,972.6,981,
+seedream_4_5,0.04,16.6,16.6,55.66328800000001,,1048.96,1035,964.2
+seedream_4_0,0.03,12.2,12.2,55.241443999999994,,1050.41,1035,
+p_image_2_ideogram_low_1k,0.0075,2.59,1.55,54.827397,53.2%,1000.07,1009,1103.9
+#p_image_2_ideogram_low_2k,0.016,5.17,4.25,53.971723,48.4%,1074.91,1003,
+qwen_image_2_0_pro,0.035,35.5,35.5,56.181776,,1033.25,1017,959.9
+juggernaut_base_flux,0.035,4.115834823496698,3.84,49.474676,46.4%,1038.22,972,
+flux_2_pro,N/A,N/A,N/A,54.43423900000002,,993.54,1019,1011.6
+z_image,0.005,1.5122045120006078,1.22,49.946227999999984,48.1%,1028.1,1001,
+p_image_2_ideogram_high_1k,0.015,4.28,3.07,55.754507999999994,52.1%,972.92,1022,1104.0
+#p_image_2_ideogram_high_2k,0.03,8.69,7.12,54.757842000000004,49.4%,1074.91,1007,
+qwen_image_2512,0.02,19.1,19.1,51.677326,,1029.6,1009,
+flux_2_max,0.07,26.4,26.4,54.047976,60.9%,955.19,1028,1001.6
+flux_1_1_pro_ultra,0.06,9.026992494000297,6.20,50.358445,52.0%,979.56,995,
+juggernaut_pro_flux,0.055,3.699636150500737,3.36,49.741183,46.4%,972.14,964,
+flux_dev,0.025,1.6931055715031107,1.49,48.241479,42.1%,925.04,940,
+p_image_2_ideogram_very_low_1k,0.003,2.56,1.49,53.51894200000001,48.1%,959.86,995,1071.7
+#p_image_2_ideogram_very_low_2k,0.006,3.94,3.05,53.68248699999998,47.2%,962.38,980,
+#p_image_2_ideogram_very_low_1k_no_upsampling,0.005,0.82,,46.33,,,,1071.7
+#p_image_2_ideogram_very_low_2k_no_upsampling,0.005,2.18,,46.39,,,,1071.7
+#p_image_2_ideogram_low_1k_no_upsampling,0.01,3.33,,45.32,,,,1103.9
+#p_image_2_ideogram_low_2k_no_upsampling,0.01,3.34,,46.25,,,,1103.9
+#p_image_2_ideogram_medium_1k_no_upsampling,0.015,2.13,,46.88,,,,1115.1
+#p_image_2_ideogram_medium_2k_no_upsampling,0.015,5.55,,47.11,,,,1115.1
+#p_image_2_ideogram_high_1k_no_upsampling,0.03,3.11,,46.88,,,,1104.0
+#p_image_2_ideogram_high_2k_no_upsampling,0.03,5.55,,47.06,,,,1104.0
+hidream_i1_full,0.014,6.008430051002506,5.69,46.82559300000002,36.9%,955.46,944,
+flux_2_dev,0.025,4.310278721997747,4.03,52.71644489795918,53.1%,1007.6,1021,942.0
+imagen_4_0,0.04,14.1,14.1,52.08996199999999,53.5%,979.62,1005,
+wan_2_2_image,0.02,3.005390542501118,2.96,48.19959399999999,,944.87,960,
+flux_krea,0.025,1.7150160090022837,1.7150160090022837,50.35734,50.0%,919.73,975,
+p_image_2_ideogram_medium_1k,0.01,3.06,2.05,54.719193000000004,51.7%,941.46,1002,1115.1
+#p_image_2_ideogram_medium_2k,0.02,7.44,6.47,54.23124444444446,48.1%,949.35,1000,
+glm_image,0.05,188.2,188.2,51.42623399999999,,923.35,953,
+p_image,0.005,1.0640762715011078,0.95,48.75217099999999,44.8%,924.37,961,1098.7
+hunyuanimage_3_0,0.09,41.0,41.0,52.32440099999998,52.4%,1009.61,979,765.3
+juggernaut_lightning_flux,0.006,1.1787893719956628,0.93,48.30471699999998,40.5%,916.68,929,
+flux_1_1_pro,0.04,3.0104645500032348,2.34,49.92882700000001,50.6%,925.04,984,
+flux_schnell,0.003,0.8411653029907029,0.80,46.685981818181816,34.8%,892.32,915,
+kling_v2_1,N/A,N/A,N/A,51.044512,,870.04,981,
+#p_image_2_ideogram_very_high_high_1k,0.075,9.34,7.25,58.45,,,1025,
+#p_image_2_ideogram_very_high_low_1k,0.0375,26.17,10.44,57.59,,,1024,
+#p_image_2_ideogram_very_high_medium_1k,0.05,9.28,5.96,57.88,,,1020,
+#p_image_2_ideogram_very_high_very_low_1k,0.015,26.52,10.60,56.92,,,1002,
+#p_image_2_ideogram_final_1k,0.0375,11.17,5.55,58.28,,,,
+#p_image_2_ideogram_final_2k,0.075,14.34,9.96,57.68,,,,

requirements.txt CHANGED Viewed

	@@ -1 +1,2 @@
1	- gradio-leaderboard


1	+ gradio-leaderboard
2	+ plotly

ui.py ADDED Viewed

	@@ -0,0 +1,1123 @@

+from html import escape
+import random
+import gradio as gr
+import pandas as pd
+import plotly.graph_objects as go
+FILTER_COLUMNS = ["Platform", "Endpoint Owner", "Model", "Optimized"]
+MAX_COMPARE_MODELS = 4
+DEFAULT_COMPARE_PROMPTS = 3
+MAX_COMPARE_PROMPTS = 8
+ALL_LEADERBOARD_NOTE = """
+> **💡 Note:** Each efficiency metric and quality metric captures only one dimension of
+> model capacity. Rankings may vary when considering other metrics. "Endpoint Owner"
+> refers to the publisher of the evaluated endpoint and may differ from the original
+> model creator. "Optimized" indicates that the endpoint uses an explicit optimization
+> mode or serves a modified/accelerated variant.
+"""
+ONEIG_SCORES_CONTENT = """
+### Reported OneIG scores
+- **Anime Alignment** — alignment for anime and stylization prompts
+- **Human Alignment** — alignment for portrait prompts
+- **Object Alignment** — alignment for general-object prompts
+The leaderboard's **OneIG Overall Score** is the mean of each model's available
+category scores.
+"""
+ONEIG_RUN_DETAILS_CONTENT = """
+### Reading the results
+Each row records the evaluation date, generation time, price per image, and links to
+the source evaluation runs when available. Use the filters in the leaderboard to
+compare providers, models, and optimized variants.
+"""
+P_JUDGE_SCORES_CONTENT = """
+### Reported P-Judge scores
+- **P-Judge Overall** — automatic preference / quality score for generated images
+Higher scores indicate stronger performance on the P-Judge evaluation.
+"""
+P_JUDGE_RUN_DETAILS_CONTENT = """
+### Reading the results
+Each row records the evaluation date, generation time, and price per image when
+available. Rankings are ordered by **P-Judge Overall**.
+"""
+DATAPOINT_ELO_SCORES_CONTENT = """
+### Reported Datapoint Elo scores
+- **Datapoint Elo** — human-preference Elo rating from pairwise comparisons
+Elo values move whenever new models are added and re-evaluated against the field,
+so the Date column reflects the leaderboard snapshot date.
+"""
+DATAPOINT_ELO_RUN_DETAILS_CONTENT = """
+### Reading the results
+Each row records the snapshot date, generation time, and price per image when
+available. Rankings are ordered by **Datapoint Elo**.
+"""
+RAPIDATA_ELO_SCORES_CONTENT = """
+### Reported Rapidata Elo scores
+- **Rapidata Elo** — Elo rating from the Rapidata evaluation suite
+Higher Elo indicates stronger relative performance on that suite.
+"""
+RAPIDATA_ELO_RUN_DETAILS_CONTENT = """
+### Reading the results
+Each row records generation time and price per image when available. Rankings are
+ordered by **Rapidata Elo**.
+"""
+BENCHMARK_AI_ELO_SCORES_CONTENT = """
+### Reported Benchmark.ai Elo scores
+- **Benchmark.ai Elo** — Elo rating from the Benchmark.ai leaderboard
+This score is not from the Qwen Image Bench prompt suite; it is shown alongside for
+cross-benchmark comparison.
+"""
+BENCHMARK_AI_ELO_RUN_DETAILS_CONTENT = """
+### Reading the results
+Each row records generation time and price per image when available. Rankings for
+this metric alone are ordered by **Benchmark.ai Elo**.
+"""
+ABOUT_OVERVIEW_CONTENT = """
+# About InferBench
+InferBench compares **text-to-image models** on quality, preference, latency, and
+price. Results are organized by prompt suite (benchmark), not by a single opaque
+score.
+## What you can do here
+- **Home** — snapshot of each prompt suite and unique model count.
+- **Benchmarks** — open a prompt suite to see its metric columns, graphs (including a
+  quality-vs-price Pareto frontier), and side-by-side sample generations.
+- **About** — this page.
+## Current prompt suites
+### OneIG Alignment
+The **alignment** slice of OneIG (not the full OneIG suite), across three categories:
+- Anime / stylization
+- Human / portrait
+- General object
+The table reports category **alignment scores**, **Datapoint Elo** columns (Anime /
+Human / Object), median / min generation time, and price per image. Rankings use the
+mean of each model's available category alignment scores (missing categories are
+skipped for that model).
+### Qwen Image Bench
+A shared prompt suite with multiple evaluation tracks shown as columns:
+- **P-Judge Overall** — automatic preference / quality score
+- **Datapoint Elo** — human-preference Elo (default sort key)
+- **Rapidata Elo** — Elo from the Rapidata evaluation on this suite
+Plus latency and price metadata, and combined generations for visual comparison.
+## How to read the numbers
+Quality metrics from different suites are **not interchangeable** — a high OneIG
+Alignment score is not the same quantity as a Datapoint Elo. Prefer comparing models
+*within* a benchmark column, and use price / generation time when you care about
+efficiency.
+The Pareto plot highlights models that are not dominated on both **higher score** and
+**lower price**.
+"""
+ABOUT_DETAILS_CONTENT = """
+# Data & caveats
+- Endpoint **price** and **generation time** come from the evaluation snapshots linked
+  in each table where available.
+- Some models are missing individual metric columns; empty cells mean that track was
+  not run (or not reported) for that model.
+- Elo ratings can shift when the comparison pool changes — treat them as relative
+  rankings for the snapshot, not absolute constants.
+- Optimized / accelerated endpoints (when labeled) may differ from the base model
+  publisher's default serving stack.
+Built by [Pruna AI](https://www.pruna.ai/). Contributions and new evaluation runs welcome.
+"""
+COMMUNITY_CONTENT = """
+<div class="community-footer">
+  <h3>Join the Pruna AI community</h3>
+  <div class="community-footer-links">
+    <a rel="nofollow" href="https://x.com/PrunaAI" target="_blank">X / Twitter</a>
+    <a rel="nofollow" href="https://github.com/PrunaAI/pruna" target="_blank">GitHub</a>
+    <a rel="nofollow" href="https://www.linkedin.com/company/pruna-ai" target="_blank">LinkedIn</a>
+    <a rel="nofollow" href="https://discord.com/invite/rskEr4BZJx" target="_blank">Discord</a>
+    <a rel="nofollow" href="https://www.reddit.com/r/PrunaAI/" target="_blank">Reddit</a>
+    <a rel="nofollow" href="https://www.pruna.ai/" target="_blank">pruna.ai</a>
+  </div>
+</div>
+"""
+CITATION_CONTENT = """
+```bibtex
+@misc{InferBench,
+    title={InferBench: A Leaderboard for Text-to-Image Models},
+    author={PrunaAI},
+    year={2026},
+    howpublished={\\url{https://huggingface.co/spaces/PrunaAI/InferBench}}
+}
+```
+"""
+def render_header():
+    gr.HTML(
+        """
+        <div style="text-align: center;">
+            <h1>InferBench</h1>
+            <h2>Compare text-to-image models on quality, speed, and price</h2>
+        </div>
+        """
+    )
+def _top_models(data, score_column, n=3):
+    if score_column not in data.columns or "Model" not in data.columns:
+        return []
+    ranked = (
+        data[["Model", score_column]]
+        .dropna(subset=[score_column])
+        .loc[lambda df: ~df["Model"].astype(str).str.startswith("#")]
+        .sort_values(score_column, ascending=False)
+        .head(n)
+    )
+    return [
+        (str(row["Model"]), float(row[score_column]))
+        for _, row in ranked.iterrows()
+    ]
+def _home_highlights(benchmarks):
+    """Quality leaders per suite — more relevant than cheapest/fastest outliers."""
+    highlights = []
+    unique_models = set()
+    for benchmark in benchmarks:
+        data = benchmark.get("data")
+        if data is None or "Model" not in getattr(data, "columns", []):
+            continue
+        active = data[~data["Model"].astype(str).str.startswith("#")]
+        unique_models.update(active["Model"].astype(str).tolist())
+        score_column = benchmark.get("overall_column")
+        score_columns = benchmark.get("score_columns") or []
+        if not score_column or score_column not in data.columns:
+            score_column = score_columns[0] if score_columns else None
+        top = _top_models(data, score_column, n=1) if score_column else []
+        if not top:
+            continue
+        model, score = top[0]
+        highlights.append(
+            {
+                "label": f"BEST {benchmark['title'].upper()}",
+                "model": model,
+                "detail": f"{_display_label(score_column)} · {_format_score(score)}",
+            }
+        )
+    if unique_models:
+        highlights.append(
+            {
+                "label": "MODELS SCORED",
+                "model": str(len(unique_models)),
+                "detail": "unique across prompt suites",
+            }
+        )
+    return highlights
+def render_home(benchmarks):
+    highlights = _home_highlights(benchmarks)
+    gr.Markdown(
+        """
+        InferBench is organized by **prompt suite**. There is no single global
+        “best model” score — open a benchmark for full tables, graphs, and sample
+        comparisons.
+        """
+    )
+    if highlights:
+        callout_bits = [
+            f"<div><span>{escape(item['label'])}</span>"
+            f"<strong>{escape(item['model'])}</strong>"
+            f"<em>{escape(item['detail'])}</em></div>"
+            for item in highlights
+        ]
+        gr.HTML(f'<div class="home-callouts">{"".join(callout_bits)}</div>')
+    gr.Markdown("### Benchmark snapshots")
+    with gr.Row(equal_height=True, elem_classes="benchmark-catalogue-row"):
+        for benchmark in benchmarks:
+            data = benchmark["data"]
+            score_column = benchmark.get("overall_column")
+            score_columns = benchmark.get("score_columns") or []
+            if not score_column or score_column not in data.columns:
+                score_column = score_columns[0] if score_columns else None
+            top = _top_models(data, score_column, n=3) if score_column else []
+            score_label = _display_label(score_column) if score_column else "Score"
+            rows_html = "".join(
+                f"<li><span class='home-rank'>{idx}</span>"
+                f"<span class='home-model'>{escape(model)}</span>"
+                f"<span class='home-score'>{_format_score(score)}</span></li>"
+                for idx, (model, score) in enumerate(top, start=1)
+            ) or "<li class='home-empty'>No scores yet.</li>"
+            with gr.Column(scale=1, min_width=280):
+                gr.HTML(
+                    f"""
+                    <div class="home-benchmark-card">
+                      <div class="home-benchmark-title">
+                        {escape(benchmark.get("emoji", "📊"))}
+                        {escape(benchmark["title"])}
+                      </div>
+                      <p class="home-benchmark-blurb">
+                        {escape(benchmark.get("card_description", ""))}
+                      </p>
+                      <div class="home-top-label">Top 3 by {escape(score_label)}</div>
+                      <ol class="home-top-list">{rows_html}</ol>
+                    </div>
+                    """
+                )
+def _format_leaderboard_cell(column, value):
+    if pd.isna(value) or value is None or value == "":
+        return "—"
+    label = str(column).lower()
+    if label == "rank":
+        return str(int(value))
+    if "price" in label:
+        return _format_price(value)
+    if "time" in label or "generation" in label:
+        try:
+            return f"{float(value):.2f}"
+        except (TypeError, ValueError):
+            return escape(str(value))
+    if label in {"model", "platform", "endpoint owner", "optimized"}:
+        return escape(str(value))
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return escape(str(value))
+    if abs(number) >= 100:
+        return f"{number:.1f}"
+    return f"{number:.4f}".rstrip("0").rstrip(".")
+def _leaderboard_sort_value(column, value):
+    """Raw value used by client-side column sorting."""
+    if pd.isna(value) or value is None or value == "":
+        return ""
+    label = str(column).lower()
+    if label in {"model", "platform", "endpoint owner", "optimized", "date"}:
+        return str(value).casefold()
+    try:
+        return f"{float(value):.10g}"
+    except (TypeError, ValueError):
+        return str(value).casefold()
+def _leaderboard_sort_type(column):
+    label = str(column).lower()
+    if label in {"model", "platform", "endpoint owner", "optimized", "date"}:
+        return "text"
+    return "number"
+def _leaderboard_html(data, columns, score_columns, overall_column):
+    leaderboard = _leaderboard_dataframe(
+        data, columns, score_columns, overall_column
+    )
+    if leaderboard.empty:
+        return (
+            '<div class="ranking-table-scroll">'
+            '<div class="ranking-table empty-state">No models match the current filters.</div>'
+            "</div>"
+        )
+    header_cells = []
+    for index, column in enumerate(leaderboard.columns):
+        sort_type = _leaderboard_sort_type(column)
+        header_cells.append(
+            f'<th class="sortable-col" data-col="{index}" data-type="{sort_type}" '
+            f'title="Sort by {escape(str(column))}">{escape(str(column))}</th>'
+        )
+    body_rows = []
+    for _, row in leaderboard.iterrows():
+        cells = []
+        for column in leaderboard.columns:
+            css = "rank" if column == "Rank" else "metric-score"
+            sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)
+            if column == "Model":
+                css = "model-cell"
+                cells.append(
+                    f'<td class="{css}" data-sort-value="{sort_value}">'
+                    f"<strong>{_format_leaderboard_cell(column, row[column])}</strong></td>"
+                )
+            else:
+                cells.append(
+                    f'<td class="{css}" data-sort-value="{sort_value}">'
+                    f"{_format_leaderboard_cell(column, row[column])}</td>"
+                )
+        body_rows.append(f"<tr>{''.join(cells)}</tr>")
+    return f"""
+    <div class="ranking-table-scroll">
+      <div class="ranking-table">
+        <table>
+          <thead><tr>{''.join(header_cells)}</tr></thead>
+          <tbody>{''.join(body_rows)}</tbody>
+        </table>
+      </div>
+    </div>
+    """
+def render_leaderboard(
+    data,
+    columns,
+    note=None,
+    score_columns=None,
+    overall_column=None,
+):
+    score_columns = list(score_columns or _infer_score_columns(columns))
+    overall_column = overall_column or _default_overall_column(score_columns)
+    platform_choices = _filter_choices(data, "Platform")
+    owner_choices = _filter_choices(data, "Endpoint Owner")
+    optimized_choices = _filter_choices(data, "Optimized")
+    if note:
+        gr.Markdown(note)
+    filter_inputs = []
+    with gr.Row(elem_classes="leaderboard-controls"):
+        search = gr.Textbox(
+            label="Search models",
+            placeholder="Search by model or provider",
+            scale=3,
+        )
+        filter_inputs.append(search)
+        platform = None
+        owner = None
+        optimized = None
+        if platform_choices:
+            platform = gr.Dropdown(
+                choices=platform_choices,
+                value=[],
+                label="Providers",
+                multiselect=True,
+                scale=1,
+            )
+            filter_inputs.append(platform)
+        if owner_choices:
+            owner = gr.Dropdown(
+                choices=owner_choices,
+                value=[],
+                label="Endpoint owners",
+                multiselect=True,
+                scale=1,
+            )
+            filter_inputs.append(owner)
+        if optimized_choices:
+            optimized = gr.Dropdown(
+                choices=optimized_choices,
+                value=[],
+                label="Optimized",
+                multiselect=True,
+                scale=1,
+            )
+            filter_inputs.append(optimized)
+    ranking = gr.HTML(
+        _leaderboard_html(data, columns, score_columns, overall_column),
+        elem_classes="ranking-table-host",
+    )
+    def update_ranking(
+        search_term,
+        platform_value=None,
+        owner_value=None,
+        optimized_value=None,
+    ):
+        filtered_data = _filter_leaderboard(
+            data,
+            search_term,
+            platform_value or [],
+            owner_value or [],
+            optimized_value or [],
+        )
+        return _leaderboard_html(
+            filtered_data, columns, score_columns, overall_column
+        )
+    # Wire only the filters that actually exist for this table.
+    change_inputs = [search]
+    if platform is not None:
+        change_inputs.append(platform)
+    if owner is not None:
+        change_inputs.append(owner)
+    if optimized is not None:
+        change_inputs.append(optimized)
+    for component in filter_inputs:
+        component.change(
+            update_ranking,
+            inputs=change_inputs,
+            outputs=ranking,
+        )
+def _infer_score_columns(columns):
+    return [column for column in columns if column.startswith("OneIG (")]
+def _default_overall_column(score_columns):
+    if len(score_columns) == 1:
+        return score_columns[0]
+    return "OneIG Overall Score"
+def _filter_choices(data, column):
+    if column not in data.columns:
+        return []
+    return sorted(data[column].dropna().astype(str).unique().tolist())
+def _filter_leaderboard(data, search_term, platform, owner, optimized):
+    filtered = data.copy()
+    if search_term:
+        search_columns = [
+            column
+            for column in ["Model", "Platform", "Endpoint Owner"]
+            if column in filtered.columns
+        ]
+        matches = pd.Series(False, index=filtered.index)
+        for column in search_columns:
+            matches |= filtered[column].astype(str).str.contains(
+                search_term, case=False, na=False
+            )
+        filtered = filtered[matches]
+    for column, values in [
+        ("Platform", platform),
+        ("Endpoint Owner", owner),
+        ("Optimized", optimized),
+    ]:
+        if values and column in filtered.columns:
+            filtered = filtered[filtered[column].astype(str).isin(values)]
+    return filtered
+def _leaderboard_dataframe(data, columns, score_columns, overall_column):
+    # Honor the caller-provided column list so extra metrics (e.g. Elo) are not
+    # dropped just because they are not part of the ranking score_columns.
+    skip_columns = {"URL", "Rank"}
+    preferred_prefix = [
+        column
+        for column in ["Model", "Platform", "Endpoint Owner", "Optimized"]
+        if column in data.columns
+    ]
+    preferred_suffix = [
+        column
+        for column in [
+            "Median Generation Time (s)",
+            "Min Generation Time (s)",
+            "Price / Image (USD)",
+            "Evaluation Date (UTC)",
+            "Date",
+        ]
+        if column in data.columns
+    ]
+    # Keep overall_column visible when the caller includes it (e.g. Datapoint Elo).
+    # Synthetic aggregates like OneIG Overall Score are simply omitted from `columns`.
+    middle = [
+        column
+        for column in columns
+        if column in data.columns
+        and column not in skip_columns
+        and column not in preferred_prefix
+        and column not in preferred_suffix
+    ]
+    ordered_columns = []
+    seen = set()
+    for column in [*preferred_prefix, *middle, *preferred_suffix]:
+        if column not in seen:
+            seen.add(column)
+            ordered_columns.append(column)
+    leaderboard = data[ordered_columns].copy()
+    # Rank by overall when available, even if that column is not displayed.
+    if overall_column and overall_column in data.columns:
+        leaderboard = (
+            leaderboard.assign(_sort_key=data[overall_column])
+            .sort_values("_sort_key", ascending=False, na_position="last")
+            .drop(columns=["_sort_key"])
+            .reset_index(drop=True)
+        )
+    else:
+        leaderboard = leaderboard.reset_index(drop=True)
+    leaderboard.insert(0, "Rank", leaderboard.index + 1)
+    return leaderboard.rename(columns=_display_label)
+def _display_label(column):
+    labels = {
+        "_overall_score": "Overall score",
+        "OneIG Overall Score": "Overall",
+        "OneIG (Anime Alignment)": "Anime",
+        "OneIG (Human Alignment)": "Human",
+        "OneIG (Object Alignment)": "Object",
+        "OneIG Anime Elo": "Anime Elo (Datapoint)",
+        "OneIG Human Elo": "Human Elo (Datapoint)",
+        "OneIG Object Elo": "Object Elo (Datapoint)",
+        "P-Judge Overall": "P-Judge",
+        "Datapoint Elo": "Datapoint Elo",
+        "Rapidata Elo": "Rapidata Elo",
+        "Benchmark.ai Elo": "Benchmark.ai Elo",
+        "Raw Win Rate": "Raw win rate",
+        "Median Generation Time (s)": "Median generation time",
+        "Min Generation Time (s)": "Min generation time",
+        "Price / Image (USD)": "Price per image",
+        "Evaluation Date (UTC)": "Date",
+        "Date": "Date",
+    }
+    return labels.get(column, column)
+def _text_value(value):
+    return "—" if pd.isna(value) or value is None else escape(str(value))
+def _format_score(value):
+    return "—" if pd.isna(value) or value is None else f"{float(value):.3f}"
+def _format_price(value):
+    return "—" if pd.isna(value) or value is None else f"${float(value):.3f}"
+def render_benchmark_detail(benchmark):
+    gr.Markdown(
+        f"""
+        # {benchmark["title"]}
+        {benchmark["intro"]}
+        """
+    )
+    with gr.Tabs(elem_classes="subtabs", selected=0) as detail_tabs:
+        with gr.TabItem("Leaderboard"):
+            render_leaderboard(
+                benchmark["data"],
+                benchmark["columns"],
+                note=benchmark.get("note"),
+                score_columns=benchmark.get("score_columns"),
+                overall_column=benchmark.get("overall_column"),
+            )
+        with gr.TabItem("Graphs"):
+            render_benchmark_graphs(benchmark)
+        with gr.TabItem("Compare samples"):
+            render_compare_samples(benchmark)
+    return detail_tabs
+def render_compare_samples(benchmark):
+    samples = benchmark.get("samples")
+    if not samples:
+        gr.Markdown(
+            """
+            Sample comparison is not available for this benchmark yet.
+            When generations are linked, you will be able to pick models and browse
+            side-by-side outputs for the same prompts.
+            """
+        )
+        return
+    models = samples["models"]
+    default_models = models[: min(2, len(models))]
+    gr.Markdown(
+        f"""
+        <p class="compare-samples-help">
+        Pick up to <strong>{MAX_COMPARE_MODELS}</strong> models, then browse shared
+        prompts side by side. Images come from the public generation URLs for this
+        benchmark.
+        </p>
+        """
+    )
+    with gr.Row(elem_classes="leaderboard-controls"):
+        model_picker = gr.Dropdown(
+            choices=models,
+            value=default_models,
+            multiselect=True,
+            max_choices=MAX_COMPARE_MODELS,
+            label="Models",
+            info=f"Select 1–{MAX_COMPARE_MODELS} models to compare",
+            scale=3,
+        )
+        prompt_count = gr.Slider(
+            minimum=1,
+            maximum=MAX_COMPARE_PROMPTS,
+            value=DEFAULT_COMPARE_PROMPTS,
+            step=1,
+            label="Prompts to show",
+            scale=1,
+        )
+        shuffle_button = gr.Button("Shuffle prompts", scale=1)
+    gallery = gr.HTML(
+        value=_build_compare_samples_html(
+            samples,
+            default_models,
+            DEFAULT_COMPARE_PROMPTS,
+            seed=0,
+        )
+    )
+    seed_state = gr.State(0)
+    def update_gallery(selected_models, num_prompts, seed):
+        return _build_compare_samples_html(
+            samples,
+            selected_models,
+            int(num_prompts),
+            seed=int(seed or 0),
+        )
+    def shuffle_gallery(selected_models, num_prompts, seed):
+        next_seed = int(seed or 0) + 1
+        return next_seed, _build_compare_samples_html(
+            samples,
+            selected_models,
+            int(num_prompts),
+            seed=next_seed,
+        )
+    model_picker.change(
+        update_gallery,
+        inputs=[model_picker, prompt_count, seed_state],
+        outputs=gallery,
+    )
+    prompt_count.change(
+        update_gallery,
+        inputs=[model_picker, prompt_count, seed_state],
+        outputs=gallery,
+    )
+    shuffle_button.click(
+        shuffle_gallery,
+        inputs=[model_picker, prompt_count, seed_state],
+        outputs=[seed_state, gallery],
+    )
+def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
+    selected_models = [
+        model
+        for model in (selected_models or [])
+        if model in samples["images"]
+    ][:MAX_COMPARE_MODELS]
+    if not selected_models:
+        return (
+            '<div class="compare-empty">'
+            "Select at least one model to compare samples."
+            "</div>"
+        )
+    shared_prompt_ids = None
+    for model in selected_models:
+        model_prompt_ids = set(samples["images"][model])
+        shared_prompt_ids = (
+            model_prompt_ids
+            if shared_prompt_ids is None
+            else shared_prompt_ids & model_prompt_ids
+        )
+    shared_prompt_ids = sorted(shared_prompt_ids or [])
+    if not shared_prompt_ids:
+        return (
+            '<div class="compare-empty">'
+            "No shared prompts found for the selected models."
+            "</div>"
+        )
+    rng = random.Random(seed)
+    prompt_pool = list(shared_prompt_ids)
+    rng.shuffle(prompt_pool)
+    chosen = prompt_pool[: max(1, min(int(num_prompts), len(prompt_pool)))]
+    columns = len(selected_models)
+    blocks = []
+    for index, prompt_id in enumerate(chosen, start=1):
+        prompt_text = escape(samples["prompts"].get(prompt_id, ""))
+        cells = []
+        for model in selected_models:
+            image_url = escape(samples["images"][model][prompt_id], quote=True)
+            cells.append(
+                f"""
+                <div class="compare-cell">
+                    <div class="compare-model-label">{escape(model)}</div>
+                    <a href="{image_url}" target="_blank" rel="noopener noreferrer">
+                        <img src="{image_url}" alt="{escape(model)} sample" loading="lazy" />
+                    </a>
+                </div>
+                """
+            )
+        blocks.append(
+            f"""
+            <div class="compare-prompt-block">
+                <div class="compare-prompt-meta">
+                    <span>Prompt {index}</span>
+                    <span>{escape(prompt_id)}</span>
+                </div>
+                <p class="compare-prompt-text">{prompt_text}</p>
+                <div class="compare-row" style="grid-template-columns: repeat({columns}, minmax(0, 1fr));">
+                    {''.join(cells)}
+                </div>
+            </div>
+            """
+        )
+    return "\n".join(blocks)
+def render_benchmarks(benchmarks):
+    """Catalogue cards + detail pages; back button returns to the catalogue."""
+    open_buttons = []
+    detail_entries = []
+    with gr.Column(visible=True, elem_classes="benchmark-catalogue") as catalogue:
+        gr.Markdown(
+            """
+            # Benchmarks
+            Choose a prompt suite. Each one has a **Leaderboard** table, **Graphs**,
+            and **Compare samples**.
+            """
+        )
+        card_rows = [benchmarks[i : i + 2] for i in range(0, len(benchmarks), 2)]
+        for row in card_rows:
+            with gr.Row(equal_height=True, elem_classes="benchmark-catalogue-row"):
+                for benchmark in row:
+                    with gr.Column(scale=1, min_width=280):
+                        with gr.Group(elem_classes="benchmark-card"):
+                            gr.Markdown(
+                                f"""
+                                ## {benchmark.get("emoji", "📊")} {benchmark["title"]}
+                                {benchmark["card_description"]}
+                                """
+                            )
+                            open_buttons.append(
+                                (
+                                    benchmark["id"],
+                                    gr.Button("View benchmark →", variant="primary"),
+                                )
+                            )
+    for benchmark in benchmarks:
+        with gr.Column(visible=False) as detail:
+            back_button = gr.Button("← All benchmarks", size="sm")
+            render_benchmark_detail(benchmark)
+            detail_entries.append((benchmark["id"], detail, back_button))
+    nav_outputs = [catalogue, *[detail for _, detail, _ in detail_entries]]
+    def show_catalogue(_evt=None):
+        return (
+            gr.Column(visible=True),
+            *[gr.Column(visible=False) for _ in detail_entries],
+        )
+    def show_detail(selected_id):
+        return (
+            gr.Column(visible=False),
+            *[
+                gr.Column(visible=(benchmark_id == selected_id))
+                for benchmark_id, _, _ in detail_entries
+            ],
+        )
+    for benchmark_id, button in open_buttons:
+        button.click(
+            lambda selected_id=benchmark_id: show_detail(selected_id),
+            outputs=nav_outputs,
+        )
+    for _, _, back_button in detail_entries:
+        back_button.click(show_catalogue, outputs=nav_outputs)
+    return show_catalogue, nav_outputs
+def _pareto_frontier_mask(x_values, scores):
+    """True for non-dominated points when maximizing score and minimizing x."""
+    n = len(x_values)
+    mask = [True] * n
+    for i in range(n):
+        for j in range(n):
+            if i == j:
+                continue
+            better_or_equal = x_values[j] <= x_values[i] and scores[j] >= scores[i]
+            strictly_better = x_values[j] < x_values[i] or scores[j] > scores[i]
+            if better_or_equal and strictly_better:
+                mask[i] = False
+                break
+    return mask
+def _build_pareto_figure(
+    data,
+    score_column,
+    x_column,
+    x_title,
+    x_hover_prefix="",
+    x_hover_suffix="",
+):
+    scatter = (
+        data[["Model", score_column, x_column]]
+        .dropna()
+        .copy()
+        .reset_index(drop=True)
+    )
+    if scatter.empty:
+        return None
+    x_values = scatter[x_column].astype(float).tolist()
+    scores = scatter[score_column].astype(float).tolist()
+    on_frontier = _pareto_frontier_mask(x_values, scores)
+    dominated = scatter.loc[[not flag for flag in on_frontier]]
+    frontier = scatter.loc[on_frontier].sort_values(x_column)
+    hover = (
+        "<b>%{text}</b><br>"
+        f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
+        "<br>Score: %{y:.4f}<extra></extra>"
+    )
+    fig = go.Figure()
+    if not dominated.empty:
+        fig.add_trace(
+            go.Scatter(
+                x=dominated[x_column],
+                y=dominated[score_column],
+                mode="markers",
+                name="Below frontier",
+                text=dominated["Model"],
+                hovertemplate=hover,
+                marker={
+                    "size": 9,
+                    "color": "#c4b5fd",
+                    "opacity": 0.75,
+                    "line": {"width": 0},
+                },
+            )
+        )
+    if not frontier.empty:
+        fig.add_trace(
+            go.Scatter(
+                x=frontier[x_column],
+                y=frontier[score_column],
+                mode="lines+markers",
+                name="On frontier",
+                text=frontier["Model"],
+                hovertemplate=hover,
+                line={"color": "#7c3aed", "width": 2.5},
+                marker={
+                    "size": 12,
+                    "color": "#db2777",
+                    "line": {"width": 1.5, "color": "#7c3aed"},
+                },
+            )
+        )
+    score_label = _display_label(score_column)
+    fig.update_layout(
+        title=None,
+        xaxis_title=x_title,
+        yaxis_title=score_label,
+        autosize=True,
+        height=420,
+        margin={"l": 56, "r": 28, "t": 28, "b": 80},
+        legend={
+            "orientation": "h",
+            "yanchor": "top",
+            "y": -0.24,
+            "xanchor": "center",
+            "x": 0.5,
+            "bgcolor": "rgba(0,0,0,0)",
+            "font": {"color": "#e9d5ff", "size": 12},
+        },
+        # Dark-theme plot: soft purple panel + light text (readable, not a white flash).
+        plot_bgcolor="#1e1b4b",
+        paper_bgcolor="#17153b",
+        font={"color": "#e9d5ff", "size": 13},
+    )
+    axis_font = {"color": "#f3e8ff", "size": 13}
+    tick_font = {"color": "#ddd6fe", "size": 12}
+    fig.update_xaxes(
+        showgrid=True,
+        gridcolor="rgba(167, 139, 250, 0.28)",
+        zeroline=False,
+        title_font=axis_font,
+        tickfont=tick_font,
+        color="#e9d5ff",
+    )
+    fig.update_yaxes(
+        showgrid=True,
+        gridcolor="rgba(167, 139, 250, 0.28)",
+        zeroline=False,
+        title_font=axis_font,
+        tickfont=tick_font,
+        color="#e9d5ff",
+    )
+    return fig
+def render_benchmark_graphs(benchmark):
+    data = benchmark["data"]
+    score_columns = [
+        column
+        for column in (benchmark.get("score_columns") or [])
+        if column in data.columns
+    ]
+    overall_column = benchmark.get("overall_column")
+    if not score_columns and overall_column and overall_column in data.columns:
+        score_columns = [overall_column]
+    if not score_columns:
+        gr.Markdown("No score data is available yet.")
+        return
+    # Pareto every displayed quality metric vs price.
+    # Skip only synthetic aggregates (e.g. OneIG mean), not real sort keys like Datapoint Elo.
+    pareto_skip = {
+        "Model",
+        "Platform",
+        "Endpoint Owner",
+        "Optimized",
+        "URL",
+        "Rank",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "Price / Image (USD)",
+        "Evaluation Date (UTC)",
+        "Date",
+        "Raw Win Rate",
+        "OneIG Overall Score",
+    }
+    display_columns = benchmark.get("columns") or []
+    pareto_columns = []
+    for column in [*score_columns, *display_columns]:
+        if (
+            column in data.columns
+            and column not in pareto_skip
+            and column not in pareto_columns
+            and pd.api.types.is_numeric_dtype(data[column])
+        ):
+            pareto_columns.append(column)
+    price_column = "Price / Image (USD)"
+    time_column = "Min Generation Time (s)"
+    price_figures = []
+    time_figures = []
+    for plot_column in pareto_columns:
+        if price_column in data.columns:
+            price_fig = _build_pareto_figure(
+                data,
+                plot_column,
+                x_column=price_column,
+                x_title="Price per image (USD)",
+                x_hover_prefix="$",
+            )
+            if price_fig is not None:
+                price_figures.append((plot_column, price_fig))
+        if time_column in data.columns:
+            time_fig = _build_pareto_figure(
+                data,
+                plot_column,
+                x_column=time_column,
+                x_title="Min generation time (s)",
+                x_hover_suffix="s",
+            )
+            if time_fig is not None:
+                time_figures.append((plot_column, time_fig))
+    if price_figures or time_figures:
+        gr.Markdown(
+            "### Pareto frontiers\n\n"
+            "<span style='color:#4c1d95;font-size:0.95rem;'>"
+            "Pink = on the frontier (lower cost or time at the same or better score). "
+            "Light purple = below the frontier."
+            "</span>"
+        )
+        with gr.Row(equal_height=False):
+            with gr.Column(scale=1, min_width=320):
+                gr.Markdown("#### Price vs score")
+                if not price_figures:
+                    gr.Markdown("_No price data available._")
+                for plot_column, pareto_fig in price_figures:
+                    gr.Markdown(f"**{_display_label(plot_column)}**")
+                    gr.Plot(
+                        value=pareto_fig,
+                        show_label=False,
+                        elem_classes="pareto-plot",
+                    )
+            with gr.Column(scale=1, min_width=320):
+                gr.Markdown("#### Min generation time vs score")
+                if not time_figures:
+                    gr.Markdown("_No min generation time data available._")
+                for plot_column, pareto_fig in time_figures:
+                    gr.Markdown(f"**{_display_label(plot_column)}**")
+                    gr.Plot(
+                        value=pareto_fig,
+                        show_label=False,
+                        elem_classes="pareto-plot",
+                    )
+def render_about():
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(ABOUT_OVERVIEW_CONTENT)
+        with gr.Column():
+            gr.Markdown(ABOUT_DETAILS_CONTENT)
+def render_footer():
+    gr.HTML(COMMUNITY_CONTENT)
+    with gr.Accordion("Citation", open=False):
+        gr.Markdown(CITATION_CONTENT)