Upload app.py with huggingface_hub

app.py

@@ -1,4 +1,8 @@
+import re
+
 import gradio as gr
+import pandas as pd
+import plotly.express as px
 
 from app.utils import (
     add_rank_and_format,
@@ -161,25 +165,11 @@ def main():
                 ></iframe>
                 """
                 )
+
             with gr.TabItem("ViDoRe V3 (Pipeline)", id="vidore-v3-pipeline"):
                 gr.Markdown("# ViDoRe V3 (Pipeline Evaluation): Retrieval Performance for Complex Pipelines ⚙️")
                 gr.Markdown("### Assessing retrieval performance, latency, and compute costs of complex retrieval pipelines")
 
-                # gr.Markdown(
-                #     """
-                # This leaderboard displays abstract pipeline evaluation results for ViDoRe V3 on **english-only queries**.
-                # Unlike model-only evaluations, these results showcase what full retrieval pipelines can achieve.
-
-                # Metrics regarding time complexity are also provided in the form of ("Indexing latency (second/doc)" and "Search latency (second/query)").
-                # Those are self-reported, highly hardware dependent, and should not be considered absolute.
-                # Nevertheless we felt it was quite important to ensure transparency and provide users with a rough understanding of the computational demands of each pipeline.
-
-                # Results are sourced from the [vidore-benchmark repository](https://github.com/illuin-tech/vidore-benchmark/tree/vidore_v3_pipeline/results).
-
-                # Those results are not directly comparable to the base ViDoRe v3 results, since they are computed on a different set of queries (english-only queries).
-                # """
-                # )
-
                 gr.Markdown(
                     """
                     This leaderboard ranks full retrieval pipelines on **English-only queries** for **ViDoRe V3**. Instead of just testing standalone models, we evaluate real-world, multi-step retrieval systems. This includes everything from basic retrievers to advanced setups using AI agents, query reformulation, hybrid search, and any other creative retrieval pipeline one can imagine.
@@ -223,6 +213,79 @@ def main():
                         datatype_pipeline = ["number", "markdown", "number", "number", "number"] + ["number"] * len(datasets_columns_pipeline)
                         dataframe_pipeline = gr.Dataframe(data_pipeline, datatype=datatype_pipeline, type="pandas", elem_id="pipeline-table")
 
+                    def clean_pipeline_name(name):
+                        if not isinstance(name, str):
+                            return str(name)
+                        # Remove Markdown links [text](url) -> text
+                        name = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', name)
+                        # Remove HTML tags <a href="...">text</a> -> text
+                        name = re.sub(r'<[^>]+>', '', name)
+                        return name.strip()
+
+                    def create_pipeline_plot(df, latency_col):
+                        if df is None or len(df) == 0:
+                            return None
+
+                        # Ensure expected columns exist
+                        if latency_col not in df.columns or "Average Score" not in df.columns or "Pipeline" not in df.columns:
+                            return None
+
+                        # Clean the dataframe for plotting
+                        plot_df = df.copy()
+
+                        # Strip HTML and Markdown for clean hover text
+                        plot_df["Cleaned Pipeline"] = plot_df["Pipeline"].apply(clean_pipeline_name)
+
+                        # Force numerical conversion (coerces weird strings to NaN)
+                        plot_df[latency_col] = pd.to_numeric(plot_df[latency_col], errors='coerce')
+                        plot_df["Average Score"] = pd.to_numeric(plot_df["Average Score"], errors='coerce')
+
+                        # Drop NaNs
+                        plot_df = plot_df.dropna(subset=[latency_col, "Average Score"])
+
+                        # Filter out non-positive values to prevent log-scale math errors
+                        plot_df = plot_df[plot_df[latency_col] > 0]
+
+                        # Sort by the x-axis to prevent Plotly from drawing out-of-order categorical ticks
+                        plot_df = plot_df.sort_values(by=latency_col)
+
+                        if len(plot_df) == 0:
+                            return None
+
+                        fig = px.scatter(
+                            plot_df,
+                            x=latency_col,
+                            y="Average Score",
+                            hover_name="Cleaned Pipeline", # Use the clean text!
+                            title=f"Mean Performance vs {latency_col}",
+                            color="orange",
+                        )
+                        fig.update_layout(
+                            plot_bgcolor='white'
+                        )
+
+                        # Explicitly force the axis to be log type so Plotly handles the spacing natively
+                        fig.update_layout(
+                            xaxis_type="log",
+                            xaxis_title=latency_col,
+                            yaxis_title="Average Score"
+                        )
+
+                        # Style the points
+                        fig.update_traces(marker=dict(size=12, opacity=0.8, line=dict(width=1, color='DarkSlateGrey')))
+                        return fig
+
+                    with gr.Row():
+                        latency_radio = gr.Radio(
+                            choices=["Search latency (s/query)", "Indexing latency (s/doc)"],
+                            value="Search latency (s/query)",
+                            label="Select Latency Metric for X-Axis"
+                        )
+
+                    with gr.Row():
+                        initial_fig = create_pipeline_plot(data_pipeline, "Search latency (s/query)")
+                        performance_plot = gr.Plot(value=initial_fig)
+
                     def update_data_pipeline(metric, search_term, selected_columns):
                         pipeline_handler.get_pipeline_data()
                         data = pipeline_handler.render_df(metric, "english")
@@ -247,6 +310,10 @@ def main():
                             inputs=[metric_dropdown_pipeline],
                             outputs=dataframe_pipeline,
                             concurrency_limit=20,
+                        ).then(
+                            fn=create_pipeline_plot,
+                            inputs=[dataframe_pipeline, latency_radio],
+                            outputs=performance_plot
                         )
 
                     with gr.Row():
@@ -264,20 +331,42 @@ def main():
                         df = pipeline_handler.render_df(metric, "english")
                         return add_rank_and_format(df, benchmark_version=3, is_pipeline=True)
 
+                    # Update dataframe and then update the plot
                     metric_dropdown_pipeline.change(
                         refresh_pipeline_data,
                         inputs=[metric_dropdown_pipeline],
                         outputs=dataframe_pipeline,
+                    ).then(
+                        fn=create_pipeline_plot,
+                        inputs=[dataframe_pipeline, latency_radio],
+                        outputs=performance_plot
                     )
+
                     research_textbox_pipeline.submit(
                         lambda metric, search_term, selected_columns: update_data_pipeline(metric, search_term, selected_columns),
                         inputs=[metric_dropdown_pipeline, research_textbox_pipeline, column_checkboxes_pipeline],
                         outputs=dataframe_pipeline,
+                    ).then(
+                        fn=create_pipeline_plot,
+                        inputs=[dataframe_pipeline, latency_radio],
+                        outputs=performance_plot
                     )
+
                     column_checkboxes_pipeline.change(
                         lambda metric, search_term, selected_columns: update_data_pipeline(metric, search_term, selected_columns),
                         inputs=[metric_dropdown_pipeline, research_textbox_pipeline, column_checkboxes_pipeline],
                         outputs=dataframe_pipeline,
+                    ).then(
+                        fn=create_pipeline_plot,
+                        inputs=[dataframe_pipeline, latency_radio],
+                        outputs=performance_plot
+                    )
+
+                    # Update plot when the radio button changes
+                    latency_radio.change(
+                        fn=create_pipeline_plot,
+                        inputs=[dataframe_pipeline, latency_radio],
+                        outputs=performance_plot
                     )
 
                     gr.Markdown(
@@ -297,7 +386,7 @@ def main():
                       eprint={2407.01449},
                       archivePrefix={arXiv},
                       primaryClass={cs.IR},
-                      url={https://arxiv.org/abs/2407.01449},
+                      url={[https://arxiv.org/abs/2407.01449](https://arxiv.org/abs/2407.01449)},
                     }
                     @misc{loison2026vidore,
                       title={ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},