Update same as GradioCompararion 2 and set tmp dir if not existent

app.py

@@ -3,28 +3,28 @@ import pandas as pd
 from transformers import AutoModelForSequenceClassification, AutoTokenizer
 from datasets import load_dataset
 from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix
-from sklearn.cluster import KMeans
 import torch
 from sentence_transformers import SentenceTransformer
 import umap
 from sklearn.manifold import TSNE
 import matplotlib.pyplot as plt
-import seaborn as sns
 import numpy as np
+import seaborn as sns
 import tempfile
 from collections import Counter
 import os
-import tempfile
 
-temp_dir = tempfile.gettempdir()
+temp_dir = '/tmp/gradio_tmp'
+os.makedirs(temp_dir, exist_ok=True)  # Creates the directory if it does not exist
 os.environ['GRADIO_TEMP_DIR'] = temp_dir
 
+
 # Load the models and their tokenizers
 model_paths = {
     "roberta-base-offensive": "./models/roberta-base-offensive",
     "distilbert-base-uncased-offensive": "./models/distilbert-base-uncased-offensive",
-    "bert-offensive": "./models/bert-offensive",
-    "deberta-offensive": "./models/deberta-offensive"
+    "bert-offensive":"./models/bert-offensive",
+    "deberta-offensive":"./models/deberta-offensive"
 }
 
 models = {name: AutoModelForSequenceClassification.from_pretrained(path) for name, path in model_paths.items()}
@@ -67,10 +67,15 @@ def generate_confusion_matrix(conf_matrix, model_name):
 def generate_embeddings_and_plot(categories):
     all_texts = sum(categories.values(), [])
     embeddings = model_embedding.encode(all_texts)
+
+    # UMAP reduction
     umap_reducer = umap.UMAP(n_neighbors=15, n_components=2, metric='cosine')
     umap_embeddings = umap_reducer.fit_transform(embeddings)
+
+    # t-SNE reduction
     tsne_embeddings = TSNE(n_components=2, perplexity=30).fit_transform(embeddings)
 
+    # Plotting helper function to avoid repetition
     def plot_embeddings(embeddings, title, file_suffix):
         plt.figure(figsize=(10, 8))
         colors = {"correct_both": "green", "incorrect_both": "red", "correct_model1_only": "blue", "correct_model2_only": "orange"}
@@ -81,13 +86,16 @@ def generate_embeddings_and_plot(categories):
         plt.title(title)
         plt.xlabel('Component 1')
         plt.ylabel('Component 2')
+
         temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=f'_{file_suffix}.png')
         plt.savefig(temp_file.name)
         plt.close()
         return temp_file.name
 
+    # Generate and save plots
     umap_plot_path = plot_embeddings(umap_embeddings, "UMAP Projection of Text Categories", "umap")
     tsne_plot_path = plot_embeddings(tsne_embeddings, "t-SNE Projection of Text Categories", "tsne")
+
     return umap_plot_path, tsne_plot_path
 
 def compare_models(model1, model2):
@@ -138,6 +146,55 @@ def compare_models(model1, model2):
 
     return metrics_df, conf_matrix_path1, conf_matrix_path2, umap_plot_path, tsne_plot_path, categories
 
+
+from sklearn.cluster import KMeans
+
+def generate_embeddings_and_cluster(categories):
+    all_texts = sum(categories.values(), [])
+    embeddings = model_embedding.encode(all_texts)
+
+    # Category labels for all texts
+    category_labels = [cat for cat, texts in categories.items() for _ in range(len(texts))]
+
+    # Calculate overall category distribution
+    overall_distribution = Counter(category_labels)
+    overall_distribution_percent = {k: v / len(category_labels) * 100 for k, v in overall_distribution.items()}
+
+    # K-means clustering
+    kmeans = KMeans(n_clusters=3, random_state=42).fit(embeddings)
+    labels = kmeans.labels_
+
+    # Map each text to its cluster and category
+    cluster_categories = [[] for _ in range(3)]  # Assuming 3 clusters
+    for label, category in zip(labels, category_labels):
+        cluster_categories[label].append(category)
+
+    # Calculate category distribution within each cluster
+    cluster_distributions = []
+    for i, cluster in enumerate(cluster_categories):
+        distribution = Counter(cluster)
+        distribution_percent = {k: v / len(cluster) * 100 for k, v in distribution.items()}
+        cluster_distributions.append(distribution_percent)
+
+    # Perform UMAP dimensionality reduction for visualization
+    umap_reducer = umap.UMAP(n_neighbors=15, n_components=2, metric='cosine')
+    reduced_embeddings = umap_reducer.fit_transform(embeddings)
+
+    # Visualization
+    plt.figure(figsize=(10, 8))
+    scatter = plt.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c=labels, cmap='viridis', alpha=0.6)
+    plt.legend(*scatter.legend_elements(), title="Clusters")
+    plt.title("K-means Clustering of Text Embeddings")
+    plt.xlabel('UMAP 1')
+    plt.ylabel('UMAP 2')
+
+    # Save the plot
+    cluster_plot_path = tempfile.NamedTemporaryFile(delete=False, suffix='_cluster.png').name
+    plt.savefig(cluster_plot_path)
+    plt.close()
+    
+    return cluster_plot_path, overall_distribution_percent, cluster_distributions
+
 def setup_gradio_interface():
     with gr.Blocks() as demo:
         gr.Markdown("## Model Comparison and Text Analysis")
@@ -155,19 +212,39 @@ def setup_gradio_interface():
         with gr.Row():
             umap_visualization_output = gr.Image(label="UMAP Text Categorization Visualization")
             tsne_visualization_output = gr.Image(label="t-SNE Text Categorization Visualization")
+        
+        clustering_visualization_output = gr.Image(label="K-means Clustering Visualization")
+        
+        category_distribution_output = gr.Dataframe(label="Category Distribution Comparison")
+        
 
         def update_interface(model1, model2):
-            metrics_df, conf_matrix1, conf_matrix2 = compare_models(model1, model2)
-            umap_plot_path, tsne_plot_path = generate_embeddings_and_plot(metrics_df)
-            return metrics_df, conf_matrix1, conf_matrix2, umap_plot_path, tsne_plot_path
+            metrics_df, cm_path1, cm_path2, umap_viz_path, tsne_viz_path, categories = compare_models(model1, model2)
+            cluster_viz_path, overall_distribution_percent, cluster_distributions = generate_embeddings_and_cluster(categories)
+    
+            # Prepare DataFrame for category distribution comparison
+            distribution_data = []
+            for cluster_index, cluster_distribution in enumerate(cluster_distributions, start=1):
+                for category, percent in cluster_distribution.items():
+                    distribution_data.append({
+                        "Cluster": f"Cluster {cluster_index}",
+                        "Category": category,
+                        "Percentage": f"{percent:.2f}%",
+                        "Difference from Overall": f"{percent - overall_distribution_percent.get(category, 0):.2f}%"
+                })
+            distribution_df = pd.DataFrame(distribution_data)
+    
+            return metrics_df, cm_path1, cm_path2, umap_viz_path, tsne_viz_path, cluster_viz_path, distribution_df
 
+        
+        
         submit_button.click(
             update_interface,
             inputs=[model1_input, model2_input],
-            outputs=[metrics_output, model1_cm_output, model2_cm_output, umap_visualization_output, tsne_visualization_output]
+            outputs=[metrics_output, model1_cm_output, model2_cm_output, umap_visualization_output, tsne_visualization_output, clustering_visualization_output, category_distribution_output]
         )
         
     return demo
 
 demo = setup_gradio_interface()
-demo.launch(share=True)
+demo.launch(share=True)