Update app.py

app.py

@@ -1,11 +1,14 @@
-import gradio as gr
+from typing import Dict, List, Union, Optional, Tuple
+from urllib.parse import urlparse
 import requests
 import time
 import json
 import re
 import ast
-from typing import Dict, List, Union, Optional, Tuple
-from urllib.parse import urlparse
+import gradio as gr
+
+# Import the scraper function
+from scrape_and_monitor_hf_mcp_servers import scrape_and_monitor_hf_mcp_servers, SORT_OPTIONS
 
 def parse_huggingface_url(url: str) -> str:
     """
@@ -32,16 +35,14 @@ def parse_huggingface_url(url: str) -> str:
         if '.hf.space' in base_url:
             domain_match = re.search(r'https://([^.]+)\.hf\.space', base_url)
             if domain_match:
-                space_name = domain_match.group(1)
-                # Convert domain format back to HF spaces format
-                # Example: {username}-{space-name} -> {username}/{space-name}
-                parts = space_name.split('-')
-                if len(parts) >= 2:
-                    username = parts[0]
-                    repo_name = '-'.join(parts[1:])
-                    hf_spaces_url = f"https://huggingface.co/spaces/{username}/{repo_name}"
+                space_identifier = domain_match.group(1)
+                # Try to split username-spacename
+                if '-' in space_identifier:
+                    parts = space_identifier.split('-', 1)
+                    username, space_name = parts[0], parts[1]
+                    hf_spaces_url = f"https://huggingface.co/spaces/{username}/{space_name}"
                 else:
-                    hf_spaces_url = f"https://huggingface.co/spaces/{space_name}"
+                    hf_spaces_url = "unknown"
             else:
                 hf_spaces_url = "unknown"
         else:
@@ -55,14 +56,14 @@ def parse_huggingface_url(url: str) -> str:
         # Extract space name and create HF spaces URL
         domain_match = re.search(r'https://([^.]+)\.hf\.space', url)
         if domain_match:
-            space_name = domain_match.group(1)
-            parts = space_name.split('-')
-            if len(parts) >= 2:
-                username = parts[0]
-                repo_name = '-'.join(parts[1:])
-                hf_spaces_url = f"https://huggingface.co/spaces/{username}/{repo_name}"
+            space_identifier = domain_match.group(1)
+            # Try to split username-spacename  
+            if '-' in space_identifier:
+                parts = space_identifier.split('-', 1)
+                username, space_name = parts[0], parts[1]
+                hf_spaces_url = f"https://huggingface.co/spaces/{username}/{space_name}"
             else:
-                hf_spaces_url = f"https://huggingface.co/spaces/{space_name}"
+                hf_spaces_url = "unknown"
         else:
             hf_spaces_url = "unknown"
     
@@ -73,12 +74,8 @@ def parse_huggingface_url(url: str) -> str:
         # Extract username and space name
         spaces_match = re.search(r'huggingface\.co/spaces/([^/]+)/([^/?]+)', url)
         if spaces_match:
-            username = spaces_match.group(1)
-            space_name = spaces_match.group(2)
-            
-            # Convert to hf.space domain format
-            space_domain = f"{username}-{space_name}".lower()
-            space_url = f"https://{space_domain}.hf.space"
+            username, space_name = spaces_match.groups()
+            space_url = f"https://{username}-{space_name}.hf.space"
             mcp_endpoint = f"{space_url}/gradio_api/mcp/sse"
         else:
             space_url = "unknown"
@@ -101,11 +98,21 @@ def parse_huggingface_url(url: str) -> str:
     
     return json.dumps(result, indent=2)
 
-def parse_huggingface_url_with_summary(url: str) -> tuple:
+def parse_huggingface_url_with_summary(url: str, progress=gr.Progress()) -> tuple:
     """Parse URL and return both markdown summary and JSON."""
+    progress(0, desc="🔍 Starting URL parsing...")
+    
+    if not url.strip():
+        return "# ❌ No URL Provided\n\nPlease enter a URL to parse.", "{}"
+    
+    progress(0.3, desc="🔍 Analyzing URL format...")
     json_result = parse_huggingface_url(url)
+    
+    progress(0.7, desc="🔍 Generating summary...")
     parsed_info = json.loads(json_result)
     md_summary = format_url_summary(parsed_info)
+    
+    progress(1.0, desc="✅ URL parsing complete!")
     return md_summary, json_result
 
 def format_url_summary(parsed_info: dict) -> str:
@@ -117,9 +124,9 @@ def format_url_summary(parsed_info: dict) -> str:
         md += "✅ **Status:** Valid URL format\n\n"
         md += "## 📋 Extracted URLs\n\n"
         if parsed_info['hf_spaces_url'] != "unknown":
-            md += f"- **HuggingFace Space:** [{parsed_info['hf_spaces_url']}]({parsed_info['hf_spaces_url']})\n"
-        md += f"- **Space URL:** [{parsed_info['space_url']}]({parsed_info['space_url']})\n"
-        md += f"- **MCP Endpoint:** [{parsed_info['mcp_endpoint']}]({parsed_info['mcp_endpoint']})\n\n"
+            md += f"- **HF Spaces URL:** `{parsed_info['hf_spaces_url']}`\n"
+        md += f"- **Space URL:** `{parsed_info['space_url']}`\n"
+        md += f"- **MCP Endpoint:** `{parsed_info['mcp_endpoint']}`\n\n"
         
         md += "## ⚙️ MCP Client Configuration\n\n"
         md += "Copy this configuration for your MCP client:\n\n"
@@ -138,7 +145,7 @@ def format_url_summary(parsed_info: dict) -> str:
     
     return md
 
-def check_single_server_health(url: str) -> tuple:
+def check_single_server_health(url: str, progress=gr.Progress()) -> tuple:
     """
     Check health of a single MCP server from any URL format.
     
@@ -148,6 +155,12 @@ def check_single_server_health(url: str) -> tuple:
     Returns:
         tuple: (markdown_summary, json_data)
     """
+    progress(0, desc="🏥 Starting health check...")
+    
+    if not url.strip():
+        return "# ❌ No URL Provided\n\nPlease enter a URL to check.", "{}"
+    
+    progress(0.1, desc="🏥 Parsing URL...")
     parsed_info = json.loads(parse_huggingface_url(url))
     
     if not parsed_info["is_valid"]:
@@ -169,10 +182,11 @@ def check_single_server_health(url: str) -> tuple:
     }
     
     # Test 1: Check space URL health
+    progress(0.3, desc="🌐 Checking space accessibility...")
     if parsed_info["space_url"] != "unknown":
         start_time = time.time()
         try:
-            response = requests.get(parsed_info["space_url"], timeout=10)
+            response = requests.get(parsed_info["space_url"], timeout=8)
             response_time = round((time.time() - start_time) * 1000, 2)
             
             results["space_health"] = {
@@ -192,6 +206,7 @@ def check_single_server_health(url: str) -> tuple:
             }
     
     # Test 2: Check MCP endpoint health
+    progress(0.6, desc="🔧 Checking MCP endpoint...")
     start_time = time.time()
     try:
         response = requests.get(parsed_info["mcp_endpoint"], timeout=8, stream=True)
@@ -213,6 +228,8 @@ def check_single_server_health(url: str) -> tuple:
             "error": str(e)
         }
     
+    progress(0.9, desc="📊 Analyzing results...")
+    
     # Determine overall status
     space_ok = results["space_health"] is None or results["space_health"]["accessible"]
     mcp_ok = results["mcp_health"]["accessible"]
@@ -229,6 +246,7 @@ def check_single_server_health(url: str) -> tuple:
     # Generate markdown summary
     md = format_health_summary(results)
     
+    progress(1.0, desc="✅ Health check complete!")
     return md, json.dumps(results, indent=2)
 
 def format_health_summary(results: dict) -> str:
@@ -250,7 +268,7 @@ def format_health_summary(results: dict) -> str:
         sh = results["space_health"]
         status_icon = "✅" if sh["accessible"] else "❌"
         md += f"## 🌐 Space Health {status_icon}\n\n"
-        md += f"- **URL:** [{sh['url']}]({sh['url']})\n"
+        md += f"- **URL:** `{sh['url']}`\n"
         md += f"- **Status Code:** {sh.get('status_code', 'N/A')}\n"
         md += f"- **Response Time:** {sh['response_time_ms']}ms\n"
         if "error" in sh:
@@ -261,7 +279,7 @@ def format_health_summary(results: dict) -> str:
     mh = results["mcp_health"]
     status_icon = "✅" if mh["accessible"] else "❌"
     md += f"## 🔧 MCP Endpoint Health {status_icon}\n\n"
-    md += f"- **URL:** [{mh['url']}]({mh['url']})\n"
+    md += f"- **URL:** `{mh['url']}`\n"
     md += f"- **Status Code:** {mh.get('status_code', 'N/A')}\n"
     md += f"- **Response Time:** {mh['response_time_ms']}ms\n"
     if "error" in mh:
@@ -299,21 +317,13 @@ def extract_functions_from_source(source_code: str) -> List[Tuple[str, str, List
         
         for node in ast.walk(tree):
             if isinstance(node, ast.FunctionDef):
-                # Skip private functions and common non-tool functions
-                if node.name.startswith('_') or node.name in ['main', 'launch', 'setup']:
-                    continue
-                
-                # Get function name
                 func_name = node.name
+                docstring = ast.get_docstring(node) or "No docstring available"
                 
-                # Get docstring
-                docstring = ast.get_docstring(node, clean=True) or ""
-                
-                # Get parameters (exclude 'self' for methods)
+                # Extract parameter names
                 parameters = []
                 for arg in node.args.args:
-                    if arg.arg != 'self':
-                        parameters.append(arg.arg)
+                    parameters.append(arg.arg)
                 
                 functions.append((func_name, docstring, parameters))
                 
@@ -323,7 +333,7 @@ def extract_functions_from_source(source_code: str) -> List[Tuple[str, str, List
     
     return functions
 
-def discover_server_tools(url: str) -> tuple:
+def discover_server_tools(url: str, progress=gr.Progress()) -> tuple:
     """
     Discover available MCP tools from a server.
     
@@ -333,50 +343,49 @@ def discover_server_tools(url: str) -> tuple:
     Returns:
         tuple: (markdown_summary, json_data)
     """
+    progress(0, desc="🛠️ Starting tools discovery...")
+    
+    if not url.strip():
+        return "# ❌ No URL Provided\n\nPlease enter a URL to discover tools.", "{}"
+    
+    progress(0.1, desc="🛠️ Parsing URL...")
     parsed_info = json.loads(parse_huggingface_url(url))
     
     if not parsed_info["is_valid"]:
         result = {
             "original_url": url,
             "status": "invalid_url",
-            "error": "Could not parse URL format",
-            "tools": [],
-            "tool_count": 0
+            "error": "Could not parse URL format"
         }
-        md = "# ❌ Tools Discovery Failed\n\nCould not parse URL format. Please check the URL."
+        md = "# ❌ Tools Discovery Failed\n\nCould not parse URL format."
         return md, json.dumps(result, indent=2)
     
     tools = []
     discovery_methods = []
     
     # Method: Analyze app.py source code
+    progress(0.3, desc="🛠️ Fetching source code...")
     try:
+        # Try to get app.py from HF spaces
         if parsed_info["hf_spaces_url"] != "unknown":
-            app_py_url = f"{parsed_info['hf_spaces_url']}/raw/main/app.py"
-            response = requests.get(app_py_url, timeout=8)
-            discovery_methods.append(f"App.py source code: HTTP {response.status_code}")
-            
+            app_url = f"{parsed_info['hf_spaces_url']}/raw/main/app.py"
+            progress(0.5, desc="🛠️ Analyzing app.py...")
+            response = requests.get(app_url, timeout=10)
             if response.status_code == 200:
-                source_code = response.text
-                
-                # Use AST to extract functions
-                functions = extract_functions_from_source(source_code)
-                
-                if functions:
-                    discovery_methods.append(f"Found {len(functions)} functions in source code")
-                    
-                    for func_name, docstring, parameters in functions:
-                        tool_data = {
-                            "name": func_name,
-                            "description": docstring if docstring else "No description available",
-                            "parameters": parameters,
-                            "parameter_count": len(parameters)
-                        }
-                        tools.append(tool_data)
-                else:
-                    discovery_methods.append("No functions found in source code")
+                progress(0.7, desc="🛠️ Extracting functions...")
+                functions = extract_functions_from_source(response.text)
+                for func_name, docstring, params in functions:
+                    tools.append({
+                        "name": func_name,
+                        "description": docstring,
+                        "parameters": params,
+                        "source": "app.py_analysis"
+                    })
+                discovery_methods.append("Analyzed app.py source code")
     except Exception as e:
-        discovery_methods.append(f"Source code analysis failed: {str(e)}")
+        discovery_methods.append(f"Failed to analyze app.py: {str(e)}")
+    
+    progress(0.9, desc="🛠️ Preparing results...")
     
     # Prepare result
     result = {
@@ -390,12 +399,12 @@ def discover_server_tools(url: str) -> tuple:
     }
     
     if not tools:
-        result["error"] = "No tools could be discovered"
-        result["note"] = "Check if the space has functions with proper definitions"
+        result["message"] = "No tools discovered. Server may not expose MCP tools or may be private."
     
     # Generate markdown summary
     md = format_tools_summary(result)
     
+    progress(1.0, desc="✅ Tools discovery complete!")
     return md, json.dumps(result, indent=2)
 
 def format_tools_summary(result: dict) -> str:
@@ -405,52 +414,27 @@ def format_tools_summary(result: dict) -> str:
     if result["status"] == "success":
         md += f"✅ **Status:** Found {result['tool_count']} tools\n\n"
         
-        if result["tools"]:
-            md += "## 🛠️ Available Tools\n\n"
-            for i, tool in enumerate(result["tools"], 1):
-                md += f"### {i}. `{tool['name']}`\n\n"
-                if tool["description"].strip():
-                    # Clean up description and limit length
-                    desc = tool["description"][:200] + "..." if len(tool["description"]) > 200 else tool["description"]
-                    md += f"**Description:** {desc}\n\n"
-                else:
-                    md += "**Description:** *No description available*\n\n"
-                
-                if tool["parameters"]:
-                    md += f"**Parameters:** `{', '.join(tool['parameters'])}`\n\n"
-                else:
-                    md += "**Parameters:** *None*\n\n"
-            
-            md += "## 📋 Quick Reference\n\n"
-            md += f"**Tool Names:** {', '.join([f'`{name}`' for name in result['tool_names']])}\n\n"
-            
-            md += "## ⚙️ MCP Client Configuration\n\n"
-            md += "Add this to your MCP client config:\n\n"
-            md += "```json\n"
-            md += "{\n"
-            md += '  "mcpServers": {\n'
-            md += '    "gradio_server": {\n'
-            md += f'      "url": "{result["mcp_endpoint"]}"\n'
-            md += '    }\n'
-            md += '  }\n'
-            md += "}\n"
-            md += "```\n"
+        md += "## 🛠️ Available Tools\n\n"
+        for i, tool in enumerate(result["tools"], 1):
+            md += f"### {i}. {tool['name']}\n"
+            md += f"**Description:** {tool['description'][:200]}{'...' if len(tool['description']) > 200 else ''}\n"
+            md += f"**Parameters:** {', '.join(tool['parameters'])}\n\n"
         
     else:
         md += "❌ **Status:** No tools found\n\n"
-        if "error" in result:
-            md += f"**Error:** {result['error']}\n\n"
-        if "note" in result:
-            md += f"**Note:** {result['note']}\n\n"
+        md += "This could mean:\n"
+        md += "- The server doesn't expose MCP tools\n"
+        md += "- The server is private or requires authentication\n"
+        md += "- The server is not running\n\n"
     
     if result.get("discovery_methods"):
-        md += "## 🔍 Discovery Methods\n\n"
+        md += "## 🔍 Discovery Methods Used\n\n"
         for method in result["discovery_methods"]:
             md += f"- {method}\n"
     
     return md
 
-def monitor_multiple_servers(urls_text: str) -> tuple:
+def monitor_multiple_servers(urls_text: str, progress=gr.Progress()) -> tuple:
     """
     Monitor health and tools of multiple MCP servers simultaneously.
     
@@ -460,45 +444,63 @@ def monitor_multiple_servers(urls_text: str) -> tuple:
     Returns:
         tuple: (markdown_summary, json_data)
     """
+    progress(0, desc="📊 Starting multi-server monitoring...")
+    
     if not urls_text.strip():
-        result = {"error": "No URLs provided", "servers": []}
-        md = "# ❌ No URLs Provided\n\nPlease provide at least one URL to monitor."
+        result = {
+            "error": "No URLs provided",
+            "servers": [],
+            "total_servers": 0
+        }
+        md = "# ❌ No URLs Provided\n\nPlease enter URLs to monitor."
         return md, json.dumps(result, indent=2)
     
+    progress(0.1, desc="📊 Parsing URL list...")
     urls = [url.strip() for url in urls_text.strip().split('\n') if url.strip()]
     
     if not urls:
-        result = {"error": "No valid URLs found", "servers": []}
-        md = "# ❌ No Valid URLs Found\n\nPlease check your URL format."
+        result = {
+            "error": "No valid URLs found",
+            "servers": [],
+            "total_servers": 0
+        }
+        md = "# ❌ No Valid URLs\n\nPlease check the URL format."
         return md, json.dumps(result, indent=2)
     
     results = []
     
-    for url in urls:
+    for i, url in enumerate(urls, 1):
+        progress_pct = 0.1 + (i / len(urls)) * 0.8  # Reserve 10% for setup, 10% for final processing
+        progress(progress_pct, desc=f"📊 Checking server {i}/{len(urls)}: {url[:50]}...")
+        
         try:
-            # Get health info
+            print(f"🔍 Checking server {i}/{len(urls)}: {url}")
+            
             _, health_json = check_single_server_health(url)
-            health_info = json.loads(health_json)
+            health_data = json.loads(health_json)
             
-            # Get tools info
             _, tools_json = discover_server_tools(url)
-            tools_info = json.loads(tools_json)
+            tools_data = json.loads(tools_json)
             
-            # Combine information
             server_result = {
                 "url": url,
-                "health": health_info,
-                "tools": tools_info
+                "health": health_data,
+                "tools": tools_data,
+                "combined_status": health_data.get("overall_status", "unknown")
             }
-            
             results.append(server_result)
             
         except Exception as e:
+            print(f"❌ Error checking {url}: {str(e)}")
             results.append({
                 "url": url,
-                "error": str(e)
+                "health": {"error": str(e)},
+                "tools": {"error": str(e)},
+                "combined_status": "error"
             })
     
+    progress(0.95, desc="📊 Generating report...")
+    
     final_result = {
         "servers": results,
         "total_servers": len(urls),
@@ -508,6 +510,7 @@ def monitor_multiple_servers(urls_text: str) -> tuple:
     # Generate markdown summary
     md = format_multiple_servers_summary(final_result)
     
+    progress(1.0, desc="✅ Multi-server monitoring complete!")
     return md, json.dumps(final_result, indent=2)
 
 def format_multiple_servers_summary(result: dict) -> str:
@@ -520,45 +523,18 @@ def format_multiple_servers_summary(result: dict) -> str:
     total_tools = 0
     
     for i, server in enumerate(result["servers"], 1):
-        md += f"## {i}. Server Report\n\n"
-        md += f"**URL:** `{server['url']}`\n\n"
-        
-        if "error" in server:
-            md += f"❌ **Error:** {server['error']}\n\n"
-            continue
-        
-        # Health summary
-        health = server.get("health", {})
-        overall_status = health.get("overall_status", "unknown")
-        status_icons = {
-            "healthy": "🟢",
-            "mcp_only": "🟡", 
-            "space_only": "🟠",
-            "unreachable": "🔴"
-        }
-        icon = status_icons.get(overall_status, "❓")
-        
-        md += f"**Health:** {icon} {overall_status.replace('_', ' ').title()}\n"
-        
-        if overall_status == "healthy":
+        status = server.get("combined_status", "unknown")
+        if status == "healthy":
             healthy_count += 1
         
-        # MCP endpoint response time
-        mcp_health = health.get("mcp_health", {})
-        if mcp_health.get("response_time_ms"):
-            md += f"**MCP Response Time:** {mcp_health['response_time_ms']}ms\n"
-        
-        # Tools summary
-        tools = server.get("tools", {})
-        tool_count = tools.get("tool_count", 0)
-        total_tools += tool_count
-        
-        if tool_count > 0:
-            md += f"**Tools:** {tool_count} available (`{', '.join(tools.get('tool_names', []))}`)\n"
-        else:
-            md += f"**Tools:** None found\n"
+        tools_count = server.get("tools", {}).get("tool_count", 0)
+        total_tools += tools_count
         
-        md += "\n"
+        status_icon = "🟢" if status == "healthy" else "🔴"
+        md += f"## {status_icon} Server {i}\n\n"
+        md += f"**URL:** `{server['url']}`\n"
+        md += f"**Status:** {status.replace('_', ' ').title()}\n"
+        md += f"**Tools Found:** {tools_count}\n\n"
     
     # Overall summary
     md += "## 📈 Summary\n\n"
@@ -566,96 +542,211 @@ def format_multiple_servers_summary(result: dict) -> str:
     md += f"- **Total Tools Available:** {total_tools}\n"
     
     if healthy_count > 0:
-        md += f"- **Success Rate:** {(healthy_count/result['total_servers']*100):.1f}%\n"
+        md += f"- **Success Rate:** {round(healthy_count/result['total_servers']*100, 1)}%\n"
     
     return md
 
-def validate_mcp_endpoint(url: str) -> tuple:
+def validate_mcp_endpoint(url: str, progress=gr.Progress()) -> tuple:
     """
-    Validate that a URL is a working MCP endpoint.
+    Validate that a URL is a working MCP endpoint by checking its schema.
     
     Args:
-        url (str): URL to validate as MCP endpoint
+        url (str): URL to validate as MCP endpoint (can be space URL or direct MCP endpoint)
     
     Returns:
         tuple: (markdown_summary, json_data)
     """
+    progress(0, desc="✅ Starting MCP validation...")
+    
+    if not url.strip():
+        return "# ❌ No URL Provided\n\nPlease enter a URL to validate.", "{}"
+    
+    progress(0.1, desc="✅ Parsing URL...")
     parsed_info = json.loads(parse_huggingface_url(url))
     
     validation_result = {
         "original_url": url,
         "is_valid_mcp": False,
-        "connection_config": None
+        "mcp_endpoint_url": parsed_info.get("mcp_endpoint"),
+        "mcp_schema_url": None,
+        "connection_config": None,
+        "error": None,
+        "schema_details": None
     }
     
-    if not parsed_info["is_valid"]:
-        validation_result["error"] = "Invalid URL format"
-        md = "# ❌ MCP Validation Failed\n\nInvalid URL format. Please check the URL."
+    if not parsed_info["is_valid"] or validation_result["mcp_endpoint_url"] == "unknown":
+        validation_result["error"] = "Invalid URL format or could not determine MCP endpoint."
+        md = f"# ❌ Invalid URL\n\nCould not parse the provided URL format to find an MCP endpoint: `{url}`"
         return md, json.dumps(validation_result, indent=2)
+
+    mcp_endpoint = validation_result["mcp_endpoint_url"]
+    
+    progress(0.3, desc="✅ Building schema URL...")
+    
+    # Construct schema URL from MCP SSE endpoint
+    # Example: https://user-space.hf.space/gradio_api/mcp/sse -> https://user-space.hf.space/gradio_api/mcp/schema
+    if mcp_endpoint.endswith("/sse"):
+        mcp_schema_url = mcp_endpoint[:-4] + "/schema"
+        validation_result["mcp_schema_url"] = mcp_schema_url
+    else:
+        # If it's not an SSE endpoint, we might not be able to reliably find the schema
+        validation_result["error"] = f"MCP endpoint does not end with /sse, cannot determine schema URL: {mcp_endpoint}"
+        md = f"# ⚠️ MCP Validation Warning\n\nCould not determine schema URL from MCP endpoint: `{mcp_endpoint}`. Validation might be incomplete."
+        return md, json.dumps(validation_result, indent=2)
+
+    print(f"ℹ️ Validating MCP: Original URL='{url}', Endpoint='{mcp_endpoint}', Schema='{mcp_schema_url}'")
+
+    progress(0.5, desc="✅ Testing schema endpoint...")
     
-    # Test MCP endpoint
+    # Test MCP schema endpoint
     try:
-        response = requests.get(parsed_info["mcp_endpoint"], timeout=8, stream=True)
+        headers = {'User-Agent': 'MCP-Validator/1.0'}
+        response = requests.get(mcp_schema_url, timeout=10, headers=headers)
         
+        validation_result["schema_http_status"] = response.status_code
+
+        progress(0.7, desc="✅ Processing schema response...")
+
         if response.status_code == 200:
-            validation_result["is_valid_mcp"] = True
-            validation_result["connection_config"] = {
-                "mcp_endpoint": parsed_info["mcp_endpoint"],
-                "mcp_client_config": {
+            try:
+                schema_data = response.json()
+                validation_result["is_valid_mcp"] = True
+                validation_result["connection_config"] = {
                     "mcpServers": {
-                        "gradio_server": {
-                            "url": parsed_info["mcp_endpoint"]
+                        "gradio_server": { # Default key, user might change
+                            "url": mcp_endpoint
                         }
                     }
                 }
-            }
+                # Store some basic schema info if available
+                if isinstance(schema_data, dict) and "tools" in schema_data:
+                     validation_result["schema_details"] = {
+                         "tool_count": len(schema_data["tools"]),
+                         "tool_names": [tool.get("name") for tool in schema_data["tools"]]
+                     }
+                elif isinstance(schema_data, list): # Sometimes schema is a list of tools
+                     validation_result["schema_details"] = {
+                         "tool_count": len(schema_data),
+                         "tool_names": [tool.get("name") for tool in schema_data]
+                     }
+                else:
+                    validation_result["schema_details"] = "Schema format not recognized or no tools found."
+                print(f"✅ MCP Schema valid for {mcp_schema_url}")
+
+            except json.JSONDecodeError:
+                validation_result["error"] = "Schema endpoint returned 200 OK, but response is not valid JSON."
+                print(f"❌ MCP Schema JSON decode error for {mcp_schema_url}")
+            except Exception as e_json:
+                validation_result["error"] = f"Schema endpoint returned 200 OK, but error processing JSON: {str(e_json)}"
+                print(f"❌ MCP Schema JSON processing error for {mcp_schema_url}: {str(e_json)}")
+        elif response.status_code == 401 or response.status_code == 403:
+            validation_result["error"] = f"Schema endpoint access denied (HTTP {response.status_code}). Private space may require auth token."
+            print(f"⚠️ MCP Schema access denied for {mcp_schema_url} (HTTP {response.status_code})")
         else:
-            validation_result["error"] = f"MCP endpoint returned HTTP {response.status_code}"
-    except Exception as e:
-        validation_result["error"] = f"Cannot connect to MCP endpoint: {str(e)}"
+            validation_result["error"] = f"Schema endpoint returned HTTP {response.status_code}."
+            print(f"❌ MCP Schema request failed for {mcp_schema_url} (HTTP {response.status_code})")
+            
+    except requests.exceptions.Timeout:
+        validation_result["error"] = f"Request to schema endpoint timed out: {mcp_schema_url}"
+        print(f"❌ MCP Schema request timeout for {mcp_schema_url}")
+    except requests.exceptions.RequestException as e:
+        validation_result["error"] = f"Request to schema endpoint failed: {str(e)}"
+        print(f"❌ MCP Schema request failed for {mcp_schema_url}: {str(e)}")
+    except Exception as e_gen:
+        validation_result["error"] = f"An unexpected error occurred during validation: {str(e_gen)}"
+        print(f"❌ Unexpected error during MCP validation for {mcp_schema_url}: {str(e_gen)}")
+
+    progress(0.9, desc="✅ Generating validation report...")
     
     # Generate markdown summary
     md = format_validation_summary(validation_result)
     
+    progress(1.0, desc="✅ MCP validation complete!")
     return md, json.dumps(validation_result, indent=2)
 
 def format_validation_summary(result: dict) -> str:
     """Generate markdown summary for MCP validation results."""
     md = f"# ✅ MCP Endpoint Validation\n\n"
-    md += f"**URL:** `{result['original_url']}`\n\n"
+    md += f"**Original URL:** `{result['original_url']}`\n"
+    if result.get('mcp_endpoint_url'):
+        md += f"**Attempted MCP Endpoint:** `{result['mcp_endpoint_url']}`\n"
+    if result.get('mcp_schema_url'):
+        md += f"**Attempted MCP Schema URL:** `{result['mcp_schema_url']}`\n\n"
     
     if result["is_valid_mcp"]:
-        md += "🟢 **Status:** Valid MCP Endpoint\n\n"
-        md += "This URL is a working MCP endpoint and can be used with MCP clients.\n\n"
+        md += "## ✅ **Status: Valid MCP Endpoint**\n\n"
+        md += "The server appears to be a functional MCP endpoint based on schema accessibility.\n\n"
         
-        md += "## ⚙️ Configuration\n\n"
-        md += "Copy this configuration for your MCP client:\n\n"
+        if result.get("schema_details"):
+            md += "### 📋 Schema Details:\n"
+            if isinstance(result["schema_details"], dict):
+                md += f"- **Tools Found:** {result['schema_details'].get('tool_count', 'N/A')}\n"
+                if result['schema_details'].get('tool_names'):
+                    md += f"- **Tool Names:** {', '.join(filter(None, result['schema_details']['tool_names']))}\n"
+            else:
+                md += f"- {result['schema_details']}\n"
+            md += "\n"
+
+        md += "### 🔧 Configuration for MCP Client\n\n"
+        md += "You can likely use the following configuration (ensure the key like `gradio_server` is appropriate for your client):\n"
         md += "```json\n"
-        config = result["connection_config"]["mcp_client_config"]
-        md += json.dumps(config, indent=2)
-        md += "\n```\n\n"
-        
-        md += "## 📁 Configuration File Locations\n\n"
-        md += "Save the configuration in the appropriate location for your MCP client:\n\n"
-        md += "### Claude Desktop\n"
-        md += "- **macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json`\n"
-        md += "- **Windows:** `%APPDATA%\\Claude\\claude_desktop_config.json`\n\n"
-        md += "### Cursor IDE\n"
-        md += "- **macOS:** `~/Library/Application Support/Cursor/User/globalStorage/anysphere.cursor-mcp/settings.json`\n"
-        md += "- **Windows:** `%APPDATA%\\Cursor\\User\\globalStorage\\anysphere.cursor-mcp\\settings.json`\n\n"
-        md += "### Cline (VS Code Extension)\n"
-        md += "- Configure through VS Code settings or the Cline extension settings\n\n"
-        md += "### Other MCP Clients\n"
-        md += "- Refer to your specific MCP client's documentation for configuration file location\n"
+        md += json.dumps(result["connection_config"], indent=2)
+        md += "\n```\n"
         
     else:
-        md += "❌ **Status:** Invalid MCP Endpoint\n\n"
-        if "error" in result:
-            md += f"**Error:** {result['error']}\n\n"
-        md += "This URL is not a valid MCP endpoint. Please check the URL or ensure the server supports MCP.\n"
-    
+        md += "## ❌ **Status: Invalid or Inaccessible MCP Endpoint**\n\n"
+        if result.get("error"):
+            md += f"**Reason:** {result['error']}\n\n"
+        else:
+            md += "Could not confirm MCP functionality.\n\n"
+        
+        md += "### 💡 Troubleshooting Tips:\n"
+        md += "- Ensure the URL is correct and the Hugging Face Space is running.\n"
+        md += "- Verify the Space has `mcp_server=True` in its `launch()` method (if it's a Gradio app).\n"
+        md += "- For private Spaces, your MCP client might need an `Authorization: Bearer <HF_TOKEN>` header.\n"
+        md += "- Check the Space logs for any errors if you own the Space.\n"
+    
+    if result.get("schema_http_status"):
+        md += f"\n**Schema HTTP Status:** {result['schema_http_status']}\n"
+        
     return md
 
+def scrape_hf_spaces_with_progress(max_pages: int, sort_by: str, progress=gr.Progress()) -> tuple:
+    """Wrapper function for scraping with progress feedback."""
+    progress(0, desc="🕷️ Starting HuggingFace scraping...")
+    
+    # Validate sort option
+    if sort_by not in SORT_OPTIONS:
+        sort_by = "relevance"
+    
+    progress(0.1, desc=f"🕷️ Using sort method: {SORT_OPTIONS[sort_by]['label']}")
+    
+    # Update the scraper to accept a progress callback
+    def progress_callback(message):
+        # Extract progress percentage from message if possible
+        if "page" in message.lower():
+            try:
+                # Try to extract current page from message
+                import re
+                match = re.search(r'page (\d+)(/(\d+))?', message.lower())
+                if match:
+                    current = int(match.group(1))
+                    total = int(match.group(3)) if match.group(3) else max_pages
+                    progress_pct = min(0.9, (current / total) * 0.8 + 0.1)  # Reserve 10% for final processing
+                    progress(progress_pct, desc=f"🕷️ {message}")
+                else:
+                    progress(None, desc=f"🕷️ {message}")
+            except:
+                progress(None, desc=f"🕷️ {message}")
+        else:
+            progress(None, desc=f"🕷️ {message}")
+    
+    # Call the scraper with progress callback and sort option
+    md, json_data = scrape_and_monitor_hf_mcp_servers(max_pages, sort_by)
+    
+    progress(1.0, desc="✅ HuggingFace scraping complete!")
+    return md, json_data
+
 # Default URLs for testing
 DEFAULT_URLS = """https://huggingface.co/spaces/abidlabs/mcp-tools"""
 
@@ -665,125 +756,169 @@ with gr.Blocks(title="🚀 MCP Server Health Monitor") as demo:
     gr.Markdown("Monitor and analyze Hugging Face Spaces configured as MCP servers")
     
     with gr.Tabs():
-        # Multi-Server Monitor Tab
-        with gr.Tab("📊 Multi-Server Monitor"):
-            gr.Markdown("### Monitor multiple MCP servers simultaneously")
+        # Tab 1: Single Server Health Check
+        with gr.Tab("🏥 Single Server Health"):
+            gr.Markdown("### Check the health of a single MCP server")
             
-            with gr.Column():
-                multi_input = gr.Textbox(
-                    label="Server URLs (one per line)",
-                    lines=5,
-                    value=DEFAULT_URLS,
-                    placeholder="Enter HF Space URLs, one per line\nExample:\nhttps://huggingface.co/spaces/abidlabs/mcp-tools\nhttps://user-space.hf.space"
+            with gr.Row():
+                single_url = gr.Textbox(
+                    label="Server URL",
+                    placeholder="Enter any HF Space URL format...",
+                    value="https://huggingface.co/spaces/abidlabs/mcp-tools"
                 )
-                multi_submit = gr.Button("🔍 Monitor Servers", variant="primary", size="lg")
-                
-            with gr.Column():
-                multi_md = gr.Markdown(label="Summary Report")
-                
-            with gr.Column():
-                multi_json = gr.JSON(label="Detailed JSON Data")
+                check_health_btn = gr.Button("Check Health", variant="primary")
             
-            multi_submit.click(
-                fn=monitor_multiple_servers,
-                inputs=multi_input,
-                outputs=[multi_md, multi_json]
+            health_output = gr.Markdown(label="Health Report")
+            health_json = gr.JSON(label="Detailed Results", visible=False)
+            
+            check_health_btn.click(
+                check_single_server_health,
+                inputs=[single_url],
+                outputs=[health_output, health_json]
             )
         
-        # Single Server Health Tab
-        with gr.Tab("🏥 Single Server Health"):
-            gr.Markdown("### Check health of a single MCP server")
+        # Tab 2: URL Parser
+        with gr.Tab("🔍 URL Parser"):
+            gr.Markdown("### Parse and validate HuggingFace Space URLs")
             
-            with gr.Column():
-                single_input = gr.Textbox(
-                    label="Server URL",
-                    value="https://huggingface.co/spaces/abidlabs/mcp-tools",
-                    placeholder="Enter HF Space URL (any format supported)"
+            with gr.Row():
+                parse_url = gr.Textbox(
+                    label="URL to Parse",
+                    placeholder="Enter any HF Space URL format...",
+                    value="https://huggingface.co/spaces/abidlabs/mcp-tools"
                 )
-                single_submit = gr.Button("🩺 Check Health", variant="primary", size="lg")
-                
-            with gr.Column():
-                single_md = gr.Markdown(label="Health Summary")
-                
-            with gr.Column():
-                single_json = gr.JSON(label="Detailed JSON Data")
+                parse_btn = gr.Button("Parse URL", variant="primary")
             
-            single_submit.click(
-                fn=check_single_server_health,
-                inputs=single_input,
-                outputs=[single_md, single_json]
+            parse_output = gr.Markdown(label="Parsing Results")
+            parse_json = gr.JSON(label="JSON Output", visible=False)
+            
+            parse_btn.click(
+                parse_huggingface_url_with_summary,
+                inputs=[parse_url],
+                outputs=[parse_output, parse_json]
             )
         
-        # Tools Discovery Tab
-        with gr.Tab("🔧 Tools Discovery"):
+        # Tab 3: Tools Discovery
+        with gr.Tab("🛠️ Tools Discovery"):
             gr.Markdown("### Discover available MCP tools from a server")
             
-            with gr.Column():
-                tools_input = gr.Textbox(
+            with gr.Row():
+                tools_url = gr.Textbox(
                     label="Server URL",
-                    value="https://abidlabs-mcp-tools.hf.space",
-                    placeholder="Enter HF Space URL to analyze for MCP tools"
+                    placeholder="Enter HF Space URL...",
+                    value="https://huggingface.co/spaces/abidlabs/mcp-tools"
                 )
-                tools_submit = gr.Button("🛠️ Discover Tools", variant="primary", size="lg")
-                
-            with gr.Column():
-                tools_md = gr.Markdown(label="Tools Summary")
-                
-            with gr.Column():
-                tools_json = gr.JSON(label="Detailed JSON Data")
+                discover_btn = gr.Button("Discover Tools", variant="primary")
+            
+            tools_output = gr.Markdown(label="Tools Report")
+            tools_json = gr.JSON(label="Tools Data", visible=False)
             
-            tools_submit.click(
-                fn=discover_server_tools,
-                inputs=tools_input,
-                outputs=[tools_md, tools_json]
+            discover_btn.click(
+                discover_server_tools,
+                inputs=[tools_url],
+                outputs=[tools_output, tools_json]
             )
         
-        # MCP Validator Tab
-        with gr.Tab("✅ MCP Validator"):
-            gr.Markdown("### Validate if a URL is a working MCP endpoint")
+        # Tab 4: Multi-Server Monitor
+        with gr.Tab("📊 Multi-Server Monitor"):
+            gr.Markdown("### Monitor multiple MCP servers simultaneously")
             
-            with gr.Column():
-                validator_input = gr.Textbox(
-                    label="URL to Validate",
-                    value="https://huggingface.co/spaces/abidlabs/mcp-tools",
-                    placeholder="Enter URL to validate as MCP endpoint"
-                )
-                validator_submit = gr.Button("✅ Validate Endpoint", variant="primary", size="lg")
+            multi_urls = gr.Textbox(
+                label="Server URLs (one per line)",
+                placeholder="Enter multiple URLs, one per line...",
+                lines=8,
+                value=DEFAULT_URLS
+            )
+            monitor_btn = gr.Button("Monitor All Servers", variant="primary")
+            
+            multi_output = gr.Markdown(label="Multi-Server Report")
+            multi_json = gr.JSON(label="Detailed Results", visible=False)
+            
+            monitor_btn.click(
+                monitor_multiple_servers,
+                inputs=[multi_urls],
+                outputs=[multi_output, multi_json]
+            )
+        
+        # Tab 5: HF Spaces Scraper (ENHANCED)
+        with gr.Tab("🕷️ HF Spaces Scraper"):
+            gr.Markdown("### Discover MCP servers on HuggingFace Spaces")
+            gr.Markdown("Scrape HuggingFace to find all spaces tagged with 'mcp-server' using different sorting methods")
+            
+            with gr.Row():
+                with gr.Column(scale=1):
+                    max_pages = gr.Slider(
+                        minimum=1,
+                        maximum=50,  # Increased from 100 to 50 for reasonable limits
+                        value=5,     # Increased default from 3 to 5
+                        step=1,
+                        label="Maximum Pages to Scrape",
+                        info="Each page contains ~24 spaces. Total pages available: ~48+"
+                    )
+                    
+                    # Create dropdown for sort options
+                    sort_choices = [(SORT_OPTIONS[key]["label"], key) for key in SORT_OPTIONS.keys()]
+                    sort_dropdown = gr.Dropdown(
+                        choices=sort_choices,
+                        value="relevance",
+                        label="Sort Method",
+                        info="Choose how to sort the search results"
+                    )
                 
-            with gr.Column():
-                validator_md = gr.Markdown(label="Validation Summary")
+                with gr.Column(scale=1):
+                    scrape_btn = gr.Button("🕷️ Scrape HF Spaces", variant="primary", size="lg")
+            
+            # Add info about sort methods and pagination
+            with gr.Accordion("ℹ️ Scraping Information", open=False):
+                gr.Markdown("""
+                **Sort Methods Explained:**
+                
+                - **🎯 Relevance (Default):** HuggingFace's default relevance ranking
+                - **📈 Trending:** Currently popular and active spaces  
+                - **❤️ Most Likes:** Spaces with the highest community appreciation
+                - **🆕 Recently Created:** Newest spaces, great for discovering latest tools
+                - **🔄 Recently Updated:** Recently modified spaces, likely actively maintained
                 
-            with gr.Column():
-                validator_json = gr.JSON(label="Detailed JSON Data")
+                **Pagination Information:**
+                - Each page contains approximately 24 spaces
+                - Current total: 48+ pages available (and growing!)
+                - The scraper will automatically stop if it encounters 3 consecutive empty pages
+                - Different sort methods may reveal different sets of MCP servers
+                
+                **Tips:**
+                - Start with 5-10 pages for a good sample
+                - Try multiple sort methods for comprehensive discovery
+                - Higher page counts will take longer but find more servers
+                """)
+            
+            scrape_output = gr.Markdown(label="Scraping Results")
+            scrape_json = gr.JSON(label="Scraped Data", visible=False)
             
-            validator_submit.click(
-                fn=validate_mcp_endpoint,
-                inputs=validator_input,
-                outputs=[validator_md, validator_json]
+            scrape_btn.click(
+                scrape_hf_spaces_with_progress,
+                inputs=[max_pages, sort_dropdown],
+                outputs=[scrape_output, scrape_json]
             )
         
-        # URL Parser Tab
-        with gr.Tab("🔍 URL Parser"):
-            gr.Markdown("### Parse and analyze HuggingFace URL formats")
+        # Tab 6: MCP Validator
+        with gr.Tab("✅ MCP Validator"):
+            gr.Markdown("### Validate MCP endpoint connectivity")
             
-            with gr.Column():
-                parser_input = gr.Textbox(
-                    label="URL to Parse",
-                    value="https://huggingface.co/spaces/abidlabs/mcp-tools",
-                    placeholder="Enter any HF Space URL format to parse"
+            with gr.Row():
+                validate_url = gr.Textbox(
+                    label="URL to Validate",
+                    placeholder="Enter URL to validate as MCP endpoint...",
+                    value="https://abidlabs-mcp-tools.hf.space/gradio_api/mcp/sse"
                 )
-                parser_submit = gr.Button("🔍 Parse URL", variant="primary", size="lg")
-                
-            with gr.Column():
-                parser_md = gr.Markdown(label="Parsing Summary")
-                
-            with gr.Column():
-                parser_json = gr.JSON(label="Detailed JSON Data")
+                validate_btn = gr.Button("Validate Endpoint", variant="primary")
+            
+            validate_output = gr.Markdown(label="Validation Results")
+            validate_json = gr.JSON(label="Validation Data", visible=False)
             
-            parser_submit.click(
-                fn=parse_huggingface_url_with_summary,
-                inputs=parser_input,
-                outputs=[parser_md, parser_json]
+            validate_btn.click(
+                validate_mcp_endpoint,
+                inputs=[validate_url],
+                outputs=[validate_output, validate_json]
             )
 
 if __name__ == "__main__":