added custom tools

tools.py

@@ -1,4 +1,9 @@
 from typing import List
+from tools.describe_image_tool import DescribeImageTool
+from tools.openai_speech_to_text_tool import OpenAISpeechToTextTool
+from tools.read_file_tool import ReadFileTool
+from tools.youtube_transcription_tool import YouTubeTranscriptionTool
+from tools.table_extractor_tool import TableExtractorTool
 
 from smolagents import (
     DuckDuckGoSearchTool,
@@ -20,5 +25,10 @@ def get_tools() -> List[Tool]:
         PythonInterpreterTool(),
         WikipediaSearchTool(),
         VisitWebpageTool(),
+        DescribeImageTool(),
+        OpenAISpeechToTextTool(),
+        ReadFileTool(),
+        YouTubeTranscriptionTool(),
+        TableExtractorTool(),
     ]
     return tools

tools/describe_image_tool.py

@@ -0,0 +1,111 @@
+import base64
+import os
+from openai import OpenAI
+from smolagents import Tool
+
+client = OpenAI()
+
+
+class DescribeImageTool(Tool):
+    """
+    Tool to analyze and describe any image using GPT-4 Vision API.
+
+    Args:
+        image_path (str): Path to the image file.
+        description_type (str): Type of description to generate. Options:
+            - "general": General description of the image
+            - "detailed": Detailed analysis of the image
+            - "chess": Analysis of a chess position
+            - "text": Extract and describe text from the image
+            - "custom": Custom description based on user prompt
+
+    Returns:
+        str: Description of the image based on the requested type.
+    """
+
+    name = "describe_image"
+    description = "Analyzes and describes images using GPT-4 Vision API"
+    inputs = {
+        "image_path": {"type": "string", "description": "Path to the image file"},
+        "description_type": {
+            "type": "string",
+            "description": "Type of description to generate (general, detailed, chess, text, custom)",
+            "nullable": True,
+        },
+        "custom_prompt": {
+            "type": "string",
+            "description": "Custom prompt for description (only used when description_type is 'custom')",
+            "nullable": True,
+        },
+    }
+    output_type = "string"
+
+    def encode_image(self, image_path: str) -> str:
+        """Encode image to base64 string."""
+        with open(image_path, "rb") as image_file:
+            return base64.b64encode(image_file.read()).decode("utf-8")
+
+    def get_prompt(self, description_type: str, custom_prompt: str = None) -> str:
+        """Get appropriate prompt based on description type."""
+        prompts = {
+            "general": "Provide a general description of this image. Focus on the main subjects, colors, and overall scene.",
+            "detailed": """Analyze this image in detail. Include:
+            1. Main subjects and their relationships
+            2. Colors, lighting, and composition
+            3. Any text or symbols present
+            4. Context or possible meaning
+            5. Notable details or interesting elements""",
+            "chess": """Analyze this chess position and provide a detailed description including:
+            1. List of pieces on the board for both white and black
+            2. Whose turn it is to move
+            3. Basic evaluation of the position
+            4. Any immediate tactical opportunities or threats
+            5. Suggested next moves with brief explanations""",
+            "text": "Extract and describe any text present in this image. If there are multiple pieces of text, organize them clearly.",
+        }
+        return (
+            custom_prompt
+            if description_type == "custom"
+            else prompts.get(description_type, prompts["general"])
+        )
+
+    def forward(
+        self,
+        image_path: str,
+        description_type: str = "general",
+        custom_prompt: str = None,
+    ) -> str:
+        try:
+            if not os.path.exists(image_path):
+                return f"Error: Image file not found at {image_path}"
+
+            # Encode the image
+            base64_image = self.encode_image(image_path)
+
+            # Get appropriate prompt
+            prompt = self.get_prompt(description_type, custom_prompt)
+
+            # Make the API call
+            response = client.chat.completions.create(
+                model="gpt-4.1",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{base64_image}"
+                                },
+                            },
+                        ],
+                    }
+                ],
+                max_tokens=1000,
+            )
+
+            return response.choices[0].message.content
+
+        except Exception as e:
+            return f"Error analyzing image: {str(e)}"

tools/openai_speech_to_text_tool.py

@@ -0,0 +1,34 @@
+import os
+import whisper
+from smolagents import Tool
+
+
+class OpenAISpeechToTextTool(Tool):
+    """
+    Tool to convert speech to text using OpenAI's Whisper model.
+
+    Args:
+        audio_path (str): Path to the audio file.
+
+    Returns:
+        str: Transcribed text from the audio file.
+    """
+
+    name = "transcribe_audio"
+    description = "Transcribes audio to text and returns the text"
+    inputs = {
+        "audio_path": {"type": "string", "description": "Path to the audio file"},
+    }
+    output_type = "string"
+
+    def forward(self, audio_path: str) -> str:
+        try:
+            model = whisper.load_model("small")
+
+            if not os.path.exists(audio_path):
+                return f"Error: Audio file not found at {audio_path}"
+
+            result = model.transcribe(audio_path)
+            return result["text"]
+        except Exception as e:
+            return f"Error transcribing audio: {str(e)}"

tools/read_file_tool.py

@@ -0,0 +1,26 @@
+from smolagents import Tool
+
+class ReadFileTool(Tool):
+    """
+    Tool to read a file and return its content.
+
+    Args:
+        file_path (str): Path to the file to read.
+
+    Returns:
+        str: Content of the file or error message.
+    """
+
+    name = "read_file"
+    description = "Reads a file and returns its content"
+    inputs = {
+        "file_path": {"type": "string", "description": "Path to the file to read"},
+    }
+    output_type = "string"
+
+    def forward(self, file_path: str) -> str:
+        try:
+            with open(file_path, "r") as file:
+                return file.read()
+        except Exception as e:
+            return f"Error reading file: {str(e)}"

tools/table_extractor_tool.py

@@ -0,0 +1,106 @@
+from smolagents import Tool
+from tabula import read_pdf
+import pandas as pd
+from typing import Optional, Dict, Any
+
+class TableExtractorTool(Tool):
+    """
+    Tool to extract tables from PDFs/webpages and answer queries about them.
+
+    Args:
+        file_path (str): Path to PDF file (optional)
+        url (str): URL of webpage containing tables (optional)
+        query (str): Natural language question about the table data (optional)
+
+    Returns:
+        str: Extracted table data or answer to query
+    """
+
+    name = "extract_table"
+    description = "Extracts tables from PDFs or webpages and answers questions about the data"
+    
+    inputs = {
+        "file_path": {
+            "type": "string", 
+            "description": "Path to PDF file (either file_path or url required)",
+            "required": False
+        },
+        "url": {
+            "type": "string",
+            "description": "URL of webpage containing tables (either file_path or url required)",
+            "required": False
+        },
+        "query": {
+            "type": "string",
+            "description": "Natural language question about the table data",
+            "required": False
+        }
+    }
+    
+    output_type = "string"
+
+    def forward(self, file_path: Optional[str] = None, 
+               url: Optional[str] = None, 
+               query: Optional[str] = None) -> str:
+        
+        # Validate input
+        if not file_path and not url:
+            return "Error: Either file_path or url must be provided"
+        
+        try:
+            # Case 1: Extract from PDF
+            if file_path and file_path.endswith(".pdf"):
+                tables = read_pdf(file_path, pages="all", multiple_tables=True)
+                df = pd.concat(tables) if tables else None
+            
+            # Case 2: Extract from HTML (webpage)
+            elif url:
+                dfs = pd.read_html(url)
+                df = dfs[0] if dfs else None
+            
+            if df is None:
+                return "No tables found in the input source"
+            
+            # Answer query if provided
+            if query:
+                return self._answer_query(df, query)
+            return df.to_string()
+            
+        except Exception as e:
+            return f"Error processing table data: {str(e)}"
+
+    def _answer_query(self, df: pd.DataFrame, query: str) -> str:
+        """Helper method to answer questions about the table data"""
+        try:
+            query = query.lower()
+            
+            # Example simple queries - you could expand this or integrate an LLM
+            if "total" in query and "sum" in query:
+                if "revenue" in query:
+                    col = "Revenue"
+                elif "sales" in query:
+                    col = "Sales"
+                else:
+                    # Try to find a numeric column
+                    numeric_cols = df.select_dtypes(include=['number']).columns
+                    col = numeric_cols[0] if len(numeric_cols) > 0 else None
+                
+                if col:
+                    return f"Total {col}: {df[col].sum()}"
+            
+            elif "average" in query or "mean" in query:
+                # Find the most likely column referenced in query
+                for col in df.columns:
+                    if col.lower() in query:
+                        return f"Average {col}: {df[col].mean():.2f}"
+                
+                # Default to first numeric column
+                numeric_cols = df.select_dtypes(include=['number']).columns
+                if len(numeric_cols) > 0:
+                    return f"Average {numeric_cols[0]}: {df[numeric_cols[0]].mean():.2f}"
+            
+            # Fallback: return the table
+            return f"Here's the table data:\n{df.to_string()}\n\nQuery '{query}' not fully understood."
+            
+        except Exception as e:
+            return f"Error answering query: {str(e)}\nTable data:\n{df.to_string()}"

tools/youtube_transcription_tool.py

@@ -0,0 +1,26 @@
+from smolagents import Tool
+from youtube_transcript_api import YouTubeTranscriptApi
+
+
+class YouTubeTranscriptionTool(Tool):
+    """
+    Tool to fetch the transcript of a YouTube video given its URL.
+
+    Args:
+        video_url (str): YouTube video URL.
+
+    Returns:
+        str: Transcript of the video as a single string.
+    """
+
+    name = "youtube_transcription"
+    description = "Fetches the transcript of a YouTube video given its URL"
+    inputs = {
+        "video_url": {"type": "string", "description": "YouTube video URL"},
+    }
+    output_type = "string"
+
+    def forward(self, video_url: str) -> str:
+        video_id = video_url.strip().split("v=")[-1]
+        transcript = YouTubeTranscriptApi.get_transcript(video_id)
+        return " ".join([entry["text"] for entry in transcript])