Upload 78 files

.gitignore

@@ -0,0 +1,51 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Dependencies
+node_modules/
+
+# Build
+dist/
+dist-ssr/
+*.local
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+.env
+.venv
+env/
+venv/
+
+# IDE
+.vscode/
+.idea/
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+# Misc
+.gradio/
+flagged/

Dockerfile

@@ -0,0 +1,103 @@
+# Stage 1: Build React frontend
+FROM node:20-slim AS frontend-builder
+
+WORKDIR /app/frontend
+
+# Copy package files
+COPY package*.json ./
+
+# Install dependencies
+RUN npm ci
+
+# Copy source files
+COPY . .
+
+# Build the React app
+RUN npm run build
+
+# Stage 2: Python backend + serve frontend
+FROM python:3.12-slim-trixie
+
+# Copy uv from the official distroless image (recommended approach)
+COPY --from=ghcr.io/astral-sh/uv:0.9.15 /uv /uvx /bin/
+
+# Install system dependencies for Playwright and nginx
+RUN apt-get update && apt-get install -y \
+    nginx \
+    supervisor \
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libxkbcommon0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libasound2 \
+    libpango-1.0-0 \
+    libpangocairo-1.0-0 \
+    libcairo2 \
+    libatspi2.0-0 \
+    xvfb \
+    fonts-liberation \
+    libappindicator3-1 \
+    libu2f-udev \
+    libvulkan1 \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create a new user named "user" with user ID 1000 (required for HF Spaces)
+RUN useradd -m -u 1000 user
+
+# Create necessary directories with proper permissions for nginx (before switching user)
+RUN mkdir -p /var/log/nginx /var/lib/nginx /var/cache/nginx /run \
+    && chown -R user:user /var/log/nginx /var/lib/nginx /var/cache/nginx /run \
+    && chmod -R 755 /var/log/nginx /var/lib/nginx /var/cache/nginx /run
+
+# Configure nginx (needs root for /etc/nginx)
+COPY nginx.conf /etc/nginx/nginx.conf
+
+# Configure supervisor
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+# Allow user to run supervisor
+RUN chown -R user:user /etc/supervisor
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy backend code and sync with locked dependencies
+COPY --chown=user:user backend/ ./backend/
+RUN cd backend && uv sync
+
+# Copy FARA source
+COPY --chown=user:user fara/ ./fara/
+
+# Activate the virtual environment by adding it to PATH
+ENV PATH="$HOME/app/backend/.venv/bin:$PATH"
+
+# Install Playwright browsers
+RUN playwright install chromium
+
+# Copy built frontend from Stage 1
+COPY --chown=user:user --from=frontend-builder /app/frontend/dist ./static
+
+# Expose port
+EXPOSE 7860
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+
+# Start supervisor (manages nginx + python backend)
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]

README.md

@@ -1,12 +1,49 @@
----
-title: Fara BrowserUse
-emoji: 🔥
-colorFrom: indigo
-colorTo: pink
-sdk: docker
-pinned: false
-license: mit
-short_description: Microsoft Fara-7B Browser-Use demo inspired by CUA2
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: FARA - Computer Use Agent
+emoji: 🤖
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+license: mit
+app_port: 7860
+suggested_hardware: cpu-upgrade
+tags:
+  - computer-use
+  - browser-automation
+  - ai-agent
+  - vision-language-model
+---
+
+# 🤖 FARA - Computer Use Agent Demo
+
+FARA (Fara Agent for Real-world Automation) is an AI agent that can browse the web and complete tasks autonomously.
+
+## Features
+
+- 🌐 **Autonomous Web Navigation** - The agent can browse websites on its own
+- 🔍 **Web Search** - Search for information across the web
+- 📝 **Form Filling** - Fill out forms automatically
+- 🖱️ **Point and Click** - Click buttons, links, and elements
+- ⌨️ **Text Input** - Type text into fields
+- 📜 **Page Scrolling** - Scroll through content
+
+## How to Use
+
+1. Enter a task in natural language (e.g., "Search for the latest news about AI")
+2. Click "Run Task" and watch the agent work!
+3. View the screenshots to see each step the agent takes
+
+## Powered By
+
+- **Microsoft Fara-7B** - Vision-Language Model for computer use
+- **Playwright** - Browser automation framework
+- **Modal** - Model hosting and inference
+
+## Links
+
+- [GitHub Repository](https://github.com/microsoft/fara)
+
+## License
+
+MIT License

backend/.env.example

@@ -0,0 +1,16 @@
+# Backend Environment Variables
+
+# FARA Model Configuration
+FARA_MODEL_NAME=microsoft/Fara-7B
+FARA_ENDPOINT_URL=https://your-workspace--fara-vllm-serve.modal.run/v1
+FARA_API_KEY=not-needed
+
+# Modal Proxy Auth Configuration
+# These credentials are used for BOTH the vLLM endpoint AND trace storage
+# Create a Proxy Auth Token at: https://modal.com/settings/proxy-auth-tokens
+MODAL_TOKEN_ID=wk-xxxxxxxx
+MODAL_TOKEN_SECRET=ws-xxxxxxxx
+
+# Modal Trace Storage Endpoint URL
+# Get the URL from `modal deploy backend/modal_fara_vllm.py`
+MODAL_TRACE_STORAGE_URL=https://your-workspace--fara-vllm-store-trace.modal.run

backend/.python-version

@@ -0,0 +1 @@
+3.12

backend/README.md

@@ -0,0 +1,163 @@
+# FaraCUA Backend
+
+The backend server for FaraCUA - a Computer Use Agent (CUA) demo powered by Microsoft's Fara-7B vision-language model and Modal for serverless GPU inference.
+
+## Overview
+
+This backend provides:
+
+- **WebSocket API** - Real-time communication with the React frontend for streaming agent actions
+- **REST API** - Model listing, random question generation, and trace storage
+- **FARA Agent Integration** - Runs the Fara agent with Playwright for browser automation
+- **Modal Integration** - Proxies requests to Modal's vLLM endpoint and trace storage
+
+## Architecture
+
+```
+┌─────────────┐     WebSocket      ┌─────────────┐     HTTP      ┌─────────────┐
+│   Frontend  │ ◄───────────────► │   Backend   │ ◄───────────► │    Modal    │
+│   (React)   │                    │  (FastAPI)  │               │   (vLLM)    │
+└─────────────┘                    └─────────────┘               └─────────────┘
+                                         │
+                                         │ Playwright
+                                         ▼
+                                   ┌─────────────┐
+                                   │   Browser   │
+                                   │  (Headless) │
+                                   └─────────────┘
+```
+
+## Files
+
+| File | Description |
+|------|-------------|
+| `server.py` | Main FastAPI server with WebSocket and REST endpoints |
+| `modal_fara_vllm.py` | Modal deployment for vLLM inference and trace storage |
+| `pyproject.toml` | Python dependencies |
+| `.env.example` | Example environment configuration |
+
+## Setup
+
+### 1. Install Dependencies
+
+```bash
+# Using uv (recommended)
+uv sync
+
+# Or using pip
+pip install -e .
+```
+
+### 2. Install Playwright
+
+```bash
+playwright install chromium
+```
+
+### 3. Deploy Modal Endpoints
+
+```bash
+modal deploy backend/modal_fara_vllm.py
+```
+
+This deploys:
+- **vLLM Server** - GPU-accelerated inference for Fara-7B at `https://<workspace>--fara-vllm-serve.modal.run`
+- **Trace Storage** - Endpoint for storing task traces at `https://<workspace>--fara-vllm-store-trace.modal.run`
+
+### 4. Configure Environment
+
+Copy `.env.example` to `.env` and fill in your values:
+
+```bash
+cp .env.example .env
+```
+
+Required variables:
+
+| Variable | Description |
+|----------|-------------|
+| `FARA_MODEL_NAME` | Model name (default: `microsoft/Fara-7B`) |
+| `FARA_ENDPOINT_URL` | Modal vLLM endpoint URL (from deploy output) |
+| `FARA_API_KEY` | API key (default: `not-needed` for Modal) |
+| `MODAL_TOKEN_ID` | Modal proxy auth token ID |
+| `MODAL_TOKEN_SECRET` | Modal proxy auth token secret |
+| `MODAL_TRACE_STORAGE_URL` | Modal trace storage endpoint URL |
+
+Get Modal proxy auth tokens at: https://modal.com/settings/proxy-auth-tokens
+
+### 5. Run the Server
+
+```bash
+# Development mode
+uvicorn backend.server:app --host 0.0.0.0 --port 8000 --reload
+
+# Or directly
+python -m backend.server
+```
+
+## API Endpoints
+
+### WebSocket
+
+- `ws://localhost:8000/ws` - Real-time agent communication
+  - **Receives**: `user_task`, `stop_task`, `ping`
+  - **Sends**: `heartbeat`, `agent_start`, `agent_progress`, `agent_complete`, `agent_error`
+
+### REST
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| GET | `/api/health` | Health check |
+| GET | `/api/models` | List available models |
+| GET | `/api/random-question` | Get a random example task |
+| POST | `/api/traces` | Store a trace (proxies to Modal) |
+
+## Trace Storage
+
+Task traces are automatically uploaded to Modal volumes for research purposes. Traces include:
+
+- Task instruction and model used
+- Step-by-step agent actions with screenshots
+- Token usage and timing metrics
+- User evaluation (success/failed)
+
+Duplicate traces (same ID and instruction) are automatically overwritten to capture the latest evaluation.
+
+## Docker
+
+The backend is designed to run in Docker alongside the frontend. See the root `Dockerfile` for the combined deployment.
+
+```bash
+# Build from root
+docker build -t fara-cua .
+
+# Run with env file
+docker run -d --name fara-cua -p 7860:7860 --env-file backend/.env fara-cua
+```
+
+## Development
+
+### Running Locally
+
+For local development, you can run the backend separately:
+
+```bash
+cd backend
+uvicorn server:app --host 0.0.0.0 --port 8000 --reload
+```
+
+Make sure the frontend is configured to connect to `http://localhost:8000`.
+
+### Testing Modal Endpoints
+
+```bash
+# Test vLLM endpoint
+modal run backend/modal_fara_vllm.py::test
+
+# Check deployment status
+modal app list
+```
+
+## License
+
+See the root LICENSE file for license information.

backend/__init__.py

@@ -0,0 +1 @@
+# Backend package

backend/modal_fara_vllm.py

@@ -0,0 +1,231 @@
+import json
+from typing import Any
+
+import aiohttp
+import modal
+
+vllm_image = (
+    modal.Image.from_registry("nvidia/cuda:12.8.0-devel-ubuntu22.04", add_python="3.12")
+    .entrypoint([])
+    .uv_pip_install(
+        "vllm==0.11.2",
+        "huggingface-hub==0.36.0",
+        "flashinfer-python==0.5.2",
+    )
+    .env({"HF_XET_HIGH_PERFORMANCE": "1"})  # faster model transfers
+)
+
+# Lightweight image for the trace storage endpoint (doesn't need CUDA/vllm)
+trace_storage_image = modal.Image.debian_slim(python_version="3.12").uv_pip_install(
+    "fastapi", "uvicorn"
+)
+
+MODEL_NAME = "microsoft/Fara-7B"
+MODEL_REVISION = None  # Use latest if no specific revision
+
+hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True)
+vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True)
+traces_vol = modal.Volume.from_name("fara-traces", create_if_missing=True)
+
+FAST_BOOT = True  # Set to False for better performance if not cold-starting frequently
+
+app = modal.App("fara-vllm")
+
+MINUTES = 60  # seconds
+VLLM_PORT = 5000  # Changed to 5000 as per user specification
+N_GPU = 1
+
+
+@app.function(
+    image=vllm_image,
+    gpu="L40S",
+    scaledown_window=2 * MINUTES,
+    timeout=10 * MINUTES,
+    volumes={
+        "/root/.cache/huggingface": hf_cache_vol,
+        "/root/.cache/vllm": vllm_cache_vol,
+    },
+)
+@modal.concurrent(max_inputs=32)
+@modal.web_server(
+    port=VLLM_PORT, startup_timeout=10 * MINUTES, requires_proxy_auth=True
+)
+def serve():
+    import subprocess
+
+    cmd = [
+        "vllm",
+        "serve",
+        "--uvicorn-log-level=info",
+        MODEL_NAME,
+        "--served-model-name",
+        MODEL_NAME,
+        "--host",
+        "0.0.0.0",
+        "--port",
+        str(VLLM_PORT),
+        "--dtype",
+        "auto",  # As per user specification
+        "--max-model-len",
+        "32768",  # Limit context length to fit in GPU memory (default 128000 too large)
+    ]
+
+    if MODEL_REVISION:
+        cmd += ["--revision", MODEL_REVISION]
+
+    # enforce-eager disables both Torch compilation and CUDA graph capture
+    cmd += ["--enforce-eager" if FAST_BOOT else "--no-enforce-eager"]
+
+    # assume multiple GPUs are for splitting up large matrix multiplications
+    cmd += ["--tensor-parallel-size", str(N_GPU)]
+
+    print(cmd)
+
+    subprocess.Popen(" ".join(cmd), shell=True)
+
+
+@app.function(
+    image=trace_storage_image,
+    volumes={"/traces": traces_vol},
+    timeout=2 * MINUTES,
+)
+@modal.fastapi_endpoint(method="POST", requires_proxy_auth=True)
+def store_trace(trace_data: dict) -> dict:
+    """
+    Store a task trace JSON in the Modal volume.
+    If a trace with the same ID and instruction already exists, it will be overwritten.
+
+    Expected trace_data structure:
+    {
+        "trace": { id, timestamp, instruction, modelId, isRunning },
+        "completion": { status, message, finalAnswer },
+        "metadata": { traceId, inputTokensUsed, outputTokensUsed, ... user_evaluation },
+        "steps": [...],
+        "exportedAt": "ISO timestamp"
+    }
+    """
+    import glob
+    import os
+    from datetime import datetime
+
+    try:
+        # Extract trace ID and instruction for duplicate detection
+        trace_id = trace_data.get("trace", {}).get("id", "unknown")
+        instruction = trace_data.get("trace", {}).get("instruction", "")
+
+        # Create organized directory structure: /traces/YYYY-MM/
+        date_folder = datetime.now().strftime("%Y-%m")
+        trace_dir = f"/traces/{date_folder}"
+        os.makedirs(trace_dir, exist_ok=True)
+
+        # Check for existing trace with same ID (in all monthly folders)
+        existing_file = None
+        for monthly_dir in glob.glob("/traces/*/"):
+            for filepath in glob.glob(f"{monthly_dir}*_{trace_id}.json"):
+                # Found an existing file with this trace ID
+                # Verify it's the same trace by checking instruction
+                try:
+                    with open(filepath, "r") as f:
+                        existing_data = json.load(f)
+                    existing_instruction = existing_data.get("trace", {}).get(
+                        "instruction", ""
+                    )
+                    if existing_instruction == instruction:
+                        existing_file = filepath
+                        break
+                except (json.JSONDecodeError, IOError):
+                    # If we can't read the file, skip it
+                    continue
+            if existing_file:
+                break
+
+        if existing_file:
+            # Overwrite the existing file
+            filepath = existing_file
+            print(f"Overwriting existing trace: {filepath}")
+        else:
+            # Generate new filename
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"{timestamp}_{trace_id}.json"
+            filepath = f"{trace_dir}/{filename}"
+
+        # Write trace to file
+        with open(filepath, "w") as f:
+            json.dump(trace_data, f, indent=2, default=str)
+
+        # Commit volume changes
+        traces_vol.commit()
+
+        return {
+            "success": True,
+            "message": "Trace stored successfully"
+            if not existing_file
+            else "Trace updated successfully",
+            "filepath": filepath,
+            "trace_id": trace_id,
+            "was_overwritten": existing_file is not None,
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+        }
+
+
+@app.local_entrypoint()
+async def test(test_timeout=10 * MINUTES, content=None, twice=True):
+    url = serve.get_web_url()
+
+    system_prompt = {
+        "role": "system",
+        "content": "You are an AI assistant specialized in computer use tasks.",
+    }
+    if content is None:
+        content = "Hello, what can you do to help with computer tasks?"
+
+    messages = [  # OpenAI chat format
+        system_prompt,
+        {"role": "user", "content": content},
+    ]
+
+    async with aiohttp.ClientSession(base_url=url) as session:
+        print(f"Running health check for server at {url}")
+        async with session.get("/health", timeout=test_timeout - 1 * MINUTES) as resp:
+            up = resp.status == 200
+        assert up, f"Failed health check for server at {url}"
+        print(f"Successful health check for server at {url}")
+
+        print(f"Sending messages to {url}:", *messages, sep="\n\t")
+        await _send_request(session, MODEL_NAME, messages)
+        if twice:
+            messages[0]["content"] = "You are a helpful assistant."
+            print(f"Sending messages to {url}:", *messages, sep="\n\t")
+            await _send_request(session, MODEL_NAME, messages)
+
+
+async def _send_request(
+    session: aiohttp.ClientSession, model: str, messages: list
+) -> None:
+    # \`stream=True\` tells an OpenAI-compatible backend to stream chunks
+    payload: dict[str, Any] = {"messages": messages, "model": model, "stream": True}
+
+    headers = {"Content-Type": "application/json", "Accept": "text/event-stream"}
+
+    async with session.post(
+        "/v1/chat/completions", json=payload, headers=headers, timeout=1 * MINUTES
+    ) as resp:
+        async for raw in resp.content:
+            resp.raise_for_status()
+            # extract new content and stream it
+            line = raw.decode().strip()
+            if not line or line == "data: [DONE]":
+                continue
+            if line.startswith("data: "):  # SSE prefix
+                line = line[len("data: ") :]
+
+            chunk = json.loads(line)
+            assert (
+                chunk["object"] == "chat.completion.chunk"
+            )  # or something went horribly wrong
+            print(chunk["choices"][0]["delta"]["content"], end="")
+    print()

backend/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+name = "backend"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "fastapi>=0.109.0",
+    "jsonschema>=4.25.1",
+    "openai>=2.8.1",
+    "pillow>=12.0.0",
+    "playwright==1.51",
+    "pyyaml>=6.0.3",
+    "tenacity>=9.1.2",
+    "uvicorn[standard]>=0.27.0",
+    "websockets>=12.0",
+]

backend/server.py

@@ -0,0 +1,588 @@
+"""
+FARA Backend Server for HuggingFace Space
+Provides WebSocket communication and REST API for the React frontend
+"""
+
+import asyncio
+import base64
+import logging
+import os
+
+# Import FARA components
+import sys
+import tempfile
+import uuid
+from datetime import datetime
+from typing import Dict, Optional
+
+import httpx
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from playwright._impl._errors import TargetClosedError
+
+sys.path.insert(0, "/app")
+from fara import FaraAgent
+from fara.browser.browser_bb import BrowserBB
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Modal trace storage configuration
+MODAL_TRACE_STORAGE_URL = os.environ.get("MODAL_TRACE_STORAGE_URL", "")
+MODAL_TOKEN_ID = os.environ.get("MODAL_TOKEN_ID", "")
+MODAL_TOKEN_SECRET = os.environ.get("MODAL_TOKEN_SECRET", "")
+
+# Modal vLLM endpoint configuration (from environment variables for HF Spaces)
+# Includes proxy auth headers for authenticated Modal endpoints
+ENDPOINT_CONFIG = {
+    "model": os.environ.get("FARA_MODEL_NAME", "microsoft/Fara-7B"),
+    "base_url": os.environ.get("FARA_ENDPOINT_URL"),
+    "api_key": os.environ.get("FARA_API_KEY", "not-needed"),
+    "default_headers": {
+        "Modal-Key": MODAL_TOKEN_ID,
+        "Modal-Secret": MODAL_TOKEN_SECRET,
+    }
+    if MODAL_TOKEN_ID and MODAL_TOKEN_SECRET
+    else None,
+}
+
+# Available models (for the frontend dropdown)
+AVAILABLE_MODELS = ["microsoft/Fara-7B"]
+
+app = FastAPI(title="FARA Backend")
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Store active connections and their sessions
+active_connections: Dict[str, WebSocket] = {}
+active_sessions: Dict[str, "FaraSession"] = {}
+
+
+class FaraSession:
+    """Manages a single FARA agent session"""
+
+    def __init__(self, trace_id: str, websocket: WebSocket):
+        self.trace_id = trace_id
+        self.websocket = websocket
+        self.agent: Optional[FaraAgent] = None
+        self.browser_manager: Optional[BrowserBB] = None
+        self.screenshots_dir: Optional[str] = None
+        self.is_running = False
+        self.should_stop = False
+        self.step_count = 0
+        self.start_time: Optional[datetime] = None
+        self.total_input_tokens = 0
+        self.total_output_tokens = 0
+
+    async def initialize(self, start_page: str = "https://www.bing.com/"):
+        """Initialize the browser and agent"""
+        # Create temp directory for screenshots
+        self.screenshots_dir = tempfile.mkdtemp(prefix="fara_screenshots_")
+
+        # Initialize browser manager (headless for HF Space)
+        self.browser_manager = BrowserBB(
+            headless=True,
+            viewport_height=900,
+            viewport_width=1440,
+            page_script_path=None,
+            browser_channel="chromium",
+            browser_data_dir=None,
+            downloads_folder=self.screenshots_dir,
+            to_resize_viewport=True,
+            single_tab_mode=True,
+            animate_actions=False,
+            use_browser_base=False,
+            logger=logger,
+        )
+
+        self.agent = FaraAgent(
+            browser_manager=self.browser_manager,
+            client_config=ENDPOINT_CONFIG,
+            start_page=start_page,
+            downloads_folder=self.screenshots_dir,
+            save_screenshots=True,
+            max_rounds=50,
+        )
+
+        await self.agent.initialize()
+        return True
+
+    async def send_event(self, event: dict):
+        """Send event to the connected WebSocket"""
+        try:
+            await self.websocket.send_json(event)
+        except Exception as e:
+            logger.error(f"Error sending event: {e}")
+
+    async def get_screenshot_base64(self) -> Optional[str]:
+        """Get the current browser screenshot as base64"""
+        if self.agent:
+            try:
+                # Get the current active page from the browser context
+                page = self._get_active_page()
+                if page:
+                    screenshot_bytes = (
+                        await self.agent._playwright_controller.get_screenshot(page)
+                    )
+                    return f"data:image/png;base64,{base64.b64encode(screenshot_bytes).decode()}"
+            except TargetClosedError:
+                logger.warning(
+                    "Page closed while getting screenshot, attempting recovery..."
+                )
+                page = self._get_active_page()
+                if page:
+                    try:
+                        screenshot_bytes = (
+                            await self.agent._playwright_controller.get_screenshot(page)
+                        )
+                        return f"data:image/png;base64,{base64.b64encode(screenshot_bytes).decode()}"
+                    except Exception as e:
+                        logger.error(f"Recovery screenshot failed: {e}")
+            except Exception as e:
+                logger.error(f"Error getting screenshot: {e}")
+        return None
+
+    def _get_active_page(self):
+        """Get the currently active page from the browser context"""
+        if (
+            self.agent
+            and self.agent.browser_manager
+            and self.agent.browser_manager._context
+        ):
+            pages = self.agent.browser_manager._context.pages
+            if pages:
+                # Return the last (most recent) page, or the one marked as active
+                return pages[-1]
+        return self.agent._page if self.agent else None
+
+    async def run_task(self, instruction: str, model_id: str):
+        """Run a task and stream results via WebSocket"""
+        self.is_running = True
+        self.should_stop = False
+        self.step_count = 0
+        self.start_time = datetime.now()
+        self.total_input_tokens = 0
+        self.total_output_tokens = 0
+
+        try:
+            # Send agent_start event
+            await self.send_event(
+                {
+                    "type": "agent_start",
+                    "agentTrace": {
+                        "id": self.trace_id,
+                        "instruction": instruction,
+                        "modelId": model_id,
+                        "timestamp": self.start_time.isoformat(),
+                        "isRunning": True,
+                        "traceMetadata": {
+                            "traceId": self.trace_id,
+                            "inputTokensUsed": 0,
+                            "outputTokensUsed": 0,
+                            "duration": 0,
+                            "numberOfSteps": 0,
+                            "maxSteps": 50,
+                            "completed": False,
+                        },
+                    },
+                }
+            )
+
+            # Initialize agent
+            await self.initialize()
+
+            # Get initial screenshot
+            initial_screenshot = await self.get_screenshot_base64()
+
+            # Run the agent with custom loop to stream progress
+            await self._run_agent_with_streaming(instruction)
+
+        except Exception as e:
+            logger.exception("Error running agent task")
+            await self.send_event({"type": "agent_error", "error": str(e)})
+        finally:
+            self.is_running = False
+            await self.close()
+
+    async def _run_agent_with_streaming(self, user_message: str):
+        """Run the agent and stream each step to the frontend"""
+        agent = self.agent
+
+        # Initialize if not already done
+        await agent.initialize()
+        assert agent._page is not None, "Page should be initialized"
+
+        # Get initial screenshot
+        scaled_screenshot = await agent._get_scaled_screenshot()
+
+        if agent.save_screenshots:
+            await agent._playwright_controller.get_screenshot(
+                agent._page,
+                path=os.path.join(
+                    agent.downloads_folder, f"screenshot{agent._num_actions}.png"
+                ),
+            )
+
+        # Add user message to chat history
+        from fara.types import ImageObj, UserMessage
+
+        agent._chat_history.append(
+            UserMessage(
+                content=[ImageObj.from_pil(scaled_screenshot), user_message],
+                is_original=True,
+            )
+        )
+
+        final_answer = "<no_answer>"
+        is_stop_action = False
+
+        for i in range(agent.max_rounds):
+            if self.should_stop:
+                # User requested stop
+                await self.send_event(
+                    {
+                        "type": "agent_complete",
+                        "traceMetadata": self._get_metadata(),
+                        "final_state": "stopped",
+                    }
+                )
+                return
+
+            is_first_round = i == 0
+            step_start_time = datetime.now()
+
+            # Wait for captcha if needed
+            if not agent.browser_manager._captcha_event.is_set():
+                logger.info("Waiting 60s for captcha to finish...")
+                captcha_solved = await agent.wait_for_captcha_with_timeout(60)
+                if (
+                    not captcha_solved
+                    and not agent.browser_manager._captcha_event.is_set()
+                ):
+                    raise RuntimeError("Captcha timed out")
+
+            try:
+                # Generate model response
+                function_call, raw_response = await agent.generate_model_call(
+                    is_first_round, scaled_screenshot if is_first_round else None
+                )
+
+                # Parse response
+                thoughts, action_dict = agent._parse_thoughts_and_action(raw_response)
+                action_args = action_dict.get("arguments", {})
+                action = action_args["action"]
+
+                logger.info(
+                    f"\nThought #{i + 1}: {thoughts}\nAction #{i + 1}: {action}"
+                )
+
+                # Execute action with recovery for page changes
+                try:
+                    (
+                        is_stop_action,
+                        new_screenshot,
+                        action_description,
+                    ) = await agent.execute_action(function_call)
+                except TargetClosedError as e:
+                    logger.warning(
+                        "Page closed during action execution, attempting recovery..."
+                    )
+                    # Try to recover the page reference
+                    new_page = self._get_active_page()
+                    if new_page and new_page != agent._page:
+                        logger.info("Recovered with new active page")
+                        agent._page = new_page
+                        # Wait for the page to stabilize
+                        await asyncio.sleep(1)
+                        action_description = (
+                            "Action completed (page navigation occurred)"
+                        )
+                        is_stop_action = False
+                        new_screenshot = None
+                    else:
+                        raise e
+
+                # Sync the agent's page reference with the active page
+                active_page = self._get_active_page()
+                if active_page and active_page != agent._page:
+                    logger.info("Updating agent page reference to active page")
+                    agent._page = active_page
+
+                # Get screenshot for this step
+                screenshot_base64 = await self.get_screenshot_base64()
+
+            except TargetClosedError as e:
+                logger.error(f"Unrecoverable page error: {e}")
+                await self.send_event(
+                    {
+                        "type": "agent_error",
+                        "error": f"Browser page closed unexpectedly: {str(e)}",
+                    }
+                )
+                return
+            except Exception as e:
+                logger.exception(f"Error in agent step {i + 1}")
+                await self.send_event({"type": "agent_error", "error": str(e)})
+                return
+
+            # Calculate step duration and tokens (estimated)
+            step_duration = (datetime.now() - step_start_time).total_seconds()
+            step_input_tokens = 1000  # Estimated
+            step_output_tokens = len(raw_response) // 4  # Rough estimate
+
+            self.total_input_tokens += step_input_tokens
+            self.total_output_tokens += step_output_tokens
+            self.step_count += 1
+
+            # Create step object
+            step = {
+                "stepId": str(uuid.uuid4()),
+                "traceId": self.trace_id,
+                "stepNumber": self.step_count,
+                "thought": thoughts,
+                "actions": [
+                    {
+                        "function_name": action,
+                        "description": action_description,
+                        "parameters": action_args,
+                    }
+                ],
+                "image": screenshot_base64,
+                "duration": step_duration,
+                "inputTokensUsed": step_input_tokens,
+                "outputTokensUsed": step_output_tokens,
+                "timestamp": datetime.now().isoformat(),
+            }
+
+            # Send progress event
+            await self.send_event(
+                {
+                    "type": "agent_progress",
+                    "agentStep": step,
+                    "traceMetadata": self._get_metadata(),
+                }
+            )
+
+            if is_stop_action:
+                final_answer = thoughts
+                break
+
+        # Send completion event
+        final_state = "success" if is_stop_action else "max_steps_reached"
+        await self.send_event(
+            {
+                "type": "agent_complete",
+                "traceMetadata": self._get_metadata(completed=True),
+                "final_state": final_state,
+            }
+        )
+
+    def _get_metadata(self, completed: bool = False) -> dict:
+        """Get current trace metadata"""
+        duration = 0
+        if self.start_time:
+            duration = (datetime.now() - self.start_time).total_seconds()
+
+        return {
+            "traceId": self.trace_id,
+            "inputTokensUsed": self.total_input_tokens,
+            "outputTokensUsed": self.total_output_tokens,
+            "duration": duration,
+            "numberOfSteps": self.step_count,
+            "maxSteps": 50,
+            "completed": completed,
+        }
+
+    async def stop(self):
+        """Request the agent to stop"""
+        self.should_stop = True
+
+    async def close(self):
+        """Clean up resources"""
+        if self.agent:
+            try:
+                await self.agent.close()
+            except Exception as e:
+                logger.error(f"Error closing agent: {e}")
+            self.agent = None
+            self.browser_manager = None
+
+        if self.screenshots_dir and os.path.exists(self.screenshots_dir):
+            import shutil
+
+            try:
+                shutil.rmtree(self.screenshots_dir)
+            except Exception as e:
+                logger.error(f"Error cleaning up screenshots: {e}")
+            self.screenshots_dir = None
+
+
+@app.get("/api/models")
+async def get_models():
+    """Return available models"""
+    return JSONResponse(content=AVAILABLE_MODELS)
+
+
+@app.post("/api/traces")
+async def store_trace(trace_data: dict):
+    """
+    Store a task trace by forwarding to the Modal trace storage endpoint.
+    This keeps Modal credentials on the server side.
+    """
+    if not MODAL_TRACE_STORAGE_URL:
+        logger.warning("Modal trace storage URL not configured")
+        return JSONResponse(
+            status_code=503,
+            content={"success": False, "error": "Trace storage not configured"},
+        )
+
+    if not MODAL_TOKEN_ID or not MODAL_TOKEN_SECRET:
+        logger.warning("Modal proxy auth credentials not configured")
+        return JSONResponse(
+            status_code=503,
+            content={"success": False, "error": "Modal auth not configured"},
+        )
+
+    try:
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.post(
+                MODAL_TRACE_STORAGE_URL,
+                json=trace_data,
+                headers={
+                    "Content-Type": "application/json",
+                    "Modal-Key": MODAL_TOKEN_ID,
+                    "Modal-Secret": MODAL_TOKEN_SECRET,
+                },
+            )
+
+            if response.status_code == 200:
+                result = response.json()
+                logger.info(
+                    f"Trace stored successfully: {result.get('trace_id', 'unknown')}"
+                )
+                return JSONResponse(content=result)
+            else:
+                error_text = response.text
+                logger.error(
+                    f"Failed to store trace: {response.status_code} - {error_text}"
+                )
+                return JSONResponse(
+                    status_code=response.status_code,
+                    content={
+                        "success": False,
+                        "error": f"Modal API error: {error_text}",
+                    },
+                )
+    except httpx.TimeoutException:
+        logger.error("Timeout storing trace to Modal")
+        return JSONResponse(
+            status_code=504,
+            content={"success": False, "error": "Timeout connecting to trace storage"},
+        )
+    except Exception as e:
+        logger.exception("Error storing trace")
+        return JSONResponse(
+            status_code=500, content={"success": False, "error": str(e)}
+        )
+
+
+@app.get("/api/random-question")
+async def get_random_question():
+    """Return a random example question"""
+    questions = [
+        "Search for the latest news about AI agents",
+        "Find the weather forecast for San Francisco",
+        "Go to GitHub and search for 'computer use agent'",
+        "Find the top trending repositories on GitHub today",
+        "Search for Python tutorials on YouTube",
+        "Look up the current stock price of Microsoft",
+        "Find the schedule for upcoming SpaceX launches",
+        "Search for healthy breakfast recipes",
+    ]
+    import random
+
+    return JSONResponse(content={"question": random.choice(questions)})
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    """WebSocket endpoint for real-time communication"""
+    await websocket.accept()
+
+    # Generate a unique connection ID
+    connection_id = str(uuid.uuid4())
+    active_connections[connection_id] = websocket
+
+    # Send heartbeat with the connection ID (used as trace ID base)
+    trace_id = str(uuid.uuid4())
+    await websocket.send_json(
+        {"type": "heartbeat", "uuid": trace_id, "timestamp": datetime.now().isoformat()}
+    )
+
+    try:
+        while True:
+            # Wait for messages from the client
+            data = await websocket.receive_json()
+            message_type = data.get("type")
+
+            if message_type == "user_task":
+                # Extract task details
+                trace = data.get("trace", {})
+                trace_id = trace.get("id", str(uuid.uuid4()))
+                instruction = trace.get("instruction", "")
+                model_id = trace.get("modelId", "microsoft/Fara-7B")
+
+                # Create and start session
+                session = FaraSession(trace_id, websocket)
+                active_sessions[trace_id] = session
+
+                # Run the task in the background
+                asyncio.create_task(session.run_task(instruction, model_id))
+
+            elif message_type == "stop_task":
+                # Stop the running task
+                trace_id = data.get("trace_id")
+                if trace_id and trace_id in active_sessions:
+                    await active_sessions[trace_id].stop()
+
+            elif message_type == "ping":
+                await websocket.send_json({"type": "pong"})
+
+    except WebSocketDisconnect:
+        logger.info(f"WebSocket disconnected: {connection_id}")
+    except Exception as e:
+        logger.exception(f"WebSocket error: {e}")
+    finally:
+        # Clean up
+        if connection_id in active_connections:
+            del active_connections[connection_id]
+
+        # Clean up any sessions for this connection
+        sessions_to_remove = []
+        for trace_id, session in active_sessions.items():
+            if session.websocket == websocket:
+                await session.close()
+                sessions_to_remove.append(trace_id)
+        for trace_id in sessions_to_remove:
+            del active_sessions[trace_id]
+
+
+@app.get("/api/health")
+async def health_check():
+    """Health check endpoint"""
+    return {"status": "healthy"}
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000)

eslint.config.js

@@ -0,0 +1,26 @@
+import js from "@eslint/js";
+import globals from "globals";
+import reactHooks from "eslint-plugin-react-hooks";
+import reactRefresh from "eslint-plugin-react-refresh";
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  { ignores: ["dist"] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ["**/*.{ts,tsx}"],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      "react-hooks": reactHooks,
+      "react-refresh": reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      "react-refresh/only-export-components": ["warn", { allowConstantExport: true }],
+      "@typescript-eslint/no-unused-vars": "off",
+    },
+  },
+);

fara/__init__.py

@@ -0,0 +1,4 @@
+from .fara_agent import FaraAgent
+from .browser.playwright_controller import PlaywrightController
+
+__all__ = ["FaraAgent", "PlaywrightController"]

fara/_prompts.py

@@ -0,0 +1,276 @@
+import math
+
+from typing import Union, Tuple
+
+from .qwen_helpers.base_tool import BaseTool
+from .qwen_helpers.fncall_prompt import NousFnCallPrompt
+from .qwen_helpers.schema import (
+    ContentItem,
+    Message,
+)
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+
+# @register_tool("computer_use")
+class FaraComputerUse(BaseTool):
+    name = "computer_use"
+
+    @property
+    def description(self):
+        return f"""
+Use a mouse and keyboard to interact with a computer, and take screenshots.
+* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.
+* The screen's resolution is {self.display_width_px}x{self.display_height_px}.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
+* When a separate scrollable container prominently overlays the webpage, if you want to scroll within it, you typically need to mouse_move() over it first and then scroll().
+* If a popup window appears that you want to close, if left_click() on the 'X' or close button doesn't work, try key(keys=['Escape']) to close it.
+* On some search bars, when you type(), you may need to press_enter=False and instead separately call left_click() on the search button to submit the search query. This is especially true of search bars that have auto-suggest popups for e.g. locations
+* For calendar widgets, you usually need to left_click() on arrows to move between months and left_click() on dates to select them; type() is not typically used to input dates there.
+""".strip()
+
+    parameters = {
+        "properties": {
+            "action": {
+                "description": """
+The action to perform. The available actions are:
+* `key`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order. Includes "Enter", "Alt", "Shift", "Tab", "Control", "Backspace", "Delete", "Escape", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "PageDown", "PageUp", "Shift", etc.
+* `type`: Type a string of text on the keyboard.
+* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.
+* `left_click`: Click the left mouse button.
+* `scroll`: Performs a scroll of the mouse scroll wheel.
+* `visit_url`: Visit a specified URL.
+* `web_search`: Perform a web search with a specified query.
+* `history_back`: Go back to the previous page in the browser history.
+* `pause_and_memorize_fact`: Pause and memorize a fact for future reference.
+* `wait`: Wait specified seconds for the change to happen.
+* `terminate`: Terminate the current task and report its completion status.
+""".strip(),
+                "enum": [
+                    "key",
+                    "type",
+                    "mouse_move",
+                    "left_click",
+                    "scroll",
+                    "visit_url",
+                    "web_search",
+                    "history_back",
+                    "pause_and_memorize_fact",
+                    "wait",
+                    "terminate",
+                ],
+                "type": "string",
+            },
+            "keys": {
+                "description": "Required only by `action=key`.",
+                "type": "array",
+            },
+            "text": {
+                "description": "Required only by `action=type`.",
+                "type": "string",
+            },
+            "press_enter": {
+                "description": "Whether to press the Enter key after typing. Required only by `action=type`.",
+                "type": "boolean",
+            },
+            "delete_existing_text": {
+                "description": "Whether to delete existing text before typing. Required only by `action=type`.",
+                "type": "boolean",
+            },
+            "coordinate": {
+                "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=left_click`, `action=mouse_move`, and `action=type`.",
+                "type": "array",
+            },
+            "pixels": {
+                "description": "The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll`.",
+                "type": "number",
+            },
+            "url": {
+                "description": "The URL to visit. Required only by `action=visit_url`.",
+                "type": "string",
+            },
+            "query": {
+                "description": "The query to search for. Required only by `action=web_search`.",
+                "type": "string",
+            },
+            "fact": {
+                "description": "The fact to remember for the future. Required only by `action=pause_and_memorize_fact`.",
+                "type": "string",
+            },
+            "time": {
+                "description": "The seconds to wait. Required only by `action=wait`.",
+                "type": "number",
+            },
+            "status": {
+                "description": "The status of the task. Required only by `action=terminate`.",
+                "type": "string",
+                "enum": ["success", "failure"],
+            },
+        },
+        "required": ["action"],
+        "type": "object",
+    }
+
+    def __init__(self, cfg=None):
+        self.display_width_px = cfg["display_width_px"]
+        self.display_height_px = cfg["display_height_px"]
+        include_input_text_key_args = cfg.pop("include_input_text_key_args", False)
+        if not include_input_text_key_args:
+            self.parameters["properties"].pop("press_enter", None)
+            self.parameters["properties"].pop("delete_existing_text", None)
+        super().__init__(cfg)
+
+    def call(self, params: Union[str, dict], **kwargs):
+        params = self._verify_json_format_args(params)
+        action = params["action"]
+        if action == "key":
+            return self._key(params["text"])
+        elif action == "click":
+            return self._click(coordinate=params["coordinate"])
+        elif action == "long_press":
+            return self._long_press(
+                coordinate=params["coordinate"], time=params["time"]
+            )
+        elif action == "swipe":
+            return self._swipe(
+                coordinate=params["coordinate"], coordinate2=params["coordinate2"]
+            )
+        elif action == "type":
+            return self._type(params["text"])
+        elif action == "system_button":
+            return self._system_button(params["button"])
+        elif action == "open":
+            return self._open(params["text"])
+        elif action == "wait":
+            return self._wait(params["time"])
+        elif action == "terminate":
+            return self._terminate(params["status"])
+        else:
+            raise ValueError(f"Unknown action: {action}")
+
+    def _key(self, text: str):
+        raise NotImplementedError()
+
+    def _click(self, coordinate: Tuple[int, int]):
+        raise NotImplementedError()
+
+    def _long_press(self, coordinate: Tuple[int, int], time: int):
+        raise NotImplementedError()
+
+    def _swipe(self, coordinate: Tuple[int, int], coordinate2: Tuple[int, int]):
+        raise NotImplementedError()
+
+    def _type(self, text: str):
+        raise NotImplementedError()
+
+    def _system_button(self, button: str):
+        raise NotImplementedError()
+
+    def _open(self, text: str):
+        raise NotImplementedError()
+
+    def _wait(self, time: int):
+        raise NotImplementedError()
+
+    def _terminate(self, status: str):
+        raise NotImplementedError()
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = IMAGE_FACTOR,
+    min_pixels: int = MIN_PIXELS,
+    max_pixels: int = MAX_PIXELS,
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+    return h_bar, w_bar
+
+
+def get_computer_use_system_prompt(
+    image,
+    processor_im_cfg,
+    include_input_text_key_args=False,
+    fn_call_template="default",
+):
+    patch_size = processor_im_cfg["patch_size"]
+    merge_size = processor_im_cfg["merge_size"]
+    min_pixels = processor_im_cfg["min_pixels"]
+    max_pixels = processor_im_cfg["max_pixels"]
+
+    resized_height, resized_width = smart_resize(
+        image.height,
+        image.width,
+        factor=patch_size * merge_size,
+        min_pixels=min_pixels,
+        max_pixels=max_pixels,
+    )
+
+    computer_use = FaraComputerUse(
+        cfg={
+            "display_width_px": resized_width,
+            "display_height_px": resized_height,
+            "include_input_text_key_args": include_input_text_key_args,
+        }
+    )
+
+    conversation = NousFnCallPrompt(
+        template_name=fn_call_template
+    ).preprocess_fncall_messages(
+        messages=[
+            Message(
+                role="system",
+                content=[ContentItem(text="You are a helpful assistant.")],
+            ),
+        ],
+        functions=[computer_use.function],
+        lang=None,
+    )
+
+    return {
+        "conversation": [msg.model_dump() for msg in conversation],
+        "im_size": (resized_width, resized_height),
+    }

fara/browser/browser_bb.py

@@ -0,0 +1,353 @@
+import asyncio
+import atexit
+import logging
+import os
+import platform
+import signal
+import subprocess
+import time
+from typing import Any, Callable, Dict, Optional
+
+from playwright.async_api import (
+    BrowserContext,
+    Download,
+    Page,
+    Playwright,
+    async_playwright,
+)
+
+from .playwright_controller import PlaywrightController
+
+
+class BrowserBB:
+    """Manages browser instance, context, and page lifecycle."""
+
+    def __init__(
+        self,
+        viewport_height: int,
+        viewport_width: int,
+        headless: bool,
+        page_script_path: str,
+        browser_channel: str = "firefox",
+        browser_data_dir: str | None = None,
+        downloads_folder: str | None = None,
+        to_resize_viewport: bool = True,
+        single_tab_mode: bool = True,
+        animate_actions: bool = False,
+        use_browser_base: bool = False,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.headless = headless
+        self.page_script_path = page_script_path
+        self.browser_channel = browser_channel
+        self.browser_data_dir = browser_data_dir
+        self.downloads_folder = downloads_folder
+        self.to_resize_viewport = to_resize_viewport
+        self.animate_actions = animate_actions
+        self.single_tab_mode = single_tab_mode
+        self.use_browser_base = use_browser_base
+        self.logger = logger or logging.getLogger("browser_manager")
+
+        self._viewport_height = viewport_height
+        self._viewport_width = viewport_width
+
+        # check _viewport_width and _viewport_height are positive integers
+        if not isinstance(self._viewport_width, int) or self._viewport_width <= 0:
+            raise ValueError(
+                f"Error: Browser_manager.Browser: Invalid viewport width: {self._viewport_width}. Must be a positive integer."
+            )
+        if not isinstance(self._viewport_height, int) or self._viewport_height <= 0:
+            raise ValueError(
+                f"Error: Browser_manager.Browser:Invalid viewport height: {self._viewport_height}. Must be a positive integer."
+            )
+        assert isinstance(self.headless, bool), (
+            f"Error: Browser_manager.Browser: headless must be a boolean, got {type(self.headless)}"
+        )
+        if page_script_path is None:
+            page_script_path = os.path.join(
+                os.path.abspath(os.path.dirname(__file__)), "page_script.js"
+            )
+            self.page_script_path = page_script_path
+        assert isinstance(page_script_path, str), (
+            f"Error: Browser_manager.Browser: page_script_path must be a string, got {type(self.page_script_path)}"
+        )
+        assert os.path.exists(self.page_script_path), (
+            f"Error: Browser_manager.Browser: page_script_path does not exist: {self.page_script_path}"
+        )
+
+        assert isinstance(self.browser_channel, str) and (
+            self.browser_channel in ["chromium", "firefox", "webkit"]
+        ), (
+            f"Error: Browser_manager.Browser: browser_channel must be one of ['chromium', 'firefox', 'webkit'], got {self.browser_channel}"
+        )
+
+        # Browser-related instances
+        self._playwright: Playwright | None = None
+        self._context: BrowserContext | None = None
+        self._page: Page | None = None
+        self.browser = None
+        self.session = None
+        self.xvfb_process = None
+
+        # Events and handlers
+        self._captcha_event = asyncio.Event()
+        self._captcha_event.set()  # Initially set (no captcha)
+        self._download_handler: Callable[[Download], None] | None = None
+
+        self._playwright_controller = PlaywrightController(
+            animate_actions=self.animate_actions,
+            downloads_folder=self.downloads_folder,
+            viewport_width=self._viewport_width,
+            viewport_height=self._viewport_height,
+            _download_handler=self._download_handler,
+            to_resize_viewport=self.to_resize_viewport,
+            single_tab_mode=self.single_tab_mode,
+            logger=self.logger,
+        )
+
+    def set_download_handler(self, handler: Callable[[Download], None]) -> None:
+        """Set the download handler for the browser."""
+        self._download_handler = handler
+        self._playwright_controller._download_handler = handler
+
+    def set_captcha_solved_callback(self, callback: Callable[[bool], None]) -> None:
+        """Set callback to be called when captcha status changes."""
+        self._captcha_solved_callback = callback
+
+    async def init(
+        self,
+        start_page: str,
+        shared_data_point=None,  # For captcha tracking
+    ) -> None:
+        """Initialize the browser, context, and page."""
+        self._playwright = await async_playwright().start()
+        self.shared_data_point = shared_data_point
+
+        if self.use_browser_base:
+            await self._init_browser_base(self.shared_data_point)
+        elif self.browser_data_dir is None:
+            await self._init_regular_browser(channel=self.browser_channel)
+        else:
+            await self._init_persistent_browser()
+
+        # Common setup for all browser types
+        await self._setup_common_browser_features(start_page)
+
+    async def _init_browser_base(self, shared_data_point) -> None:
+        """Initialize BrowserBase connection, defaults to chromium."""
+        # Lazy import browserbase only when needed
+        import browserbase
+        from browserbase import Browserbase
+
+        self.logger.info("Initializing BrowserBase session...")
+        self.bb = Browserbase(api_key=os.environ["BROWSERBASE_API_KEY"])
+
+        while True:  # Wait indefinitely until we get a session
+            try:
+                self.session = self.bb.sessions.create(
+                    project_id=os.environ["BROWSERBASE_PROJECT_ID"],
+                    proxies=True,
+                    browser_settings={"advanced_stealth": True},
+                    keep_alive=True,
+                    timeout=7200,  # 2 hour timeout
+                    region="us-east-1",
+                )
+                break
+            except browserbase.RateLimitError:
+                self.logger.warning(
+                    "Rate limit exceeded while trying to create BrowserBase session. Retrying in 10 seconds..."
+                )
+                await asyncio.sleep(10)
+
+        assert self.session.id is not None
+        assert self.session.status == "RUNNING", (
+            f"Session status is {self.session.status}"
+        )
+
+        chromium = self._playwright.chromium
+        self.browser = await chromium.connect_over_cdp(self.session.connect_url)
+        self.logger.info(
+            f"Connected to BrowserBase session: https://browserbase.com/sessions/{self.session.id}"
+        )
+
+        self._context = self.browser.contexts[0]
+        assert len(self._context.pages) == 1
+        self._page = self._context.pages[0]
+
+        # Set up captcha handling
+        def handle_console(msg):
+            """Handle captcha detection and solving."""
+            if msg.text == "browserbase-solving-started":
+                self.logger.info("Captcha Solving In Progress!!")
+                if shared_data_point:
+                    shared_data_point.set_encountered_captcha(True)
+                self._captcha_event.clear()  # Block execution
+            elif msg.text == "browserbase-solving-finished":
+                self.logger.info("Captcha Solving Completed!!")
+
+                async def delayed_resume():
+                    await asyncio.sleep(3)  # Wait for navigation to settle
+                    await self._page.wait_for_load_state("networkidle")
+                    self._captcha_event.set()
+
+                asyncio.create_task(delayed_resume())
+
+        self._context.on("console", handle_console)
+        self._page.on("console", handle_console)
+
+    async def _init_regular_browser(self, channel: str = "chromium") -> None:
+        """Initialize regular browser according to the specified channel."""
+        if not self.headless:
+            self.start_xvfb()
+
+        launch_args: Dict[str, Any] = {"headless": self.headless}
+
+        if channel == "chromium":
+            self.browser = await self._playwright.chromium.launch(**launch_args)
+        elif channel == "firefox":
+            self.browser = await self._playwright.firefox.launch(**launch_args)
+        elif channel == "webkit":
+            self.browser = await self._playwright.webkit.launch(**launch_args)
+        else:
+            raise ValueError(
+                f"Unsupported browser channel: {channel}. Supported channels are 'chromium', 'firefox', and 'webkit'."
+            )
+
+        self._context = await self.browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"
+        )
+
+        self._page = await self._context.new_page()
+
+    async def _init_persistent_browser(self) -> None:
+        """Initialize persistent browser with data directory."""
+        if not self.headless:
+            self.start_xvfb()
+
+        launch_args: Dict[str, Any] = {"headless": self.headless}
+        self._context = await self._playwright.chromium.launch_persistent_context(
+            self.browser_data_dir, **launch_args
+        )
+        self._page = await self._context.new_page()
+
+    async def _setup_common_browser_features(self, start_page: str) -> None:
+        """Set up features common to all browser types."""
+        self._context.set_default_timeout(60000)  # One minute
+        await self._playwright_controller.on_new_page(self._page)
+        assert self._page is not None
+
+        # Set up new page handling for single tab mode
+        if self.single_tab_mode:
+            self._context.on(
+                "page", lambda new_pg: self._handle_new_page_safe(new_pg, self._page)
+            )
+
+        # Set up download handler
+        if self._download_handler:
+            self._page.on("download", self._download_handler)
+
+        # Set viewport and add init script
+        await self._page.set_viewport_size(
+            {"width": self._viewport_width, "height": self._viewport_height}
+        )
+
+        await self._page.add_init_script(path=self.page_script_path)
+
+        # Navigate to start page
+        await self._page.goto(start_page)
+        await self._page.wait_for_load_state()
+
+    async def _handle_new_page_safe(self, new_pg: Page, main_page: Page) -> None:
+        """Safely handle new pages in single tab mode."""
+        try:
+            await new_pg.wait_for_load_state("domcontentloaded")
+
+            # Do not close if new_pg is the current page
+            if new_pg == main_page or new_pg.url == main_page.url:
+                self.logger.info("New tab is same as current page, not closing.")
+                return
+
+            new_url = new_pg.url
+            await new_pg.close()
+            await self._playwright_controller.visit_page(main_page, new_url)
+        except Exception as e:
+            self.logger.warning(f"Error in handle_new_page_safe: {e}")
+
+    def start_xvfb(self) -> None:
+        """Start Xvfb virtual display server (Linux only)."""
+        # Xvfb is only needed on Linux - macOS and Windows have native display support
+        if platform.system() != "Linux":
+            return
+
+        display_num = 99  # Choose a display number unlikely to be in use
+        self.xvfb_process = subprocess.Popen(
+            ["Xvfb", f":{display_num}", "-screen", "0", "1280x1024x24", "-ac"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        os.environ["DISPLAY"] = f":{display_num}"
+
+        # Make sure Xvfb has time to start
+        time.sleep(1)
+
+        # Register cleanup function
+        atexit.register(self.stop_xvfb)
+
+    def stop_xvfb(self) -> None:
+        """Stop the Xvfb process if it's running."""
+        if self.xvfb_process:
+            self.xvfb_process.send_signal(signal.SIGTERM)
+            self.xvfb_process.wait()
+            self.xvfb_process = None
+
+    async def wait_for_captcha_resolution(self) -> None:
+        """Wait for captcha to be resolved if one is being solved."""
+        await self._captcha_event.wait()
+
+    @property
+    def page(self) -> Page | None:
+        """Get the current page."""
+        return self._page
+
+    @page.setter
+    def page(self, value):
+        self._page = value
+
+    @property
+    def context(self) -> BrowserContext | None:
+        """Get the browser context."""
+        return self._context
+
+    @property
+    def playwright_controller(self):
+        """Get the playwright controller."""
+        return self._playwright_controller
+
+    async def close(self) -> None:
+        """Close the browser and clean up resources."""
+        self.logger.info("Closing browser...")
+
+        if self._page is not None:
+            await self._page.close()
+            self._page = None
+
+        if self._context is not None:
+            await self._context.close()
+            self._context = None
+
+        if self._playwright is not None:
+            await self._playwright.stop()
+            self._playwright = None
+
+        if self.browser:
+            if self.use_browser_base and self.session:
+                self.bb.sessions.update(
+                    self.session.id,
+                    status="REQUEST_RELEASE",
+                    project_id=os.environ["BROWSERBASE_PROJECT_ID"],
+                )
+            await self.browser.close()
+            self.browser = None
+
+        if not self.headless:
+            self.stop_xvfb()

fara/browser/page_script.js

@@ -0,0 +1,610 @@
+var MultimodalWebSurfer = MultimodalWebSurfer || (function() {
+    let nextLabel = 10;
+
+    let roleMapping = {
+        "a": "link",
+        "area": "link",
+        "button": "button",
+        "input, type=button": "button",
+        "input, type=checkbox": "checkbox",
+        "input, type=email": "textbox",
+        "input, type=number": "spinbutton",
+        "input, type=radio": "radio",
+        "input, type=range": "slider",
+        "input, type=reset": "button",
+        "input, type=search": "searchbox",
+        "input, type=submit": "button",
+        "input, type=tel": "textbox",
+        "input, type=text": "textbox",
+        "input, type=url": "textbox",
+        "search": "search",
+        "select": "combobox",
+        "option": "option",
+        "textarea": "textbox"
+    };
+
+    let getCursor = function (elm) {
+        return window.getComputedStyle(elm)["cursor"];
+    };
+
+    let isVisible = function (element) {
+        return !!(element.offsetWidth || element.offsetHeight || element.getClientRects().length);
+    };
+
+    /**
+     * Finds interactive elements in the regular DOM (excluding Shadow DOM)
+     * Looks for elements that are:
+     * 1. Standard interactive elements (inputs, buttons, links)
+     * 2. Elements with ARIA roles indicating interactivity
+     * 3. Elements with cursor styles suggesting interactivity
+     *
+     * @returns {Array} Array of DOM elements that are deemed interactive
+     */
+    let getInteractiveElementsNoShaddow = function () {
+        let results = []
+        let roles = ["scrollbar", "searchbox", "slider", "spinbutton", "switch", "tab", "treeitem", "button", "checkbox", "gridcell", "link", "menuitem", "menuitemcheckbox", "menuitemradio", "option", "progressbar", "radio", "textbox", "combobox", "menu", "tree", "treegrid", "grid", "listbox", "radiogroup", "widget"];
+        let inertCursors = ["auto", "default", "none", "text", "vertical-text", "not-allowed", "no-drop"];
+
+        // Get the main interactive elements
+        let nodeList = document.querySelectorAll("input, select, textarea, button, [href], [onclick], [contenteditable], [tabindex]:not([tabindex='-1'])");
+        for (let i = 0; i < nodeList.length; i++) { // Copy to something mutable
+            // make sure not disabled and visible
+            if (nodeList[i].disabled || !isVisible(nodeList[i])) {
+                continue;
+            }
+            results.push(nodeList[i]);
+        }
+
+        // Anything not already included that has a suitable role
+        nodeList = document.querySelectorAll("[role]");
+        for (let i = 0; i < nodeList.length; i++) { // Copy to something mutable
+            // make sure not disabled and visible
+            if (nodeList[i].disabled || !isVisible(nodeList[i])) {
+                continue;
+            }
+            if (results.indexOf(nodeList[i]) == -1) {
+                let role = nodeList[i].getAttribute("role");
+                if (roles.indexOf(role) > -1) {
+                    results.push(nodeList[i]);
+                }
+            }
+        }
+
+        // Any element that changes the cursor to something implying interactivity
+        nodeList = document.querySelectorAll("*");
+        for (let i = 0; i < nodeList.length; i++) {
+            let node = nodeList[i];
+            if (node.disabled || !isVisible(node)) {
+                continue;
+            }
+
+            // Cursor is default, or does not suggest interactivity
+            let cursor = getCursor(node);
+            if (inertCursors.indexOf(cursor) >= 0) {
+                continue;
+            }
+
+            // Move up to the first instance of this cursor change
+            let parent = node.parentNode;
+            while (parent && getCursor(parent) == cursor) {
+                node = parent;
+                parent = node.parentNode;
+            }
+
+            // Add the node if it is new
+            if (results.indexOf(node) == -1) {
+                results.push(node);
+            }
+        }
+
+        return results;
+    };
+
+    /**
+     * Recursively gathers elements matching specified roles from both regular DOM and Shadow DOM
+     * @param {Array} roles - Array of role selectors to match
+     * @param {Document|ShadowRoot} root - Root element to start search from
+     * @returns {Array} Array of matching elements
+     */
+    function gatherAllElements(roles, root = document) {
+        const elements = [];
+        const stack = [root];
+        const selector = roles.join(",");
+
+        while (stack.length > 0) {
+            const currentRoot = stack.pop();
+
+            // Add elements at current level
+            elements.push(...Array.from(currentRoot.querySelectorAll(selector)));
+
+            // Add shadow roots to stack
+            currentRoot.querySelectorAll("*").forEach(el => {
+                if (el.shadowRoot && el.shadowRoot.mode === "open") {
+                    stack.push(el.shadowRoot);
+                }
+            });
+        }
+
+        return elements;
+    }
+
+    /**
+     * Gets all interactive elements from both regular DOM and Shadow DOM
+     * Filters elements to ensure they are visible and accessible
+     * @returns {Array} Array of interactive elements
+     */
+    let getInteractiveElements = function () {
+        // Get all elements that are interactive without the shadow DOM
+        const interactive_roles = ["input", "option", "select", "textarea", "button", "href", "onclick", "contenteditable", "tabindex:not([tabindex='-1'])"];
+
+        let results = [];
+
+        let elements_no_shaddow = getInteractiveElementsNoShaddow();
+        for (let i = 0; i < elements_no_shaddow.length; i++) {
+            if (results.indexOf(elements_no_shaddow[i]) == -1) {
+                // check if it has a rect
+                let rects = elements_no_shaddow[i].getClientRects();
+                for (const rect of rects) {
+                    let x = rect.left + rect.width / 2;
+                    let y = rect.top + rect.height / 2;
+                    if (isTopmost(elements_no_shaddow[i], x, y)) {
+                        results.push(elements_no_shaddow[i]);
+                        break;
+                    }
+                }
+            }
+        }
+
+        // From the shadow DOM get all interactive elements and options that are not in the no shadow list
+        let elements_all = gatherAllElements(interactive_roles);
+
+        // Filter and process interactive elements
+        elements_all.forEach(element => {
+            // if file, auto add
+            if (element.tagName.toLowerCase() === "input" && element.getAttribute("type") == "file") {
+                results.push(element);
+                return;
+            }
+            // if option  auto add
+            if (element.tagName.toLowerCase() === "option") {
+                results.push(element);
+                return;
+            }
+            if (element.disabled || !isVisible(element)) {
+                return;
+            }
+
+            if (interactive_roles.includes(element.tagName.toLowerCase())) {
+                results.push(element);
+            }
+        });
+
+        return results;
+    };
+
+    /**
+     * Assigns unique identifiers to interactive elements
+     * @param {Array} elements - Array of elements to label
+     * @returns {Array} Updated array of interactive elements
+     */
+    let labelElements = function (elements) {
+        for (let i = 0; i < elements.length; i++) {
+            if (!elements[i].hasAttribute("__elementId")) {
+                elements[i].setAttribute("__elementId", "" + (nextLabel++));
+            }
+        }
+        return getInteractiveElements();
+    };
+
+    /**
+     * Checks if an element is the topmost element at given coordinates
+     * @param {Element} element - Element to check
+     * @param {number} x - X coordinate
+     * @param {number} y - Y coordinate
+     * @returns {boolean} True if element is topmost at coordinates
+     */
+    let isTopmost = function (element, x, y) {
+        let hit = document.elementFromPoint(x, y);
+
+        // Hack to handle elements outside the viewport
+        if (hit === null) {
+            return true;
+        }
+
+        while (hit) {
+            if (hit == element) return true;
+            hit = hit.parentNode;
+        }
+        return false;
+    };
+
+    let getFocusedElementId = function () {
+        let elm = document.activeElement;
+        while (elm) {
+            if (elm.hasAttribute && elm.hasAttribute("__elementId")) {
+                return elm.getAttribute("__elementId");
+            }
+            elm = elm.parentNode;
+        }
+        return null;
+    };
+
+    let trimmedInnerText = function (element) {
+        if (!element) {
+            return "";
+        }
+        let text = element.innerText;
+        if (!text) {
+            return "";
+        }
+        return text.trim();
+    };
+
+    let getApproximateAriaName = function (element) {
+        if (element.hasAttribute("aria-label")) {
+            return element.getAttribute("aria-label");
+        }
+
+        // check if element has span that is called label and grab the inner text
+        if (element.querySelector("span.label")) {
+            return element.querySelector("span.label").innerText;
+        }
+
+        // Check for aria labels
+        if (element.hasAttribute("aria-labelledby")) {
+            let buffer = "";
+            let ids = element.getAttribute("aria-labelledby").split(" ");
+            for (let i = 0; i < ids.length; i++) {
+                let label = document.getElementById(ids[i]);
+                if (label) {
+                    buffer = buffer + " " + trimmedInnerText(label);
+                }
+            }
+            return buffer.trim();
+        }
+
+        if (element.hasAttribute("aria-label")) {
+            return element.getAttribute("aria-label");
+        }
+
+        // Check for labels
+        if (element.hasAttribute("id")) {
+            let label_id = element.getAttribute("id");
+            let label = "";
+            try {
+                // Escape special characters in the ID
+                let escaped_id = CSS.escape(label_id);
+                let labels = document.querySelectorAll(`label[for="${escaped_id}"]`);
+                for (let j = 0; j < labels.length; j++) {
+                    label += labels[j].innerText + " ";
+                }
+                label = label.trim();
+                if (label != "") {
+                    return label;
+                }
+            } catch (e) {
+                console.warn("Error finding label for element:", e);
+            }
+        }
+
+        if (element.hasAttribute("name")) {
+            return element.getAttribute("name");
+        }
+
+        if (element.parentElement && element.parentElement.tagName == "LABEL") {
+            return element.parentElement.innerText;
+        }
+
+        // Check for alt text or titles
+        if (element.hasAttribute("alt")) {
+            return element.getAttribute("alt")
+        }
+
+        if (element.hasAttribute("title")) {
+            return element.getAttribute("title")
+        }
+
+        return trimmedInnerText(element);
+    };
+
+    let getApproximateAriaRole = function (element) {
+        let tag = element.tagName.toLowerCase();
+        if (tag == "input" && element.hasAttribute("type")) {
+            tag = tag + ", type=" + element.getAttribute("type");
+        }
+
+        if (element.hasAttribute("role")) {
+            return [element.getAttribute("role"), tag];
+        }
+        else if (tag in roleMapping) {
+            return [roleMapping[tag], tag];
+        }
+        else {
+            return ["", tag];
+        }
+    };
+
+    /**
+     * Gets information about all interactive elements including their:
+     * - Position and dimensions
+     * - ARIA roles and names
+     * - Tag names
+     * - Scrollability
+     *
+     * @returns {Object} Map of element IDs to their properties
+     */
+    let getInteractiveRects = function () {
+        let elements = labelElements(getInteractiveElements());
+        let results = {};
+        for (let i = 0; i < elements.length; i++) {
+            let key = elements[i].getAttribute("__elementId");
+            let rects = elements[i].getBoundingClientRect();
+
+            // Skip options unless their select is focused
+            if (elements[i].tagName.toLowerCase() === "option") {
+
+                let select_focused = false;
+                let select = elements[i].closest("select");
+                if (select && select.hasAttribute("__elementId") &&
+                    getFocusedElementId() === select.getAttribute("__elementId")) {
+                    select_focused = true;
+                }
+                // check if option is visible without select being focused
+                let option_visible = false;
+                if (isVisible(elements[i])) {
+                    option_visible = true;
+                }
+                // check if select is expanded even if not focused
+                let select_expanded = false;
+                if (select && select.hasAttribute("open")) {
+                    select_expanded = true;
+                }
+                if (!(select_focused || option_visible || select_expanded)) {
+                    continue;
+                }
+            }
+
+            let ariaRole = getApproximateAriaRole(elements[i]);
+            let ariaName = getApproximateAriaName(elements[i]);
+            let vScrollable = elements[i].scrollHeight - elements[i].clientHeight >= 1;
+
+            let record = {
+                "tag_name": ariaRole[1],
+                "role": ariaRole[0],
+                "aria-name": ariaName,
+                "v-scrollable": vScrollable,
+                "rects": []
+            };
+
+            if (rects.length > 0) {
+                for (const rect of rects) {
+                    let x = rect.left + rect.width / 2;
+                    let y = rect.top + rect.height / 2;
+                    if (isTopmost(elements[i], x, y)) {
+                        record["rects"].push(JSON.parse(JSON.stringify(rect)));
+                    }
+                }
+            }
+            else {
+                record["rects"].push(JSON.parse(JSON.stringify(rects)));
+            }
+
+            results[key] = record;
+        }
+        return results;
+    };
+
+    /**
+     * Gets current viewport information including dimensions and scroll positions
+     * @returns {Object} Viewport properties
+     */
+    let getVisualViewport = function () {
+        let vv = window.visualViewport;
+        let de = document.documentElement;
+        return {
+            "height": vv ? vv.height : 0,
+            "width": vv ? vv.width : 0,
+            "offsetLeft": vv ? vv.offsetLeft : 0,
+            "offsetTop": vv ? vv.offsetTop : 0,
+            "pageLeft": vv ? vv.pageLeft : 0,
+            "pageTop": vv ? vv.pageTop : 0,
+            "scale": vv ? vv.scale : 0,
+            "clientWidth": de ? de.clientWidth : 0,
+            "clientHeight": de ? de.clientHeight : 0,
+            "scrollWidth": de ? de.scrollWidth : 0,
+            "scrollHeight": de ? de.scrollHeight : 0
+        };
+    };
+
+    let _getMetaTags = function () {
+        let meta = document.querySelectorAll("meta");
+        let results = {};
+        for (let i = 0; i < meta.length; i++) {
+            let key = null;
+            if (meta[i].hasAttribute("name")) {
+                key = meta[i].getAttribute("name");
+            }
+            else if (meta[i].hasAttribute("property")) {
+                key = meta[i].getAttribute("property");
+            }
+            else {
+                continue;
+            }
+            if (meta[i].hasAttribute("content")) {
+                results[key] = meta[i].getAttribute("content");
+            }
+        }
+        return results;
+    };
+
+    let _getJsonLd = function () {
+        let jsonld = [];
+        let scripts = document.querySelectorAll('script[type="application/ld+json"]');
+        for (let i = 0; i < scripts.length; i++) {
+            jsonld.push(scripts[i].innerHTML.trim());
+        }
+        return jsonld;
+    };
+
+    // From: https://www.stevefenton.co.uk/blog/2022/12/parse-microdata-with-javascript/
+    let _getMicrodata = function () {
+        function sanitize(input) {
+            return input.replace(/\s/gi, ' ').trim();
+        }
+
+        function addValue(information, name, value) {
+            if (information[name]) {
+                if (typeof information[name] === 'array') {
+                    information[name].push(value);
+                } else {
+                    const arr = [];
+                    arr.push(information[name]);
+                    arr.push(value);
+                    information[name] = arr;
+                }
+            } else {
+                information[name] = value;
+            }
+        }
+
+        function traverseItem(item, information) {
+            const children = item.children;
+
+            for (let i = 0; i < children.length; i++) {
+                const child = children[i];
+
+                if (child.hasAttribute('itemscope')) {
+                    if (child.hasAttribute('itemprop')) {
+                        const itemProp = child.getAttribute('itemprop');
+                        const itemType = child.getAttribute('itemtype');
+
+                        const childInfo = {
+                            itemType: itemType
+                        };
+
+                        traverseItem(child, childInfo);
+
+                        itemProp.split(' ').forEach(propName => {
+                            addValue(information, propName, childInfo);
+                        });
+                    }
+
+                } else if (child.hasAttribute('itemprop')) {
+                    const itemProp = child.getAttribute('itemprop');
+                    itemProp.split(' ').forEach(propName => {
+                        if (propName === 'url') {
+                            addValue(information, propName, child.href);
+                        } else {
+                            addValue(information, propName, sanitize(child.getAttribute("content") || child.content || child.textContent || child.src || ""));
+                        }
+                    });
+                    traverseItem(child, information);
+                } else {
+                    traverseItem(child, information);
+                }
+            }
+        }
+
+        const microdata = [];
+
+        document.querySelectorAll("[itemscope]").forEach(function (elem, i) {
+            const itemType = elem.getAttribute('itemtype');
+            const information = {
+                itemType: itemType
+            };
+            traverseItem(elem, information);
+            microdata.push(information);
+        });
+
+        return microdata;
+    };
+
+    let getPageMetadata = function () {
+        let jsonld = _getJsonLd();
+        let metaTags = _getMetaTags();
+        let microdata = _getMicrodata();
+        let results = {}
+        if (jsonld.length > 0) {
+            try {
+                results["jsonld"] = JSON.parse(jsonld);
+            }
+            catch (e) {
+                results["jsonld"] = jsonld;
+            }
+        }
+        if (microdata.length > 0) {
+            results["microdata"] = microdata;
+        }
+        for (let key in metaTags) {
+            if (metaTags.hasOwnProperty(key)) {
+                results["meta_tags"] = metaTags;
+                break;
+            }
+        }
+        return results;
+    };
+
+    /**
+     * Extracts all visible text content from the viewport
+     * Preserves basic formatting with newlines for block elements
+     * @returns {string} Visible text content
+     */
+    let getVisibleText = function () {
+        // Get the window's current viewport boundaries
+        const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
+        const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
+
+        let textInView = "";
+        const walker = document.createTreeWalker(
+            document.body,
+            NodeFilter.SHOW_TEXT,
+            null,
+            false
+        );
+
+        while (walker.nextNode()) {
+            const textNode = walker.currentNode;
+            // Create a range to retrieve bounding rectangles of the current text node
+            const range = document.createRange();
+            range.selectNodeContents(textNode);
+
+            const rects = range.getClientRects();
+
+            // Check if any rect is inside (or partially inside) the viewport
+            for (const rect of rects) {
+                const isVisible =
+                    rect.width > 0 &&
+                    rect.height > 0 &&
+                    rect.bottom >= 0 &&
+                    rect.right >= 0 &&
+                    rect.top <= viewportHeight &&
+                    rect.left <= viewportWidth;
+
+                if (isVisible) {
+                    textInView += textNode.nodeValue.replace(/\s+/g, " ");
+                    // Is the parent a block element?
+                    if (textNode.parentNode) {
+                        const parent = textNode.parentNode;
+                        const style = window.getComputedStyle(parent);
+                        if (["inline", "hidden", "none"].indexOf(style.display) === -1) {
+                            textInView += "\n";
+                        }
+                    }
+                    break; // No need to check other rects once found visible
+                }
+            }
+        }
+
+        // Remove blank lines from textInView
+        textInView = textInView.replace(/^\s*\n/gm, "").trim().replace(/\n+/g, "\n");
+        return textInView;
+    };
+
+    // Public API
+    return {
+        getInteractiveRects: getInteractiveRects,
+        getVisualViewport: getVisualViewport,
+        getFocusedElementId: getFocusedElementId,
+        getPageMetadata: getPageMetadata,
+    };
+})();

fara/browser/playwright_controller.py

@@ -0,0 +1,581 @@
+import asyncio
+import base64
+import os
+import random
+import logging
+import functools
+from typing import Any, Callable, Optional, Tuple, Union, TypeVar, Awaitable
+
+from playwright._impl._errors import Error as PlaywrightError
+from playwright._impl._errors import TimeoutError, TargetClosedError
+from playwright.async_api import Download, Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+
+# Adapted from Magentic-UI
+# Some of the Code for clicking coordinates and keypresses adapted from https://github.com/openai/openai-cua-sample-app/blob/main/computers/base_playwright.py
+# Copyright 2025 OpenAI - MIT License
+CUA_KEY_TO_PLAYWRIGHT_KEY = {
+    "/": "Divide",
+    "\\": "Backslash",
+    "alt": "Alt",
+    "arrowdown": "ArrowDown",
+    "arrowleft": "ArrowLeft",
+    "arrowright": "ArrowRight",
+    "arrowup": "ArrowUp",
+    "backspace": "Backspace",
+    "capslock": "CapsLock",
+    "cmd": "Meta",
+    "ctrl": "Control",
+    "delete": "Delete",
+    "end": "End",
+    "enter": "Enter",
+    "esc": "Escape",
+    "home": "Home",
+    "insert": "Insert",
+    "option": "Alt",
+    "pagedown": "PageDown",
+    "pageup": "PageUp",
+    "shift": "Shift",
+    "space": " ",
+    "super": "Meta",
+    "tab": "Tab",
+    "win": "Meta",
+}
+
+F = TypeVar("F", bound=Callable[..., Awaitable[Any]])
+
+
+def handle_target_closed(max_retries: int = 2, timeout_secs: int = 30):
+    """
+    Decorator to handle TargetClosedError and tunnel connection errors by attempting to recover the page.
+
+    Args:
+        max_retries: Maximum number of retry attempts
+        timeout_secs: Timeout for page operations during recovery
+    """
+
+    def decorator(func: F) -> F:
+        @functools.wraps(func)
+        async def wrapper(*args, **kwargs):
+            # Extract the page object - assume it's the first argument after self
+            logger = args[0].logger
+            page = None
+            if len(args) >= 2 and hasattr(
+                args[1], "url"
+            ):  # Check if second arg looks like a Page
+                page = args[1]
+
+            retries = 0
+            last_error = None
+
+            while retries <= max_retries:
+                try:
+                    return await func(*args, **kwargs)
+                except (TargetClosedError, PlaywrightError) as e:
+                    # Check if this is a tunnel connection error
+                    is_tunnel_error = "net::ERR_TUNNEL_CONNECTION_FAILED" in str(e)
+                    is_target_closed = isinstance(
+                        e, TargetClosedError
+                    ) or "Target page, context or browser has been closed" in str(e)
+
+                    if not (is_tunnel_error or is_target_closed):
+                        # Not an error we handle, re-raise
+                        raise e
+
+                    last_error = e
+                    retries += 1
+
+                    if retries > max_retries:
+                        raise e
+
+                    if page is None:
+                        # Can't recover without page reference
+                        raise e
+
+                    error_type = (
+                        "tunnel connection" if is_tunnel_error else "target closed"
+                    )
+                    logger.warning(
+                        f"{error_type} error in {func.__name__}, attempting recovery (retry {retries}/{max_retries})"
+                    )
+
+                    try:
+                        # Attempt to recover the page
+                        await _recover_page(page, timeout_secs, logger)
+                        # Small delay before retry
+                        await asyncio.sleep(0.5)
+                    except Exception as recovery_error:
+                        logger.error(f"Page recovery failed: {recovery_error}")
+                        # If recovery fails, raise the original error
+                        raise e from recovery_error
+
+            # This shouldn't be reached, but just in case
+            raise last_error
+
+        return wrapper
+
+    return decorator
+
+
+async def _recover_page(page: Page, timeout_secs: int = 30, logger=None) -> None:
+    """
+    Attempt to recover a closed page by reloading it.
+
+    Args:
+        page: The Playwright page object to recover
+        timeout_secs: Timeout for recovery operations
+    """
+    logger = logger or logging.getLogger("playwright_controller")
+    try:
+        # First, try to check if the page is still responsive
+        await page.evaluate("1", timeout=1000)
+        # If we get here, the page is actually fine
+        return
+    except Exception:
+        # Page is indeed problematic, attempt recovery
+        pass
+
+    try:
+        # Stop any ongoing navigation
+        await page.evaluate("window.stop()", timeout=2000)
+    except Exception:
+        # Ignore errors from window.stop()
+        pass
+
+    try:
+        # Try to reload the page
+        await page.reload(timeout=timeout_secs * 1000)
+        await page.wait_for_load_state("load", timeout=timeout_secs * 1000)
+        logger.info("playwright_controller._recover_page(): Page recovery successful")
+    except Exception as e:
+        logger.error(f"playwright_controller._recover_page(): Page reload failed: {e}")
+
+        # Try alternative recovery: navigate to current URL
+        try:
+            current_url = page.url
+            if current_url and current_url != "about:blank":
+                await page.goto(current_url, timeout=timeout_secs * 1000)
+                await page.wait_for_load_state("load", timeout=timeout_secs * 1000)
+                logger.info(
+                    "playwright_controller._recover_page(): Page recovery via goto successful"
+                )
+            else:
+                raise Exception(
+                    "playwright_controller._recover_page(): No valid URL to navigate to"
+                )
+        except Exception as goto_error:
+            raise Exception(
+                f"playwright_controller._recover_page(): All recovery methods failed. Reload error: {e}, Goto error: {goto_error}"
+            )
+
+
+# Enhanced version that can handle browser context recreation
+def handle_target_closed_with_context(max_retries: int = 2, timeout_secs: int = 30):
+    """
+    Enhanced decorator that can also handle browser context recreation.
+    Use this for critical operations where you have access to the browser context.
+    """
+
+    def decorator(func: F) -> F:
+        @functools.wraps(func)
+        async def wrapper(*args, **kwargs):
+            logger = args[0].logger
+            page = None
+            if len(args) >= 2 and hasattr(args[1], "url"):
+                page = args[1]
+
+            retries = 0
+            last_error = None
+
+            while retries <= max_retries:
+                try:
+                    return await func(*args, **kwargs)
+                except (TargetClosedError, PlaywrightError) as e:
+                    # Check if this is a tunnel connection error
+                    is_tunnel_error = "net::ERR_TUNNEL_CONNECTION_FAILED" in str(e)
+                    is_target_closed = isinstance(
+                        e, TargetClosedError
+                    ) or "Target page, context or browser has been closed" in str(e)
+
+                    if not (is_tunnel_error or is_target_closed):
+                        # Not an error we handle, re-raise
+                        raise e
+
+                    last_error = e
+                    retries += 1
+
+                    if retries > max_retries:
+                        raise e
+
+                    if page is None:
+                        raise e
+
+                    error_type = (
+                        "tunnel connection" if is_tunnel_error else "target closed"
+                    )
+                    logger.warning(
+                        f"playwright_controller.handle_target_closed_with_context(): {error_type} error in {func.__name__}, attempting enhanced recovery (retry {retries}/{max_retries})"
+                    )
+
+                    try:
+                        # Check if the browser context is still alive
+                        context = page.context
+                        browser = context.browser
+
+                        if browser and not browser.is_connected():
+                            # Browser connection is lost - this is a more serious issue
+                            logger.error(
+                                "playwright_controller.handle_target_closed_with_context(): Browser connection lost - cannot recover automatically"
+                            )
+                            raise e
+
+                        # Try basic recovery first
+                        await _recover_page(page, timeout_secs)
+                        await asyncio.sleep(0.5)
+
+                    except Exception as recovery_error:
+                        logger.error(
+                            f"playwright_controller.handle_target_closed_with_context(): Enhanced page recovery failed: {recovery_error}"
+                        )
+                        raise e from recovery_error
+
+            raise last_error
+
+        return wrapper
+
+    return decorator
+
+
+class PlaywrightController:
+    def __init__(
+        self,
+        animate_actions: bool = False,
+        downloads_folder: Optional[str] = None,
+        viewport_width: int = 1440,
+        viewport_height: int = 900,
+        _download_handler: Optional[Callable[[Download], None]] = None,
+        to_resize_viewport: bool = True,
+        single_tab_mode: bool = False,
+        sleep_after_action: int = 10,
+        timeout_load: int = 1,
+        logger=None,
+    ) -> None:
+        """
+        A controller for Playwright to interact with web pages.
+        animate_actions: If True, actions will be animated.
+        downloads_folder: The folder to save downloads to.
+        viewport_width: The width of the viewport.
+        viewport_height: The height of the viewport.
+        _download_handler: A handler for downloads.
+        to_resize_viewport: If True, the viewport will be resized.
+        single_tab_mode (bool): If True, forces navigation to happen in the same tab rather than opening new tabs/windows.
+
+        """
+        self.animate_actions = animate_actions
+        self.downloads_folder = downloads_folder
+        self.viewport_width = viewport_width
+        self.viewport_height = viewport_height
+        self._download_handler = _download_handler
+        self.to_resize_viewport = to_resize_viewport
+        self.single_tab_mode = single_tab_mode
+        self._sleep_after_action = sleep_after_action
+        self._timeout_load = timeout_load
+        self.logger = logger or logging.getLogger("playwright_controller")
+
+        # Set up the download handler
+        self.last_cursor_position: Tuple[float, float] = (0.0, 0.0)
+
+    async def sleep(self, page: Page, duration: Union[int, float]) -> None:
+        await asyncio.sleep(duration)
+
+    @handle_target_closed()
+    async def on_new_page(self, page: Page) -> None:
+        assert page is not None
+        # bring page to front just in case
+        await page.bring_to_front()
+        page.on("download", self._download_handler)  # type: ignore
+        if self.to_resize_viewport and self.viewport_width and self.viewport_height:
+            await page.set_viewport_size(
+                {"width": self.viewport_width, "height": self.viewport_height}
+            )
+        await self.sleep(page, 0.2)
+        try:
+            await page.wait_for_load_state(timeout=30000)
+        except PlaywrightTimeoutError:
+            self.logger.error("WARNING: Page load timeout, page might not be loaded")
+            # stop page loading
+            await page.evaluate("window.stop()")
+
+    @handle_target_closed()
+    async def _ensure_page_ready(self, page: Page) -> None:
+        assert page is not None
+        await self.on_new_page(page)
+
+    @handle_target_closed()
+    async def get_screenshot(self, page: Page, path: str | None = None) -> bytes:
+        """
+        Capture a screenshot of the current page.
+
+        Args:
+            page (Page): The Playwright page object.
+            path (str, optional): The file path to save the screenshot. If None, the screenshot will be returned as bytes. Default: None
+        """
+        await self._ensure_page_ready(page)
+        try:
+            screenshot = await page.screenshot(path=path, timeout=15000)
+            return screenshot
+        except Exception:
+            await page.evaluate("window.stop()")
+            # try again
+            screenshot = await page.screenshot(path=path, timeout=15000)
+            return screenshot
+
+    @handle_target_closed()
+    async def back(self, page: Page) -> None:
+        await self._ensure_page_ready(page)
+        await page.go_back()
+
+    @handle_target_closed()
+    async def visit_page(self, page: Page, url: str) -> Tuple[bool, bool]:
+        await self._ensure_page_ready(page)
+        reset_prior_metadata_hash = False
+        reset_last_download = False
+        try:
+            # Regular webpage
+            await page.goto(url)
+            await page.wait_for_load_state()
+            reset_prior_metadata_hash = True
+        except Exception as e_outer:
+            # Downloaded file
+            if self.downloads_folder and "net::ERR_ABORTED" in str(e_outer):
+                async with page.expect_download() as download_info:
+                    try:
+                        await page.goto(url)
+                    except Exception as e_inner:
+                        if "net::ERR_ABORTED" in str(e_inner):
+                            pass
+                        else:
+                            raise e_inner
+                    download = await download_info.value
+                    fname = os.path.join(
+                        self.downloads_folder, download.suggested_filename
+                    )
+                    await download.save_as(fname)
+                    message = f"<body style=\"margin: 20px;\"><h1>Successfully downloaded '{download.suggested_filename}' to local path:<br><br>{fname}</h1></body>"
+                    await page.goto(
+                        "data:text/html;base64,"
+                        + base64.b64encode(message.encode("utf-8")).decode("utf-8")
+                    )
+                    reset_last_download = True
+            else:
+                raise e_outer
+        return reset_prior_metadata_hash, reset_last_download
+
+    @handle_target_closed()
+    async def page_down(
+        self, page: Page, amount: int = 400, full_page: bool = False
+    ) -> None:
+        await self._ensure_page_ready(page)
+        if full_page:
+            await page.mouse.wheel(0, self.viewport_height - 50)
+        else:
+            await page.mouse.wheel(0, amount)
+
+    @handle_target_closed()
+    async def page_up(
+        self, page: Page, amount: int = 400, full_page: bool = False
+    ) -> None:
+        await self._ensure_page_ready(page)
+        if full_page:
+            await page.mouse.wheel(0, -self.viewport_height + 50)
+        else:
+            await page.mouse.wheel(0, -amount)
+
+    async def gradual_cursor_animation(
+        self, page: Page, start_x: float, start_y: float, end_x: float, end_y: float
+    ) -> None:
+        # animation helper
+        # Create the red cursor if it doesn't exist
+        await page.evaluate("""
+            (function() {
+                if (!document.getElementById('red-cursor')) {
+                    let cursor = document.createElement('div');
+                    cursor.id = 'red-cursor';
+                    cursor.style.width = '10px';
+                    cursor.style.height = '10px';
+                    cursor.style.backgroundColor = 'red';
+                    cursor.style.position = 'absolute';
+                    cursor.style.borderRadius = '50%';
+                    cursor.style.zIndex = '10000';
+                    document.body.appendChild(cursor);
+                }
+            })();
+        """)
+
+        steps = 20
+        for step in range(steps):
+            x = start_x + (end_x - start_x) * (step / steps)
+            y = start_y + (end_y - start_y) * (step / steps)
+            # await page.mouse.move(x, y, steps=1)
+            await page.evaluate(f"""
+                (function() {{
+                    let cursor = document.getElementById('red-cursor');
+                    if (cursor) {{
+                        cursor.style.left = '{x}px';
+                        cursor.style.top = '{y}px';
+                    }}
+                }})();
+            """)
+            await asyncio.sleep(0.05)
+
+        self.last_cursor_position = (end_x, end_y)
+        await asyncio.sleep(1.0)
+
+    @handle_target_closed()
+    async def click_coords(self, page: Page, x: float, y: float) -> None:
+        new_page: Page | None = None
+        await self._ensure_page_ready(page)
+
+        if self.animate_actions:
+            # Move cursor to the box slowly
+            start_x, start_y = self.last_cursor_position
+            await self.gradual_cursor_animation(page, start_x, start_y, x, y)
+            await asyncio.sleep(0.1)
+
+            try:
+                # Give it a chance to open a new page
+                async with page.expect_event("popup", timeout=1000) as page_info:  # type: ignore
+                    await page.mouse.click(x, y, delay=10)
+                    new_page = await page_info.value  # type: ignore
+                    assert isinstance(new_page, Page)
+                    await self.on_new_page(new_page)
+            except TimeoutError:
+                pass
+        else:
+            try:
+                # Give it a chance to open a new page
+                async with page.expect_event("popup", timeout=1000) as page_info:  # type: ignore
+                    await page.mouse.click(x, y, delay=10)
+                    new_page = await page_info.value  # type: ignore
+                    assert isinstance(new_page, Page)
+                    await self.on_new_page(new_page)
+            except TimeoutError:
+                pass
+        return new_page
+
+    @handle_target_closed()
+    async def hover_coords(self, page: Page, x: float, y: float) -> None:
+        """
+        Hovers the mouse over the specified coordinates.
+
+        Args:
+            page (Page): The Playwright page object.
+            x (float): The x coordinate to hover over.
+            y (float): The y coordinate to hover over.
+        """
+        await self._ensure_page_ready(page)
+
+        if self.animate_actions:
+            # Move cursor to the coordinates slowly
+            start_x, start_y = self.last_cursor_position
+            await self.gradual_cursor_animation(page, start_x, start_y, x, y)
+            await asyncio.sleep(0.1)
+
+        await page.mouse.move(x, y)
+
+    @handle_target_closed()
+    async def fill_coords(
+        self,
+        page: Page,
+        x: float,
+        y: float,
+        value: str,
+        press_enter: bool = True,
+        delete_existing_text: bool = False,
+    ) -> None:
+        await self._ensure_page_ready(page)
+        new_page: Page | None = None
+
+        if self.animate_actions:
+            # Move cursor to the box slowly
+            start_x, start_y = self.last_cursor_position
+            await self.gradual_cursor_animation(page, start_x, start_y, x, y)
+            await asyncio.sleep(0.1)
+
+        await page.mouse.click(x, y)
+
+        if delete_existing_text:
+            await page.keyboard.press("ControlOrMeta+A")
+            await page.keyboard.press("Backspace")
+
+        # fill char by char to mimic human speed for short text and type fast for long text
+        if len(value) < 100:
+            delay_typing_speed = 50 + 100 * random.random()
+        else:
+            delay_typing_speed = 10
+
+        if self.animate_actions:
+            try:
+                # Give it a chance to open a new page
+                async with page.expect_event("popup", timeout=1000) as page_info:  # type: ignore
+                    try:
+                        await page.keyboard.type(value)
+                    except PlaywrightError:
+                        await page.keyboard.type(value, delay=delay_typing_speed)
+                    if press_enter:
+                        await page.keyboard.press("Enter")
+                    new_page = await page_info.value  # type: ignore
+                    assert isinstance(new_page, Page)
+                    await self.on_new_page(new_page)
+            except TimeoutError:
+                pass
+        else:
+            try:
+                # Give it a chance to open a new page
+                async with page.expect_event("popup", timeout=1000) as page_info:  # type: ignore
+                    try:
+                        await page.keyboard.type(value)
+                    except PlaywrightError:
+                        await page.keyboard.type(value, delay=delay_typing_speed)
+                    if press_enter:
+                        await page.keyboard.press("Enter")
+                    new_page = await page_info.value  # type: ignore
+                    assert isinstance(new_page, Page)
+                    await self.on_new_page(new_page)
+            except TimeoutError:
+                pass
+
+        return new_page
+
+    async def keypress(self, page: Page, keys: list[str]) -> None:
+        """
+        Press specified keys in sequence.
+
+        Args:
+            page (Page): The Playwright page object
+            keys (List[str]): List of keys to press
+        """
+        await self._ensure_page_ready(page)
+        mapped_keys = [CUA_KEY_TO_PLAYWRIGHT_KEY.get(key.lower(), key) for key in keys]
+        try:
+            for key in mapped_keys:
+                await page.keyboard.down(key)
+            for key in reversed(mapped_keys):
+                await page.keyboard.up(key)
+        except Exception as e:
+            raise RuntimeError(
+                f"I tried to keypress(keys={keys}), but I got an error: {e}"
+            ) from None
+
+    @handle_target_closed()
+    async def wait_for_load_state(
+        self, page: Page, state: str = "load", timeout: Optional[int] = None
+    ) -> None:
+        """Wait for the page to reach a specific load state."""
+        await page.wait_for_load_state(state, timeout=timeout)
+
+    @handle_target_closed()
+    async def get_page_url(self, page: Page) -> str:
+        """Get the current page URL."""
+        await self._ensure_page_ready(page)
+        return page.url

fara/fara_agent.py

@@ -0,0 +1,602 @@
+import ast
+import asyncio
+import io
+import json
+import logging
+import os
+from typing import Any, Dict, List, Tuple
+from urllib.parse import quote_plus
+
+from openai import AsyncOpenAI
+from PIL import Image
+from playwright.async_api import BrowserContext, Download, Page
+from tenacity import before_sleep_log, retry, stop_after_attempt, wait_exponential
+
+from ._prompts import get_computer_use_system_prompt
+from .browser.playwright_controller import PlaywrightController
+from .types import (
+    AssistantMessage,
+    FunctionCall,
+    ImageObj,
+    LLMMessage,
+    ModelResponse,
+    SystemMessage,
+    UserMessage,
+    WebSurferEvent,
+    message_to_openai_format,
+)
+from .utils import get_trimmed_url
+
+
+class FaraAgent:
+    DEFAULT_START_PAGE = "https://www.bing.com/"
+
+    MLM_PROCESSOR_IM_CFG = {
+        "min_pixels": 3136,
+        "max_pixels": 12845056,
+        "patch_size": 14,
+        "merge_size": 2,
+    }
+
+    SCREENSHOT_TOKENS = 1105
+    USER_MESSAGE = "Here is the next screenshot. Think about what to do next."
+    MAX_URL_LENGTH = 100
+
+    def __init__(
+        self,
+        browser_manager: Any,
+        client_config: dict,
+        downloads_folder: str | None = None,
+        start_page: str | None = "about:blank",
+        animate_actions: bool = False,
+        single_tab_mode: bool = True,
+        max_n_images: int = 3,
+        fn_call_template: str = "default",
+        model_call_timeout: int = 20,
+        max_rounds: int = 10,
+        save_screenshots: bool = False,
+        logger: logging.Logger | None = None,
+    ):
+        self.downloads_folder = downloads_folder
+        if not os.path.exists(self.downloads_folder or "") and self.downloads_folder:
+            os.makedirs(self.downloads_folder)
+        self.single_tab_mode = single_tab_mode
+        self.start_page = start_page or self.DEFAULT_START_PAGE
+        self.animate_actions = animate_actions
+        self.browser_manager = browser_manager
+        self.client_config = client_config
+        self.max_n_images = max_n_images
+        self.fn_call_template = fn_call_template
+        self.model_call_timeout = model_call_timeout
+        self.max_rounds = max_rounds
+        self.max_url_chars = self.MAX_URL_LENGTH
+        if save_screenshots and self.downloads_folder is None:
+            assert False, "downloads_folder must be set if save_screenshots is True"
+        self.save_screenshots = save_screenshots
+        self._facts = []
+        self._task_summary = None
+        self._num_actions = 0
+        self.logger = logger or logging.getLogger(__name__)
+        self._mlm_width = 1440
+        self._mlm_height = 900
+        self.viewport_height = 900
+        self.viewport_width = 1440
+        self.include_input_text_key_args = True
+
+        def _download_handler(download: Download) -> None:
+            self._last_download = download
+
+        self._download_handler = _download_handler
+        self.did_initialize = False
+
+        # OpenAI client will be initialized in initialize()
+        self._openai_client: AsyncOpenAI | None = None
+        self._chat_history: List[LLMMessage] = []
+
+    async def initialize(self) -> None:
+        if self.did_initialize:
+            return
+        self._last_download = None
+        self._prior_metadata_hash = None
+
+        # Initialize OpenAI client
+        self._openai_client = AsyncOpenAI(
+            api_key=self.client_config.get("api_key"),
+            base_url=self.client_config.get("base_url"),
+            default_headers=self.client_config.get("default_headers"),
+        )
+
+        # Set up download handler
+        self.browser_manager.set_download_handler(self._download_handler)
+
+        # Initialize browser
+        await self.browser_manager.init(self.start_page)
+        self.did_initialize = True
+
+    @property
+    def _page(self) -> Page | None:
+        """Get the current page from browser manager."""
+        return self.browser_manager.page if self.browser_manager else None
+
+    @_page.setter
+    def _page(self, value):
+        if self.browser_manager:
+            self.browser_manager.page = value
+        else:
+            raise ValueError("Browser manager is not initialized. Cannot set page.")
+
+    @property
+    def context(self) -> BrowserContext | None:
+        """Get the browser context from browser manager."""
+        return self.browser_manager.context if self.browser_manager else None
+
+    @property
+    def _playwright_controller(self) -> PlaywrightController | None:
+        """Get the playwright controller from browser manager."""
+        return (
+            self.browser_manager.playwright_controller if self.browser_manager else None
+        )
+
+    async def wait_for_captcha_with_timeout(
+        self, timeout_seconds=300
+    ):  # 5 minutes default
+        """Wait for captcha to be solved with timeout"""
+        try:
+            await asyncio.wait_for(
+                self.browser_manager.wait_for_captcha_resolution(),
+                timeout=timeout_seconds,
+            )
+            return True  # Captcha solved in time
+        except asyncio.TimeoutError:
+            self.logger.warning(f"Captcha timeout after {timeout_seconds} seconds!")
+            # Force resume execution
+            self.browser_manager._captcha_event.set()
+            return False  # Captcha timed out
+
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_exponential(multiplier=5.0, min=5.0, max=60),
+        before_sleep=before_sleep_log(logging.getLogger(__name__), logging.WARNING),
+        reraise=True,
+    )
+    async def _make_model_call(
+        self,
+        history: List[LLMMessage],
+        extra_create_args: Dict[str, Any] | None = None,
+    ) -> ModelResponse:
+        """Make a model call using OpenAI client"""
+        openai_messages = [message_to_openai_format(msg) for msg in history]
+        request_params = {
+            "model": self.client_config.get("model", "gpt-4o"),
+            "messages": openai_messages,
+        }
+        if extra_create_args:
+            request_params.update(extra_create_args)
+
+        response = await self._openai_client.chat.completions.create(**request_params)
+        content = response.choices[0].message.content
+        usage = {}
+        if response.usage:
+            usage = {
+                "prompt_tokens": response.usage.prompt_tokens,
+                "completion_tokens": response.usage.completion_tokens,
+                "total_tokens": response.usage.total_tokens,
+            }
+        return ModelResponse(content=content, usage=usage)
+
+    def remove_screenshot_from_message(self, msg: List[Dict[str, Any]] | Any) -> Any:
+        """Remove the screenshot from the message content."""
+        if isinstance(msg.content, list):
+            new_content = []
+            for c in msg.content:
+                if not isinstance(c, ImageObj):
+                    new_content.append(c)
+            msg.content = new_content
+        elif isinstance(msg.content, ImageObj):
+            msg = None
+        return msg
+
+    def maybe_remove_old_screenshots(
+        self, history: List[LLMMessage], includes_current: bool = False
+    ) -> List[LLMMessage]:
+        """Remove old screenshots from the chat history. Assuming we have not yet added the current screenshot message.
+
+        Note: Original user messages (marked with is_original=True) have their TEXT preserved,
+        but their images may be removed if we exceed max_n_images. Boilerplate messages can be
+        completely removed.
+        """
+        if self.max_n_images <= 0:
+            return history
+
+        max_n_images = self.max_n_images if includes_current else self.max_n_images - 1
+        new_history: List[LLMMessage] = []
+        n_images = 0
+        for i in range(len(history) - 1, -1, -1):
+            msg = history[i]
+
+            is_original_user_message = isinstance(msg, UserMessage) and getattr(
+                msg, "is_original", False
+            )
+
+            if i == 0 and n_images >= max_n_images:
+                # First message is always the task so we keep it and remove the screenshot if necessary
+                msg = self.remove_screenshot_from_message(msg)
+                if msg is None:
+                    continue
+
+            if isinstance(msg.content, list):
+                # Check if the message contains an image. Assumes 1 image per message.
+                has_image = False
+                for c in msg.content:
+                    if isinstance(c, ImageObj):
+                        has_image = True
+                        break
+                if has_image:
+                    if n_images < max_n_images:
+                        new_history.append(msg)
+                    elif is_original_user_message:
+                        # Original user message but over limit: keep text, remove image
+                        msg = self.remove_screenshot_from_message(msg)
+                        if msg is not None:
+                            new_history.append(msg)
+                    n_images += 1
+                else:
+                    new_history.append(msg)
+            elif isinstance(msg.content, ImageObj):
+                if n_images < max_n_images:
+                    new_history.append(msg)
+                n_images += 1
+            else:
+                new_history.append(msg)
+
+        new_history = new_history[::-1]
+
+        return new_history
+
+    async def _get_scaled_screenshot(self) -> Image.Image:
+        """Get current screenshot and scale it for the model."""
+        screenshot = await self._playwright_controller.get_screenshot(self._page)
+        screenshot = Image.open(io.BytesIO(screenshot))
+        _, scaled_screenshot = self._get_system_message(screenshot)
+        return scaled_screenshot
+
+    def _get_system_message(
+        self, screenshot: ImageObj | Image.Image
+    ) -> Tuple[List[SystemMessage], Image.Image]:
+        system_prompt_info = get_computer_use_system_prompt(
+            screenshot,
+            self.MLM_PROCESSOR_IM_CFG,
+            include_input_text_key_args=self.include_input_text_key_args,
+            fn_call_template=self.fn_call_template,
+        )
+        self._mlm_width, self._mlm_height = system_prompt_info["im_size"]
+        scaled_screenshot = screenshot.resize((self._mlm_width, self._mlm_height))
+
+        system_message = []
+        for msg in system_prompt_info["conversation"]:
+            tmp_content = ""
+            for content in msg["content"]:
+                tmp_content += content["text"]
+
+            system_message.append(SystemMessage(content=tmp_content))
+
+        return system_message, scaled_screenshot
+
+    def _parse_thoughts_and_action(self, message: str) -> Tuple[str, Dict[str, Any]]:
+        try:
+            tmp = message.split("<tool_call>\n")
+            thoughts = tmp[0].strip()
+            action_text = tmp[1].split("\n</tool_call>")[0]
+            try:
+                action = json.loads(action_text)
+            except json.decoder.JSONDecodeError:
+                self.logger.error(f"Invalid action text: {action_text}")
+                action = ast.literal_eval(action_text)
+
+            return thoughts, action
+        except Exception as e:
+            self.logger.error(
+                f"Error parsing thoughts and action: {message}", exc_info=True
+            )
+            raise e
+
+    def convert_resized_coords_to_original(
+        self, coords: List[float], rsz_w: int, rsz_h: int, og_w: int, og_h: int
+    ) -> List[float]:
+        scale_x = og_w / rsz_w
+        scale_y = og_h / rsz_h
+        return [coords[0] * scale_x, coords[1] * scale_y]
+
+    def proc_coords(
+        self,
+        coords: List[float] | None,
+        im_w: int,
+        im_h: int,
+        og_im_w: int | None = None,
+        og_im_h: int | None = None,
+    ) -> List[float] | None:
+        if not coords:
+            return coords
+
+        if og_im_w is None:
+            og_im_w = im_w
+        if og_im_h is None:
+            og_im_h = im_h
+
+        tgt_x, tgt_y = coords
+        return self.convert_resized_coords_to_original(
+            [tgt_x, tgt_y], im_w, im_h, og_im_w, og_im_h
+        )
+
+    async def run(self, user_message: str) -> Tuple:
+        """Run the agent with a user message."""
+        # Initialize if not already done
+        await self.initialize()
+
+        # Ensure page is ready after initialization
+        assert self._page is not None, "Page should be initialized"
+
+        # Get initial screenshot and add user message with image to chat history
+        scaled_screenshot = await self._get_scaled_screenshot()
+
+        if self.save_screenshots:
+            await self._playwright_controller.get_screenshot(
+                self._page,
+                path=os.path.join(
+                    self.downloads_folder, f"screenshot{self._num_actions}.png"
+                ),
+            )
+
+        self._chat_history.append(
+            UserMessage(
+                content=[ImageObj.from_pil(scaled_screenshot), user_message],
+                is_original=True,
+            )
+        )
+
+        all_actions = []
+        all_observations = []
+        final_answer = "<no_answer>"
+        is_stop_action = False
+        for i in range(self.max_rounds):
+            is_first_round = i == 0
+            if not self.browser_manager._captcha_event.is_set():
+                self.logger.info("Waiting 60s for captcha to finish...")
+                captcha_solved = await self.wait_for_captcha_with_timeout(60)
+                if (
+                    not captcha_solved
+                    and not self.browser_manager._captcha_event.is_set()
+                ):
+                    raise RuntimeError(
+                        "Captcha timed out, unable to proceed with web surfing."
+                    )
+
+            function_call, raw_response = await self.generate_model_call(
+                is_first_round, scaled_screenshot if is_first_round else None
+            )
+            assert isinstance(raw_response, str)
+            all_actions.append(raw_response)
+            thoughts, action_dict = self._parse_thoughts_and_action(raw_response)
+            action_args = action_dict.get("arguments", {})
+            action = action_args["action"]
+            self.logger.info(
+                f"\nThought #{i + 1}: {thoughts}\nAction #{i + 1}: executing tool '{action}' with arguments {json.dumps(action_args)}"
+            )
+
+            (
+                is_stop_action,
+                new_screenshot,
+                action_description,
+            ) = await self.execute_action(function_call)
+            all_observations.append(action_description)
+            self.logger.info(f"Observation#{i + 1}: {action_description}")
+            if is_stop_action:
+                final_answer = thoughts
+                break
+        return final_answer, all_actions, all_observations
+
+    async def generate_model_call(
+        self, is_first_round: bool, first_screenshot: Image.Image | None = None
+    ) -> Tuple[List[FunctionCall], str]:
+        history = self.maybe_remove_old_screenshots(self._chat_history)
+
+        screenshot_for_system = first_screenshot
+        if not is_first_round:
+            # Get screenshot and add new user message for subsequent rounds
+            scaled_screenshot = await self._get_scaled_screenshot()
+            screenshot_for_system = scaled_screenshot
+
+            text_prompt = self.USER_MESSAGE
+            curr_url = await self._playwright_controller.get_page_url(self._page)
+            trimmed_url = get_trimmed_url(curr_url, max_len=self.max_url_chars)
+            text_prompt = f"Current URL: {trimmed_url}\n" + text_prompt
+
+            curr_message = UserMessage(
+                content=[ImageObj.from_pil(scaled_screenshot), text_prompt]
+            )
+            self._chat_history.append(curr_message)
+            history.append(curr_message)
+
+        # Generate system message using the screenshot
+        system_message, _ = self._get_system_message(screenshot_for_system)
+        history = system_message + history
+        response = await self._make_model_call(
+            history, extra_create_args={"temperature": 0}
+        )
+        message = response.content
+
+        self._chat_history.append(AssistantMessage(content=message))
+        thoughts, action = self._parse_thoughts_and_action(message)
+        action["arguments"]["thoughts"] = thoughts
+
+        function_call = [FunctionCall(id="dummy", **action)]
+        return function_call, message
+
+    async def execute_action(
+        self,
+        function_call: List[FunctionCall],
+    ) -> Tuple[bool, bytes, str]:
+        name = function_call[0].name
+        args = function_call[0].arguments
+        action_description = ""
+        assert self._page is not None
+        self.logger.debug(
+            WebSurferEvent(
+                source="FaraAgent",
+                url=await self._playwright_controller.get_page_url(self._page),
+                action=name,
+                arguments=args,
+                message=f"{name}( {json.dumps(args)} )",
+            )
+        )
+        if "coordinate" in args:
+            args["coordinate"] = self.proc_coords(
+                args["coordinate"],
+                self._mlm_width,
+                self._mlm_height,
+                self.viewport_width,
+                self.viewport_height,
+            )
+
+        is_stop_action = False
+
+        if args["action"] == "visit_url":
+            url = str(args["url"])
+            action_description = f"I typed '{url}' into the browser address bar."
+            # Check if the argument starts with a known protocol
+            if url.startswith(("https://", "http://", "file://", "about:")):
+                (
+                    reset_prior_metadata,
+                    reset_last_download,
+                ) = await self._playwright_controller.visit_page(self._page, url)
+            # If the argument contains a space, treat it as a search query
+            elif " " in url:
+                (
+                    reset_prior_metadata,
+                    reset_last_download,
+                ) = await self._playwright_controller.visit_page(
+                    self._page,
+                    f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH",
+                )
+            # Otherwise, prefix with https://
+            else:
+                (
+                    reset_prior_metadata,
+                    reset_last_download,
+                ) = await self._playwright_controller.visit_page(
+                    self._page, "https://" + url
+                )
+            if reset_last_download and self._last_download is not None:
+                self._last_download = None
+            if reset_prior_metadata and self._prior_metadata_hash is not None:
+                self._prior_metadata_hash = None
+        elif args["action"] == "history_back":
+            action_description = "I clicked the browser back button."
+            await self._playwright_controller.back(self._page)
+        elif args["action"] == "web_search":
+            query = args.get("query")
+            action_description = f"I typed '{query}' into the browser search bar."
+            encoded_query = quote_plus(query)
+            (
+                reset_prior_metadata,
+                reset_last_download,
+            ) = await self._playwright_controller.visit_page(
+                self._page, f"https://www.bing.com/search?q={encoded_query}&FORM=QBLH"
+            )
+            if reset_last_download and self._last_download is not None:
+                self._last_download = None
+            if reset_prior_metadata and self._prior_metadata_hash is not None:
+                self._prior_metadata_hash = None
+        elif args["action"] == "scroll":
+            pixels = int(args.get("pixels", 0))
+            if pixels > 0:
+                action_description = "I scrolled up one page in the browser."
+                await self._playwright_controller.page_up(self._page)
+            elif pixels < 0:
+                action_description = "I scrolled down one page in the browser."
+                await self._playwright_controller.page_down(self._page)
+        elif args["action"] == "keypress" or args["action"] == "key":
+            keys = args.get("keys", [])
+            action_description = f"I pressed the following keys: {keys}"
+            await self._playwright_controller.keypress(self._page, keys)
+        elif args["action"] == "hover" or args["action"] == "mouse_move":
+            if "coordinate" in args:
+                tgt_x, tgt_y = args["coordinate"]
+                await self._playwright_controller.hover_coords(self._page, tgt_x, tgt_y)
+
+        elif args["action"] == "sleep" or args["action"] == "wait":
+            duration = args.get("duration", 3.0)
+            duration = args.get("time", duration)
+            action_description = (
+                "I am waiting a short period of time before taking further action."
+            )
+            await self._playwright_controller.sleep(self._page, duration)
+        elif args["action"] == "click" or args["action"] == "left_click":
+            if "coordinate" in args:
+                tgt_x, tgt_y = args["coordinate"]
+                action_description = f"I clicked at coordinates ({tgt_x}, {tgt_y})."
+                new_page_tentative = await self._playwright_controller.click_coords(
+                    self._page, tgt_x, tgt_y
+                )
+
+            if new_page_tentative is not None:
+                self._page = new_page_tentative
+                self._prior_metadata_hash = None
+
+        elif args["action"] == "input_text" or args["action"] == "type":
+            text_value = str(args.get("text", args.get("text_value")))
+            action_description = f"I typed '{text_value}'."
+            press_enter = args.get("press_enter", True)
+            delete_existing_text = args.get("delete_existing_text", False)
+
+            if "coordinate" in args:
+                tgt_x, tgt_y = args["coordinate"]
+                new_page_tentative = await self._playwright_controller.fill_coords(
+                    self._page,
+                    tgt_x,
+                    tgt_y,
+                    text_value,
+                    press_enter=press_enter,
+                    delete_existing_text=delete_existing_text,
+                )
+                if new_page_tentative is not None:
+                    self._page = new_page_tentative
+                    self._prior_metadata_hash = None
+
+        elif args["action"] == "pause_and_memorize_fact":
+            fact = str(args.get("fact"))
+            self._facts.append(fact)
+            action_description = f"I memorized the following fact: {fact}"
+        elif args["action"] == "stop" or args["action"] == "terminate":
+            action_description = args.get("thoughts")
+            is_stop_action = True
+
+        else:
+            raise ValueError(f"Unknown tool: {args['action']}")
+
+        await self._playwright_controller.wait_for_load_state(self._page)
+        await self._playwright_controller.sleep(self._page, 3)
+
+        # Get new screenshot after action
+        self._num_actions += 1
+        if self.save_screenshots:
+            new_screenshot = await self._playwright_controller.get_screenshot(
+                self._page,
+                path=os.path.join(
+                    self.downloads_folder, f"screenshot{self._num_actions}.png"
+                ),
+            )
+        else:
+            new_screenshot = await self._playwright_controller.get_screenshot(
+                self._page
+            )
+        return is_stop_action, new_screenshot, action_description
+
+    async def close(self) -> None:
+        """
+        Close the browser and the page.
+        Should be called when the agent is no longer needed.
+        """
+        if self._page is not None:
+            self._page = None
+        await self.browser_manager.close()

fara/qwen_helpers/base_tool.py

@@ -0,0 +1,156 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/tools/base.py
+
+import json
+from abc import ABC, abstractmethod
+from typing import List, Optional, Union
+
+from .schema import ContentItem
+from .utils import has_chinese_chars, json_loads
+
+
+def is_tool_schema(obj: dict) -> bool:
+    """
+    Check if obj is a valid JSON schema describing a tool compatible with OpenAI's tool calling.
+    Example valid schema:
+    {
+      "name": "get_current_weather",
+      "description": "Get the current weather in a given location",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "location": {
+            "type": "string",
+            "description": "The city and state, e.g. San Francisco, CA"
+          },
+          "unit": {
+            "type": "string",
+            "enum": ["celsius", "fahrenheit"]
+          }
+        },
+        "required": ["location"]
+      }
+    }
+    """
+    import jsonschema
+
+    try:
+        assert set(obj.keys()) == {"name", "description", "parameters"}
+        assert isinstance(obj["name"], str)
+        assert obj["name"].strip()
+        assert isinstance(obj["description"], str)
+        assert isinstance(obj["parameters"], dict)
+
+        assert set(obj["parameters"].keys()) == {"type", "properties", "required"}
+        assert obj["parameters"]["type"] == "object"
+        assert isinstance(obj["parameters"]["properties"], dict)
+        assert isinstance(obj["parameters"]["required"], list)
+        assert set(obj["parameters"]["required"]).issubset(
+            set(obj["parameters"]["properties"].keys())
+        )
+    except AssertionError:
+        return False
+    try:
+        jsonschema.validate(instance={}, schema=obj["parameters"])
+    except jsonschema.exceptions.SchemaError:
+        return False
+    except jsonschema.exceptions.ValidationError:
+        pass
+    return True
+
+
+class BaseTool(ABC):
+    name: str = ""
+    description: str = ""
+    parameters: Union[List[dict], dict] = []
+
+    def __init__(self, cfg: Optional[dict] = None):
+        self.cfg = cfg or {}
+        if not self.name:
+            raise ValueError(
+                f"You must set {self.__class__.__name__}.name, either by @register_tool(name=...) or explicitly setting {self.__class__.__name__}.name"
+            )
+        if isinstance(self.parameters, dict):
+            if not is_tool_schema(
+                {
+                    "name": self.name,
+                    "description": self.description,
+                    "parameters": self.parameters,
+                }
+            ):
+                raise ValueError(
+                    "The parameters, when provided as a dict, must confirm to a valid openai-compatible JSON schema."
+                )
+
+    @abstractmethod
+    def call(
+        self, params: Union[str, dict], **kwargs
+    ) -> Union[str, list, dict, List[ContentItem]]:
+        """The interface for calling tools.
+
+        Each tool needs to implement this function, which is the workflow of the tool.
+
+        Args:
+            params: The parameters of func_call.
+            kwargs: Additional parameters for calling tools.
+
+        Returns:
+            The result returned by the tool, implemented in the subclass.
+        """
+        raise NotImplementedError
+
+    def _verify_json_format_args(
+        self, params: Union[str, dict], strict_json: bool = False
+    ) -> dict:
+        """Verify the parameters of the function call"""
+        if isinstance(params, str):
+            try:
+                if strict_json:
+                    params_json: dict = json.loads(params)
+                else:
+                    params_json: dict = json_loads(params)
+            except json.decoder.JSONDecodeError:
+                raise ValueError("Parameters must be formatted as a valid JSON!")
+        else:
+            params_json: dict = params
+        if isinstance(self.parameters, list):
+            for param in self.parameters:
+                if "required" in param and param["required"]:
+                    if param["name"] not in params_json:
+                        raise ValueError("Parameters %s is required!" % param["name"])
+        elif isinstance(self.parameters, dict):
+            import jsonschema
+
+            jsonschema.validate(instance=params_json, schema=self.parameters)
+        else:
+            raise ValueError
+        return params_json
+
+    @property
+    def function(self) -> dict:  # Bad naming. It should be `function_info`.
+        return {
+            # 'name_for_human': self.name_for_human,
+            "name": self.name,
+            "description": self.description,
+            "parameters": self.parameters,
+            # 'args_format': self.args_format
+        }
+
+    @property
+    def name_for_human(self) -> str:
+        return self.cfg.get("name_for_human", self.name)
+
+    @property
+    def args_format(self) -> str:
+        fmt = self.cfg.get("args_format")
+        if fmt is None:
+            if has_chinese_chars(
+                [self.name_for_human, self.name, self.description, self.parameters]
+            ):
+                fmt = "此工具的输入应为JSON对象。"
+            else:
+                fmt = "Format the arguments as a JSON object."
+        return fmt
+
+    @property
+    def file_access(self) -> bool:
+        return False

fara/qwen_helpers/fncall_prompt.py

@@ -0,0 +1,212 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/fncall_prompts/nous_fncall_prompt.py
+
+import copy
+import json
+import os
+
+from typing import List, Literal, Union
+
+from .schema import ASSISTANT, FUNCTION, SYSTEM, USER, ContentItem, Message
+
+
+class NousFnCallPrompt:
+    def __init__(self, template_name: str = "default"):
+        """Initialize NousFnCallPrompt with a specific template.
+
+        Args:
+            template_name: Name of the template to use. Options:
+                          "default", "qwen", "with_ci"
+        """
+        self.template_name = template_name
+        self.template_map = {
+            "default": FN_CALL_TEMPLATE,
+            "qwen": FN_CALL_TEMPLATE_QWEN,
+            "with_ci": FN_CALL_TEMPLATE_WITH_CI,
+        }
+
+        if template_name not in self.template_map:
+            raise ValueError(
+                f"Unknown template_name: {template_name}. "
+                f"Available options: {list(self.template_map.keys())}"
+            )
+
+    def preprocess_fncall_messages(
+        self,
+        messages: List[Message],
+        functions: List[dict],
+        lang: Literal["en", "zh"],
+        parallel_function_calls: bool = True,
+        function_choice: Union[Literal["auto"], str] = "auto",
+    ) -> List[Message]:
+        del lang  # ignored
+        del parallel_function_calls  # ignored
+        if function_choice != "auto":
+            raise NotImplementedError
+
+        ori_messages = messages
+
+        # Change function_call responses to plaintext responses:
+        messages = []
+        for msg in copy.deepcopy(ori_messages):
+            role, content, reasoning_content = (
+                msg.role,
+                msg.content,
+                msg.reasoning_content,
+            )
+            if role in (SYSTEM, USER):
+                messages.append(msg)
+            elif role == ASSISTANT:
+                content = content or []
+                fn_call = msg.function_call
+                if fn_call:
+                    if (not SPECIAL_CODE_MODE) or (
+                        CODE_TOOL_PATTERN not in fn_call.name
+                    ):
+                        fc = {
+                            "name": fn_call.name,
+                            "arguments": json.loads(fn_call.arguments),
+                        }
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n</tool_call>"
+                    else:
+                        para = json.loads(fn_call.arguments)
+                        code = para["code"]
+                        para["code"] = ""
+                        fc = {"name": fn_call.name, "arguments": para}
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n<code>\n{code}\n</code>\n</tool_call>"
+
+                    content.append(ContentItem(text=fc))
+                if messages[-1].role == ASSISTANT:
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    # TODO: Assuming there will only be one continuous reasoning_content here
+                    messages.append(
+                        Message(
+                            role=role,
+                            content=content,
+                            reasoning_content=reasoning_content,
+                        )
+                    )
+            elif role == FUNCTION:
+                assert isinstance(content, list)
+                assert len(content) == 1
+                assert content[0].text
+                fc = f"<tool_response>\n{content[0].text}\n</tool_response>"
+                content = [ContentItem(text=fc)]
+                if messages[-1].role == USER:
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    messages.append(Message(role=USER, content=content))
+            else:
+                raise TypeError
+
+        tool_descs = [{"type": "function", "function": f} for f in functions]
+        tool_names = [
+            function.get("name_for_model", function.get("name", ""))
+            for function in functions
+        ]
+        tool_descs = "\n".join([json.dumps(f, ensure_ascii=False) for f in tool_descs])
+
+        # Select template based on configuration
+        if SPECIAL_CODE_MODE and any([CODE_TOOL_PATTERN in x for x in tool_names]):
+            selected_template = FN_CALL_TEMPLATE_WITH_CI
+        else:
+            selected_template = self.template_map[self.template_name]
+
+        tool_system = selected_template.format(tool_descs=tool_descs)
+        if messages[0].role == SYSTEM:
+            messages[0].content.append(ContentItem(text="\n\n" + tool_system))
+        else:
+            messages = [
+                Message(role=SYSTEM, content=[ContentItem(text=tool_system)])
+            ] + messages
+        return messages
+
+
+FN_CALL_TEMPLATE_QWEN = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+FN_CALL_TEMPLATE = """You are a web automation agent that performs actions on websites to fulfill user requests by calling various tools.
+* You should stop execution at Critical Points. A Critical Point would be encountered in tasks like 'Checkout', 'Book', 'Purchase', 'Call', 'Email', 'Order', etc where a binding transaction/agreement would require the user's permission/personal or sensitive information (name, email, credit card, address, payment information, resume, etc) in order to complete a transaction (purchase, reservation, sign-up etc), or to communicate in a way that a human would be expected to do (call, email, apply to a job, etc).
+* Solve the task as far as you can up until a Critical Point:
+    - For example, if the task is to "call a restaurant to make a reservation", you should not actually make the call but should navigate to the restaurant's page and find the phone number.
+    - Similarly, if the task is to "order new size 12 running shoes" you should not actually place the order but should instead search for the right shoes that meet the criteria and add them to the cart.
+    - Some tasks, like answering questions, may not encounter a Critical Point at all.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+
+SPECIAL_CODE_MODE = os.getenv("SPECIAL_CODE_MODE", "false").lower() == "true"
+CODE_TOOL_PATTERN = "code_interpreter"
+FN_CALL_TEMPLATE_WITH_CI = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>
+For code parameters, use placeholders first, and then put the code within <code></code> XML tags, such as:
+<tool_call>
+{{"name": <function-name>, "arguments": {{"code": ""}}}}
+<code>
+Here is the code.
+</code>
+</tool_call>"""
+
+
+# Mainly for removing incomplete special tokens when streaming the output
+# This assumes that '<tool_call>\n{"name": "' is the special token for the NousFnCallPrompt
+def remove_incomplete_special_tokens(text: str) -> str:
+    if text in '<tool_call>\n{"name": "':
+        text = ""
+    return text
+
+
+def extract_fn(text: str):
+    fn_name, fn_args = "", ""
+    fn_name_s = '"name": "'
+    fn_name_e = '", "'
+    fn_args_s = '"arguments": '
+    i = text.find(fn_name_s)
+    k = text.find(fn_args_s)
+    if i > 0:
+        _text = text[i + len(fn_name_s) :]
+        j = _text.find(fn_name_e)
+        if j > -1:
+            fn_name = _text[:j]
+    if k > 0:
+        fn_args = text[k + len(fn_args_s) :]
+
+    if len(fn_args) > 5:
+        fn_args = fn_args[:-5]
+    else:
+        fn_args = ""
+    return fn_name, fn_args

fara/qwen_helpers/schema.py

@@ -0,0 +1,166 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/schema.py
+
+from typing import List, Literal, Optional, Tuple, Union
+
+from pydantic import BaseModel, field_validator, model_validator
+
+
+DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant."
+
+ROLE = "role"
+CONTENT = "content"
+REASONING_CONTENT = "reasoning_content"
+NAME = "name"
+
+SYSTEM = "system"
+USER = "user"
+ASSISTANT = "assistant"
+FUNCTION = "function"
+
+FILE = "file"
+IMAGE = "image"
+AUDIO = "audio"
+VIDEO = "video"
+
+
+class BaseModelCompatibleDict(BaseModel):
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def model_dump(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump(**kwargs)
+
+    def model_dump_json(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump_json(**kwargs)
+
+    def get(self, key, default=None):
+        try:
+            value = getattr(self, key)
+            if value:
+                return value
+            else:
+                return default
+        except AttributeError:
+            return default
+
+    def __str__(self):
+        return f"{self.model_dump()}"
+
+
+class FunctionCall(BaseModelCompatibleDict):
+    name: str
+    arguments: str
+
+    def __init__(self, name: str, arguments: str):
+        super().__init__(name=name, arguments=arguments)
+
+    def __repr__(self):
+        return f"FunctionCall({self.model_dump()})"
+
+
+class ContentItem(BaseModelCompatibleDict):
+    text: Optional[str] = None
+    image: Optional[str] = None
+    file: Optional[str] = None
+    audio: Optional[Union[str, dict]] = None
+    video: Optional[Union[str, list]] = None
+
+    def __init__(
+        self,
+        text: Optional[str] = None,
+        image: Optional[str] = None,
+        file: Optional[str] = None,
+        audio: Optional[Union[str, dict]] = None,
+        video: Optional[Union[str, list]] = None,
+    ):
+        super().__init__(text=text, image=image, file=file, audio=audio, video=video)
+
+    @model_validator(mode="after")
+    def check_exclusivity(self):
+        provided_fields = 0
+        if self.text is not None:
+            provided_fields += 1
+        if self.image:
+            provided_fields += 1
+        if self.file:
+            provided_fields += 1
+        if self.audio:
+            provided_fields += 1
+        if self.video:
+            provided_fields += 1
+
+        if provided_fields != 1:
+            raise ValueError(
+                "Exactly one of 'text', 'image', 'file', 'audio', or 'video' must be provided."
+            )
+        return self
+
+    def __repr__(self):
+        return f"ContentItem({self.model_dump()})"
+
+    def get_type_and_value(
+        self,
+    ) -> Tuple[Literal["text", "image", "file", "audio", "video"], str]:
+        ((t, v),) = self.model_dump().items()
+        assert t in ("text", "image", "file", "audio", "video")
+        return t, v
+
+    @property
+    def type(self) -> Literal["text", "image", "file", "audio", "video"]:
+        t, v = self.get_type_and_value()
+        return t
+
+    @property
+    def value(self) -> str:
+        t, v = self.get_type_and_value()
+        return v
+
+
+class Message(BaseModelCompatibleDict):
+    role: str
+    content: Union[str, List[ContentItem]]
+    reasoning_content: Optional[Union[str, List[ContentItem]]] = None
+    name: Optional[str] = None
+    function_call: Optional[FunctionCall] = None
+    extra: Optional[dict] = None
+
+    def __init__(
+        self,
+        role: str,
+        content: Union[str, List[ContentItem]],
+        reasoning_content: Optional[Union[str, List[ContentItem]]] = None,
+        name: Optional[str] = None,
+        function_call: Optional[FunctionCall] = None,
+        extra: Optional[dict] = None,
+        **kwargs,
+    ):
+        if content is None:
+            content = ""
+        if reasoning_content is None:
+            reasoning_content = ""
+        super().__init__(
+            role=role,
+            content=content,
+            reasoning_content=reasoning_content,
+            name=name,
+            function_call=function_call,
+            extra=extra,
+        )
+
+    def __repr__(self):
+        return f"Message({self.model_dump()})"
+
+    @field_validator("role")
+    def role_checker(cls, value: str) -> str:
+        if value not in [USER, ASSISTANT, SYSTEM, FUNCTION]:
+            raise ValueError(
+                f'{value} must be one of {",".join([USER, ASSISTANT, SYSTEM, FUNCTION])}'
+            )
+        return value

fara/qwen_helpers/utils.py

@@ -0,0 +1,24 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/utils/utils.py
+
+import json
+import re
+
+from typing import Any
+
+
+CHINESE_CHAR_RE = re.compile(r"[\u4e00-\u9fff]")
+
+
+def has_chinese_chars(data: Any) -> bool:
+    text = f"{data}"
+    return bool(CHINESE_CHAR_RE.search(text))
+
+
+def json_loads(text: str) -> dict:
+    text = text.strip("\n")
+    if text.startswith("```") and text.endswith("\n```"):
+        text = "\n".join(text.split("\n")[1:-1])
+    try:
+        return json.loads(text)
+    except json.decoder.JSONDecodeError as json_err:
+        raise json_err

fara/run_fara.py

@@ -0,0 +1,193 @@
+import asyncio
+import argparse
+import os
+from fara import FaraAgent
+from fara.browser.browser_bb import BrowserBB
+import logging
+from typing import Dict
+from pathlib import Path
+import json
+
+
+# Configure logging to only show logs from fara.fara_agent
+logging.basicConfig(
+    level=logging.CRITICAL,  
+    format="%(message)s",  
+)
+
+# Enable INFO level only for fara.fara_agent
+fara_agent_logger = logging.getLogger("fara.fara_agent")
+fara_agent_logger.setLevel(logging.INFO)
+
+# Add a handler to ensure fara_agent logs are shown
+handler = logging.StreamHandler()
+handler.setLevel(logging.INFO)
+handler.setFormatter(logging.Formatter("%(message)s"))
+fara_agent_logger.addHandler(handler)
+fara_agent_logger.propagate = False  # Don't propagate to root logger
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_ENDPOINT_CONFIG = {
+    "model": "microsoft/Fara-7B",
+    "base_url": "http://localhost:5000/v1",
+    "api_key": "not-needed",
+}
+
+
+async def run_fara_agent(
+    initial_task: str = None,
+    endpoint_config: Dict[str, str] = None,
+    start_page: str = "https://www.bing.com/",
+    headless: bool = True,
+    downloads_folder: str = None,
+    save_screenshots: bool = True,
+    max_rounds: int = 100,
+    use_browser_base: bool = False,
+):
+    # Initialize browser manager
+    print("Initializing Browser...")
+    browser_manager = BrowserBB(
+        headless=headless,
+        viewport_height=900,
+        viewport_width=1440,
+        page_script_path=None,
+        browser_channel="firefox",
+        browser_data_dir=None,
+        downloads_folder=downloads_folder,
+        to_resize_viewport=True,
+        single_tab_mode=True,
+        animate_actions=False,
+        use_browser_base=use_browser_base,
+        logger=logger,
+    )
+    print("Browser Running... Starting Fara Agent...")
+
+    agent = FaraAgent(
+        browser_manager=browser_manager,
+        client_config=endpoint_config,
+        start_page=start_page,
+        downloads_folder=downloads_folder,
+        save_screenshots=save_screenshots,
+        max_rounds=max_rounds,
+    )
+
+    try:
+        await agent.initialize()
+
+        # Interactive loop
+        task = initial_task
+        first_round = True
+
+        while True:
+            if task is None:
+                if first_round:
+                    task = input("Enter task: ").strip()
+                else:
+                    task = input(
+                        "\nEnter another task (or press Enter to exit): "
+                    ).strip()
+
+                if not task:
+                    print("Exiting...")
+                    break
+
+            print("##########################################")
+            print(f"Task: {task}")
+            print("##########################################")
+
+            try:
+                print("Running Fara...\n")
+                final_answer, all_actions, all_observations = await agent.run(task)
+                print(f"\nFinal Answer: {final_answer}")
+            except Exception as e:
+                print(f"Error occurred: {e}")
+            task = None
+            first_round = False
+
+    finally:
+        # Close the agent and browser
+        await agent.close()
+
+
+def main():
+    """CLI entry point for fara command."""
+    parser = argparse.ArgumentParser(description="Run FARA agent interactively")
+    parser.add_argument(
+        "--task",
+        type=str,
+        required=False,
+        help="Initial task for the FARA agent (optional)",
+    )
+    parser.add_argument(
+        "--start_page",
+        type=str,
+        default="https://www.bing.com/",
+        help="The starting page",
+    )
+    parser.add_argument(
+        "--headful",
+        action="store_true",
+        help="Run the browser in headful mode (show GUI, default is headless)",
+    )
+    parser.add_argument(
+        "--downloads_folder",
+        type=str,
+        default=None,
+        help="Folder to save screenshots and downloads",
+    )
+    parser.add_argument(
+        "--save_screenshots",
+        action="store_true",
+        help="Whether to save screenshots during the agent's operation",
+    )
+    parser.add_argument(
+        "--max_rounds",
+        type=int,
+        default=100,
+        help="Maximum number of rounds for the agent to run",
+    )
+    parser.add_argument(
+        "--browserbase",
+        action="store_true",
+        help="Whether to use BrowserBase for browser management",
+    )
+    parser.add_argument(
+        "--endpoint_config",
+        type=Path,
+        default=None,
+        help="Path to the endpoint configuration JSON file. By default, tries local vllm on 5000 port",
+    )
+
+    args = parser.parse_args()
+
+    if args.browserbase:
+        assert os.environ.get("BROWSERBASE_API_KEY"), (
+            "BROWSERBASE_API_KEY environment variable must be set to use browserbase"
+        )
+        assert os.environ.get("BROWSERBASE_PROJECT_ID"), (
+            "BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID environment variables must be set to use browserbase"
+        )
+
+    endpoint_config = DEFAULT_ENDPOINT_CONFIG
+    if args.endpoint_config:
+        with open(args.endpoint_config, "r") as f:
+            endpoint_config = json.load(f)
+
+    asyncio.run(
+        run_fara_agent(
+            initial_task=args.task,
+            endpoint_config=endpoint_config,
+            start_page=args.start_page,
+            headless=not args.headful,
+            downloads_folder=args.downloads_folder,
+            save_screenshots=args.save_screenshots,
+            max_rounds=args.max_rounds,
+            use_browser_base=args.browserbase,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

fara/types.py

@@ -0,0 +1,119 @@
+import io
+import base64
+from dataclasses import dataclass, field
+from typing import Any, List, Tuple, Dict
+from PIL import Image
+
+
+@dataclass
+class LLMMessage:
+    content: str | List[Dict[str, Any]]
+    source: str = "user"
+
+
+@dataclass
+class SystemMessage(LLMMessage):
+    def __init__(self, content: str, source: str = "system"):
+        self.content = content
+        self.source = source
+
+
+@dataclass
+class UserMessage(LLMMessage):
+    def __init__(
+        self,
+        content: str | List[Dict[str, Any]],
+        source: str = "user",
+        is_original: bool = False,
+    ):
+        self.content = content
+        self.source = source
+        self.is_original = is_original
+
+
+@dataclass
+class AssistantMessage(LLMMessage):
+    def __init__(self, content: str, source: str = "assistant"):
+        self.content = content
+        self.source = source
+
+
+@dataclass
+class ImageObj:
+    """Image wrapper for handling screenshots and images"""
+
+    image: Image.Image
+
+    @classmethod
+    def from_pil(cls, image: Image.Image) -> "ImageObj":
+        return cls(image=image)
+
+    def to_base64(self) -> str:
+        """Convert PIL image to base64 string"""
+        buffered = io.BytesIO()
+        self.image.save(buffered, format="PNG")
+        return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+    def resize(self, size: Tuple[int, int]) -> Image.Image:
+        """Resize the image"""
+        return self.image.resize(size)
+
+
+@dataclass
+class ModelResponse:
+    """Response from model call"""
+
+    content: str
+    usage: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class FunctionCall:
+    """Represents a function call with arguments"""
+
+    id: str
+    name: str
+    arguments: Dict[str, Any]
+
+
+def message_to_openai_format(message: LLMMessage) -> Dict[str, Any]:
+    """Convert our LLMMessage to OpenAI API format"""
+    role = (
+        "system"
+        if isinstance(message, SystemMessage)
+        else "assistant"
+        if isinstance(message, AssistantMessage)
+        else "user"
+    )
+
+    # Handle multimodal content (text + images)
+    if isinstance(message.content, list):
+        content_parts = []
+        for item in message.content:
+            if isinstance(item, ImageObj):
+                # Convert image to base64 data URL
+                base64_image = item.to_base64()
+                content_parts.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/png;base64,{base64_image}"},
+                    }
+                )
+            elif isinstance(item, str):
+                content_parts.append({"type": "text", "text": item})
+            elif isinstance(item, dict):
+                # Already in proper format
+                content_parts.append(item)
+        return {"role": role, "content": content_parts}
+    else:
+        # Simple text content
+        return {"role": role, "content": message.content}
+
+
+@dataclass
+class WebSurferEvent:
+    source: str
+    message: str
+    url: str
+    action: str | None = None
+    arguments: Dict[str, Any] | None = None

fara/utils.py

@@ -0,0 +1,9 @@
+def strip_url_query(url):
+    return url.split("?", 1)[0]
+
+
+def get_trimmed_url(url, max_len):
+    trimmed_url = strip_url_query(url)
+    if len(trimmed_url) > max_len:
+        trimmed_url = trimmed_url[:max_len] + " ..."
+    return trimmed_url

index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Fara CUA</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

nginx.conf

@@ -0,0 +1,79 @@
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /tmp/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+    keepalive_timeout 65;
+
+    # Temp paths for non-root user
+    client_body_temp_path /tmp/client_temp;
+    proxy_temp_path /tmp/proxy_temp_path;
+    fastcgi_temp_path /tmp/fastcgi_temp;
+    uwsgi_temp_path /tmp/uwsgi_temp;
+    scgi_temp_path /tmp/scgi_temp;
+
+    upstream backend {
+        server 127.0.0.1:8000;
+    }
+
+    server {
+        listen 7860;
+        server_name localhost;
+
+        root /app/static;
+        index index.html;
+
+        # API endpoints proxy to Python backend
+        location /api/ {
+            proxy_pass http://backend/api/;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 300s;
+            proxy_connect_timeout 75s;
+        }
+
+        # WebSocket endpoint
+        location /ws {
+            proxy_pass http://backend/ws;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 86400;
+            proxy_send_timeout 86400;
+        }
+
+        # Serve static files and SPA fallback
+        location / {
+            try_files $uri $uri/ /index.html;
+        }
+
+        # Cache static assets
+        location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
+            expires 1y;
+            add_header Cache-Control "public, immutable";
+        }
+    }
+}

package.json

@@ -0,0 +1,41 @@
+{
+  "name": "fara-cua-front",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "build:dev": "vite build --mode development",
+    "lint": "eslint src/ --config eslint.config.js",
+    "type-check": "tsc --noEmit --project tsconfig.json",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@emotion/react": "^11.14.0",
+    "@emotion/styled": "^11.14.1",
+    "@mui/icons-material": "^7.3.4",
+    "@mui/lab": "^7.0.1-beta.19",
+    "@mui/material": "^7.3.4",
+    "gifshot": "^0.4.5",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-router-dom": "^6.30.1",
+    "ulid": "^3.0.1",
+    "zustand": "^5.0.8"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.38.0",
+    "@types/node": "^22.16.5",
+    "@types/react": "^18.3.23",
+    "@types/react-dom": "^18.3.7",
+    "@vitejs/plugin-react-swc": "^3.11.0",
+    "autoprefixer": "^10.4.21",
+    "eslint": "^9.32.0",
+    "eslint-plugin-react-hooks": "^5.2.0",
+    "eslint-plugin-react-refresh": "^0.4.20",
+    "globals": "^15.15.0",
+    "typescript-eslint": "^8.38.0",
+    "vite": "^5.4.19"
+  }
+}

src/App.tsx

@@ -0,0 +1,35 @@
+import { CssBaseline, ThemeProvider } from '@mui/material';
+import { useMemo } from 'react';
+import { BrowserRouter, Route, Routes } from "react-router-dom";
+import { getWebSocketUrl } from './config/api';
+import { useAgentWebSocket } from './hooks/useAgentWebSocket';
+import Task from "./pages/Task";
+import Welcome from "./pages/Welcome";
+import { selectIsDarkMode, useAgentStore } from './stores/agentStore';
+import getTheme from './theme';
+
+const App = () => {
+  const isDarkMode = useAgentStore(selectIsDarkMode);
+  const theme = useMemo(() => getTheme(isDarkMode ? 'dark' : 'light'), [isDarkMode]);
+
+  // Initialize WebSocket connection at app level so it persists across route changes
+  const { stopCurrentTask } = useAgentWebSocket({ url: getWebSocketUrl() });
+
+  // Store functions in window for global access
+  (window as Window & { __stopCurrentTask?: () => void }).__stopCurrentTask = stopCurrentTask;
+
+
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <BrowserRouter>
+        <Routes>
+          <Route path="/" element={<Welcome />} />
+          <Route path="/task" element={<Task />} />
+        </Routes>
+      </BrowserRouter>
+    </ThemeProvider>
+  );
+};
+
+export default App;

src/components/ConnectionStatus.tsx

@@ -0,0 +1,55 @@
+import React from 'react';
+import { Box, Chip, keyframes } from '@mui/material';
+import CircleIcon from '@mui/icons-material/Circle';
+
+interface ConnectionStatusProps {
+  isConnected: boolean;
+}
+
+// Pulse animation for connected indicator
+const pulse = keyframes`
+  0%, 100% {
+    opacity: 1;
+  }
+  50% {
+    opacity: 0.5;
+  }
+`;
+
+export const ConnectionStatus: React.FC<ConnectionStatusProps> = ({ isConnected }) => {
+  return (
+    <Chip
+      label={isConnected ? 'Backend Online' : 'Backend Offline'}
+      deleteIcon={
+        <CircleIcon
+          sx={{
+            fontSize: 6,
+            animation: isConnected ? `${pulse} 2s ease-in-out infinite` : 'none',
+          }}
+        />
+      }
+      onDelete={() => {}} // Required for deleteIcon to show
+      size="small"
+      sx={{
+        backgroundColor: 'action.hover',
+        border: '1px solid',
+        borderColor: 'divider',
+        color: 'text.primary',
+        fontSize: '0.7rem',
+        fontWeight: 500,
+        height: 'auto',
+        '& .MuiChip-label': {
+          px: 1,
+          py: 0.5,
+        },
+        '& .MuiChip-deleteIcon': {
+          color: isConnected ? '#10b981' : '#ef4444',
+          marginRight: 0.5,
+          '&:hover': {
+            color: isConnected ? '#10b981' : '#ef4444',
+          },
+        },
+      }}
+    />
+  );
+};

src/components/Header.tsx

@@ -0,0 +1,450 @@
+import React, { useState, useEffect, useRef } from 'react';
+import { AppBar, Toolbar, Box, Typography, Chip, IconButton, CircularProgress, keyframes, Button } from '@mui/material';
+import ArrowBackIcon from '@mui/icons-material/ArrowBack';
+import LightModeOutlined from '@mui/icons-material/LightModeOutlined';
+import DarkModeOutlined from '@mui/icons-material/DarkModeOutlined';
+import CheckIcon from '@mui/icons-material/Check';
+import CloseIcon from '@mui/icons-material/Close';
+import AccessTimeIcon from '@mui/icons-material/AccessTime';
+import InputIcon from '@mui/icons-material/Input';
+import OutputIcon from '@mui/icons-material/Output';
+import SmartToyIcon from '@mui/icons-material/SmartToy';
+import FormatListNumberedIcon from '@mui/icons-material/FormatListNumbered';
+import HourglassEmptyIcon from '@mui/icons-material/HourglassEmpty';
+import StopCircleIcon from '@mui/icons-material/StopCircle';
+import { useAgentStore, selectTrace, selectError, selectIsDarkMode, selectMetadata, selectIsConnectingToE2B, selectFinalStep } from '@/stores/agentStore';
+
+interface HeaderProps {
+  isAgentProcessing: boolean;
+  onBackToHome?: () => void;
+}
+
+// Animation for the running task border - smooth oscillation (primary)
+const borderPulse = keyframes`
+  0%, 100% {
+    border-color: rgba(79, 134, 198, 0.5);
+    box-shadow: 0 0 0 0 rgba(79, 134, 198, 0.3);
+  }
+  50% {
+    border-color: rgba(79, 134, 198, 1);
+    box-shadow: 0 0 8px 2px rgba(79, 134, 198, 0.4);
+  }
+`;
+
+// Animation for the background glow (primary)
+const backgroundPulse = keyframes`
+  0%, 100% {
+    background-color: rgba(79, 134, 198, 0.08);
+  }
+  50% {
+    background-color: rgba(79, 134, 198, 0.15);
+  }
+`;
+
+// Animation for token flash - smooth glow effect
+const tokenFlash = keyframes`
+  0% {
+    filter: brightness(1);
+    text-shadow: none;
+  }
+  25% {
+    filter: brightness(1.4);
+    text-shadow: 0 0 8px rgba(79, 134, 198, 0.6);
+  }
+  100% {
+    filter: brightness(1);
+    text-shadow: none;
+  }
+`;
+
+// Animation for token icon flash
+const iconFlash = keyframes`
+  0% {
+    filter: brightness(1);
+    transform: scale(1);
+  }
+  25% {
+    filter: brightness(1.6);
+    transform: scale(1.15);
+  }
+  100% {
+    filter: brightness(1);
+    transform: scale(1);
+  }
+`;
+
+export const Header: React.FC<HeaderProps> = ({ isAgentProcessing, onBackToHome }) => {
+  const trace = useAgentStore(selectTrace);
+  const error = useAgentStore(selectError);
+  const finalStep = useAgentStore(selectFinalStep);
+  const isDarkMode = useAgentStore(selectIsDarkMode);
+  const toggleDarkMode = useAgentStore((state) => state.toggleDarkMode);
+  const metadata = useAgentStore(selectMetadata);
+  const isConnectingToE2B = useAgentStore(selectIsConnectingToE2B);
+  const [elapsedTime, setElapsedTime] = useState(0);
+  const [inputTokenFlash, setInputTokenFlash] = useState(false);
+  const [outputTokenFlash, setOutputTokenFlash] = useState(false);
+  const prevInputTokens = useRef(0);
+  const prevOutputTokens = useRef(0);
+
+  // Update elapsed time every 100ms when agent is processing
+  useEffect(() => {
+    if (isAgentProcessing && trace?.timestamp) {
+      const interval = setInterval(() => {
+        const now = new Date();
+        const startTime = new Date(trace.timestamp);
+        const elapsed = (now.getTime() - startTime.getTime()) / 1000;
+        setElapsedTime(elapsed);
+      }, 100);
+
+      return () => clearInterval(interval);
+    } else if (metadata && metadata.duration > 0) {
+      setElapsedTime(metadata.duration);
+    }
+  }, [isAgentProcessing, trace?.timestamp, metadata]);
+
+  // Detect token changes and trigger flash animation
+  useEffect(() => {
+    if (metadata) {
+      // Input tokens changed
+      if (metadata.inputTokensUsed > prevInputTokens.current && prevInputTokens.current > 0) {
+        setInputTokenFlash(true);
+        setTimeout(() => setInputTokenFlash(false), 800);
+      }
+      prevInputTokens.current = metadata.inputTokensUsed;
+
+      // Output tokens changed
+      if (metadata.outputTokensUsed > prevOutputTokens.current && prevOutputTokens.current > 0) {
+        setOutputTokenFlash(true);
+        setTimeout(() => setOutputTokenFlash(false), 800);
+      }
+      prevOutputTokens.current = metadata.outputTokensUsed;
+    }
+  }, [metadata?.inputTokensUsed, metadata?.outputTokensUsed]);
+
+  // Determine task status - Use finalStep as source of truth
+  const getTaskStatus = () => {
+    // If we have a final step, use its type
+    if (finalStep) {
+      switch (finalStep.type) {
+        case 'failure':
+          return { label: 'Task failed', color: 'error', icon: <CloseIcon sx={{ fontSize: 16, color: 'error.main' }} /> };
+        case 'stopped':
+          return { label: 'Task stopped', color: 'warning', icon: <StopCircleIcon sx={{ fontSize: 16, color: 'warning.main' }} /> };
+        case 'max_steps_reached':
+          return { label: 'Max steps reached', color: 'warning', icon: <HourglassEmptyIcon sx={{ fontSize: 16, color: 'warning.main' }} /> };
+        case 'success':
+          return { label: 'Completed', color: 'success', icon: <CheckIcon sx={{ fontSize: 16, color: 'success.main' }} /> };
+      }
+    }
+    // Otherwise check running states
+    if (isConnectingToE2B) return { label: 'Starting...', color: 'primary', icon: <CircularProgress size={16} thickness={5} sx={{ color: 'primary.main' }} /> };
+    if (isAgentProcessing || trace?.isRunning) return { label: 'Running', color: 'primary', icon: <CircularProgress size={16} thickness={5} sx={{ color: 'primary.main' }} /> };
+    return { label: 'Ready', color: 'default', icon: <CheckIcon sx={{ fontSize: 16, color: 'text.secondary' }} /> };
+  };
+
+  const taskStatus = getTaskStatus();
+
+  // Extract model name from modelId (e.g., "Qwen/Qwen3-VL-8B-Instruct" -> "Qwen3-VL-8B-Instruct")
+  const modelName = trace?.modelId?.split('/').pop() || 'Unknown Model';
+
+  // Handler for emergency stop
+  const handleEmergencyStop = () => {
+    const stopTask = (window as Window & { __stopCurrentTask?: () => void }).__stopCurrentTask;
+    if (stopTask) {
+      stopTask();
+    }
+  };
+
+  return (
+    <AppBar
+      position="static"
+      elevation={0}
+      sx={{
+        backgroundColor: 'background.paper',
+        borderBottom: '1px solid',
+        borderColor: 'divider',
+      }}
+    >
+      <Toolbar disableGutters sx={{ px: 2, py: 2.5, flexDirection: 'column', alignItems: 'stretch', gap: 0 }}>
+        {/* First row: Back button + Task info + Connection Status */}
+        <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', width: '100%', gap: 3 }}>
+          {/* Left side: Back button + Task info */}
+          <Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5, flex: 1, minWidth: 0 }}>
+            <IconButton
+              onClick={onBackToHome}
+              size="small"
+              sx={{
+                color: 'primary.main',
+                backgroundColor: 'primary.50',
+                border: '1px solid',
+                borderColor: 'primary.200',
+                cursor: 'pointer',
+                '&:hover': {
+                  backgroundColor: 'primary.100',
+                  borderColor: 'primary.main',
+                },
+              }}
+            >
+              <ArrowBackIcon fontSize="small" />
+            </IconButton>
+            <Typography
+              variant="body2"
+              sx={{
+                color: 'text.primary',
+                fontWeight: 700,
+                fontSize: '1rem',
+                overflow: 'hidden',
+                textOverflow: 'ellipsis',
+                whiteSpace: 'nowrap',
+              }}
+            >
+              {trace?.instruction || 'No task running'}
+            </Typography>
+          </Box>
+
+          {/* Right side: Emergency Stop + Dark Mode */}
+          <Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
+            {/* Emergency Stop Button - Only show when agent is processing */}
+            {isAgentProcessing && (
+              <Button
+                onClick={handleEmergencyStop}
+                variant="outlined"
+                size="small"
+                startIcon={<StopCircleIcon />}
+                sx={{
+                  color: 'error.main',
+                  borderColor: 'error.main',
+                  backgroundColor: 'transparent',
+                  fontWeight: 600,
+                  fontSize: '0.8rem',
+                  px: 1.5,
+                  py: 0.5,
+                  textTransform: 'none',
+                  '&:hover': {
+                    backgroundColor: 'error.50',
+                    borderColor: 'error.dark',
+                  },
+                }}
+              >
+                Stop
+              </Button>
+            )}
+
+            <IconButton
+              onClick={toggleDarkMode}
+              size="small"
+              sx={{
+                color: 'primary.main',
+                backgroundColor: 'primary.50',
+                border: '1px solid',
+                borderColor: 'primary.200',
+                '&:hover': {
+                  backgroundColor: 'primary.100',
+                  borderColor: 'primary.main',
+                },
+              }}
+            >
+              {isDarkMode ? <LightModeOutlined fontSize="small" /> : <DarkModeOutlined fontSize="small" />}
+            </IconButton>
+          </Box>
+        </Box>
+
+        {/* Second row: Status + Model + Metadata - Only show when we have trace data */}
+        {trace && (
+          <Box
+            sx={{
+              display: 'flex',
+              alignItems: 'center',
+              gap: 1.5,
+              pl: 5.5,
+              pr: 1,
+              pt: .5,
+              mt: .5,
+            }}
+          >
+            {/* Status Badge - Compact */}
+            <Box
+              sx={{
+                display: 'flex',
+                alignItems: 'center',
+                gap: 0.5,
+                px: 1,
+                py: 0.25,
+                borderRadius: 1,
+                backgroundColor:
+                  taskStatus.color === 'primary' ? 'primary.50' :
+                  taskStatus.color === 'success' ? 'success.50' :
+                  taskStatus.color === 'error' ? 'error.50' :
+                  taskStatus.color === 'warning' ? 'warning.50' :
+                  'action.hover',
+                border: '1px solid',
+                borderColor:
+                  taskStatus.color === 'primary' ? 'primary.main' :
+                  taskStatus.color === 'success' ? 'success.main' :
+                  taskStatus.color === 'error' ? 'error.main' :
+                  taskStatus.color === 'warning' ? 'warning.main' :
+                  'divider',
+              }}
+            >
+              {taskStatus.icon}
+              <Typography
+                variant="caption"
+                sx={{
+                  fontSize: '0.7rem',
+                  fontWeight: 700,
+                  color:
+                    taskStatus.color === 'primary' ? 'primary.main' :
+                    taskStatus.color === 'success' ? 'success.main' :
+                    taskStatus.color === 'error' ? 'error.main' :
+                    taskStatus.color === 'warning' ? 'warning.main' :
+                    'text.primary',
+                }}
+              >
+                {taskStatus.label}
+              </Typography>
+            </Box>
+
+            {/* Divider */}
+            <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+
+            {/* Model */}
+            <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+              <SmartToyIcon sx={{ fontSize: '0.85rem', color: 'primary.main' }} />
+              <Typography
+                variant="caption"
+                sx={{
+                  fontSize: '0.75rem',
+                  fontWeight: 600,
+                  color: 'text.primary',
+                }}
+              >
+                {modelName}
+              </Typography>
+            </Box>
+
+            {/* Steps Count */}
+            {metadata && (
+              <>
+                <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+                <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+                  <Typography
+                    variant="caption"
+                    sx={{
+                      fontSize: '0.75rem',
+                      fontWeight: 700,
+                      color: 'text.primary',
+                      mr: 0.5,
+                    }}
+                  >
+                    {metadata.numberOfSteps}
+                  </Typography>
+                  <Typography
+                    variant="caption"
+                    sx={{
+                      fontSize: '0.7rem',
+                      fontWeight: 400,
+                      color: 'text.secondary',
+                    }}
+                  >
+                    {metadata.numberOfSteps === 1 ? 'Step' : 'Steps'}
+                  </Typography>
+                </Box>
+              </>
+            )}
+
+            {/* Time */}
+            {(isAgentProcessing || metadata) && (
+              <>
+                <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+                <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+                  <AccessTimeIcon sx={{ fontSize: '0.85rem', color: 'primary.main' }} />
+                  <Typography
+                    variant="caption"
+                    sx={{
+                      fontSize: '0.75rem',
+                      fontWeight: 700,
+                      color: 'text.primary',
+                      minWidth: '45px',
+                      textAlign: 'left',
+                    }}
+                  >
+                    {elapsedTime.toFixed(1)}s
+                  </Typography>
+                </Box>
+              </>
+            )}
+
+            {/* Input Tokens */}
+            {metadata && metadata.inputTokensUsed > 0 && (
+              <>
+                <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+                <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+                  <InputIcon
+                    sx={{
+                      fontSize: '0.85rem',
+                      color: 'primary.main',
+                      transition: 'all 0.2s ease',
+                      animation: inputTokenFlash ? `${iconFlash} 0.8s ease-out` : 'none',
+                    }}
+                  />
+                  <Box
+                    sx={{
+                      transition: 'all 0.2s ease',
+                      animation: inputTokenFlash ? `${tokenFlash} 0.8s ease-out` : 'none',
+                    }}
+                  >
+                    <Typography
+                      variant="caption"
+                      sx={{
+                        fontSize: '0.75rem',
+                        fontWeight: 700,
+                        color: 'text.primary',
+                      }}
+                    >
+                      {metadata.inputTokensUsed.toLocaleString()}
+                    </Typography>
+                  </Box>
+                </Box>
+              </>
+            )}
+
+            {/* Output Tokens */}
+            {metadata && metadata.outputTokensUsed > 0 && (
+              <>
+                <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+                <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+                  <OutputIcon
+                    sx={{
+                      fontSize: '0.85rem',
+                      color: 'primary.main',
+                      transition: 'all 0.2s ease',
+                      animation: outputTokenFlash ? `${iconFlash} 0.8s ease-out` : 'none',
+                    }}
+                  />
+                  <Box
+                    sx={{
+                      transition: 'all 0.2s ease',
+                      animation: outputTokenFlash ? `${tokenFlash} 0.8s ease-out` : 'none',
+                    }}
+                  >
+                    <Typography
+                      variant="caption"
+                      sx={{
+                        fontSize: '0.75rem',
+                        fontWeight: 700,
+                        color: 'text.primary',
+                      }}
+                    >
+                      {metadata.outputTokensUsed.toLocaleString()}
+                    </Typography>
+                  </Box>
+                </Box>
+              </>
+            )}
+          </Box>
+        )}
+      </Toolbar>
+    </AppBar>
+  );
+};

src/components/ProcessingIndicator.tsx

@@ -0,0 +1,31 @@
+import React from 'react';
+import { Box, CircularProgress, Typography } from '@mui/material';
+
+interface ProcessingIndicatorProps {
+  isAgentProcessing: boolean;
+}
+
+export const ProcessingIndicator: React.FC<ProcessingIndicatorProps> = ({ isAgentProcessing }) => {
+  if (!isAgentProcessing) return null;
+
+  return (
+    <Box
+      sx={{
+        display: 'flex',
+        alignItems: 'center',
+        gap: 2,
+        backgroundColor: 'rgba(255, 255, 255, 0.9)',
+        px: 2,
+        py: 1,
+        borderRadius: 2,
+        backdropFilter: 'blur(10px)',
+        border: '1px solid rgba(0, 0, 0, 0.1)',
+      }}
+    >
+      <CircularProgress size={20} thickness={4} />
+      <Typography variant="body2" sx={{ fontWeight: 600, color: 'text.primary' }}>
+        Agent is running...
+      </Typography>
+    </Box>
+  );
+};

src/components/WelcomeScreen.tsx

@@ -0,0 +1,521 @@
+import { fetchAvailableModels, generateRandomQuestion } from '@/services/api';
+import { selectAvailableModels, selectIsDarkMode, selectIsLoadingModels, selectSelectedModelId, useAgentStore } from '@/stores/agentStore';
+import DarkModeOutlined from '@mui/icons-material/DarkModeOutlined';
+import LightModeOutlined from '@mui/icons-material/LightModeOutlined';
+import SendIcon from '@mui/icons-material/Send';
+import ShuffleIcon from '@mui/icons-material/Shuffle';
+import SmartToyIcon from '@mui/icons-material/SmartToy';
+import { Box, Button, CircularProgress, Container, FormControl, IconButton, InputLabel, MenuItem, Paper, Select, TextField, Typography } from '@mui/material';
+import React, { useEffect, useRef, useState } from 'react';
+
+interface WelcomeScreenProps {
+  onStartTask: (instruction: string, modelId: string) => void;
+  isConnected: boolean;
+}
+
+export const WelcomeScreen: React.FC<WelcomeScreenProps> = ({ onStartTask, isConnected }) => {
+  const [customTask, setCustomTask] = useState('');
+  const [isTyping, setIsTyping] = useState(false);
+  const [isGeneratingQuestion, setIsGeneratingQuestion] = useState(false);
+  const typingIntervalRef = useRef<NodeJS.Timeout | null>(null);
+
+  const isDarkMode = useAgentStore(selectIsDarkMode);
+  const toggleDarkMode = useAgentStore((state) => state.toggleDarkMode);
+  const selectedModelId = useAgentStore(selectSelectedModelId);
+  const setSelectedModelId = useAgentStore((state) => state.setSelectedModelId);
+  const availableModels = useAgentStore(selectAvailableModels);
+  const isLoadingModels = useAgentStore(selectIsLoadingModels);
+  const setAvailableModels = useAgentStore((state) => state.setAvailableModels);
+  const setIsLoadingModels = useAgentStore((state) => state.setIsLoadingModels);
+
+  // Load available models on mount
+  useEffect(() => {
+    const loadModels = async () => {
+      setIsLoadingModels(true);
+      try {
+        const models = await fetchAvailableModels();
+        setAvailableModels(models);
+
+        // Set first model as default if current selection is not in the list
+        if (models.length > 0 && !models.includes(selectedModelId)) {
+          setSelectedModelId(models[0]);
+        }
+      } catch (error) {
+        console.error('Failed to load models:', error);
+        // Fallback to empty array on error
+        setAvailableModels([]);
+      } finally {
+        setIsLoadingModels(false);
+      }
+    };
+
+    loadModels();
+  }, []); // eslint-disable-line react-hooks/exhaustive-deps
+
+  // Clean up typing interval on unmount
+  useEffect(() => {
+    return () => {
+      if (typingIntervalRef.current) {
+        clearInterval(typingIntervalRef.current);
+      }
+    };
+  }, []);
+
+  const handleWriteRandomTask = async () => {
+    // Clear any existing typing interval
+    if (typingIntervalRef.current) {
+      clearInterval(typingIntervalRef.current);
+      typingIntervalRef.current = null;
+    }
+
+    setIsGeneratingQuestion(true);
+    try {
+      const randomTask = await generateRandomQuestion();
+
+      // Clear current text
+      setCustomTask('');
+      setIsTyping(true);
+
+      // Type effect
+      let currentIndex = 0;
+      typingIntervalRef.current = setInterval(() => {
+        if (currentIndex < randomTask.length) {
+          setCustomTask(randomTask.substring(0, currentIndex + 1));
+          currentIndex++;
+        } else {
+          if (typingIntervalRef.current) {
+            clearInterval(typingIntervalRef.current);
+            typingIntervalRef.current = null;
+          }
+          setIsTyping(false);
+        }
+      }, 10); // 10ms per character
+    } catch (error) {
+      console.error('Failed to generate question:', error);
+      setIsTyping(false);
+    } finally {
+      setIsGeneratingQuestion(false);
+    }
+  };
+
+  const handleCustomTask = () => {
+    if (customTask.trim() && !isTyping) {
+      onStartTask(customTask.trim(), selectedModelId);
+    }
+  };
+
+  return (
+    <>
+      {/* Dark Mode Toggle - Top Right (Absolute to viewport) */}
+      <Box sx={{ position: 'absolute', top: 24, right: 24, zIndex: 1000 }}>
+        <IconButton
+          onClick={toggleDarkMode}
+          size="medium"
+          sx={{
+            color: 'text.primary',
+            backgroundColor: 'background.paper',
+            border: '1px solid',
+            borderColor: 'divider',
+            '&:hover': {
+              backgroundColor: 'action.hover',
+              borderColor: 'primary.main',
+            },
+          }}
+        >
+          {isDarkMode ? <LightModeOutlined /> : <DarkModeOutlined />}
+        </IconButton>
+      </Box>
+
+      <Container
+        maxWidth="md"
+        sx={{
+          display: 'flex',
+          flexDirection: 'column',
+          alignItems: 'center',
+          justifyContent: 'center',
+          minHeight: '100vh',
+          textAlign: 'center',
+          py: 8,
+        }}
+      >
+        {/* Title */}
+        <Typography
+          variant="h2"
+          sx={{
+            fontWeight: 800,
+            mb: 1,
+            color: 'text.primary',
+          }}
+        >
+          FARA Agent
+        </Typography>
+
+        {/* Powered by Microsoft */}
+        <Box
+          sx={{
+            display: 'flex',
+            alignItems: 'center',
+            gap: 1,
+            mb: 2,
+            flexWrap: 'wrap',
+            justifyContent: 'center',
+          }}
+        >
+          <Typography
+            variant="body2"
+            sx={{
+              color: 'text.secondary',
+              fontWeight: 500,
+            }}
+          >
+            Powered by
+          </Typography>
+
+          {/* Microsoft Fara link */}
+          <Box
+            component="a"
+            href="https://github.com/microsoft/fara"
+            target="_blank"
+            rel="noopener noreferrer"
+            sx={{
+              display: 'flex',
+              alignItems: 'center',
+              gap: 0.75,
+              textDecoration: 'none',
+              transition: 'all 0.2s ease',
+              '&:hover': {
+                '& .fara-text': {
+                  textDecoration: 'underline',
+                },
+              },
+            }}
+          >
+            <Typography
+              className="fara-text"
+              sx={{
+                color: 'primary.main',
+                fontWeight: 700,
+                fontSize: '1rem',
+              }}
+            >
+              Microsoft Fara-7B
+            </Typography>
+          </Box>
+
+          {/* Separator */}
+          <Typography
+            variant="body2"
+            sx={{
+              color: 'text.secondary',
+              mx: 0.5,
+            }}
+          >
+            &
+          </Typography>
+
+          {/* Modal link */}
+          <Box
+            component="a"
+            href="https://modal.com/"
+            target="_blank"
+            rel="noopener noreferrer"
+            sx={{
+              display: 'flex',
+              alignItems: 'center',
+              gap: 0.75,
+              textDecoration: 'none',
+              transition: 'all 0.2s ease',
+              '&:hover': {
+                '& .modal-text': {
+                  textDecoration: 'underline',
+                },
+              },
+            }}
+          >
+            <Typography
+              className="modal-text"
+              sx={{
+                color: 'primary.main',
+                fontWeight: 700,
+                fontSize: '1rem',
+              }}
+            >
+              Modal
+            </Typography>
+          </Box>
+        </Box>
+
+        {/* Subtitle */}
+        <Typography
+          variant="h6"
+          sx={{
+            color: 'text.secondary',
+            fontWeight: 500,
+            mb: 1,
+          }}
+        >
+          AI-Powered Browser Automation
+        </Typography>
+
+        {/* Description */}
+        <Typography
+          variant="body1"
+          sx={{
+            color: 'text.secondary',
+            maxWidth: '650px',
+            mb: 3,
+            lineHeight: 1.7,
+          }}
+        >
+          Experience the future of AI automation as FARA operates your browser in real time to complete complex on-screen tasks.
+          Built with{' '}
+          <Box
+            component="a"
+            href="https://github.com/microsoft/fara"
+            target="_blank"
+            rel="noopener noreferrer"
+            sx={{
+              color: 'primary.main',
+              textDecoration: 'none',
+              fontWeight: 700,
+              '&:hover': {
+                textDecoration: 'underline',
+              },
+            }}
+          >
+            Microsoft Fara-7B
+          </Box>
+          , a vision-language model specifically designed for <strong>computer use and GUI automation</strong>.
+        </Typography>
+
+        {/* Task Input Section */}
+        <Paper
+          elevation={0}
+          sx={{
+            maxWidth: '725px',
+            width: '100%',
+            p: 2.5,
+            border: '2px solid',
+            borderColor: isConnected ? 'primary.main' : 'divider',
+            borderRadius: 2,
+            backgroundColor: 'background.paper',
+            transition: 'all 0.2s ease',
+            '&:hover': isConnected ? {
+              borderColor: 'primary.dark',
+              boxShadow: (theme) => `0 4px 16px ${theme.palette.mode === 'dark' ? 'rgba(79, 134, 198, 0.3)' : 'rgba(79, 134, 198, 0.15)'}`,
+            } : {},
+          }}
+        >
+          {/* Input Field */}
+          <TextField
+            fullWidth
+            placeholder="Describe your task here..."
+            value={customTask}
+            onChange={(e) => setCustomTask(e.target.value)}
+            onKeyPress={(e) => {
+              if (e.key === 'Enter' && !e.shiftKey && isConnected && customTask.trim() && !isTyping) {
+                handleCustomTask();
+              }
+            }}
+            disabled={!isConnected || isTyping}
+            multiline
+            rows={3}
+            sx={{
+              mb: 2,
+              '& .MuiOutlinedInput-root': {
+                borderRadius: 1.5,
+                backgroundColor: 'action.hover',
+                color: 'text.primary',
+                '& fieldset': {
+                  borderColor: 'divider',
+                },
+                '&:hover fieldset': {
+                  borderColor: 'text.secondary',
+                },
+                '&.Mui-focused fieldset': {
+                  borderColor: 'primary.main',
+                  borderWidth: '2px',
+                },
+              },
+              '& .MuiInputBase-input': {
+                color: (theme) => theme.palette.mode === 'dark' ? '#FFFFFF !important' : '#000000 !important',
+                fontWeight: 500,
+                WebkitTextFillColor: (theme) => theme.palette.mode === 'dark' ? '#FFFFFF !important' : '#000000 !important',
+              },
+              '& .MuiInputBase-input.Mui-disabled': {
+                color: (theme) => theme.palette.mode === 'dark' ? '#FFFFFF !important' : '#000000 !important',
+                WebkitTextFillColor: (theme) => theme.palette.mode === 'dark' ? '#FFFFFF !important' : '#000000 !important',
+              },
+              '& .MuiInputBase-input::placeholder': {
+                color: 'text.secondary',
+                opacity: 0.7,
+              },
+            }}
+          />
+
+          {/* Model Selection + Buttons Row */}
+          <Box sx={{ display: 'flex', gap: 1.5, alignItems: 'center', justifyContent: 'space-between' }}>
+            {/* Model Select */}
+            <FormControl size="small" sx={{ minWidth: 240 }}>
+              <InputLabel id="model-select-label">Model</InputLabel>
+              <Select
+                labelId="model-select-label"
+                value={availableModels.length > 0 && availableModels.includes(selectedModelId) ? selectedModelId : ''}
+                label="Model"
+                onChange={(e) => setSelectedModelId(e.target.value)}
+                disabled={!isConnected || isTyping || isLoadingModels}
+                sx={{
+                  borderRadius: 1.5,
+                  '& .MuiOutlinedInput-notchedOutline': {
+                    borderWidth: 2,
+                  },
+                }}
+              >
+                {isLoadingModels ? (
+                  <MenuItem disabled>
+                    <Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
+                      <CircularProgress size={16} />
+                      <Typography variant="body2">Loading models...</Typography>
+                    </Box>
+                  </MenuItem>
+                ) : availableModels.length === 0 ? (
+                  <MenuItem disabled>
+                    <Typography variant="body2" sx={{ color: 'error.main' }}>
+                      No models available
+                    </Typography>
+                  </MenuItem>
+                ) : (
+                  availableModels.map((modelId) => (
+                    <MenuItem key={modelId} value={modelId}>
+                      <Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
+                        <SmartToyIcon sx={{ fontSize: '0.9rem', color: 'primary.main' }} />
+                        <Typography variant="body2" sx={{ fontWeight: 600, fontSize: '0.875rem' }}>
+                          {modelId.split('/').pop()}
+                        </Typography>
+                      </Box>
+                    </MenuItem>
+                  ))
+                )}
+              </Select>
+            </FormControl>
+
+            {/* Buttons on the right */}
+            <Box sx={{ display: 'flex', gap: 1.5 }}>
+              <Button
+                variant="outlined"
+                onClick={handleWriteRandomTask}
+                disabled={!isConnected || isTyping || isGeneratingQuestion}
+                startIcon={isGeneratingQuestion ? <CircularProgress size={16} /> : <ShuffleIcon />}
+                sx={{
+                  borderRadius: 1.5,
+                  textTransform: 'none',
+                  fontWeight: 600,
+                  borderWidth: 2,
+                  px: 3,
+                  '&:hover': {
+                    borderWidth: 2,
+                  },
+                }}
+              >
+                {isGeneratingQuestion ? 'Generating...' : isTyping ? 'Writing...' : 'Write random task'}
+              </Button>
+
+              <Button
+                variant="contained"
+                onClick={handleCustomTask}
+                disabled={!isConnected || !customTask.trim() || isTyping}
+                sx={{
+                  borderRadius: 1.5,
+                  textTransform: 'none',
+                  fontWeight: 600,
+                  px: 4,
+                  background: 'linear-gradient(135deg, #4F86C6 0%, #2B5C94 100%)',
+                }}
+                endIcon={<SendIcon />}
+              >
+                Run Task
+              </Button>
+            </Box>
+          </Box>
+        </Paper>
+
+        {/* Research Notice */}
+        <Typography
+          variant="body2"
+          sx={{
+            color: 'text.secondary',
+            maxWidth: '700px',
+            mt: 3,
+            mb: 2,
+            lineHeight: 1.6,
+            fontStyle: 'italic',
+            opacity: 0.8,
+            textAlign: 'center',
+          }}
+        >
+          This is a demo of the FARA computer use agent. The agent will browse the web on your behalf.
+          Cold starts may take upto 1 minute for the first prompt after which each step should take 5-10s.
+          <strong> Please do not enter any personal or sensitive information.</strong>
+          {' '}Task logs will be stored for research purposes.
+        </Typography>
+
+        {/* Credits */}
+        <Typography
+          variant="caption"
+          sx={{
+            color: 'text.secondary',
+            mt: 1,
+            opacity: 0.7,
+            textAlign: 'center',
+          }}
+        >
+          Frontend based on{' '}
+          <Box
+            component="a"
+            href="https://huggingface.co/spaces/smolagents/computer-use-agent"
+            target="_blank"
+            rel="noopener noreferrer"
+            sx={{
+              color: 'primary.main',
+              textDecoration: 'none',
+              '&:hover': {
+                textDecoration: 'underline',
+              },
+            }}
+          >
+            HuggingFace smolagents/computer-use-agent
+          </Box>
+        </Typography>
+
+        {/* Connection status hint */}
+        {!isConnected && (
+          <Typography
+            variant="caption"
+            sx={{
+              mt: 2,
+              color: 'text.secondary',
+              display: 'flex',
+              alignItems: 'center',
+              gap: 1,
+            }}
+          >
+            <Box
+              sx={{
+                width: 8,
+                height: 8,
+                borderRadius: '50%',
+                backgroundColor: 'warning.main',
+                animation: 'pulse 2s ease-in-out infinite',
+                '@keyframes pulse': {
+                  '0%, 100%': { opacity: 1 },
+                  '50%': { opacity: 0.5 },
+                },
+              }}
+            />
+            Make sure the backend is running on port 8000
+          </Typography>
+        )}
+      </Container>
+    </>
+  );
+};

src/components/index.ts

@@ -0,0 +1,14 @@
+// General components
+export { Header } from './Header';
+export { ConnectionStatus } from './ConnectionStatus';
+export { ProcessingIndicator } from './ProcessingIndicator';
+export { WelcomeScreen } from './WelcomeScreen';
+
+// Sandbox components
+export { SandboxViewer, CompletionView, DownloadGifButton, DownloadJsonButton } from './sandbox';
+
+// Timeline components
+export { Timeline } from './timeline';
+
+// Steps components
+export { StepsList, StepCard, FinalStepCard, ThinkingStepCard, ConnectionStepCard } from './steps';

src/components/sandbox/SandboxViewer.tsx

@@ -0,0 +1,400 @@
+import { useGifGenerator } from '@/hooks/useGifGenerator';
+import { useJsonExporter } from '@/hooks/useJsonExporter';
+import { selectError, selectFinalStep, selectSteps, selectTrace, useAgentStore } from '@/stores/agentStore';
+import { AgentStep, AgentTraceMetadata } from '@/types/agent';
+import ImageIcon from '@mui/icons-material/Image';
+import MonitorIcon from '@mui/icons-material/Monitor';
+import PlayCircleIcon from '@mui/icons-material/PlayCircle';
+import { Box, Button, CircularProgress, keyframes, Typography } from '@mui/material';
+import React from 'react';
+import { useNavigate } from 'react-router-dom';
+import { CompletionView } from './completionview/CompletionView';
+
+// Animation for live indicator
+const livePulse = keyframes`
+  0%, 100% {
+    opacity: 1;
+    transform: scale(1);
+  }
+  50% {
+    opacity: 0.7;
+    transform: scale(1.2);
+  }
+`;
+
+interface SandboxViewerProps {
+  vncUrl: string;
+  isAgentProcessing?: boolean;
+  metadata?: AgentTraceMetadata;
+  traceStartTime?: Date;
+  selectedStep?: AgentStep | null; // The step to display in time-travel mode
+  isRunning?: boolean; // Is the agent currently running
+}
+
+export const SandboxViewer: React.FC<SandboxViewerProps> = ({
+  vncUrl,
+  isAgentProcessing = false,
+  metadata,
+  traceStartTime,
+  selectedStep,
+  isRunning = false
+}) => {
+  const navigate = useNavigate();
+  const error = useAgentStore(selectError);
+  const finalStep = useAgentStore(selectFinalStep);
+  const steps = useAgentStore(selectSteps);
+  const trace = useAgentStore(selectTrace);
+  const resetAgent = useAgentStore((state) => state.resetAgent);
+  const setSelectedStepIndex = useAgentStore((state) => state.setSelectedStepIndex);
+
+  // Get the latest screenshot from steps (for non-VNC mode)
+  const latestScreenshot = steps && steps.length > 0 ? steps[steps.length - 1].image : null;
+
+  // Hook to generate GIF
+  const { isGenerating, error: gifError, generateAndDownloadGif } = useGifGenerator({
+    steps: steps || [],
+    traceId: finalStep?.metadata.traceId || '',
+  });
+
+  // Hook to export JSON
+  const { downloadTraceAsJson } = useJsonExporter({
+    trace,
+    steps: steps || [],
+    metadata: finalStep?.metadata || metadata,
+    finalStep,
+  });
+
+  // Extract final_answer from the last step, or fallback to last thought
+  const getFinalAnswer = (): string | null => {
+    console.log('🔍 getFinalAnswer - steps:', steps);
+    if (!steps || steps.length === 0) {
+      console.log('❌ No steps available');
+      return null;
+    }
+
+    // Try to find final_answer in any step (iterate backwards)
+    for (let i = steps.length - 1; i >= 0; i--) {
+      const step = steps[i];
+
+      if (step.actions && Array.isArray(step.actions)) {
+        const finalAnswerAction = step.actions.find(
+          (action) => action.function_name === 'final_answer'
+        );
+
+        if (finalAnswerAction) {
+          // Handle both named parameter and positional argument
+          const result = finalAnswerAction?.parameters?.answer || finalAnswerAction?.parameters?.arg_0 || null;
+          console.log('✅ Final answer found in step', i + 1, ':', result);
+          return result;
+        }
+      }
+    }
+
+    console.log('🔍 No final_answer found, looking for last thought...');
+
+    // Fallback: find the last step with a thought (iterate backwards)
+    for (let i = steps.length - 1; i >= 0; i--) {
+      const step = steps[i];
+      if (step.thought) {
+        console.log('📝 Using thought from step', i + 1, 'as fallback:', step.thought);
+        return step.thought;
+      }
+    }
+
+    console.log('❌ No final answer or thought found in any step');
+    return null;
+  };
+
+  const finalAnswer = getFinalAnswer();
+  console.log('🎯 Final answer to display:', finalAnswer);
+
+  // Determine if we should show success/fail status
+  const showStatus = !isRunning && !selectedStep && finalStep;
+
+  // Handler to go back to home
+  const handleBackToHome = () => {
+    // Reset frontend state
+    useAgentStore.getState().resetAgent();
+
+    // Reload the page to reconnect websocket
+    window.location.href = '/';
+  };
+
+  // Handler to go back to live mode
+  const handleGoLive = () => {
+    setSelectedStepIndex(null);
+  };
+
+  return (
+    <Box
+      sx={{
+        flex: '1 1 auto',
+        display: 'flex',
+        flexDirection: 'column',
+        position: 'relative',
+        border: '1px solid',
+        borderColor: showStatus
+          ? ((finalStep?.type === 'failure' || finalStep?.type === 'sandbox_timeout') ? 'error.main' : 'success.main')
+          : ((vncUrl || isAgentProcessing) && !selectedStep && !showStatus ? 'primary.main' : 'divider'),
+        borderRadius: '12px',
+        backgroundColor: 'background.paper',
+        transition: 'border 0.3s ease',
+        overflow: 'hidden',
+      }}
+    >
+      {/* Live Badge or Go Live Button */}
+      {vncUrl && !showStatus && (
+        <>
+          {!selectedStep ? (
+            // Live Badge when in live mode
+            <Box
+              sx={{
+                position: 'absolute',
+                top: 12,
+                right: 12,
+                zIndex: 10,
+                display: 'flex',
+                alignItems: 'center',
+                gap: 1,
+                px: 2,
+                py: 1,
+                backgroundColor: (theme) =>
+                  theme.palette.mode === 'dark'
+                    ? 'rgba(0, 0, 0, 0.7)'
+                    : 'rgba(255, 255, 255, 0.9)',
+                backdropFilter: 'blur(8px)',
+                borderRadius: 0.75,
+                border: '1px solid',
+                borderColor: 'primary.main',
+                boxShadow: (theme) =>
+                  theme.palette.mode === 'dark'
+                    ? '0 2px 8px rgba(0, 0, 0, 0.4)'
+                    : '0 2px 8px rgba(0, 0, 0, 0.1)',
+              }}
+            >
+              <Box
+                sx={{
+                  width: 10,
+                  height: 10,
+                  borderRadius: '50%',
+                  backgroundColor: 'error.main',
+                  animation: `${livePulse} 2s ease-in-out infinite`,
+                }}
+              />
+              <Typography
+                variant="caption"
+                sx={{
+                  fontSize: '0.8rem',
+                  fontWeight: 700,
+                  color: 'text.primary',
+                  textTransform: 'uppercase',
+                  letterSpacing: '0.5px',
+                }}
+              >
+                Live
+              </Typography>
+            </Box>
+          ) : (
+            // Go Live Button when viewing a specific step
+            <Button
+              onClick={handleGoLive}
+              startIcon={<PlayCircleIcon sx={{ fontSize: 20 }} />}
+              sx={{
+                position: 'absolute',
+                top: 12,
+                right: 12,
+                zIndex: 10,
+                px: 2,
+                py: 1,
+                backgroundColor: (theme) =>
+                  theme.palette.mode === 'dark'
+                    ? 'rgba(0, 0, 0, 0.7)'
+                    : 'rgba(255, 255, 255, 0.9)',
+                backdropFilter: 'blur(8px)',
+                borderRadius: 0.75,
+                border: '1px solid',
+                borderColor: 'primary.main',
+                boxShadow: (theme) =>
+                  theme.palette.mode === 'dark'
+                    ? '0 2px 8px rgba(0, 0, 0, 0.4)'
+                    : '0 2px 8px rgba(0, 0, 0, 0.1)',
+                fontSize: '0.8rem',
+                fontWeight: 700,
+                textTransform: 'uppercase',
+                letterSpacing: '0.5px',
+                color: 'primary.main',
+                '&:hover': {
+                  backgroundColor: (theme) =>
+                    theme.palette.mode === 'dark'
+                      ? 'rgba(0, 0, 0, 0.85)'
+                      : 'rgba(255, 255, 255, 1)',
+                  borderColor: 'primary.dark',
+                },
+              }}
+            >
+              Go Live
+            </Button>
+          )}
+        </>
+      )}
+
+      <Box
+        sx={{
+          flex: 1,
+          minHeight: 0,
+          display: 'flex',
+          alignItems: 'center',
+          justifyContent: 'center',
+        }}
+      >
+        {showStatus && finalStep ? (
+          // Show success/fail status when agent has completed
+          <CompletionView
+            finalStep={finalStep}
+            trace={trace}
+            steps={steps}
+            metadata={metadata}
+            finalAnswer={finalAnswer}
+            isGenerating={isGenerating}
+            gifError={gifError}
+            onGenerateGif={generateAndDownloadGif}
+            onDownloadJson={downloadTraceAsJson}
+            onBackToHome={handleBackToHome}
+          />
+        ) : selectedStep ? (
+          // Time-travel mode: Show screenshot of selected step
+          <Box
+            sx={{
+              width: '100%',
+              height: '100%',
+              display: 'flex',
+              alignItems: 'center',
+              justifyContent: 'center',
+              overflow: 'auto',
+              backgroundColor: 'black',
+              position: 'relative',
+            }}
+          >
+            {selectedStep.image ? (
+              <img
+                src={selectedStep.image}
+                alt="Step screenshot"
+                style={{
+                  maxWidth: '100%',
+                  maxHeight: '100%',
+                  objectFit: 'contain',
+                }}
+              />
+            ) : (
+              <Box
+                sx={{
+                  textAlign: 'center',
+                  p: 4,
+                  color: 'text.secondary',
+                  width: '100%',
+                  height: '100%',
+                  display: 'flex',
+                  flexDirection: 'column',
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                }}
+              >
+                <ImageIcon sx={{ fontSize: 48, mb: 2, opacity: 0.5 }} />
+                <Typography variant="body2" sx={{ fontWeight: 600, mb: 0.5, fontSize: '0.875rem', color: 'text.primary' }}>
+                  No screenshot available
+                </Typography>
+                <Typography variant="caption" sx={{ fontSize: '0.75rem', color: 'text.secondary' }}>
+                  This step doesn't have a screenshot
+                </Typography>
+              </Box>
+            )}
+          </Box>
+        ) : vncUrl ? (
+          // Live mode: Show VNC stream
+          <iframe
+            src={vncUrl}
+            style={{ width: '100%', height: '100%', border: 'none' }}
+            title="OS Stream"
+            lang="en"
+          />
+        ) : latestScreenshot ? (
+          // Live mode without VNC: Show latest screenshot
+          <Box
+            sx={{
+              width: '100%',
+              height: '100%',
+              display: 'flex',
+              alignItems: 'center',
+              justifyContent: 'center',
+              overflow: 'auto',
+              backgroundColor: 'black',
+              position: 'relative',
+            }}
+          >
+            <img
+              src={latestScreenshot}
+              alt="Latest screenshot"
+              style={{
+                maxWidth: '100%',
+                maxHeight: '100%',
+                objectFit: 'contain',
+              }}
+            />
+          </Box>
+        ) : isAgentProcessing ? (
+          // Loading state
+          <Box
+            sx={{
+              textAlign: 'center',
+              p: 4,
+              color: 'text.secondary',
+              width: '100%',
+              height: '100%',
+              display: 'flex',
+              flexDirection: 'column',
+              alignItems: 'center',
+              justifyContent: 'center',
+            }}
+          >
+            <CircularProgress
+              size={48}
+              sx={{
+                mb: 2,
+                color: 'primary.main'
+              }}
+            />
+            <Typography variant="body2" sx={{ fontWeight: 600, mb: 0.5, fontSize: '0.875rem', color: 'text.primary' }}>
+              Starting FARA Agent...
+            </Typography>
+            <Typography variant="caption" sx={{ fontSize: '0.75rem', color: 'text.secondary' }}>
+              Initializing browser environment
+            </Typography>
+          </Box>
+        ) : (
+          // No stream available
+          <Box
+            sx={{
+              textAlign: 'center',
+              p: 4,
+              color: 'text.secondary',
+              width: '100%',
+              height: '100%',
+              display: 'flex',
+              flexDirection: 'column',
+              alignItems: 'center',
+              justifyContent: 'center',
+            }}
+          >
+            <MonitorIcon sx={{ fontSize: 48, mb: 2, opacity: 0.5 }} />
+            <Typography variant="body2" sx={{ fontWeight: 600, mb: 0.5, fontSize: '0.875rem' }}>
+              No stream available
+            </Typography>
+            <Typography variant="caption" sx={{ fontSize: '0.75rem', color: 'text.secondary' }}>
+              Stream will appear when agent starts
+            </Typography>
+          </Box>
+        )}
+      </Box>
+    </Box>
+  );
+};

src/components/sandbox/completionview/CompletionView.tsx

@@ -0,0 +1,525 @@
+import { useTraceUploader } from '@/hooks/useTraceUploader';
+import { useAgentStore } from '@/stores/agentStore';
+import { AgentStep, AgentTrace, AgentTraceMetadata, FinalStep } from '@/types/agent';
+import AccessTimeIcon from '@mui/icons-material/AccessTime';
+import AddIcon from '@mui/icons-material/Add';
+import AssignmentIcon from '@mui/icons-material/Assignment';
+import ChatBubbleOutlineIcon from '@mui/icons-material/ChatBubbleOutline';
+import CheckIcon from '@mui/icons-material/Check';
+import CloseIcon from '@mui/icons-material/Close';
+import CloudUploadIcon from '@mui/icons-material/CloudUpload';
+import FormatListNumberedIcon from '@mui/icons-material/FormatListNumbered';
+import HourglassEmptyIcon from '@mui/icons-material/HourglassEmpty';
+import InputIcon from '@mui/icons-material/Input';
+import OutputIcon from '@mui/icons-material/Output';
+import SmartToyIcon from '@mui/icons-material/SmartToy';
+import StopCircleIcon from '@mui/icons-material/StopCircle';
+import ThumbDownIcon from '@mui/icons-material/ThumbDown';
+import ThumbUpIcon from '@mui/icons-material/ThumbUp';
+import { Alert, Box, Button, Divider, IconButton, Paper, Tooltip, Typography } from '@mui/material';
+import React, { useEffect, useState, useRef, useCallback } from 'react';
+import { DownloadGifButton } from './DownloadGifButton';
+import { DownloadJsonButton } from './DownloadJsonButton';
+
+interface CompletionViewProps {
+  finalStep: FinalStep;
+  trace?: AgentTrace;
+  steps?: AgentStep[];
+  metadata?: AgentTraceMetadata;
+  finalAnswer?: string | null;
+  isGenerating: boolean;
+  gifError: string | null;
+  onGenerateGif: () => void;
+  onDownloadJson: () => void;
+  onBackToHome: () => void;
+}
+
+/**
+ * Component displaying the completion status (success or failure) of a task
+ */
+export const CompletionView: React.FC<CompletionViewProps> = ({
+  finalStep,
+  trace,
+  steps,
+  metadata,
+  finalAnswer,
+  isGenerating,
+  gifError,
+  onGenerateGif,
+  onDownloadJson,
+  onBackToHome,
+}) => {
+  const updateTraceEvaluationInStore = useAgentStore((state) => state.updateTraceEvaluation);
+  const [evaluation, setEvaluation] = useState<'success' | 'failed' | 'not_evaluated'>(
+    finalStep.metadata.user_evaluation || 'not_evaluated'
+  );
+  const [isVoting, setIsVoting] = useState(false);
+  
+  // Use refs to always have fresh values for the upload callback
+  const traceRef = useRef(trace);
+  const stepsRef = useRef(steps || []);
+  const metadataRef = useRef(metadata || finalStep.metadata);
+  const finalStepRef = useRef(finalStep);
+  
+  // Keep refs updated
+  useEffect(() => {
+    traceRef.current = trace;
+    stepsRef.current = steps || [];
+    metadataRef.current = metadata || finalStep.metadata;
+    finalStepRef.current = finalStep;
+  }, [trace, steps, metadata, finalStep]);
+  
+  // Hook for uploading traces to Modal - uses callback to get fresh data
+  const { uploadTrace, isUploading, uploadError, uploadSuccess } = useTraceUploader({
+    getTraceData: useCallback(() => ({
+      trace: traceRef.current,
+      steps: stepsRef.current,
+      metadata: metadataRef.current,
+      finalStep: finalStepRef.current,
+    }), []),
+  });
+
+  // Note: Auto-upload on task completion is now handled by useAgentWebSocket
+  // This component only handles re-uploads when user provides evaluation
+
+  const handleTraceEvaluation = async (vote: 'success' | 'failed') => {
+    if (isVoting || !trace?.id) return;
+
+    const newEvaluation = evaluation === vote ? 'not_evaluated' : vote;
+    setIsVoting(true);
+
+    try {
+      setEvaluation(newEvaluation);
+      // Update the store so the evaluation is reflected in the trace data
+      updateTraceEvaluationInStore(newEvaluation);
+      
+      // Force re-upload the full trace with evaluation included
+      // The Modal storage will overwrite the existing trace with the same ID
+      // Use a slight delay to ensure store is updated
+      setTimeout(() => {
+        uploadTrace(true); // forceUpload=true to always upload with new evaluation
+      }, 100);
+    } catch (error) {
+      console.error('Failed to update trace evaluation:', error);
+    } finally {
+      setIsVoting(false);
+    }
+  };
+
+  const getStatusConfig = () => {
+    switch (finalStep.type) {
+      case 'success':
+        return {
+          icon: <CheckIcon sx={{ fontSize: 28 }} />,
+          title: 'Task Completed Successfully!',
+          color: 'success.main',
+        };
+      case 'stopped':
+        return {
+          icon: <StopCircleIcon sx={{ fontSize: 28 }} />,
+          title: 'Task Stopped',
+          color: 'warning.main',
+        };
+      case 'max_steps_reached':
+        return {
+          icon: <HourglassEmptyIcon sx={{ fontSize: 28 }} />,
+          title: 'Maximum Steps Reached',
+          color: 'warning.main',
+        };
+      case 'sandbox_timeout':
+        return {
+          icon: <AccessTimeIcon sx={{ fontSize: 28 }} />,
+          title: 'Max Sandbox Time Reached',
+          color: 'error.main',
+        };
+      case 'failure':
+      default:
+        return {
+          icon: <CloseIcon sx={{ fontSize: 28 }} />,
+          title: 'Task Failed (Agent Internal Error)',
+          color: 'error.main',
+        };
+    }
+  };
+
+  const statusConfig = getStatusConfig();
+
+  // Format model name for display
+  const formatModelName = (modelId: string) => {
+    const parts = modelId.split('/');
+    return parts.length > 1 ? parts[1] : modelId;
+  };
+
+  return (
+    <Box
+      sx={{
+        width: '100%',
+        maxWidth: 600,
+        mx: 'auto',
+        p: 2,
+        display: 'flex',
+        flexDirection: 'column',
+        gap: 1.5,
+      }}
+    >
+      {/* Status Header - Compact */}
+      <Box sx={{ textAlign: 'center', mb: 0.5 }}>
+        <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 1.5, mb: 0.75 }}>
+          <Box
+            sx={{
+              width: 40,
+              height: 40,
+              borderRadius: '50%',
+              backgroundColor: statusConfig.color,
+              display: 'flex',
+              alignItems: 'center',
+              justifyContent: 'center',
+              boxShadow: (theme) => {
+                const rgba = finalStep.type === 'success'
+                  ? '102, 187, 106'
+                  : (finalStep.type === 'failure' || finalStep.type === 'sandbox_timeout')
+                    ? '244, 67, 54'
+                    : '255, 152, 0';
+                return `0 2px 8px ${theme.palette.mode === 'dark' ? `rgba(${rgba}, 0.3)` : `rgba(${rgba}, 0.2)`}`;
+              },
+            }}
+          >
+            {React.cloneElement(statusConfig.icon, { sx: { fontSize: 24, color: 'white' } })}
+          </Box>
+          <Typography
+            variant="h6"
+            sx={{
+              fontWeight: 700,
+              color: statusConfig.color,
+              fontSize: '1.1rem',
+              letterSpacing: '-0.5px',
+            }}
+          >
+            {statusConfig.title}
+          </Typography>
+        </Box>
+      </Box>
+
+      {/* Single Report Box - Task + Agent + Response + Metrics */}
+      <Paper
+        elevation={0}
+        sx={{
+          p: 2.5,
+          backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(255,255,255,0.03)' : 'rgba(0,0,0,0.03)',
+          borderRadius: 1.5,
+          border: '1px solid',
+          borderColor: 'divider',
+        }}
+      >
+        {/* Task */}
+        {trace?.instruction && (
+          <Box sx={{ mb: 2 }}>
+            <Box sx={{ display: 'flex', alignItems: 'flex-start', gap: 1.5 }}>
+              <AssignmentIcon sx={{ fontSize: 18, color: 'text.secondary', mt: 0.25, flexShrink: 0 }} />
+              <Box sx={{ flex: 1, minWidth: 0 }}>
+                <Typography
+                  variant="caption"
+                  sx={{
+                    fontWeight: 700,
+                    color: 'text.secondary',
+                    fontSize: '0.7rem',
+                    textTransform: 'uppercase',
+                    letterSpacing: '0.5px',
+                    display: 'block',
+                    mb: 0.5,
+                  }}
+                >
+                  Task
+                </Typography>
+                <Typography
+                  variant="body2"
+                  sx={{
+                    color: 'text.primary',
+                    fontWeight: 700,
+                    lineHeight: 1.5,
+                    fontSize: '0.85rem',
+                  }}
+                >
+                  {trace.instruction}
+                </Typography>
+              </Box>
+            </Box>
+          </Box>
+        )}
+
+        {/* Agent Response */}
+        {finalAnswer && (
+          <Box sx={{ mb: 2 }}>
+            <Box sx={{ display: 'flex', alignItems: 'flex-start', gap: 1.5 }}>
+              <ChatBubbleOutlineIcon
+                sx={{
+                  fontSize: 18,
+                  color: 'text.secondary',
+                  mt: 0.25,
+                  flexShrink: 0
+                }}
+              />
+              <Box sx={{ flex: 1, minWidth: 0 }}>
+                <Typography
+                  variant="caption"
+                  sx={{
+                    fontWeight: 700,
+                    color: 'text.secondary',
+                    fontSize: '0.7rem',
+                    textTransform: 'uppercase',
+                    letterSpacing: '0.5px',
+                    display: 'block',
+                    mb: 0.75,
+                  }}
+                >
+                  Agent Response
+                </Typography>
+                <Typography
+                  variant="body2"
+                  sx={{
+                    color: 'text.primary',
+                    lineHeight: 1.5,
+                    fontSize: '0.85rem',
+                    whiteSpace: 'pre-wrap',
+                    wordBreak: 'break-word',
+                  }}
+                >
+                  {finalAnswer}
+                </Typography>
+              </Box>
+            </Box>
+          </Box>
+        )}
+
+        {/* Trace Evaluation */}
+        <Box sx={{ mb: 2 }}>
+          <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between' }}>
+            <Typography
+              variant="caption"
+              sx={{
+                fontWeight: 700,
+                color: 'text.secondary',
+                fontSize: '0.7rem',
+                textTransform: 'uppercase',
+                letterSpacing: '0.5px',
+              }}
+            >
+              Was this task completed successfully?
+            </Typography>
+
+            {/* Evaluation buttons */}
+            <Box sx={{ display: 'flex', gap: 1 }}>
+              <Tooltip title={evaluation === 'success' ? 'Remove success rating' : 'Mark as successful'}>
+                <IconButton
+                  size="small"
+                  onClick={() => handleTraceEvaluation('success')}
+                  disabled={isVoting}
+                  sx={{
+                    padding: '4px',
+                    color: evaluation === 'success' ? 'success.main' : 'action.disabled',
+                    '&:hover': {
+                      color: 'success.main',
+                      backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(102, 187, 106, 0.1)' : 'rgba(102, 187, 106, 0.08)',
+                    },
+                  }}
+                >
+                  <ThumbUpIcon sx={{ fontSize: 18 }} />
+                </IconButton>
+              </Tooltip>
+              <Tooltip title={evaluation === 'failed' ? 'Remove failure rating' : 'Mark as failed'}>
+                <IconButton
+                  size="small"
+                  onClick={() => handleTraceEvaluation('failed')}
+                  disabled={isVoting}
+                  sx={{
+                    padding: '4px',
+                    color: evaluation === 'failed' ? 'error.main' : 'action.disabled',
+                    '&:hover': {
+                      color: 'error.main',
+                      backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(244, 67, 54, 0.1)' : 'rgba(244, 67, 54, 0.08)',
+                    },
+                  }}
+                >
+                  <ThumbDownIcon sx={{ fontSize: 18 }} />
+                </IconButton>
+              </Tooltip>
+            </Box>
+          </Box>
+        </Box>
+
+        {/* Divider before metrics */}
+        <Divider sx={{ my: 2 }} />
+
+        {/* Metrics */}
+        <Box
+          sx={{
+            display: 'flex',
+            alignItems: 'center',
+            gap: 1.5,
+            flexWrap: 'wrap',
+            justifyContent: 'center',
+          }}
+        >
+          {/* Agent */}
+          {trace?.modelId && (
+            <>
+              <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+                <SmartToyIcon sx={{ fontSize: '0.85rem', color: 'primary.main' }} />
+                <Typography
+                  variant="caption"
+                  sx={{
+                    color: 'text.primary',
+                    fontFamily: 'monospace',
+                    fontSize: '0.75rem',
+                    fontWeight: 700,
+                  }}
+                >
+                  {formatModelName(trace.modelId)}
+                </Typography>
+              </Box>
+
+              {/* Divider */}
+              <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+            </>
+          )}
+
+          {/* Steps Count */}
+          <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+            <FormatListNumberedIcon sx={{ fontSize: '0.85rem', color: 'primary.main' }} />
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.75rem',
+                fontWeight: 700,
+                color: 'text.primary',
+                mr: 0.5,
+              }}
+            >
+              {finalStep.metadata.numberOfSteps}
+            </Typography>
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.7rem',
+                fontWeight: 400,
+                color: 'text.secondary',
+              }}
+            >
+              {finalStep.metadata.numberOfSteps === 1 ? 'Step' : 'Steps'}
+            </Typography>
+          </Box>
+
+          {/* Divider */}
+          <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+
+          {/* Duration */}
+          <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+            <AccessTimeIcon sx={{ fontSize: '0.85rem', color: 'primary.main' }} />
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.75rem',
+                fontWeight: 700,
+                color: 'text.primary',
+              }}
+            >
+              {finalStep.metadata.duration.toFixed(1)}s
+            </Typography>
+          </Box>
+
+          {/* Divider */}
+          <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+
+          {/* Input Tokens */}
+          <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+            <InputIcon sx={{ fontSize: '0.85rem', color: 'primary.main' }} />
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.75rem',
+                fontWeight: 700,
+                color: 'text.primary',
+              }}
+            >
+              {finalStep.metadata.inputTokensUsed.toLocaleString()}
+            </Typography>
+          </Box>
+
+          {/* Divider */}
+          <Box sx={{ width: '1px', height: 16, backgroundColor: 'divider' }} />
+
+          {/* Output Tokens */}
+          <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+            <OutputIcon sx={{ fontSize: '0.85rem', color: 'primary.main' }} />
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.75rem',
+                fontWeight: 700,
+                color: 'text.primary',
+              }}
+            >
+              {finalStep.metadata.outputTokensUsed.toLocaleString()}
+            </Typography>
+          </Box>
+        </Box>
+      </Paper>
+
+      {/* GIF Error Alert */}
+      {gifError && (
+        <Alert severity="error" sx={{ fontSize: '0.72rem', py: 0.5 }}>
+          {gifError}
+        </Alert>
+      )}
+
+      {/* Action Buttons */}
+      <Box
+        sx={{
+          display: 'flex',
+          flexDirection: 'column',
+          gap: 1.5,
+          alignItems: 'center',
+        }}
+      >
+        {/* Download buttons */}
+        <Box
+          sx={{
+            display: 'flex',
+            gap: 1,
+            justifyContent: 'center',
+            flexWrap: 'wrap',
+          }}
+        >
+          <DownloadGifButton
+            isGenerating={isGenerating}
+            onClick={onGenerateGif}
+            disabled={!steps || steps.length === 0}
+          />
+          <DownloadJsonButton onClick={onDownloadJson} disabled={!trace} />
+        </Box>
+
+        {/* New Task button - larger and below */}
+        <Button
+          variant="contained"
+          startIcon={<AddIcon sx={{ fontSize: 20 }} />}
+          onClick={onBackToHome}
+          color="primary"
+          sx={{
+            textTransform: 'none',
+            fontWeight: 700,
+            fontSize: '0.9rem',
+            px: 3,
+            py: 1,
+            boxShadow: 2,
+            minWidth: 200,
+            '&:hover': {
+              boxShadow: 4,
+            },
+          }}
+        >
+          New Task
+        </Button>
+      </Box>
+    </Box>
+  );
+};

src/components/sandbox/completionview/DownloadGifButton.tsx

@@ -0,0 +1,64 @@
+import React from 'react';
+import { Button, CircularProgress, Tooltip } from '@mui/material';
+import GifIcon from '@mui/icons-material/Gif';
+
+interface DownloadGifButtonProps {
+  isGenerating: boolean;
+  onClick: () => void;
+  disabled?: boolean;
+}
+
+/**
+ * Button to download a GIF replay of the trace
+ */
+export const DownloadGifButton: React.FC<DownloadGifButtonProps> = ({
+  isGenerating,
+  onClick,
+  disabled = false,
+}) => {
+  return (
+    <Tooltip
+      title={
+        disabled
+          ? "No steps available"
+          : "Download GIF replay"
+      }
+    >
+      <span>
+        <Button
+          variant="outlined"
+          size="small"
+          onClick={onClick}
+          disabled={disabled || isGenerating}
+          startIcon={
+            isGenerating ? (
+              <CircularProgress size={16} />
+            ) : (
+              <GifIcon sx={{ fontSize: '1.2rem' }} />
+            )
+          }
+          sx={{
+            textTransform: 'none',
+            fontSize: '0.75rem',
+            fontWeight: 600,
+            borderRadius: 1,
+            px: 1.5,
+            py: 0.5,
+            borderColor: 'divider',
+            color: 'text.primary',
+            '&:hover': {
+              borderColor: 'primary.main',
+              backgroundColor: 'action.hover',
+            },
+            '&.Mui-disabled': {
+              borderColor: 'divider',
+              color: 'text.disabled',
+            },
+          }}
+        >
+          {isGenerating ? 'Generating...' : 'Download GIF'}
+        </Button>
+      </span>
+    </Tooltip>
+  );
+};

src/components/sandbox/completionview/DownloadJsonButton.tsx

@@ -0,0 +1,56 @@
+import React from 'react';
+import { Button, Tooltip } from '@mui/material';
+import DownloadIcon from '@mui/icons-material/Download';
+
+interface DownloadJsonButtonProps {
+  onClick: () => void;
+  disabled?: boolean;
+}
+
+/**
+ * Button to download trace as JSON
+ */
+export const DownloadJsonButton: React.FC<DownloadJsonButtonProps> = ({
+  onClick,
+  disabled = false,
+}) => {
+  return (
+    <Tooltip
+      title={
+        disabled
+          ? "No trace available"
+          : "Download trace as JSON"
+      }
+    >
+      <span>
+        <Button
+          variant="outlined"
+          size="small"
+          onClick={onClick}
+          disabled={disabled}
+          startIcon={<DownloadIcon sx={{ fontSize: '1.2rem' }} />}
+          sx={{
+            textTransform: 'none',
+            fontSize: '0.75rem',
+            fontWeight: 600,
+            borderRadius: 1,
+            px: 1.5,
+            py: 0.5,
+            borderColor: 'divider',
+            color: 'text.primary',
+            '&:hover': {
+              borderColor: 'primary.main',
+              backgroundColor: 'action.hover',
+            },
+            '&.Mui-disabled': {
+              borderColor: 'divider',
+              color: 'text.disabled',
+            },
+          }}
+        >
+          Download JSON Trace
+        </Button>
+      </span>
+    </Tooltip>
+  );
+};

src/components/sandbox/completionview/index.ts

@@ -0,0 +1,3 @@
+export { CompletionView } from './CompletionView';
+export { DownloadGifButton } from './DownloadGifButton';
+export { DownloadJsonButton } from './DownloadJsonButton';

src/components/sandbox/index.ts

@@ -0,0 +1,2 @@
+export { SandboxViewer } from './SandboxViewer';
+export { CompletionView, DownloadGifButton, DownloadJsonButton } from './completionview';

src/components/steps/ConnectionStepCard.tsx

@@ -0,0 +1,110 @@
+import React from 'react';
+import { Card, CardContent, Box, Typography, CircularProgress } from '@mui/material';
+import CableIcon from '@mui/icons-material/Cable';
+import { keyframes } from '@mui/system';
+
+// Border pulse animation
+const borderPulse = keyframes`
+  0%, 100% {
+    border-color: rgba(79, 134, 198, 0.4);
+    box-shadow: 0 2px 8px rgba(79, 134, 198, 0.15);
+  }
+  50% {
+    border-color: rgba(79, 134, 198, 0.8);
+    box-shadow: 0 2px 12px rgba(79, 134, 198, 0.3);
+  }
+`;
+
+// Background pulse animation
+const backgroundPulse = keyframes`
+  0%, 100% {
+    background-color: rgba(79, 134, 198, 0.03);
+  }
+  50% {
+    background-color: rgba(79, 134, 198, 0.08);
+  }
+`;
+
+interface ConnectionStepCardProps {
+  isConnecting: boolean;
+}
+
+export const ConnectionStepCard: React.FC<ConnectionStepCardProps> = ({ isConnecting }) => {
+  return (
+    <Card
+      elevation={0}
+      sx={{
+        backgroundColor: 'background.paper',
+        border: '2px solid',
+        borderColor: isConnecting ? 'primary.main' : 'success.main',
+        borderRadius: 1.5,
+        animation: isConnecting ? `${borderPulse} 2s ease-in-out infinite` : 'none',
+        position: 'relative',
+        overflow: 'hidden',
+        '&::before': isConnecting ? {
+          content: '""',
+          position: 'absolute',
+          top: 0,
+          left: 0,
+          right: 0,
+          bottom: 0,
+          animation: `${backgroundPulse} 2s ease-in-out infinite`,
+          zIndex: 0,
+        } : {},
+      }}
+    >
+      <CardContent sx={{ p: 1.5, '&:last-child': { pb: 1.5 }, position: 'relative', zIndex: 1 }}>
+        {/* Header with spinner or check */}
+        <Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5 }}>
+          <Box
+            sx={{
+              display: 'flex',
+              alignItems: 'center',
+              justifyContent: 'center',
+              position: 'relative',
+            }}
+          >
+            {isConnecting ? (
+              <CircularProgress
+                size={32}
+                thickness={2.5}
+                sx={{
+                  color: 'primary.main',
+                }}
+              />
+            ) : (
+              <CableIcon
+                sx={{
+                  fontSize: 28,
+                  color: 'success.main',
+                }}
+              />
+            )}
+          </Box>
+
+          <Box sx={{ flex: 1, minWidth: 0 }}>
+            <Typography
+              sx={{
+                fontSize: '0.85rem',
+                fontWeight: 700,
+                color: isConnecting ? 'primary.main' : 'success.main',
+                lineHeight: 1.3,
+              }}
+            >
+              {isConnecting ? 'Starting FARA...' : 'Browser Ready'}
+            </Typography>
+            <Typography
+              sx={{
+                fontSize: '0.7rem',
+                color: 'text.secondary',
+                lineHeight: 1.2,
+              }}
+            >
+              {isConnecting ? 'Initializing browser environment' : 'Agent ready to execute tasks'}
+            </Typography>
+          </Box>
+        </Box>
+      </CardContent>
+    </Card>
+  );
+};

src/components/steps/FinalStepCard.tsx

@@ -0,0 +1,105 @@
+import { useAgentStore } from '@/stores/agentStore';
+import { FinalStep } from '@/types/agent';
+import AccessTimeIcon from '@mui/icons-material/AccessTime';
+import CheckIcon from '@mui/icons-material/Check';
+import CloseIcon from '@mui/icons-material/Close';
+import HourglassEmptyIcon from '@mui/icons-material/HourglassEmpty';
+import StopCircleIcon from '@mui/icons-material/StopCircle';
+import { Box, Card, CardContent, Typography } from '@mui/material';
+import React from 'react';
+
+interface FinalStepCardProps {
+  finalStep: FinalStep;
+  isActive?: boolean;
+}
+
+export const FinalStepCard: React.FC<FinalStepCardProps> = ({ finalStep, isActive = false }) => {
+  const setSelectedStepIndex = useAgentStore((state) => state.setSelectedStepIndex);
+
+  const getStatusConfig = () => {
+    switch (finalStep.type) {
+      case 'success':
+        return {
+          icon: <CheckIcon sx={{ fontSize: 20, color: 'success.main' }} />,
+          label: 'Task completed',
+          color: 'success',
+        };
+      case 'stopped':
+        return {
+          icon: <StopCircleIcon sx={{ fontSize: 20, color: 'warning.main' }} />,
+          label: 'Task stopped',
+          color: 'warning',
+        };
+      case 'max_steps_reached':
+        return {
+          icon: <HourglassEmptyIcon sx={{ fontSize: 20, color: 'warning.main' }} />,
+          label: 'Max steps reached',
+          color: 'warning',
+        };
+      case 'sandbox_timeout':
+        return {
+          icon: <AccessTimeIcon sx={{ fontSize: 20, color: 'error.main' }} />,
+          label: 'Sandbox timeout',
+          color: 'error',
+        };
+      case 'failure':
+      default:
+        return {
+          icon: <CloseIcon sx={{ fontSize: 20, color: 'error.main' }} />,
+          label: 'Task failed',
+          color: 'error',
+        };
+    }
+  };
+
+  const statusConfig = getStatusConfig();
+
+  const handleClick = () => {
+    // Clicking on final step goes to live mode (null)
+    setSelectedStepIndex(null);
+  };
+
+  return (
+    <Card
+      elevation={0}
+      onClick={handleClick}
+      sx={{
+        backgroundColor: 'background.paper',
+        border: '1px solid',
+        borderColor: (theme) => `${isActive
+          ? theme.palette[statusConfig.color].main
+          : theme.palette.divider} !important`,
+        borderRadius: 1.5,
+        transition: 'all 0.2s ease',
+        cursor: 'pointer',
+        boxShadow: isActive
+          ? (theme) => `0 2px 8px ${theme.palette.mode === 'dark'
+            ? `rgba(${statusConfig.color === 'success' ? '102, 187, 106' : statusConfig.color === 'error' ? '244, 67, 54' : '255, 152, 0'}, 0.3)`
+            : `rgba(${statusConfig.color === 'success' ? '102, 187, 106' : statusConfig.color === 'error' ? '244, 67, 54' : '255, 152, 0'}, 0.2)`}`
+          : 'none',
+        '&:hover': {
+          borderColor: (theme) => `${theme.palette[statusConfig.color].main} !important`,
+          boxShadow: (theme) => `0 2px 8px ${theme.palette.mode === 'dark'
+            ? `rgba(${statusConfig.color === 'success' ? '102, 187, 106' : statusConfig.color === 'error' ? '244, 67, 54' : '255, 152, 0'}, 0.2)`
+            : `rgba(${statusConfig.color === 'success' ? '102, 187, 106' : statusConfig.color === 'error' ? '244, 67, 54' : '255, 152, 0'}, 0.1)`}`,
+        },
+      }}
+    >
+      <CardContent sx={{ p: 1.5, '&:last-child': { pb: 1.5 } }}>
+        {/* Header with icon */}
+        <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.75 }}>
+          {statusConfig.icon}
+          <Typography
+            sx={{
+              fontSize: '0.85rem',
+              fontWeight: 700,
+              color: `${statusConfig.color}.main`,
+            }}
+          >
+            {statusConfig.label}
+          </Typography>
+        </Box>
+      </CardContent>
+    </Card>
+  );
+};

src/components/steps/StepCard.tsx

@@ -0,0 +1,399 @@
+import { updateStepEvaluation } from '@/services/api';
+import { useAgentStore } from '@/stores/agentStore';
+import { AgentStep } from '@/types/agent';
+import AccessTimeIcon from '@mui/icons-material/AccessTime';
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
+import InputIcon from '@mui/icons-material/Input';
+import OutputIcon from '@mui/icons-material/Output';
+import ThumbDownIcon from '@mui/icons-material/ThumbDown';
+import ThumbUpIcon from '@mui/icons-material/ThumbUp';
+import { Accordion, AccordionDetails, AccordionSummary, Box, Card, CardContent, Chip, IconButton, Tooltip, Typography } from '@mui/material';
+import React, { useState } from 'react';
+
+interface StepCardProps {
+  step: AgentStep;
+  index: number;
+  isLatest?: boolean;
+  isActive?: boolean;
+}
+
+export const StepCard: React.FC<StepCardProps> = ({ step, index, isLatest = false, isActive = false }) => {
+  const setSelectedStepIndex = useAgentStore((state) => state.setSelectedStepIndex);
+  const updateStepEvaluationInStore = useAgentStore((state) => state.updateStepEvaluation);
+  const [thoughtExpanded, setThoughtExpanded] = useState(false);
+  const [actionsExpanded, setActionsExpanded] = useState(false);
+  const [evaluation, setEvaluation] = useState<'like' | 'dislike' | 'neutral'>(step.step_evaluation || 'neutral');
+  const [isVoting, setIsVoting] = useState(false);
+
+  const hasMultipleActions = step.actions && step.actions.length > 1;
+  const displayedActions = hasMultipleActions && !actionsExpanded
+    ? step.actions.slice(0, 1)
+    : step.actions;
+
+  const handleClick = () => {
+    setSelectedStepIndex(index);
+  };
+
+  const handleAccordionClick = (event: React.MouseEvent) => {
+    event.stopPropagation(); // Prevent propagation to avoid selecting the step
+  };
+
+  const handleVote = async (event: React.MouseEvent, vote: 'like' | 'dislike') => {
+    event.stopPropagation(); // Prevent propagation to avoid selecting the step
+
+    if (isVoting) return;
+
+    const newEvaluation = evaluation === vote ? 'neutral' : vote;
+    setIsVoting(true);
+
+    try {
+      await updateStepEvaluation(step.traceId, step.stepId, newEvaluation);
+      setEvaluation(newEvaluation);
+      // Update the store so the evaluation is reflected in JSON export
+      updateStepEvaluationInStore(step.stepId, newEvaluation);
+    } catch (error) {
+      console.error('Failed to update step evaluation:', error);
+    } finally {
+      setIsVoting(false);
+    }
+  };
+
+  return (
+    <Card
+      elevation={0}
+      onClick={handleClick}
+      sx={{
+        backgroundColor: 'background.paper',
+        border: '1px solid',
+        borderColor: (theme) => `${isActive ? theme.palette.primary.main : theme.palette.divider} !important`,
+        borderRadius: 1.5,
+        transition: 'all 0.2s ease',
+        cursor: 'pointer',
+        boxShadow: isActive ? (theme) => `0 2px 8px ${theme.palette.mode === 'dark' ? 'rgba(79, 134, 198, 0.3)' : 'rgba(79, 134, 198, 0.2)'}` : 'none',
+        '&:hover': {
+          borderColor: (theme) => `${theme.palette.primary.main} !important`,
+          boxShadow: (theme) => `0 2px 8px ${theme.palette.mode === 'dark' ? 'rgba(79, 134, 198, 0.2)' : 'rgba(79, 134, 198, 0.1)'}`,
+        },
+      }}
+    >
+      <CardContent sx={{ p: 1.5, '&:last-child': { pb: 1.5 } }}>
+        {/* Step header */}
+        <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', mb: 1.5 }}>
+          <Typography
+            sx={{
+              fontSize: '1.5rem',
+              fontWeight: 800,
+              color: isActive ? 'primary.main' : 'text.primary',
+              lineHeight: 1,
+            }}
+          >
+            {index + 1}
+          </Typography>
+          <Box sx={{ display: 'flex', gap: 0.5, alignItems: 'center' }}>
+            <Chip
+              icon={<AccessTimeIcon sx={{ fontSize: '0.7rem !important' }} />}
+              label={`${step.duration.toFixed(1)}s`}
+              size="small"
+              sx={{
+                height: 'auto',
+                py: 0.25,
+                fontSize: '0.65rem',
+                fontWeight: 600,
+                backgroundColor: 'action.hover',
+                color: 'text.primary',
+                '& .MuiChip-icon': { marginLeft: 0.5, color: 'text.secondary' },
+              }}
+            />
+            <Chip
+              icon={<InputIcon sx={{ fontSize: '0.7rem !important' }} />}
+              label={step.inputTokensUsed.toLocaleString()}
+              size="small"
+              sx={{
+                height: 'auto',
+                py: 0.25,
+                fontSize: '0.65rem',
+                fontWeight: 600,
+                backgroundColor: 'action.hover',
+                color: 'text.primary',
+                '& .MuiChip-icon': { marginLeft: 0.5, color: 'text.secondary' },
+              }}
+            />
+            <Chip
+              icon={<OutputIcon sx={{ fontSize: '0.7rem !important' }} />}
+              label={step.outputTokensUsed.toLocaleString()}
+              size="small"
+              sx={{
+                height: 'auto',
+                py: 0.25,
+                fontSize: '0.65rem',
+                fontWeight: 600,
+                backgroundColor: 'action.hover',
+                color: 'text.primary',
+                '& .MuiChip-icon': { marginLeft: 0.5, color: 'text.secondary' },
+              }}
+            />
+          </Box>
+        </Box>
+
+        {/* Step image */}
+        {step.image && (
+          <Box
+            sx={{
+              mb: 1.5,
+              borderRadius: 1,
+              overflow: 'hidden',
+              border: '1px solid',
+              borderColor: (theme) => isActive ? theme.palette.primary.main : theme.palette.divider,
+              backgroundColor: 'action.hover',
+              transition: 'border-color 0.2s ease',
+            }}
+          >
+            <img
+              src={step.image}
+              alt={`Step ${index + 1}`}
+              style={{ width: '100%', height: 'auto', display: 'block' }}
+            />
+          </Box>
+        )}
+
+        {/* Action */}
+        {step.actions && step.actions.length > 0 && (
+          <Box sx={{ mb: 1.5 }}>
+            <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5, mb: 0.75, justifyContent: 'space-between' }}>
+              <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+                <Typography
+                  variant="caption"
+                  sx={{
+                    fontWeight: 700,
+                    color: 'text.secondary',
+                    fontSize: '0.65rem',
+                    textTransform: 'uppercase',
+                    letterSpacing: '0.5px',
+                  }}
+                >
+                  Action
+                </Typography>
+                {hasMultipleActions && (
+                  <Tooltip title={actionsExpanded ? 'Show less' : `Show all ${step.actions.length} actions`}>
+                    <IconButton
+                      size="small"
+                      onClick={(e) => {
+                        e.stopPropagation();
+                        setActionsExpanded(!actionsExpanded);
+                      }}
+                      sx={{
+                        padding: '2px',
+                        color: 'text.secondary',
+                        '&:hover': {
+                          color: 'text.primary',
+                          backgroundColor: 'action.hover',
+                        },
+                      }}
+                    >
+                      <ExpandMoreIcon
+                        sx={{
+                          fontSize: 16,
+                          transform: actionsExpanded ? 'rotate(180deg)' : 'rotate(0deg)',
+                          transition: 'transform 0.2s',
+                        }}
+                      />
+                    </IconButton>
+                  </Tooltip>
+                )}
+              </Box>
+
+              {/* Vote buttons */}
+              <Box sx={{ display: 'flex', gap: 0.5 }}>
+                <Tooltip title={evaluation === 'like' ? 'Remove like' : 'Like this step'}>
+                  <IconButton
+                    size="small"
+                    onClick={(e) => handleVote(e, 'like')}
+                    disabled={isVoting}
+                    sx={{
+                      padding: '2px',
+                      color: evaluation === 'like' ? 'success.main' : 'action.disabled',
+                      '&:hover': {
+                        color: 'success.main',
+                        backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(102, 187, 106, 0.1)' : 'rgba(102, 187, 106, 0.08)',
+                      },
+                    }}
+                  >
+                    <ThumbUpIcon sx={{ fontSize: 14 }} />
+                  </IconButton>
+                </Tooltip>
+                <Tooltip title={evaluation === 'dislike' ? 'Remove dislike' : 'Dislike this step'}>
+                  <IconButton
+                    size="small"
+                    onClick={(e) => handleVote(e, 'dislike')}
+                    disabled={isVoting}
+                    sx={{
+                      padding: '2px',
+                      color: evaluation === 'dislike' ? 'error.main' : 'action.disabled',
+                      '&:hover': {
+                        color: 'error.main',
+                        backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(244, 67, 54, 0.1)' : 'rgba(244, 67, 54, 0.08)',
+                      },
+                    }}
+                  >
+                    <ThumbDownIcon sx={{ fontSize: 14 }} />
+                  </IconButton>
+                </Tooltip>
+              </Box>
+            </Box>
+            <Box component="ul" sx={{ listStyle: 'none', p: 0, m: 0 }}>
+              {displayedActions?.map((action, actionIndex) => (
+                <Box
+                  key={actionIndex}
+                  component="li"
+                  sx={{
+                    display: 'flex',
+                    alignItems: 'flex-start',
+                    fontSize: '0.75rem',
+                    color: 'text.primary',
+                    lineHeight: 1.4,
+                    mb: 0.5,
+                    '&:last-child': { mb: 0 },
+                  }}
+                >
+                  {/* <Typography
+                    component="span"
+                    sx={{
+                      mr: 0.5,
+                      color: 'text.secondary',
+                      fontWeight: 700,
+                      flexShrink: 0,
+                      fontSize: '0.75rem',
+                    }}
+                  >
+                    →
+                  </Typography> */}
+                  <Typography
+                    component="span"
+                    sx={{
+                      fontSize: '0.75rem',
+                      fontWeight: 900,
+                      wordBreak: 'break-word',
+                    }}
+                  >
+                    {action.description}
+                  </Typography>
+                </Box>
+              ))}
+            </Box>
+          </Box>
+        )}
+
+        {/* Thought - Accordion */}
+        {step.thought && (
+          <Accordion
+            expanded={thoughtExpanded}
+            onChange={(e, expanded) => setThoughtExpanded(expanded)}
+            onClick={handleAccordionClick}
+            elevation={0}
+            disableGutters
+            sx={{
+              mb: 0.5,
+              backgroundColor: 'transparent',
+              border: 'none',
+              boxShadow: 'none',
+              '&:before': { display: 'none' },
+              '&.MuiAccordion-root': {
+                backgroundColor: 'transparent',
+                boxShadow: 'none',
+                '&:before': {
+                  display: 'none',
+                },
+              },
+              '& .MuiAccordionSummary-root': {
+                minHeight: 'auto',
+                p: 0,
+                backgroundColor: 'transparent',
+                '&:hover': {
+                  backgroundColor: 'transparent',
+                },
+                '&.Mui-expanded': {
+                  minHeight: 'auto',
+                },
+              },
+              '& .MuiAccordionSummary-content': {
+                margin: '0 !important',
+              },
+              '& .MuiAccordionDetails-root': {
+                p: 0,
+                pt: 0.5,
+                pb: 0,
+                backgroundColor: 'transparent',
+              },
+            }}
+          >
+            <AccordionSummary
+              expandIcon={<ExpandMoreIcon sx={{ fontSize: 16, color: 'text.secondary' }} />}
+              sx={{
+                flexDirection: 'row',
+                border: 'none',
+                '& .MuiAccordionSummary-expandIconWrapper': {
+                  transform: 'rotate(-90deg)',
+                  transition: 'transform 0.2s',
+                  '&.Mui-expanded': {
+                    transform: 'rotate(0deg)',
+                  },
+                },
+              }}
+            >
+              <Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
+                <Typography
+                  variant="caption"
+                  sx={{
+                    fontWeight: 700,
+                    color: 'text.secondary',
+                    fontSize: '0.65rem',
+                    textTransform: 'uppercase',
+                    letterSpacing: '0.5px',
+                  }}
+                >
+                  Thought
+                </Typography>
+              </Box>
+            </AccordionSummary>
+            <AccordionDetails>
+              <Typography
+                variant="body2"
+                sx={{
+                  fontSize: '0.75rem',
+                  color: 'text.primary',
+                  lineHeight: 1.4,
+                  pl: 2.5,
+                }}
+              >
+                {step.thought}
+              </Typography>
+            </AccordionDetails>
+          </Accordion>
+        )}
+
+        {/* Error */}
+        {step.error && (
+          <Box sx={{
+            mt: 1.5,
+            p: 1,
+            borderRadius: 1,
+            backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(244, 67, 54, 0.1)' : 'rgba(244, 67, 54, 0.08)',
+            border: '1px solid',
+            borderColor: 'error.main'
+          }}>
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.7rem',
+                color: 'error.main',
+                fontWeight: 600,
+              }}
+            >
+              Error: {step.error}
+            </Typography>
+          </Box>
+        )}
+      </CardContent>
+    </Card>
+  );
+};

src/components/steps/StepsList.tsx

@@ -0,0 +1,395 @@
+import React, { useRef, useEffect } from 'react';
+import { AgentTrace } from '@/types/agent';
+import { Box, Typography, Stack, Paper } from '@mui/material';
+import { StepCard } from './StepCard';
+import { FinalStepCard } from './FinalStepCard';
+import { ThinkingStepCard } from './ThinkingStepCard';
+import { ConnectionStepCard } from './ConnectionStepCard';
+import ListAltIcon from '@mui/icons-material/ListAlt';
+import FormatListNumberedIcon from '@mui/icons-material/FormatListNumbered';
+import { useAgentStore, selectSelectedStepIndex, selectFinalStep, selectIsConnectingToE2B, selectIsAgentProcessing } from '@/stores/agentStore';
+
+interface StepsListProps {
+  trace?: AgentTrace;
+}
+
+export const StepsList: React.FC<StepsListProps> = ({ trace }) => {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const selectedStepIndex = useAgentStore(selectSelectedStepIndex);
+  const setSelectedStepIndex = useAgentStore((state) => state.setSelectedStepIndex);
+  const finalStep = useAgentStore(selectFinalStep);
+  const isConnectingToE2B = useAgentStore(selectIsConnectingToE2B);
+  const isAgentProcessing = useAgentStore(selectIsAgentProcessing);
+  const isScrollingProgrammatically = useRef(false);
+  const [showThinkingCard, setShowThinkingCard] = React.useState(false);
+  const thinkingTimeoutRef = useRef<NodeJS.Timeout | null>(null);
+  const streamStartTimeRef = useRef<number | null>(null);
+  const [showConnectionCard, setShowConnectionCard] = React.useState(false);
+  const hasConnectedRef = useRef(false);
+
+  // Check if final step is active (when selectedStepIndex is null and finalStep exists and trace is not running)
+  const isFinalStepActive = selectedStepIndex === null && finalStep && !trace?.isRunning;
+
+  // Check if thinking card is active (when in live mode and thinking card is shown)
+  const isThinkingCardActive = selectedStepIndex === null && showThinkingCard;
+
+  // Determine the active step index
+  // If a specific step is selected, use that
+  // If the final step is active, no normal step should be active
+  // If the thinking card is active, no normal step should be active
+  // Otherwise, show the last step as active
+  const activeStepIndex = selectedStepIndex !== null
+    ? selectedStepIndex
+    : isFinalStepActive
+      ? null  // When final step is active, no normal step is active
+      : isThinkingCardActive
+        ? null  // When thinking card is active, no normal step is active
+        : (trace?.steps && trace.steps.length > 0 && trace?.isRunning)
+          ? trace.steps.length - 1
+          : (trace?.steps && trace.steps.length > 0)
+            ? trace.steps.length - 1
+            : null;
+
+  // Manage ConnectionStepCard display:
+  // - Shows when isConnectingToE2B = true OR when we had a connection
+  // - Remains visible even when task is finished (if we have steps or finalStep)
+  useEffect(() => {
+    if (isConnectingToE2B || isAgentProcessing || (trace?.steps && trace.steps.length > 0) || finalStep) {
+      setShowConnectionCard(true);
+      hasConnectedRef.current = true;
+    }
+  }, [isConnectingToE2B, isAgentProcessing, trace?.steps, finalStep]);
+
+  // Manage ThinkingCard display:
+  // - Appears 5 seconds AFTER stream starts (isAgentProcessing = true, NOT during isConnectingToE2B)
+  // - Remains visible during the entire agent processing
+  // - Hides only when agent stops OR a finalStep exists
+  useEffect(() => {
+    // If stream really starts (isAgentProcessing = true and NOT connecting)
+    // And no startTime recorded yet
+    if (isAgentProcessing && !isConnectingToE2B && !streamStartTimeRef.current) {
+      streamStartTimeRef.current = Date.now();
+    }
+
+    // If agent stops OR we have a finalStep, reset and hide
+    if (!isAgentProcessing || finalStep) {
+      streamStartTimeRef.current = null;
+      setShowThinkingCard(false);
+      if (thinkingTimeoutRef.current) {
+        clearTimeout(thinkingTimeoutRef.current);
+        thinkingTimeoutRef.current = null;
+      }
+      return;
+    }
+
+    // If agent is running, not connecting, no finalStep: start 5 second timer
+    if (isAgentProcessing && !isConnectingToE2B && !finalStep && streamStartTimeRef.current) {
+      // Clean up any existing timeout
+      if (thinkingTimeoutRef.current) {
+        clearTimeout(thinkingTimeoutRef.current);
+      }
+
+      // Calculate elapsed time since stream started
+      const elapsedTime = Date.now() - streamStartTimeRef.current;
+      const remainingTime = Math.max(0, 5000 - elapsedTime);
+
+      thinkingTimeoutRef.current = setTimeout(() => {
+        setShowThinkingCard(true);
+      }, remainingTime);
+    }
+
+    // Cleanup on unmount or when dependencies change
+    return () => {
+      if (thinkingTimeoutRef.current) {
+        clearTimeout(thinkingTimeoutRef.current);
+        thinkingTimeoutRef.current = null;
+      }
+    };
+  }, [isAgentProcessing, isConnectingToE2B, finalStep]);
+
+  // Auto-scroll logic
+  useEffect(() => {
+    const container = containerRef.current;
+    if (!container) return;
+
+    isScrollingProgrammatically.current = true;
+
+    // Use setTimeout to ensure DOM has updated
+    setTimeout(() => {
+      if (!container) return;
+
+      // LIVE MODE: Always scroll to the bottom (last visible element)
+      if (selectedStepIndex === null) {
+        // Scroll to bottom
+        container.scrollTo({
+          top: container.scrollHeight,
+          behavior: 'smooth',
+        });
+      }
+      // NON-LIVE MODE: Scroll to selected step
+      else {
+        const selectedElement = container.querySelector(`[data-step-index="${selectedStepIndex}"]`);
+        if (selectedElement) {
+          selectedElement.scrollIntoView({
+            behavior: 'smooth',
+            block: 'center',
+          });
+        }
+      }
+
+      // Reset flag after scroll animation
+      setTimeout(() => {
+        isScrollingProgrammatically.current = false;
+      }, 500);
+    }, 100);
+  }, [selectedStepIndex, trace?.steps?.length, showThinkingCard, finalStep]);
+
+  // Detect which step is visible when scrolling (steps → timeline)
+  useEffect(() => {
+    const container = containerRef.current;
+    if (!container || !trace?.steps || trace.steps.length === 0) return;
+
+    const handleScroll = () => {
+      // Don't update if we're scrolling programmatically
+      if (isScrollingProgrammatically.current) return;
+
+      // Don't update if agent is running (stay in live mode)
+      if (trace?.isRunning) return;
+
+      const containerRect = container.getBoundingClientRect();
+      const containerTop = containerRect.top;
+      const containerBottom = containerRect.bottom;
+      const containerCenter = containerRect.top + containerRect.height / 2;
+
+      // Check scroll position
+      const isAtTop = container.scrollTop <= 5; // 5px tolerance
+      const isAtBottom = container.scrollTop + container.clientHeight >= container.scrollHeight - 5; // 5px tolerance
+
+      let targetStepIndex: number | null = -1;
+      let targetDistance = Infinity;
+      let isFinalStepTarget = false;
+
+      if (isAtTop) {
+        // At the top: find the highest visible step
+        let highestVisibleBottom = Infinity;
+
+        trace.steps.forEach((_, index) => {
+          const stepElement = container.querySelector(`[data-step-index="${index}"]`);
+          if (stepElement) {
+            const stepRect = stepElement.getBoundingClientRect();
+            const stepTop = stepRect.top;
+            const stepBottom = stepRect.bottom;
+            const isVisible = stepTop < containerBottom && stepBottom > containerTop;
+
+            if (isVisible && stepTop < highestVisibleBottom) {
+              highestVisibleBottom = stepTop;
+              targetStepIndex = index;
+              isFinalStepTarget = false;
+            }
+          }
+        });
+      } else if (isAtBottom) {
+        // At the bottom: find the lowest visible step
+        let lowestVisibleTop = -Infinity;
+
+        trace.steps.forEach((_, index) => {
+          const stepElement = container.querySelector(`[data-step-index="${index}"]`);
+          if (stepElement) {
+            const stepRect = stepElement.getBoundingClientRect();
+            const stepTop = stepRect.top;
+            const stepBottom = stepRect.bottom;
+            const isVisible = stepTop < containerBottom && stepBottom > containerTop;
+
+            if (isVisible && stepTop > lowestVisibleTop) {
+              lowestVisibleTop = stepTop;
+              targetStepIndex = index;
+              isFinalStepTarget = false;
+            }
+          }
+        });
+
+        // Check if final step is the lowest visible
+        if (finalStep) {
+          const finalStepElement = container.querySelector(`[data-step-index="final"]`);
+          if (finalStepElement) {
+            const finalStepRect = finalStepElement.getBoundingClientRect();
+            const finalStepTop = finalStepRect.top;
+            const finalStepBottom = finalStepRect.bottom;
+            const isVisible = finalStepTop < containerBottom && finalStepBottom > containerTop;
+
+            if (isVisible && finalStepTop > lowestVisibleTop) {
+              targetStepIndex = null;
+              isFinalStepTarget = true;
+            }
+          }
+        }
+      } else {
+        // Not at bottom: find the step closest to center
+        trace.steps.forEach((_, index) => {
+          const stepElement = container.querySelector(`[data-step-index="${index}"]`);
+          if (stepElement) {
+            const stepRect = stepElement.getBoundingClientRect();
+            const stepCenter = stepRect.top + stepRect.height / 2;
+            const distance = Math.abs(containerCenter - stepCenter);
+
+            if (distance < targetDistance) {
+              targetDistance = distance;
+              targetStepIndex = index;
+              isFinalStepTarget = false;
+            }
+          }
+        });
+
+        // Check if final step is closest to center
+        if (finalStep) {
+          const finalStepElement = container.querySelector(`[data-step-index="final"]`);
+          if (finalStepElement) {
+            const finalStepRect = finalStepElement.getBoundingClientRect();
+            const finalStepCenter = finalStepRect.top + finalStepRect.height / 2;
+            const distance = Math.abs(containerCenter - finalStepCenter);
+
+            if (distance < targetDistance) {
+              targetStepIndex = null;
+              isFinalStepTarget = true;
+            }
+          }
+        }
+      }
+
+      // Update the selected step if changed
+      if (isFinalStepTarget && selectedStepIndex !== null) {
+        setSelectedStepIndex(null);
+      } else if (!isFinalStepTarget && targetStepIndex !== -1 && targetStepIndex !== selectedStepIndex) {
+        setSelectedStepIndex(targetStepIndex);
+      }
+    };
+
+    // Throttle scroll events
+    let scrollTimeout: NodeJS.Timeout;
+    const throttledScroll = () => {
+      clearTimeout(scrollTimeout);
+      scrollTimeout = setTimeout(handleScroll, 150);
+    };
+
+    container.addEventListener('scroll', throttledScroll);
+    return () => {
+      container.removeEventListener('scroll', throttledScroll);
+      clearTimeout(scrollTimeout);
+    };
+  }, [trace?.steps, selectedStepIndex, setSelectedStepIndex, finalStep]);
+
+  return (
+    <Paper
+      elevation={0}
+      sx={{
+        width: { xs: '100%', md: 320 },
+        flexShrink: 0,
+        display: 'flex',
+        flexDirection: 'column',
+        ml: { xs: 0, md: 1.5 },
+        mt: { xs: 3, md: 0 },
+        overflow: 'hidden',
+      }}
+    >
+      <Box sx={{ px: 2, py: 1.5, borderBottom: '1px solid', borderColor: 'divider', display: 'flex', alignItems: 'center', justifyContent: 'space-between' }}>
+        <Typography variant="h6" sx={{ fontSize: '0.9rem', fontWeight: 700, color: 'text.primary' }}>
+          Steps
+        </Typography>
+        {trace?.traceMetadata && trace.traceMetadata.numberOfSteps > 0 && (
+          <Box sx={{ display: 'flex', alignItems: 'center', gap: 0 }}>
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.75rem',
+                fontWeight: 700,
+                color: 'text.primary',
+              }}
+            >
+              {trace.traceMetadata.numberOfSteps}
+            </Typography>
+            <Typography
+              variant="caption"
+              sx={{
+                fontSize: '0.75rem',
+                fontWeight: 700,
+                color: 'text.disabled',
+              }}
+            >
+              /{trace.traceMetadata.maxSteps}
+            </Typography>
+          </Box>
+        )}
+      </Box>
+      <Box
+        ref={containerRef}
+        sx={{
+          flex: 1,
+          overflowY: 'auto',
+          minHeight: 0,
+          p: 2,
+        }}
+      >
+        {(trace?.steps && trace.steps.length > 0) || finalStep || showThinkingCard || showConnectionCard ? (
+          <Stack spacing={2.5}>
+            {/* Show connection step card (first item) */}
+            {showConnectionCard && (
+              <Box data-step-index="connection">
+                <ConnectionStepCard isConnecting={isConnectingToE2B} />
+              </Box>
+            )}
+
+            {/* Show all steps */}
+            {trace?.steps && trace.steps.map((step, index) => (
+              <Box key={step.stepId} data-step-index={index}>
+                <StepCard
+                  step={step}
+                  index={index}
+                  isLatest={index === trace.steps!.length - 1}
+                  isActive={index === activeStepIndex}
+                />
+              </Box>
+            ))}
+
+            {/* Show thinking indicator after steps (appears 5 seconds after stream start) */}
+            {showThinkingCard && (
+              <Box data-step-index="thinking">
+                <ThinkingStepCard isActive={isThinkingCardActive} />
+              </Box>
+            )}
+
+            {/* Show final step card if exists */}
+            {finalStep && (
+              <Box data-step-index="final">
+                <FinalStepCard
+                  finalStep={finalStep}
+                  isActive={isFinalStepActive}
+                />
+              </Box>
+            )}
+          </Stack>
+        ) : (
+          <Box
+            sx={{
+              display: 'flex',
+              flexDirection: 'column',
+              alignItems: 'center',
+              justifyContent: 'center',
+              height: '100%',
+              color: 'text.secondary',
+              p: 3,
+              textAlign: 'center',
+            }}
+          >
+            <ListAltIcon sx={{ fontSize: 48, mb: 2, opacity: 0.5 }} />
+            <Typography variant="body1" sx={{ fontWeight: 600, mb: 0.5 }}>
+              No steps yet
+            </Typography>
+            <Typography variant="caption" sx={{ fontSize: '0.75rem' }}>
+              Steps will appear as the agent progresses
+            </Typography>
+          </Box>
+        )}
+      </Box>
+    </Paper>
+  );
+};

src/components/steps/ThinkingStepCard.tsx

@@ -0,0 +1,104 @@
+import React from 'react';
+import { Card, CardContent, Box, Typography, CircularProgress } from '@mui/material';
+import { keyframes } from '@mui/system';
+
+// Border pulse animation
+const borderPulse = keyframes`
+  0%, 100% {
+    border-color: rgba(79, 134, 198, 0.4);
+    box-shadow: 0 2px 8px rgba(79, 134, 198, 0.15);
+  }
+  50% {
+    border-color: rgba(79, 134, 198, 0.8);
+    box-shadow: 0 2px 12px rgba(79, 134, 198, 0.3);
+  }
+`;
+
+// Background pulse animation
+const backgroundPulse = keyframes`
+  0%, 100% {
+    background-color: rgba(79, 134, 198, 0.03);
+  }
+  50% {
+    background-color: rgba(79, 134, 198, 0.08);
+  }
+`;
+
+interface ThinkingStepCardProps {
+  isActive?: boolean;
+}
+
+export const ThinkingStepCard: React.FC<ThinkingStepCardProps> = ({ isActive = false }) => {
+
+  return (
+    <Card
+      elevation={0}
+      sx={{
+        backgroundColor: 'background.paper',
+        border: '1px solid',
+        borderColor: (theme) => `${isActive ? theme.palette.primary.main : theme.palette.divider} !important`,
+        borderRadius: 1.5,
+        animation: isActive ? `${borderPulse} 2s ease-in-out infinite` : 'none',
+        position: 'relative',
+        overflow: 'hidden',
+        boxShadow: isActive ? (theme) => `0 2px 8px ${theme.palette.mode === 'dark' ? 'rgba(79, 134, 198, 0.3)' : 'rgba(79, 134, 198, 0.2)'}` : 'none',
+        transition: 'all 0.2s ease',
+        '&::before': {
+          content: '""',
+          position: 'absolute',
+          top: 0,
+          left: 0,
+          right: 0,
+          bottom: 0,
+          animation: isActive ? `${backgroundPulse} 2s ease-in-out infinite` : 'none',
+          zIndex: 0,
+        },
+      }}
+    >
+      <CardContent sx={{ p: 1.5, '&:last-child': { pb: 1.5 }, position: 'relative', zIndex: 1 }}>
+        {/* Header with spinner */}
+        <Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5 }}>
+          <Box
+            sx={{
+              display: 'flex',
+              alignItems: 'center',
+              justifyContent: 'center',
+            }}
+          >
+            {/* Spinner circulaire */}
+            <CircularProgress
+              size={32}
+              thickness={3.5}
+              sx={{
+                color: 'primary.main',
+              }}
+            />
+          </Box>
+
+          <Box sx={{ flex: 1, minWidth: 0 }}>
+            <Typography
+              sx={{
+                fontSize: '0.85rem',
+                fontWeight: 700,
+                color: 'primary.main',
+                lineHeight: 1.3,
+              }}
+            >
+              Agent
+            </Typography>
+            <Typography
+              sx={{
+                fontSize: '0.7rem',
+                color: 'text.secondary',
+                lineHeight: 1.2,
+                fontStyle: 'italic',
+              }}
+            >
+              Thinking...
+            </Typography>
+          </Box>
+        </Box>
+      </CardContent>
+    </Card>
+  );
+};

src/components/steps/index.ts

@@ -0,0 +1,5 @@
+export { StepsList } from './StepsList';
+export { StepCard } from './StepCard';
+export { ThinkingStepCard } from './ThinkingStepCard';
+export { FinalStepCard } from './FinalStepCard';
+export { ConnectionStepCard } from './ConnectionStepCard';

src/components/timeline/Timeline.tsx

@@ -0,0 +1,486 @@
+import React, { useRef, useEffect } from 'react';
+import { Box, Typography, CircularProgress, Button } from '@mui/material';
+import CheckIcon from '@mui/icons-material/Check';
+import CloseIcon from '@mui/icons-material/Close';
+import StopCircleIcon from '@mui/icons-material/StopCircle';
+import HourglassEmptyIcon from '@mui/icons-material/HourglassEmpty';
+import AccessTimeIcon from '@mui/icons-material/AccessTime';
+import CableIcon from '@mui/icons-material/Cable';
+import { AgentTraceMetadata } from '@/types/agent';
+import { useAgentStore, selectSelectedStepIndex, selectFinalStep, selectIsConnectingToE2B, selectIsAgentProcessing } from '@/stores/agentStore';
+
+interface TimelineProps {
+  metadata: AgentTraceMetadata;
+  isRunning: boolean;
+}
+
+export const Timeline: React.FC<TimelineProps> = ({ metadata, isRunning }) => {
+  const timelineRef = useRef<HTMLDivElement>(null);
+  const selectedStepIndex = useAgentStore(selectSelectedStepIndex);
+  const setSelectedStepIndex = useAgentStore((state) => state.setSelectedStepIndex);
+  const finalStep = useAgentStore(selectFinalStep);
+  const isConnectingToE2B = useAgentStore(selectIsConnectingToE2B);
+  const isAgentProcessing = useAgentStore(selectIsAgentProcessing);
+
+  // Show connection indicator if connecting or if we have started processing
+  const showConnectionIndicator = isConnectingToE2B || isAgentProcessing || (metadata.numberOfSteps > 0) || finalStep;
+
+  // Generate array of steps with their status
+  // Only show completed steps + current step if running
+  const totalStepsToShow = isRunning && !isConnectingToE2B
+    ? metadata.numberOfSteps + 1  // Show completed steps + current step
+    : metadata.numberOfSteps;     // Show only completed steps when not running
+
+  // Calculate total width for the line (including finalStep if present)
+  const lineWidth = finalStep
+    ? `calc(${totalStepsToShow} * (40px + 12px) + 52px)` // Add space for finalStep (40px + 12px gap)
+    : `calc(${totalStepsToShow} * (40px + 12px))`;
+
+  const steps = Array.from({ length: totalStepsToShow }, (_, index) => ({
+    stepNumber: index + 1,
+    stepIndex: index,
+    isCompleted: index < metadata.numberOfSteps,
+    // Step is current if: we're at the right index AND running AND not connecting to E2B
+    isCurrent: (index === metadata.numberOfSteps && isRunning && !isConnectingToE2B) ||
+               (index === 0 && metadata.numberOfSteps === 0 && isRunning && !isConnectingToE2B),
+    isSelected: selectedStepIndex === index,
+  }));
+
+  // Handle step click
+  const handleStepClick = (stepIndex: number, isCompleted: boolean, isCurrent: boolean) => {
+    if (isCompleted) {
+      setSelectedStepIndex(stepIndex);
+    } else if (isCurrent) {
+      // Clicking on the current step (with animation) goes back to live mode
+      setSelectedStepIndex(null);
+    }
+  };
+
+  // Handle final step click (goes to live mode showing the final status)
+  const handleFinalStepClick = () => {
+    setSelectedStepIndex(null);
+  };
+
+  // Auto-scroll to current step while running
+  useEffect(() => {
+    if (timelineRef.current && isRunning) {
+      // Only auto-scroll while running, not when finished
+      const currentStepElement = timelineRef.current.querySelector(`[data-step="${metadata.numberOfSteps}"]`);
+      if (currentStepElement) {
+        currentStepElement.scrollIntoView({ behavior: 'smooth', inline: 'center', block: 'nearest' });
+      }
+    }
+  }, [metadata.numberOfSteps, isRunning]);
+
+  return (
+    <Box
+      sx={{
+        p: 2,
+        border: '1px solid',
+        borderColor: 'divider',
+        borderRadius: '12px',
+        backgroundColor: 'background.paper',
+        flexShrink: 0,
+      }}
+    >
+      <Box sx={{ display: 'flex', flexDirection: 'column', gap: 1.5 }}>
+        {/* Header with step count */}
+        <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between' }}>
+          <Typography variant="h6" sx={{ fontSize: '0.9rem', fontWeight: 700, color: 'text.primary' }}>
+            Timeline
+            {selectedStepIndex !== null && (
+              <Typography component="span" sx={{ ml: 1, color: 'text.secondary', fontWeight: 500, fontSize: '0.65rem' }}>
+                - Viewing step {selectedStepIndex + 1}
+              </Typography>
+            )}
+          </Typography>
+          {selectedStepIndex !== null && (
+            <Button
+              size="small"
+              variant="outlined"
+              onClick={handleFinalStepClick}
+              sx={{
+                textTransform: 'none',
+                fontSize: '0.7rem',
+                fontWeight: 600,
+                px: 1.5,
+                py: 0.25,
+                minWidth: 'auto',
+                color: 'text.secondary',
+                borderColor: 'divider',
+                '&:hover': {
+                  backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(255,255,255,0.05)' : 'rgba(0,0,0,0.03)',
+                  borderColor: 'text.secondary',
+                },
+              }}
+            >
+              Back to latest step
+            </Button>
+          )}
+        </Box>
+
+        {/* Horizontal scrollable step indicators */}
+        <Box
+          ref={timelineRef}
+          sx={{
+            display: 'flex',
+            alignItems: 'center',
+            overflowX: 'auto',
+            overflowY: 'hidden',
+            gap: 1.5,
+            py: 1.5,
+            height: 60,
+            position: 'relative',
+            // Hide scrollbar completely
+            scrollbarWidth: 'none', // Firefox
+            '&::-webkit-scrollbar': {
+              display: 'none', // Chrome, Safari, Edge
+            },
+            // Horizontal line crossing through circles
+            '&::before': {
+              content: '""',
+              position: 'absolute',
+              left: "25px",
+              // Calculate width to cover visible steps + finalStep if present
+              width: lineWidth,
+              top: '19.5px',
+              transform: 'translateY(-50%)',
+              transition: 'width 0.6s cubic-bezier(0.4, 0, 0.2, 1)',
+              height: '2px',
+              backgroundColor: (theme) => theme.palette.mode === 'dark' ? 'rgba(255, 255, 255, 0.1)' : 'rgba(0, 0, 0, 0.3)',
+              zIndex: 0,
+              pointerEvents: 'none',
+            },
+          }}
+        >
+          {/* Connection indicator (step 0) */}
+          {showConnectionIndicator && (
+            <Box
+              data-step="connection"
+              sx={{
+                display: 'flex',
+                flexDirection: 'column',
+                alignItems: 'center',
+                gap: 0.75,
+                minWidth: 40,
+                flexShrink: 0,
+                position: 'relative',
+                zIndex: 1,
+              }}
+            >
+              {/* White circle background to hide the line */}
+              <Box
+                sx={{
+                  position: 'relative',
+                  display: 'flex',
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  height: 28,
+                  width: 28,
+                }}
+              >
+                {/* White background to hide the line */}
+                <Box
+                  sx={{
+                    position: 'absolute',
+                    width: 28,
+                    height: 28,
+                    borderRadius: '50%',
+                    backgroundColor: 'background.paper',
+                    zIndex: 0,
+                  }}
+                />
+
+                {/* Connection icon */}
+                {isConnectingToE2B ? (
+                  <CircularProgress
+                    size={20}
+                    thickness={5}
+                    sx={{
+                      color: 'primary.main',
+                      position: 'relative',
+                      zIndex: 1,
+                    }}
+                  />
+                ) : (
+                  <CableIcon
+                    sx={{
+                      fontSize: 20,
+                      color: 'success.main',
+                      position: 'relative',
+                      zIndex: 1,
+                    }}
+                  />
+                )}
+              </Box>
+
+              {/* Connection label */}
+              <Typography
+                variant="caption"
+                sx={{
+                  fontSize: '0.7rem',
+                  fontWeight: 700,
+                  color: isConnectingToE2B ? 'primary.main' : 'success.main',
+                  whiteSpace: 'nowrap',
+                }}
+              >
+                {isConnectingToE2B ? 'Connecting' : 'Connected'}
+              </Typography>
+            </Box>
+          )}
+
+          {/* Render steps and insert final step at the right position */}
+          {steps.map((step, index) => (
+            <React.Fragment key={step.stepNumber}>
+              <Box
+                data-step={step.stepNumber}
+                onClick={() => handleStepClick(step.stepIndex, step.isCompleted, step.isCurrent)}
+                sx={{
+                  display: 'flex',
+                  flexDirection: 'column',
+                  alignItems: 'center',
+                  gap: 0.75,
+                  minWidth: 40,
+                  flexShrink: 0,
+                  position: 'relative',
+                  zIndex: 1,
+                  cursor: (step.isCompleted || step.isCurrent) ? 'pointer' : 'default',
+                  '&:hover': (step.isCompleted || step.isCurrent) ? {
+                    '& .step-dot': {
+                      transform: 'scale(1.15)',
+                    },
+                  } : {},
+                }}
+              >
+                {/* White circle background to hide the line */}
+                <Box
+                  sx={{
+                    position: 'relative',
+                    display: 'flex',
+                    alignItems: 'center',
+                    justifyContent: 'center',
+                    height: 28,
+                    width: 28,
+                  }}
+                >
+                  {/* White background to hide the line */}
+                  <Box
+                    sx={{
+                      position: 'absolute',
+                      width: 28,
+                      height: 28,
+                      borderRadius: '50%',
+                      backgroundColor: 'background.paper',
+                      zIndex: 0,
+                    }}
+                  />
+
+                  {/* Step dot */}
+                  {step.isCurrent ? (
+                    <Box
+                      sx={{
+                        position: 'relative',
+                        display: 'flex',
+                        alignItems: 'center',
+                        justifyContent: 'center',
+                        zIndex: 1,
+                      }}
+                    >
+                      <CircularProgress
+                        size={20}
+                        thickness={5}
+                        sx={{
+                          color: 'primary.main',
+                          position: 'absolute',
+                        }}
+                      />
+                      <Box
+                        sx={{
+                          width: 8,
+                          height: 8,
+                          borderRadius: '50%',
+                          backgroundColor: 'white',
+                          position: 'absolute',
+                          pointerEvents: 'none',
+                          boxShadow: '0 0 4px rgba(0,0,0,0.2)',
+                        }}
+                      />
+                    </Box>
+                  ) : (
+                    <Box
+                      sx={{
+                        position: 'relative',
+                        display: 'flex',
+                        alignItems: 'center',
+                        justifyContent: 'center',
+                        zIndex: 1,
+                      }}
+                    >
+                      <Box
+                        className="step-dot"
+                        sx={{
+                          width: step.isSelected ? 20 : step.isCompleted ? 14 : 12,
+                          height: step.isSelected ? 20 : step.isCompleted ? 14 : 12,
+                          borderRadius: '50%',
+                          // Always keep steps in primary color (blue)
+                          backgroundColor: step.isCompleted
+                            ? 'primary.main' // Blue for completed steps
+                            : (theme) => theme.palette.mode === 'dark' ? 'grey.800' : 'grey.300', // Light grey for future steps
+                          transition: 'all 0.2s ease',
+                          boxShadow: step.isCompleted || step.isSelected
+                            ? step.isSelected
+                              ? '0 0 8px rgba(255, 167, 38, 0.5)'
+                              : '0 2px 4px rgba(0,0,0,0.1)'
+                            : 'none',
+                        }}
+                      />
+                      {/* White dot for selected step */}
+                      {step.isSelected && (
+                        <Box
+                          sx={{
+                            width: 8,
+                            height: 8,
+                            borderRadius: '50%',
+                            backgroundColor: 'white',
+                            position: 'absolute',
+                          }}
+                        />
+                      )}
+                    </Box>
+                  )}
+                </Box>
+
+                {/* Step number - show for all steps */}
+                <Typography
+                  variant="caption"
+                  sx={{
+                    fontSize: '0.7rem',
+                    fontWeight: step.isSelected || step.isCurrent ? 900 : 400,
+                    color: step.isCurrent
+                      ? 'primary.main'
+                      : (step.isCompleted || step.isSelected
+                        ? 'text.primary'
+                        : (theme) => theme.palette.mode === 'dark' ? 'grey.700' : 'grey.400'),
+                    whiteSpace: 'nowrap',
+                    lineHeight: 1,
+                  }}
+                >
+                  {step.stepNumber}
+                </Typography>
+              </Box>
+
+              {/* Insert final step indicator right after the last completed step */}
+              {finalStep && step.stepNumber === metadata.numberOfSteps && (
+            <Box
+              data-step="final"
+              onClick={handleFinalStepClick}
+              sx={{
+                display: 'flex',
+                flexDirection: 'column',
+                alignItems: 'center',
+                gap: 0.75,
+                minWidth: 40,
+                flexShrink: 0,
+                position: 'relative',
+                zIndex: 1,
+                cursor: 'pointer',
+                '&:hover': {
+                  '& .final-step-icon': {
+                    transform: 'scale(1.15)',
+                  },
+                },
+              }}
+            >
+              {/* White circle background to hide the line */}
+              <Box
+                sx={{
+                  position: 'relative',
+                  display: 'flex',
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  height: 28,
+                  width: 28,
+                }}
+              >
+                {/* White background to hide the line */}
+                <Box
+                  sx={{
+                    position: 'absolute',
+                    width: 28,
+                    height: 28,
+                    borderRadius: '50%',
+                    backgroundColor: 'background.paper',
+                    zIndex: 0,
+                  }}
+                />
+
+                {/* Final step icon */}
+                <Box
+                  className="final-step-icon"
+                  sx={{
+                    width: selectedStepIndex === null ? 20 : 18,
+                    height: selectedStepIndex === null ? 20 : 18,
+                    borderRadius: '50%',
+                    backgroundColor:
+                      finalStep.type === 'success' ? 'success.main' :
+                      finalStep.type === 'stopped' || finalStep.type === 'max_steps_reached' ? 'warning.main' :
+                      'error.main',
+                    display: 'flex',
+                    alignItems: 'center',
+                    justifyContent: 'center',
+                    transition: 'all 0.2s ease',
+                    boxShadow: selectedStepIndex === null
+                      ? finalStep.type === 'success'
+                        ? '0 2px 8px rgba(102, 187, 106, 0.4)'
+                        : finalStep.type === 'stopped' || finalStep.type === 'max_steps_reached'
+                          ? '0 2px 8px rgba(255, 152, 0, 0.4)'
+                          : '0 2px 8px rgba(244, 67, 54, 0.4)'
+                      : '0 2px 4px rgba(0,0,0,0.1)',
+                    position: 'relative',
+                    zIndex: 1,
+                  }}
+                >
+                  {finalStep.type === 'success' ? (
+                    <CheckIcon sx={{ fontSize: 14, color: 'white' }} />
+                  ) : finalStep.type === 'stopped' ? (
+                    <StopCircleIcon sx={{ fontSize: 14, color: 'white' }} />
+                  ) : finalStep.type === 'max_steps_reached' ? (
+                    <HourglassEmptyIcon sx={{ fontSize: 14, color: 'white' }} />
+                  ) : finalStep.type === 'sandbox_timeout' ? (
+                    <AccessTimeIcon sx={{ fontSize: 14, color: 'white' }} />
+                  ) : (
+                    <CloseIcon sx={{ fontSize: 14, color: 'white' }} />
+                  )}
+                </Box>
+              </Box>
+
+              {/* Final step label */}
+              <Typography
+                variant="caption"
+                sx={{
+                  fontSize: '0.7rem',
+                  fontWeight: selectedStepIndex === null ? 700 : 500,
+                  color:
+                    finalStep.type === 'success'
+                      ? (selectedStepIndex === null ? 'text.primary' : 'text.secondary')
+                      : finalStep.type === 'stopped' || finalStep.type === 'max_steps_reached'
+                        ? 'warning.main'
+                        : 'error.main',
+                  whiteSpace: 'nowrap',
+                }}
+              >
+                {finalStep.type === 'success' ? 'End' :
+                 finalStep.type === 'stopped' ? 'Stopped' :
+                 finalStep.type === 'max_steps_reached' ? 'Max Steps' :
+                 finalStep.type === 'sandbox_timeout' ? 'Timeout' :
+                 'Failed'}
+              </Typography>
+            </Box>
+              )}
+            </React.Fragment>
+          ))}
+        </Box>
+      </Box>
+    </Box>
+  );
+};