setup_documents.py

"""
Setup script to populate LanceDB with sample documents for the Voice Bot
"""
import os
from lancedb_service import lancedb_service
import asyncio
import logging
from comprehensive_documents import COMPREHENSIVE_DOCUMENTS

logger = logging.getLogger("voicebot")

# Sample government documents content - Diverse content for different queries
SAMPLE_DOCUMENTS = [
    {
        "content": """Government employees are entitled to various types of leave during their service period. The leave rules are governed by the Central Civil Services (Leave) Rules and state-specific adaptations.

Types of Leave Available:
1. Casual Leave (CL): 8 days per calendar year
   - Can be taken for urgent personal work
   - No medical certificate required
   - Cannot be carried forward to next year

2. Earned Leave (EL): 30 days per year
   - Can be accumulated up to 300 days
   - Encashment allowed at retirement
   - Advance grant possible for valid reasons

3. Medical Leave: As per medical requirements
   - Medical certificate mandatory
   - Can be combined with EL
   - Special provisions for chronic illnesses

4. Maternity/Paternity Leave:
   - Maternity: 180 days (6 months)
   - Paternity: 15 days within 6 months of child birth

Application Process:
- Submit leave application in advance
- Get approval from competent authority
- Maintain proper leave records
- Emergency leave can be regularized later""",
        "filename": "leave_rules.txt",
        "source": "Government Leave Manual 2024"
    },
    {
        "content": """Government employee salary structure consists of multiple components designed to provide comprehensive compensation and benefits.

Salary Components:
1. Basic Pay: Core salary amount based on pay scale and grade
2. Dearness Allowance (DA): Currently 50% of basic pay (as of 2024)
3. House Rent Allowance (HRA): Varies by city classification
   - X Class cities: 24% of basic pay
   - Y Class cities: 16% of basic pay  
   - Z Class cities: 8% of basic pay

4. Transport Allowance: ₹3,600 per month for most employees
5. Medical Allowance: ₹1,000 per month

Special Allowances:
- City Compensatory Allowance (CCA)
- Special Allowance for difficult postings
- Overtime Allowance (where applicable)

Deductions:
- Income Tax (as per IT rules)
- Provident Fund: 12% of basic pay
- Group Insurance Scheme (GIS)
- Professional Tax (state-specific)

Pay Revision:
- Pay Commission recommendations every 10 years
- Annual increment: Usually 3% of basic pay
- Promotion-based pay upgrades as per rules""",
        "filename": "salary_structure.txt",
        "source": "Pay Commission Guidelines 2024"
    },
    {
        "content": """Transfer and posting policies for government employees are designed to ensure fair distribution of work, prevent corruption, and provide career development opportunities.

Transfer Rules:
1. Tenure-based Transfers:
   - Minimum tenure: 3 years in sensitive posts
   - Maximum tenure: 5 years in one location (general rule)
   - Cooling off period: 2 years before returning to same post

2. Request Transfers:
   - Can apply after completing minimum tenure
   - Valid grounds: medical, family, educational needs
   - Subject to administrative approval and replacement availability

3. Administrative Transfers:
   - Based on service requirements
   - Immediate transfer in case of disciplinary issues
   - Mutual transfers allowed with proper approvals

Posting Guidelines:
- Home district posting: After 5 years of service
- Difficult area posting: Incentives and allowances provided
- Border area posting: Special security clearance required

Transfer Process:
1. Issue of transfer order by competent authority
2. Relieving formalities at current posting
3. Journey time allowance as per rules
4. Joining at new posting within prescribed time
5. Handing/taking over of charge properly

Benefits:
- Transfer TA/DA as per entitlement
- Family accommodation assistance
- School transfer certificates for children""",
        "filename": "transfer_posting_rules.txt",
        "source": "Administrative Transfer Policy 2024"
    },
    {
        "content": """Training and skill development programs are essential for government employees to enhance their capabilities and stay updated with modern administrative practices.

Training Categories:
1. Induction Training:
   - Mandatory for all new recruits
   - Duration: 3-6 months depending on service
   - Covers service rules, conduct rules, and job-specific skills

2. In-Service Training:
   - Periodic skill upgradation programs
   - Leadership development courses
   - Technology and digital literacy training

3. Specialized Training:
   - Domain-specific technical training
   - Foreign training opportunities for senior officers
   - Research and innovation workshops

Training Institutes:
- National Academy of Administration (NAAN) - for IAS officers
- State Administrative Training Institutes
- Sector-specific training centers
- Online learning platforms (iGOT Karmayogi)

Benefits of Training:
1. Career advancement opportunities
2. Better performance and efficiency
3. Exposure to best practices
4. Networking with peers
5. Personal and professional development

Training Leave:
- Special training leave with full pay
- Study leave for higher education
- Deputation opportunities to training institutes

Digital Learning Initiative:
- Mission Karmayogi for capacity building
- Online certification courses
- Competency-based training modules
- Performance-linked training requirements

Training is considered essential for promotion to higher grades and is often a mandatory requirement for career progression in government service.""",
        "filename": "training_development.txt",
        "source": "Government Training Policy 2024"
    },
    {
        "content": """Retirement benefits for government employees include multiple components to ensure financial security post-retirement.

Retirement Benefits Package:
1. Pension: Monthly payment based on last drawn salary and service years
   - Calculation: (Last drawn basic pay + DA) × service years ÷ 70
   - Minimum pension: ₹9,000 per month
   - Maximum pension: No upper limit

2. Gratuity: Lump sum payment at retirement
   - Formula: (Basic pay + DA) × 15/26 × years of service
   - Maximum: ₹20 lakh (as of 2024)
   - Tax exemption available

3. Provident Fund (GPF/CPF):
   - Employee + Government contribution throughout service
   - Withdrawal allowed at retirement
   - Interest rate: Currently 8% per annum

4. Commutation of Pension:
   - Option to convert part of pension to lump sum
   - Up to 50% of pension can be commuted
   - Restoration after 15 years

5. Medical Benefits:
   - Continued medical facility post-retirement
   - Central Government Health Scheme (CGHS) coverage
   - Reimbursement of medical expenses

Retirement Process:
- Apply 6 months before retirement date
- Complete all service formalities
- Obtain clearances from all departments
- Submit required documents and forms

The retirement package is designed to provide comprehensive financial support and maintain dignity of life for retired government employees.""",
        "filename": "retirement_benefits.txt",
        "source": "Retirement Benefits Manual 2024"
    },
    {
        "content": """Government Pension Policies: Comprehensive Overview

India's government pension system has undergone significant reforms to ensure financial security for public sector employees. Here are the key pension policies:

**1. OLD PENSION SCHEME (OPS)**
- Available to: Government employees recruited before January 1, 2004
- Type: Defined Benefit Scheme
- Features:
  * Guaranteed pension of 50% of last drawn salary
  * Dearness Relief applicable
  * Family pension available to dependents
  * No employee contribution required

**2. NEW PENSION SCHEME (NPS)**
- Available to: Government employees recruited after January 1, 2004
- Type: Defined Contribution Scheme
- Features:
  * Employee contribution: 10% of basic salary
  * Government contribution: 14% of basic salary
  * Market-linked returns
  * Portable across jobs and sectors

**3. CONTRIBUTORY PENSION SCHEME**
- Transition scheme between OPS and NPS
- Applicable to certain categories of employees
- Combines elements of both schemes

**4. FAMILY PENSION POLICY**
- Eligibility: Spouse and unmarried children of deceased employee
- Amount: 50% of employee's pension or ₹9,000 (whichever is higher)
- Duration: Lifelong for spouse, until age 25 for children

**5. DISABILITY PENSION POLICY**
- Categories: Extraordinary, Ordinary, and Invalid pension
- Medical board assessment required
- Compensation for service-related disabilities

**6. POLICY REFORMS AND CHANGES**
Recent policy developments:
- Restoration of Old Pension Scheme in several states
- Enhanced minimum pension amounts
- Digital pension processing systems
- Automatic pension updates

**7. PENSION GOVERNANCE FRAMEWORK**
- Administered by: Department of Pension & Pensioners' Welfare
- Regulatory body: Pension Fund Regulatory & Development Authority (PFRDA)
- Implementation: Through banks and pension disbursing agencies

**8. ELIGIBILITY CRITERIA**
- Minimum service: 10 years for pension eligibility
- Superannuation age: 58-62 years (varies by service)
- Voluntary retirement: After 20 years of service

**9. RECENT POLICY INITIATIVES**
- Digital Life Certificate (Jeevan Pramaan)
- Pension through Aadhaar authentication
- Direct Benefit Transfer (DBT) for pensions
- Centralized Pension Processing Centers

These pension policies are designed to provide comprehensive social security and ensure dignified retirement for government employees across India.""",
        "filename": "pension_policies_overview.txt",
        "source": "Department of Pension & Pensioners' Welfare 2024"
    },
    {
        "content": """Government procurement policies and procedures ensure transparency, fairness, and value for money in public purchases.

Procurement Methods:
1. Open Tender:
   - Public advertisement required
   - Minimum 21 days for bid submission
   - Used for high-value procurements above ₹25 lakh

2. Limited Tender:
   - Invitation to selected vendors
   - For specialized items or urgent requirements
   - Proper justification required

3. Single Tender:
   - Direct negotiation with one vendor
   - Only in exceptional circumstances
   - Requires special approval

Key Procurement Rules:
- Preference to Make in India products
- MSME reservation: 25% of procurement
- Minimum 50% local content requirement
- GeM (Government e-Marketplace) mandatory for routine items

Tender Process:
1. Preparation of tender documents
2. Advertisement and bidder registration
3. Pre-bid meetings for clarifications
4. Bid submission and opening
5. Technical and financial evaluation
6. Contract award to lowest compliant bidder

Documentation Required:
- Technical specifications
- Terms and conditions
- Evaluation criteria
- Approval for procurement
- Committee formations for evaluation

This system ensures competitive pricing, quality products, and supports domestic manufacturing while maintaining complete transparency in government spending.""",
        "filename": "procurement_policy.txt", 
        "source": "Government Procurement Guidelines 2024"
    }
]

async def setup_sample_documents():
    """Create comprehensive government documents in the database"""
    try:
        logger.info("🗂️ Setting up comprehensive government documents...")
        
        # Combine sample documents with comprehensive documents for better coverage
        all_documents = SAMPLE_DOCUMENTS + COMPREHENSIVE_DOCUMENTS
        
        # Create document objects that match LanceDB expectations
        docs = []
        for doc_data in all_documents:
            # Create a simple document object with the expected attributes
            doc = type('Document', (), {
                'page_content': doc_data["content"],
                'metadata': {"source": doc_data["source"], "filename": doc_data["filename"]}
            })()
            docs.append(doc)
        
        # Add to government_docs knowledge base
        await lancedb_service.add_documents(
            docs=docs,
            user_id="system",
            knowledge_base="government_docs", 
            filename="sample_documents.txt"
        )
        
        # Also create a specific rajasthan_documents table entry
        # Check if rajasthan_documents table exists, create if not
        try:
            if "rajasthan_documents" not in lancedb_service.db.table_names():
                # Create the table using the same structure as documents table
                import pandas as pd
                import uuid
                from datetime import datetime
                
                sample_data = pd.DataFrame({
                    "id": [str(uuid.uuid4())],
                    "content": ["sample"],
                    "filename": ["sample"],
                    "vector": [lancedb_service.embedding_model.embed_query("sample")]
                })
                lancedb_service.db.create_table("rajasthan_documents", sample_data)
                # Delete sample data
                tbl = lancedb_service.db.open_table("rajasthan_documents")
                tbl.delete("id = 'sample'")
                
                # Now add all documents (sample + comprehensive)
                rajasthan_docs = []  
                for doc_data in all_documents:
                    embedding = lancedb_service.embedding_model.embed_query(doc_data["content"])
                    raj_doc = {
                        "id": str(uuid.uuid4()),
                        "content": doc_data["content"],
                        "filename": doc_data["filename"],
                        "vector": embedding
                    }
                    rajasthan_docs.append(raj_doc)
                
                df = pd.DataFrame(rajasthan_docs)
                tbl.add(df)
                
        except Exception as e:
            logger.warning(f"⚠️ Could not setup rajasthan_documents table: {e}")
            
        logger.info(f"✅ Added {len(all_documents)} comprehensive government documents successfully")
        
    except Exception as e:
        logger.error(f"❌ Error setting up documents: {e}")

if __name__ == "__main__":
    asyncio.run(setup_sample_documents())