Add 899999999999999999999999999

rag_service.py

@@ -431,6 +431,8 @@ async def search_documents_async(query: str, limit: int = 5) -> List[Dict[str, A
                 enhanced_query = f"{query} pension rules retirement benefits modifications"
             elif "calculation" in query_lower or "formula" in query_lower:
                 enhanced_query = f"{query} pension calculation retirement benefits formula"
+            elif any(word in query_lower for word in ["old age", "elderly", "senior", "old"]):
+                enhanced_query = f"{query} pension retirement benefits elderly old age senior citizen"
         
         # Leave queries  
         elif any(word in query_lower for word in ["leave", "casual", "earned"]):
@@ -539,7 +541,7 @@ async def search_documents_async(query: str, limit: int = 5) -> List[Dict[str, A
                                 'financial', 'fund', 'payment', 'authorization', 'delegation', 'registers'],
                 'leadership_policymaker': ['policy', 'impact', 'scenario', 'analysis', 'evidence', 'comparative',
                                          'evaluation', 'decision', 'strategic', 'implementation', 'assessment'],
-                'pension': ['pension', 'retirement', 'gratuity', 'provident fund', 'superannuation', 'benefits'],
+                'pension': ['pension', 'retirement', 'gratuity', 'provident fund', 'superannuation', 'benefits', 'old age', 'elderly', 'senior'],
                 'leave': ['leave', 'casual', 'earned', 'medical leave', 'maternity'],
                 'allowance': ['allowance', 'dearness', 'house rent', 'travel', 'da', 'hra', 'increment'],
                 'procurement': ['procurement', 'tender', 'bidding', 'contract', 'vendor', 'gem', 'purchase'],
@@ -566,21 +568,33 @@ async def search_documents_async(query: str, limit: int = 5) -> List[Dict[str, A
                 content_matches = [kw for kw in matching_content_keywords if kw in content]
                 if content_matches:
                     relevance_score += 1.5  # Strong boost for matching content
-                    logger.debug(f"✅ Content match found: {content_matches} for category: {detected_query_category}")
+                    logger.info(f"✅ Content match found: {content_matches} for category: {detected_query_category}, New score: {relevance_score:.2f}")
+                else:
+                    logger.info(f"❌ No content match for category: {detected_query_category}, checking keywords: {matching_content_keywords}")
                 
-                # Penalize documents from different categories
-                for other_category, other_keywords in content_categories.items():
-                    if other_category != detected_query_category:
-                        if any(keyword in content for keyword in other_keywords):
-                            relevance_score -= 0.8  # Moderate penalty for non-matching content
+                # Apply lighter penalties for documents from different categories
+                # Only penalize if no matching content was found for the main category
+                if not content_matches:
+                    for other_category, other_keywords in content_categories.items():
+                        if other_category != detected_query_category:
+                            if any(keyword in content for keyword in other_keywords):
+                                relevance_score -= 0.3  # Light penalty only when no main category match
                             
-                logger.debug(f"Query category: {detected_query_category}, Relevance: {relevance_score:.2f} for content: {content[:50]}...")
+                logger.info(f"📊 Query category: {detected_query_category}, Final relevance: {relevance_score:.2f} for content: {content[:50]}...")
+            
+            # Use different thresholds based on query type
+            threshold = 0.1  # Default threshold
+            if any(keyword in query_lower for keyword in ['pension', 'retirement', 'old age']):
+                threshold = -0.2  # Very inclusive for pension queries
             
-            if relevance_score > 0.1:  # More inclusive threshold for relevant documents
+            if relevance_score > threshold:  # Adaptive threshold for relevant documents
+                logger.info(f"✅ Document PASSED filter: score {relevance_score:.2f} > threshold {threshold}")
                 # Add relevance score to document (create dict if needed)
                 if hasattr(doc, 'metadata'):
                     doc.metadata['relevance_score'] = relevance_score
                 filtered_docs.append(doc)
+            else:
+                logger.info(f"❌ Document FAILED filter: score {relevance_score:.2f} <= threshold {threshold}")
         
         # Sort by relevance score and limit results
         filtered_docs = sorted(filtered_docs, key=lambda x: getattr(x, 'metadata', {}).get('relevance_score', 0), reverse=True)[:limit]