Implement comprehensive query-document matching system - Add category-based relevance scoring for all query types (pension, leave, allowance, procurement, medical, etc.) - Heavy penalties for mismatched content to prevent wrong results

rag_service.py

@@ -269,53 +269,47 @@ async def search_documents_async(query: str, limit: int = 5) -> List[Dict[str, A
             query_categories = {
                 'pension': ['pension', 'retirement', 'gratuity', 'provident fund', 'gpf', 'cpf', 'superannuation'],
                 'leave': ['leave', 'casual leave', 'earned leave', 'medical leave', 'maternity', 'paternity'],
-                'allowance': ['allowance', 'dearness allowance', 'da', 'hra', 'house rent', 'travel allowance'],
-                'procurement': ['procurement', 'tender', 'bid', 'contract', 'purchase', 'vendor'],
+                'allowance': ['allowance', 'dearness allowance', 'da', 'hra', 'house rent', 'travel allowance', 'increment'],
+                'procurement': ['procurement', 'tender', 'bid', 'contract', 'purchase', 'vendor', 'gem'],
                 'medical': ['medical', 'health', 'treatment', 'reimbursement', 'cghs', 'hospital'],
                 'transfer': ['transfer', 'posting', 'deputation', 'cadre'],
                 'promotion': ['promotion', 'seniority', 'grade', 'advancement', 'career progression'],
                 'service': ['service', 'conduct', 'discipline', 'rules', 'regulation']
             }
             
+            # Content categories - what each document type contains
+            content_categories = {
+                'pension': ['pension', 'retirement benefit', 'gratuity', 'provident fund', 'superannuation'],
+                'leave': ['leave rules', 'casual leave', 'earned leave', 'medical leave', 'maternity leave'],
+                'allowance': ['dearness allowance', 'house rent allowance', 'travel allowance', 'da', 'hra', 'increment'],
+                'procurement': ['procurement', 'tender', 'bidding', 'contract', 'vendor', 'gem', 'purchase'],
+                'medical': ['medical', 'health', 'cghs', 'reimbursement', 'treatment'],
+                'transfer': ['transfer', 'posting', 'deputation', 'cadre'],
+                'promotion': ['promotion', 'seniority', 'grade pay', 'advancement'],
+                'service': ['service rules', 'conduct', 'discipline', 'misconduct']
+            }
+            
             # Determine query category
-            query_category = None
+            detected_query_category = None
             for category, keywords in query_categories.items():
                 if any(keyword in query_lower for keyword in keywords):
-                    query_category = category
+                    detected_query_category = category
                     break
             
-            if query_category:
-                # Boost content matching the query category
-                category_keywords = query_categories[query_category]
-                if any(keyword in content for keyword in category_keywords):
-                    relevance_score += 1.0
-                
-                # Add specific boosts for each category
-                if query_category == 'pension':
-                    if any(keyword in content for keyword in ['pension rules', 'retirement benefit', 'pension scheme']):
-                        relevance_score += 0.8
-                elif query_category == 'leave':
-                    if any(keyword in content for keyword in ['leave rules', 'leave entitlement', 'leave policy']):
-                        relevance_score += 0.8
-                elif query_category == 'allowance':
-                    if any(keyword in content for keyword in ['allowance rates', 'da revision', 'allowance rules']):
-                        relevance_score += 0.8
-                elif query_category == 'procurement':
-                    if any(keyword in content for keyword in ['tender process', 'procurement rules', 'bidding']):
-                        relevance_score += 0.8
-                elif query_category == 'medical':
-                    if any(keyword in content for keyword in ['medical rules', 'reimbursement policy', 'cghs']):
-                        relevance_score += 0.8
-                elif query_category == 'transfer':
-                    if any(keyword in content for keyword in ['transfer policy', 'posting rules', 'cadre']):
-                        relevance_score += 0.8
+            # If query category detected, apply targeted scoring
+            if detected_query_category:
+                # Boost score if document content matches query category
+                matching_content_keywords = content_categories.get(detected_query_category, [])
+                if any(keyword in content for keyword in matching_content_keywords):
+                    relevance_score += 1.5  # Strong boost for matching content
                 
-                # Penalize irrelevant content for specific queries
-                other_categories = [cat for cat in query_categories.keys() if cat != query_category]
-                for other_cat in other_categories:
-                    other_keywords = query_categories[other_cat]
-                    if any(keyword in content for keyword in other_keywords[:3]):  # Check top 3 keywords
-                        relevance_score -= 0.7
+                # Penalize documents from different categories
+                for other_category, other_keywords in content_categories.items():
+                    if other_category != detected_query_category:
+                        if any(keyword in content for keyword in other_keywords):
+                            relevance_score -= 1.2  # Heavy penalty for non-matching content
+                            
+                logger.debug(f"Query category: {detected_query_category}, Relevance: {relevance_score:.2f} for content: {content[:50]}...")
             
             if relevance_score > 0.3:  # Only include relevant documents
                 # Add relevance score to document (create dict if needed)