Add 899999999

rag_service.py

@@ -205,8 +205,55 @@ async def search_docs(query: str, config: RunnableConfig) -> str:
         # Search in the specified knowledge base
         docs = await lancedb_service.similarity_search(query, userid, knowledge_base)
         if docs:
-            context = "\n\n".join([doc.page_content[:500] for doc in docs])  # Only first 500 chars per doc
-            return f"📄 Found {len(docs)} relevant documents:\n\n{context}"
+            # Advanced extractive summarization using NLTK
+            try:
+                import nltk
+                nltk.download('punkt', quiet=True)
+                from nltk.tokenize import sent_tokenize
+            except ImportError:
+                sent_tokenize = lambda x: x.split('.')
+
+            # Embedding-based chunk selection
+            try:
+                from sentence_transformers import SentenceTransformer
+                embedder = SentenceTransformer('all-MiniLM-L6-v2')
+            except ImportError:
+                embedder = None
+
+            def select_best_chunk(chunks, query):
+                if not embedder or not chunks:
+                    return chunks[0] if chunks else ""
+                chunk_embeddings = embedder.encode(chunks)
+                query_embedding = embedder.encode([query])[0]
+                # Cosine similarity
+                import numpy as np
+                scores = [np.dot(chunk_emb, query_embedding)/(np.linalg.norm(chunk_emb)*np.linalg.norm(query_embedding)) for chunk_emb in chunk_embeddings]
+                best_idx = int(np.argmax(scores))
+                return chunks[best_idx]
+
+            def extractive_summary(text, max_sentences=3):
+                sentences = sent_tokenize(text)
+                keywords = query.lower().split()
+                scored = [s for s in sentences if any(k in s.lower() for k in keywords)]
+                if scored:
+                    return ' '.join(scored[:max_sentences])
+                return ' '.join(sentences[:max_sentences])
+
+            compressed_contexts = []
+            for doc in docs:
+                # Chunking: split by paragraphs or headings
+                if hasattr(doc, 'chunks') and doc.chunks:
+                    best_chunk = select_best_chunk(doc.chunks, query)
+                    summary = extractive_summary(best_chunk)
+                else:
+                    # Fallback: split by paragraphs
+                    paragraphs = doc.page_content.split('\n\n')
+                    best_chunk = select_best_chunk(paragraphs, query) if paragraphs else doc.page_content
+                    summary = extractive_summary(best_chunk)
+                compressed_contexts.append(summary)
+
+            context = "\n\n".join(compressed_contexts)
+            return f"📄 Found {len(docs)} relevant documents (chunked & summarized):\n\n{context}"
         else:
             context = ""
             return "No relevant documents found in the knowledge base."