debug_generation.py

from imp_query_data import HybridRAGSystem
from langchain.prompts import ChatPromptTemplate

def debug_generation_step():
    """Debug exactly what the LLM is seeing and generating"""
    
    rag = HybridRAGSystem()
    
    # Test with a question we KNOW has an answer
    question = "What BLEU score did the Transformer achieve on English-German translation?"
    
    print("=" * 80)
    print("DEBUGGING GENERATION STEP")
    print("=" * 80)
    print(f"Question: {question}")
    
    # Step 1: Get retrieved documents
    docs, scores = rag.retrieve_and_rerank(question, final_k=5)
    
    print(f"\n📚 RETRIEVED {len(docs)} DOCUMENTS:")
    print("-" * 50)
    
    for i, (doc, score) in enumerate(zip(docs, scores)):
        content = doc.page_content
        print(f"\nDocument {i+1} (Score: {score:.3f}):")
        print(f"Source: {doc.metadata.get('source', 'Unknown')}")
        print(f"Content: {content[:300]}...")
        
        # Check if this document contains our target answer
        has_bleu = 'bleu' in content.lower()
        has_28_4 = '28.4' in content
        has_english_german = 'english' in content.lower() and 'german' in content.lower()
        
        print(f"Contains 'BLEU': {has_bleu}")
        print(f"Contains '28.4': {has_28_4}")
        print(f"Contains 'English-German': {has_english_german}")
        
        if has_bleu and has_28_4 and has_english_german:
            print("🎯 THIS DOCUMENT HAS THE ANSWER!")
    
    # Step 2: Create the exact prompt that will be sent to Mistral
    context, domain = rag.create_structured_context(docs, scores, question)
    
    print(f"\n📝 FINAL PROMPT SENT TO MISTRAL:")
    print("-" * 50)
    
    prompt_template = ChatPromptTemplate.from_template("""
You are an expert assistant analyzing technical documents about {domain}.

Use the following context documents to answer the question. Extract specific facts, numbers, and technical details from the provided content.

Context Documents:
{context}

Instructions:
1. Extract relevant information from the context documents above
2. Provide specific answers with concrete details (numbers, formulas, percentages)
3. Cite documents using [Document X] notation when referencing specific facts
4. If you find numerical values or technical specifications, include them in your answer
5. Synthesize information from multiple documents if needed
6. Only state "information not available" if you genuinely cannot find any relevant details

Question: {question}

Answer:""")
    
    final_prompt = prompt_template.format(
        domain=domain,
        context=context,
        question=question
    )
    
    print(final_prompt)
    print("\n" + "=" * 80)
    
    # Step 3: Get Mistral's actual response
    from langchain_ollama import OllamaLLM
    model = OllamaLLM(model="mistral", temperature=0.1, stream=False)
    
    print("🤖 MISTRAL'S RESPONSE:")
    print("-" * 50)
    response = model.invoke(final_prompt)
    print(response)
    
    return response

if __name__ == "__main__":
    debug_generation_step()