diff --git a/RAG/.gitignore b/RAG/.gitignore new file mode 100644 index 0000000..38abdb8 --- /dev/null +++ b/RAG/.gitignore @@ -0,0 +1,119 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +.chroma/* +.chroma +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# VS Code +.vscode/ +*.code-workspace + +# PyCharm +.idea/ +*.iml + +# Jupyter +.ipynb_checkpoints/ + +# macOS +.DS_Store + +# Windows +Thumbs.db \ No newline at end of file diff --git a/RAG/README.md b/RAG/README.md new file mode 100644 index 0000000..866bb61 --- /dev/null +++ b/RAG/README.md @@ -0,0 +1,115 @@ +# Playground/RAG + +Part of the [playground](https://github.com/commitBlob/playground) repository collection. + +## LangGraph Agentic RAG + +An intelligent question-answering system built with LangGraph that combines vector store retrieval and web search capabilities with sophisticated answer validation. The system uses an agent-based approach to dynamically decide the best information source and verify the quality of responses. + +### Project Location + +This project is located in the `RAG` directory of the playground repository: +``` +playground/ +└── RAG/ # This project + ├── ingestion.py + ├── main.py + └── graph/ + └── ... +``` + +## Features + +- **Smart Routing**: Automatically routes questions to either vector store or web search based on the question's content +- **Document Relevance Grading**: Evaluates retrieved documents for relevance to the question +- **Hallucination Detection**: Verifies that generated answers are grounded in the source documents +- **Answer Quality Assessment**: Ensures generated responses actually answer the user's question +- **Fallback Mechanisms**: Dynamically switches to web search when vector store results are insufficient +- **Flexible Architecture**: Built with LangGraph for clear state management and workflow control + +## Architecture + +The system uses a state-based graph architecture with several key components: + +1. **Router**: Determines whether to use vector store or web search based on the question +2. **Retriever**: Fetches relevant documents from Pinecone vector store +3. **Document Grader**: Evaluates document relevance +4. **Generator**: Creates answers based on retrieved documents +5. **Answer Grader**: Validates answers for hallucinations and relevance +6. **Web Search**: Provides additional information when needed using Tavily Search + +## Setup + +1. Clone the repository +2. Install dependencies: + ```bash + pip install langchain langchainhub langchain-community langchain-tavily langchain-pinecone langgraph python-dotenv pytest + ``` + +3. Create a `.env` file with the following keys: + ``` + OPENAI_API_KEY=your_openai_key + PINECONE_API_KEY=your_pinecone_key + TAVILY_API_KEY=your_tavily_key + LANGSMITH_API_KEY=your_langsmith_key (optional) + LANGSMITH_TRACING=true (optional) + LANGSMITH_PROJECT_NAME=your_project_name (optional) + + ``` + +## Usage + +1. Run the ingestion script to populate the vector store: + ```bash + python ingestion.py + ``` + +2. Run the main application: + ```bash + python main.py + ``` + +Example query: +```python +from graph.graph import app + +result = app.invoke(input={"question": "What is agent memory?"}) +print(result) +``` + +## Project Structure + +``` +├── ingestion.py # Document ingestion and vector store setup +├── main.py # Main application entry point +├── graph/ +│ ├── chains/ # LangChain components +│ │ ├── answer_grader.py +│ │ ├── generation.py +│ │ ├── hallucination_grader.py +│ │ ├── retrieval_grader.py +│ │ └── router.py +│ ├── nodes/ # Graph nodes implementation +│ │ ├── generate.py +│ │ ├── grade_documents.py +│ │ ├── retrieve.py +│ │ └── web_search.py +│ ├── tests/ # Test cases +│ ├── graph.py # Main graph definition +│ └── state.py # State management +``` + +## Testing + +Run the test suite: +```bash +pytest . -s -v +``` + +## Flow Visualization + +The system generates a visual representation of the workflow graph in `graph_output.png`: + +![LangGraph Workflow](graph_output.png) + +This visualization shows the complete flow of the question-answering system, including routing decisions, document retrieval, grading steps, and generation paths. \ No newline at end of file diff --git a/RAG/graph/__init__.py b/RAG/graph/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/RAG/graph/chains/__init__.py b/RAG/graph/chains/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/RAG/graph/chains/answer_grader.py b/RAG/graph/chains/answer_grader.py new file mode 100644 index 0000000..96fd2a9 --- /dev/null +++ b/RAG/graph/chains/answer_grader.py @@ -0,0 +1,25 @@ +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import RunnableSequence +from langchain_openai import ChatOpenAI +from pydantic import BaseModel, Field + + +class GradeAnswer(BaseModel): + binary_score: bool = Field( + description="Answer addresses the question, 'yes' or 'no'" + ) + +llm = ChatOpenAI(temperature=0) + +structured_llm_grader = llm.with_structured_output(GradeAnswer) + +system = """You are a grader assessing whether an answer addresses / resolves a question \n + Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.""" +answer_prompt = ChatPromptTemplate.from_messages( + [ + ("system", system), + ("human", "User question: \n\n {question} \n\n LLM generation: {generation}"), + ] +) + +answer_grader: RunnableSequence = answer_prompt | structured_llm_grader \ No newline at end of file diff --git a/RAG/graph/chains/generation.py b/RAG/graph/chains/generation.py new file mode 100644 index 0000000..d4720f1 --- /dev/null +++ b/RAG/graph/chains/generation.py @@ -0,0 +1,8 @@ +from langchain import hub +from langchain_core.output_parsers import StrOutputParser +from langchain_openai import ChatOpenAI + +llm = ChatOpenAI(temperature=0) +prompt = hub.pull("rlm/rag-prompt") + +generation_chain = prompt | llm | StrOutputParser() \ No newline at end of file diff --git a/RAG/graph/chains/hallucination_grader.py b/RAG/graph/chains/hallucination_grader.py new file mode 100644 index 0000000..d78f112 --- /dev/null +++ b/RAG/graph/chains/hallucination_grader.py @@ -0,0 +1,24 @@ +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import RunnableSequence +from langchain_openai import ChatOpenAI +from pydantic import BaseModel, Field + +llm = ChatOpenAI(temperature=0) + +class GradeHallucinations(BaseModel): + """Binary score for hallucination prensent in generation answer.""" + + binary_score: bool = Field(description="Answer is grounded in the facts, 'yes' or 'no'") + +structured_llm_grader = llm.with_structured_output(GradeHallucinations) + +system = """You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \n + Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.""" +hallucination_prompt = ChatPromptTemplate.from_messages( + [ + ("system", system), + ("human", "Set of facts: \n\n {documents} \n\n LLM generation: {generation}"), + ] +) + +hallucination_grader: RunnableSequence = hallucination_prompt | structured_llm_grader \ No newline at end of file diff --git a/RAG/graph/chains/retrieval_grader.py b/RAG/graph/chains/retrieval_grader.py new file mode 100644 index 0000000..7bfd851 --- /dev/null +++ b/RAG/graph/chains/retrieval_grader.py @@ -0,0 +1,25 @@ +from langchain_core.prompts import ChatPromptTemplate +from pydantic import BaseModel, Field +from langchain_openai import ChatOpenAI + +llm = ChatOpenAI(temperature=0) + +class GradeDocuments(BaseModel): + """Binary score for rlevance check on retrieved documents""" + + binary_score: str = Field(description="Document are relevant to the question, 'yes' or 'no'") + +structured_llm_grader = llm.with_structured_output(GradeDocuments) + +system = """You are a grader assessing relevance of a retrieved document to a user question. \n + If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \n + Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.""" + +grade_prompt = ChatPromptTemplate.from_messages( + [ + ("system", system), + ("human", "Retrieved document: \n\n {document} \n\n User question: {question}"), + ] +) + +retrieval_grader = grade_prompt | structured_llm_grader \ No newline at end of file diff --git a/RAG/graph/chains/router.py b/RAG/graph/chains/router.py new file mode 100644 index 0000000..3730d00 --- /dev/null +++ b/RAG/graph/chains/router.py @@ -0,0 +1,29 @@ +from typing import Literal + +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI +from pydantic import BaseModel, Field + + +class RouteQuery(BaseModel): + """Route a user query to the most relevant datasource.""" + + datasource: Literal["vectorstore", "websearch"] = Field( + ..., + description="Given a user question choose to route it to web search or a vectorstore.", + ) + +llm = ChatOpenAI(temperature=0) +structured_llm_router = llm.with_structured_output(RouteQuery) + +system = """You are an expert at routing a user question to a vectorstore or web search. +The vectorstore contains documents related to agents, prompt engineering, and adversarial attacks. +Use the vectorstore for questions on these topics. For all else, use web-search.""" +route_prompt = ChatPromptTemplate.from_messages( + [ + ("system", system), + ("human", "{question}"), + ] +) + +question_router = route_prompt | structured_llm_router \ No newline at end of file diff --git a/RAG/graph/consts.py b/RAG/graph/consts.py new file mode 100644 index 0000000..47eb623 --- /dev/null +++ b/RAG/graph/consts.py @@ -0,0 +1,4 @@ +RETRIEVE = "retrieve" +GRADE_DOCUMENTS = "grade_documents" +GENERATE = "generate" +WEBSEARCH = "websearch" \ No newline at end of file diff --git a/RAG/graph/graph.py b/RAG/graph/graph.py new file mode 100644 index 0000000..f38d7e3 --- /dev/null +++ b/RAG/graph/graph.py @@ -0,0 +1,99 @@ +from dotenv import load_dotenv +from langgraph.graph import END, StateGraph + +from graph.chains.answer_grader import answer_grader +from graph.chains.hallucination_grader import hallucination_grader +from graph.chains.router import RouteQuery, question_router +from graph.consts import GENERATE, GRADE_DOCUMENTS, RETRIEVE, WEBSEARCH +from graph.nodes import generate, grade_documents, retrieve, web_search +from graph.state import GraphState + +load_dotenv() + +def decide_to_generate(state): + print("assessing graded documents...") + + if state["web_search"]: + print( + "---DECISION: NOT ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, INCLUDE WEB SEARCH---" + ) + return WEBSEARCH + else: + print("---DECISION: GENERATE---") + return GENERATE + +def grade_generation_grounded_in_documents_and_question(state: GraphState) -> str: + print("check for hallucinations in generation...") + question = state["question"] + documents = state["documents"] + generation = state["generation"] + + score = hallucination_grader.invoke( + {"documents": documents, "generation": generation} + ) + + if hallucination_grade := score.binary_score: + print("decision - generation is grounded in documents") + score = answer_grader.invoke( + {"generation": generation, "question": question} + ) + if answer_grade := score.binary_score: + print("decision - generation answers the question") + return "useful" + else: + print("decision - generation does not answer the question") + return "not useful" + else: + print("decision - generation is NOT grounded in documents") + return "not supported" + +def route_question(state: GraphState) -> str: + print("routing question...") + question = state["question"] + source: RouteQuery = question_router.invoke({"question": question}) + if source.datasource == WEBSEARCH: + print("decision - route question to web search") + return WEBSEARCH + elif source.datasource == "vectorstore": + print("decision - route question to rag") + return RETRIEVE + + + +workflow = StateGraph(GraphState) +workflow.add_node(RETRIEVE, retrieve) +workflow.add_node(GRADE_DOCUMENTS, grade_documents) +workflow.add_node(GENERATE, generate) +workflow.add_node(WEBSEARCH, web_search) + +workflow.set_conditional_entry_point( + route_question, + { + RETRIEVE: RETRIEVE, + WEBSEARCH: WEBSEARCH + } +) +workflow.add_edge(RETRIEVE, GRADE_DOCUMENTS) +workflow.add_conditional_edges( + GRADE_DOCUMENTS, + decide_to_generate, + { + WEBSEARCH: WEBSEARCH, + GENERATE: GENERATE, + } +) +workflow.add_conditional_edges( + GENERATE, + grade_generation_grounded_in_documents_and_question, + { + "useful": END, + "not useful": WEBSEARCH, + "not supported": GENERATE, + } +) +workflow.add_edge(WEBSEARCH, GENERATE) +workflow.add_edge(GENERATE, END) + +app = workflow.compile() + +app.get_graph().draw_mermaid_png(output_file_path="graph_output.png") \ No newline at end of file diff --git a/RAG/graph/nodes/__init__.py b/RAG/graph/nodes/__init__.py new file mode 100644 index 0000000..cc1cbfa --- /dev/null +++ b/RAG/graph/nodes/__init__.py @@ -0,0 +1,6 @@ +from graph.nodes.generate import generate +from graph.nodes.grade_documents import grade_documents +from graph.nodes.retrieve import retrieve +from graph.nodes.web_search import web_search + +__all__ = ["generate", "grade_documents", "retrieve", "web_search"] \ No newline at end of file diff --git a/RAG/graph/nodes/generate.py b/RAG/graph/nodes/generate.py new file mode 100644 index 0000000..a346dc7 --- /dev/null +++ b/RAG/graph/nodes/generate.py @@ -0,0 +1,21 @@ +from typing import Any, Dict + +from graph.chains.generation import generation_chain +from graph.state import GraphState + +def generate(state: GraphState) -> Dict[str, Any]: + """ + Generate a response based on the question and documents in the state. + + Args: + state (GraphState): The current state of the graph. + + Returns: + Dict[str, Any]: A dictionary containing the generated response and the original question. + """ + print("Generating response...") + question = state["question"] + documents = state["documents"] + + generation = generation_chain.invoke({"context": documents, "question": question}) + return {"documents": documents, "generation": generation, "question": question} \ No newline at end of file diff --git a/RAG/graph/nodes/grade_documents.py b/RAG/graph/nodes/grade_documents.py new file mode 100644 index 0000000..4abbb1e --- /dev/null +++ b/RAG/graph/nodes/grade_documents.py @@ -0,0 +1,42 @@ +from typing import Any, Dict + +from graph import state +from graph.chains.retrieval_grader import retrieval_grader +from graph.state import GraphState + + +def grade_documents(state: GraphState) -> Dict[str, Any]: + """ + Determines whether the retrieved documents are relevant to the question + If any document is not relevant, we will set a flag to run web search + + Args: + state (dict): The current graph state + + Returns: + state (dict): Filtered out irrelevant documents and updated web_search state + """ + + print("Grading documents...") + question = state["question"] + documents = state["documents"] + + filtered_docs = [] + web_search = False + + for document in documents: + score = retrieval_grader.invoke( + {"document": document.page_content, "question": question} + ) + + grade = score.binary_score + + if grade.lower() == "yes": + print("Document is relevant") + filtered_docs.append(document) + else: + print("Document is not relevant") + web_search = True + continue + + return {"documents": filtered_docs, "web_search": web_search, "question": question} \ No newline at end of file diff --git a/RAG/graph/nodes/retrieve.py b/RAG/graph/nodes/retrieve.py new file mode 100644 index 0000000..8031adf --- /dev/null +++ b/RAG/graph/nodes/retrieve.py @@ -0,0 +1,20 @@ +from typing import Any, Dict + +from graph.state import GraphState +from ingestion import retriever + +def retrieve(state: GraphState) -> Dict[str, Any]: + """ + Retrieve documents based on the question in the state. + + Args: + state (GraphState): The current state of the graph. + + Returns: + Dict[str, Any]: A dictionary containing the retrieved documents and the original question. + """ + print("Retrieving documents...") + question = state["question"] + documents = retriever.invoke(question) + + return {"documents": documents, "question": question} \ No newline at end of file diff --git a/RAG/graph/nodes/web_search.py b/RAG/graph/nodes/web_search.py new file mode 100644 index 0000000..536c71e --- /dev/null +++ b/RAG/graph/nodes/web_search.py @@ -0,0 +1,43 @@ +from typing import Any, Dict + +from dotenv import load_dotenv +from langchain.schema import Document +from langchain_tavily import TavilySearch + +from graph.state import GraphState + +load_dotenv() +web_search_tool = TavilySearch(max_results=3) + +def web_search(state: GraphState) -> Dict[str, Any]: + """ + Perform a web search based on the question in the state. + + Args: + state (GraphState): The current state of the graph. + + Returns: + Dict[str, Any]: A dictionary containing the search results and the original question. + """ + print("Performing web search...") + question = state["question"] + if "documents" in state: + documents = state["documents"] + else: + documents = None + + tavily_results = web_search_tool.invoke({"query": question})["results"] + joined_tavily_result = "\n".join( + [tavily_result["content"] for tavily_result in tavily_results] + ) + web_results = Document(page_content=joined_tavily_result) + + if documents is not None: + documents.append(web_results) + else: + documents = [web_results] + + return {"documents": documents, "question": question} + +if __name__ == "__main__": + web_search(state={"question": "agent memory", "documents": None}) \ No newline at end of file diff --git a/RAG/graph/state.py b/RAG/graph/state.py new file mode 100644 index 0000000..2c3dc43 --- /dev/null +++ b/RAG/graph/state.py @@ -0,0 +1,18 @@ +from typing import List, TypedDict + +class GraphState(TypedDict): + """ + Represents the state of our graph. + + Attributes: + question: question + generation: LLM generation + web_search: whether to add search + documents: list of documents + """ + + question: str + generation: str + web_search: bool + documents: List[str] + \ No newline at end of file diff --git a/RAG/graph/tests/__init__.py b/RAG/graph/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/RAG/graph/tests/test_chains.py b/RAG/graph/tests/test_chains.py new file mode 100644 index 0000000..099e815 --- /dev/null +++ b/RAG/graph/tests/test_chains.py @@ -0,0 +1,76 @@ +from pprint import pprint + +from dotenv import load_dotenv + +load_dotenv() + +from graph.chains.generation import generation_chain +from graph.chains.hallucination_grader import (GradeHallucinations, + hallucination_grader) +from graph.chains.retrieval_grader import GradeDocuments, retrieval_grader +from graph.chains.router import RouteQuery, question_router +from ingestion import retriever + + +def test_retrieval_grader_answer_yes() -> None: + question = "agent memory" + documents = retriever.invoke(question) + doc_txt = documents[0].page_content + + res: GradeDocuments = retrieval_grader.invoke( + {"document": doc_txt, "question": question} + ) + + assert res.binary_score == "yes" + +def test_retrieval_grader_answer_no() -> None: + question = "agent memory" + documents = retriever.invoke(question) + doc_txt = documents[1].page_content + + res: GradeDocuments = retrieval_grader.invoke( + {"document": doc_txt, "question": "how to make pizza"} + ) + + assert res.binary_score == "no" + +def test_generation_chain() -> None: + question = "agent memory" + docs = retriever.invoke(question) + generation = generation_chain.invoke({"context": docs, "question": question}) + pprint(generation) + +def test_hallucination_grader_answer_yes() -> None: + question = "agent memory" + docs = retriever.invoke(question) + + generation = generation_chain.invoke({"context": docs, "question": question}) + res: GradeHallucinations = hallucination_grader.invoke( + {"documents": docs, "generation": generation} + ) + assert res.binary_score + +def test_hallucination_grader_answer_no() -> None: + question = "agent memory" + docs = retriever.invoke(question) + + res: GradeHallucinations = hallucination_grader.invoke( + { + "documents": docs, + "generation": "In order to make pizza we need to first start with the dough", + } + ) + assert not res.binary_score + +def test_router_to_vectorstore() -> None: + question = "agent memory" + + res: RouteQuery = question_router.invoke({"question": question}) + assert res.datasource == "vectorstore" + + +def test_router_to_websearch() -> None: + question = "how to make pizza" + + res: RouteQuery = question_router.invoke({"question": question}) + assert res.datasource == "websearch" diff --git a/RAG/graph_output.png b/RAG/graph_output.png new file mode 100644 index 0000000..6477268 Binary files /dev/null and b/RAG/graph_output.png differ diff --git a/RAG/ingestion.py b/RAG/ingestion.py new file mode 100644 index 0000000..e71a74c --- /dev/null +++ b/RAG/ingestion.py @@ -0,0 +1,35 @@ +import os +from dotenv import load_dotenv +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain_community.document_loaders import WebBaseLoader +from langchain_pinecone.vectorstores import PineconeVectorStore +from langchain_openai import OpenAIEmbeddings + +load_dotenv() + +urls = [ + "https://lilianweng.github.io/posts/2023-06-23-agent/", + "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/", + "https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/", +] + +docs = [WebBaseLoader(url).load() for url in urls] +docs_list = [item for sublist in docs for item in sublist] + +text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=250, chunk_overlap=0 +) + +doc_splits = text_splitter.split_documents(docs_list) + +vectorstore = PineconeVectorStore.from_documents( + documents=doc_splits, + embedding=OpenAIEmbeddings(), + index_name="langgraph-agentic-rag", +) + +retriever = PineconeVectorStore( + index_name="langgraph-agentic-rag", + embedding=OpenAIEmbeddings(), + text_key="text" +).as_retriever() \ No newline at end of file diff --git a/RAG/main.py b/RAG/main.py new file mode 100644 index 0000000..6c90c11 --- /dev/null +++ b/RAG/main.py @@ -0,0 +1,9 @@ +from dotenv import load_dotenv + +load_dotenv() + +from graph.graph import app + +if __name__ == "__main__": + print("Hello, LangGraph!") + print(app.invoke(input={"question": "What is agent memory?"})) \ No newline at end of file