KI-Server-RAG: Vom PDF zur abfragebereiten Wissensdatenbank

from langchain_community.document_loaders import PyPDFLoader, DocxLoader, UnstructuredMarkdownLoader

def load_document(file_path):
    if file_path.endswith('.pdf'):
        return PyPDFLoader(file_path).load()
    elif file_path.endswith('.docx'):
        return DocxLoader(file_path).load()
    elif file_path.endswith('.md'):
        return UnstructuredMarkdownLoader(file_path).load()

from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=500,      # 500 Tokens pro Chunk
    chunk_overlap=50,    # 50 Tokens Overlap
    length_function=len,
)
chunks = splitter.split_documents(docs)
# 1000 Seiten → ~500 Chunks

from langchain_community.embeddings import OllamaEmbeddings

embeddings = OllamaEmbeddings(model="nomic-embed-text")
for chunk in chunks:
    embedding = embeddings.embed_query(chunk.page_content)

from qdrant_client import QdrantClient, models

client = QdrantClient(path="./qdrant_wissen")
client.create_collection("firmen_wissen", 
                         vectors_config=models.VectorParams(
                             size=768, 
                             distance=models.Distance.COSINE
                         ))

client.upsert(
    collection_name="firmen_wissen",
    points=models.Batch(
        ids=list(range(len(chunks))),
        vectors=[embeddings.embed_query(c.page_content) for c in chunks]
    )
)

from langchain_community.llms import Ollama
from langchain.chains import RetrievalQA

llm = Ollama(model="llama3.3:70b")
qa = RetrievalQA.from_chain_type(
    llm=llm,
    retriever=client.as_retriever(search_kwargs={"k": 5})
)

result = qa.invoke("Was sagt die Dokumentation über Wartung?")
print(result["result"])