chunking.py

import os
from langchain_experimental.text_splitter import SemanticChunker
from langchain_openai import OpenAIEmbeddings
from Load_file import documents
from dotenv import load_dotenv

load_dotenv()

GITHUB_TOKEN = os.getenv("GITHUB_RAG_TOKEN")
INFERENCE_URL="https://models.github.ai/inference"

embeddings_model = OpenAIEmbeddings(
    base_url=INFERENCE_URL,
    model="openai/text-embedding-3-small",
    api_key=GITHUB_TOKEN
)

semantic_splitter = SemanticChunker(
    embeddings_model,
    breakpoint_threshold_amount=95,
    breakpoint_threshold_type="percentile"
)

semantic_chunks = semantic_splitter.split_documents(documents)

if __name__=="__main__":
    print(f" Semantic Chunks: {len(semantic_chunks)} created\n")
    for idx, chunk in enumerate(semantic_chunks):
        print(f"----- CHUNK {idx+1} ------")
        print(f"SOURCE : {chunk.metadata.get('source')}")
        print(chunk.page_content)