preprocess_json.py

import os

import joblib
import json
import pandas as pd
import requests


def create_embedding(text):
    result = requests.post("http://localhost:11434/api/embed", json= {
    "model": "bge-m3",
    "input": text
    })

    embedding = result.json()["embeddings"] #[0:5]
    return embedding

jsons = os.listdir("json")  # list all the jsons
my_dict = []
chunk_id =0
for json_file in jsons:
    with open(f"json/{json_file}") as f:
        content = json.load(f)
    print(f"Creating embeddings for {json_file}")
    embeddings = create_embedding([c['text'] for c in content['chunks']])
    for i, chunk in enumerate(content['chunks']):
        chunk['chunk_id'] = chunk_id
        chunk['embedding'] = embeddings[i]
        chunk_id +=1
        my_dict.append(chunk)

df = pd.DataFrame.from_records(my_dict)

# save dataframe

joblib.dump(df,'embeddings.joblib')