Gorse_Items_API.py

import boto3
import requests
import json
from math import ceil


def fetch_json_from_s3(bucket_name, key):
    """Fetch JSON data from S3."""
    s3 = boto3.client('s3')
    obj = s3.get_object(Bucket=bucket_name, Key=key)
    return json.loads(obj['Body'].read().decode('utf-8'))  # Parse JSON data


def load_to_gorse_rest_api(json_data, gorse_api_url):
    """Load a chunk of JSON data to Gorse REST API."""
    headers = {
        'Content-Type': 'application/json',
        'X-API-Key': 'd4e231fc-8966-44fa-9310-afe52e69636b'
    }
    response = requests.post(gorse_api_url, data=json.dumps(json_data), headers=headers)

    if response.status_code == 200:
        print("Data loaded successfully to Gorse API")
    else:
        print("Failed to load data to Gorse API. Status code:", response.status_code)
        print("Response:", response.text)


def chunk_data(data, chunk_size):
    """Split data into chunks of a specified size."""
    return [data[i:i + chunk_size] for i in range(0, len(data), chunk_size)]


def cleanup_processed_file(bucket_name, file_key):
    """Remove processed files after the Gorse API push."""
    s3 = boto3.client('s3')
    delete_response = s3.delete_object(Bucket=bucket_name, Key=file_key)
    print(delete_response)


def cleanup_temp_files(bucket_name, folder_key):
    """Remove temporary files after the Gorse API push."""
    s3 = boto3.client('s3')
    response = s3.list_objects_v2(Bucket=bucket_name, Prefix=folder_key)

    # Check if the folder contains any objects
    if 'Contents' in response:
        # Prepare a list of objects to delete
        objects_to_delete = [{'Key': obj['Key']} for obj in response['Contents']]

        # Perform the delete operation
        delete_response = s3.delete_objects(
            Bucket=bucket_name,
            Delete={'Objects': objects_to_delete}
        )

        # Print the response (or handle as needed)
        print(f"Deleted {len(delete_response['Deleted'])} objects from {bucket_name}")


if __name__ == "__main__":
    # S3 bucket details
    s3_bucket_name = 'pb-etl-athena'
    s3_key = 'Parsed_Data_Gorse/ITEM_DATA_API/processed-item_label.json'

    # Gorse API details
    gorse_api_url = 'https://sbrecommendation.wavespb.com/api/items'

    # Chunk size (adjust depending on the API limit)
    chunk_size = 100  # Change this value based on your dataset and the API's payload limit

    # Fetch JSON data from S3
    json_data = fetch_json_from_s3(s3_bucket_name, s3_key)

    # Count the number of items in the dataset
    item_id_count = len(json_data)
    print(f"Number of ItemIds: {item_id_count}")

    # Split data into chunks
    chunks = chunk_data(json_data, chunk_size)
    print(f"Total chunks to upload: {len(chunks)}")

    # Load data to Gorse API in chunks
    for i, chunk in enumerate(chunks, 1):
        print(f"Uploading chunk {i}/{len(chunks)}...")
        load_to_gorse_rest_api(chunk, gorse_api_url)

    print("Data upload complete.")

    # Cleanup processed and temporary files
    cleanup_processed_file(s3_bucket_name, s3_key)
    cleanup_temp_files(s3_bucket_name, 'item_data/')