School-Project-Optimized-AI-Algorithms/csv_loading.py at AI_Model_Optimization · MistaTechE/School-Project-Optimized-AI-Algorithms · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#csv_loading.py
import pandas as pd
from optimization_techniques import remove_correlated_features

def load_data(path):
    #load csv as dataframe
    df = pd.read_csv(path)
    #fix true/false to 0 or 1
    if "isWeekend" in df.columns:
        df["isWeekend"] = df["isWeekend"].astype(int)
    return df

def data_inspection(dataframe, stuff=''):
    #TODO REMOVE DEBUG STATMENTS
    #print("Dataframe Head:")
    #print(dataframe.head())
    #print("shape:")
    #print(dataframe.shape)
    #print("info")
    #print(dataframe.info())
    #remove duplicate rows
    #initial_rows = dataframe.shape[0]
    #dataframe.drop_duplicates(inplace=True)

    #OPTIMIZATION remove correlated features
    dataframe_features_removed = remove_correlated_features(dataframe, stuff, threshold=0.9)

    #print("*******************************************************Dataframe Head:")
    #print(dataframe_features_removed.head())
    #print("shape:")
    #print(dataframe_features_removed.shape)
    #print("info")
    #print(dataframe_features_removed.info())

    return dataframe_features_removed