7_linkage.py

import torch
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import SVC
from sklearn import svm
import pandas as pd
import numpy as np

print('Loading data...')
data = pd.read_csv('data/final_data_deserving_cat.csv')
data_claims = data[data['preds_binary'] == 1]
#data_claims = data
print('Data loaded.')

print('Train SVM...')
# SVM
# We train an SVM model. We don't provide the .csv file here. 
# The data contains the predictions of subject & objects on the training dataset.
data_train = pd.read_csv('/path/to/train/selection/subject.csv')
obj = pd.read_excel('/path/to/train/selection/objects.xlsx')
data_train = pd.merge(data_train, obj[['Message_x', 'object_pred']], on='Message_x', how='inner')
#print(len(data_train))

# Linkage
data_train['linkage'].replace({'Positive': 1, 'Negative': 0}, inplace=True)
data_train['linkage'] =data_train['linkage'].fillna(0)
data_train['linkage'] = data_train['linkage'].astype(int)

data_train = data_train[data_train['repclaim_decision'] == 1]
print('Data loaded.')

print('Training SVM...')
for col in ['Message_x', 'Page Name', 'final_decision', 'object_pred']:
    if col != 'linkage':
    #if col != 'deserving':
        data_train[col] = data_train[col].astype(str)

data_train['all_features'] = data_train['Message_x'] + ' SEP ' + data_train['Page Name'] + ' SEP ' + data_train['final_decision'] + ' SEP ' +  data_train['object_pred']
X = data_train['all_features'].values
y = data_train['linkage'].values
X_train,X_test, y_train, y_test = train_test_split(X, y, test_size = .15, random_state=42)
classifier = Pipeline([('vect', TfidfVectorizer()),('clf', SVC(kernel='linear', C=1, class_weight='balanced'))])
classifier.fit(X_train, y_train)
print('SVM trained.')

print('Making predictions...')
data_claims['preds_linkage_rule-based_BERTje'] = np.nan
data_claims['preds_linkage_rule-based_SVM'] = np.nan
def rule_based_linkage_BERTje(row):
    if row['preds_deserving_cat'] == 1 and row['preds_subject'] in ['no subject', 'same party', 'same flemish coalition', 'same federal coalition']:
        row['preds_linkage_rule-based_BERTje'] = 1
    elif row['preds_deserving_cat'] == 0 and row['preds_subject'] == 'not same party/ coalition':
        row['preds_linkage_rule-based_BERTje'] = 1
    elif row['preds_subject'] == 'Outside Belgium':
        row['preds_linkage_rule-based_BERTje'] = classifier.predict([row['all_features']])[0]
    else:
        row['preds_linkage_rule-based_BERTje'] = 0
    return row

def rule_based_linkage_SVM(row):
    if row['preds_deserving_SVM_cat'] == 1 and row['preds_subject'] in ['no subject', 'same party', 'same flemish coalition', 'same federal coalition']:
        row['preds_linkage_rule-based_SVM'] = 1
    elif row['preds_deserving_SVM_cat'] == 0 and row['preds_subject'] == 'not same party/ coalition':
        row['preds_linkage_rule-based_SVM'] = 1
    elif row['preds_subject'] == 'Outside Belgium':
        row['preds_linkage_rule-based_SVM'] = classifier.predict([row['all_features']])[0]
    else:
        row['preds_linkage_rule-based_SVM'] = 0
    return row

data_claims = data_claims.apply(rule_based_linkage_BERTje, axis=1)
data_claims = data_claims.apply(rule_based_linkage_SVM, axis=1)
print('Predictions made.')

print('Join back with non-claims...')
non_claims = data[data['preds_binary'] == 0]
non_claims['preds_linkage_rule-based_SVM'] = 0
non_claims['preds_linkage_rule-based_BERTje'] = 0
data = pd.concat([data_claims, non_claims])
#data = data_claims
print('Joined back with non-claims.')

print('Saving data...')
#print(data.head())
data.to_csv('data/FINAL_DATA.csv')
#data.to_csv('/home/igevers/repclaims_final/linkage_deservingness/data_linkage/data_with_link_rule-based.csv')
print('Data saved.')