Bank Loan Default Prediction--Predictive Modeling (Part 3)

import pandas as pd 
import numpy as np 

import matplotlib.pyplot as plt
import seaborn as sns 
sns.set(font_scale = 1.2)
%matplotlib inline

def dinstinct_values(df):
    '''
    Check the number of distinct values in each column
    INPUT:
      - df: the pandas dataframe needs to be checked
    OUTPUT:
      - None
    '''
    col = df.columns.to_list()
    for c in col:
        print('There are %d different values in the column %s.' % (len(df[c].value_counts()), c))  

dinstinct_values(data)

corr = data.corr()
plt.figure(figsize = (15, 7))
sns.heatmap(corr, cmap = 'OrRd', linewidths = 0.01)

features = data.columns.to_list()
features.remove('status')

from sklearn import svm
clf = svm.SVC(kernel = 'linear', C = 1, probability = True)
clf.fit(X_train, y_train)
clf.score(X_test, y_test) # accuracy

# check the difference between target and prediction
# if result is not 0, it means target != prediction

diff = clf.predict(X_test) - y_test
diff[diff != 0] 

# predict the probablity score for each class/category

clf.predict_proba(X_test)[:5]

# evaluation with sklearn's classification_report

from sklearn.metrics import classification_report
print(classification_report(y_test, clf.predict(X_test)))

# process the svm weights by square
svm_weights = (clf.coef_ ** 2).sum(axis = 0)
svm_weights /= svm_weights.max() # indicates how large is the svm coeff of each feature

plt.figure(figsize = (12, 5))
sns.barplot(features, svm_weights, label = 'SVM weight', alpha = 0.6)

plt.title("Feature Selection")
plt.xticks(rotation = 90) 
plt.axis('tight')
plt.legend(loc = 'upper right')

from sklearn.metrics import confusion_matrix

def plot_cf_matrix(y_test, y_pred):
    '''
    Plot the confusion matrix.
    INPUT:
      - y_test: ground truth label 
      - y_pred: model's prediction
    '''
    cf_matrix = confusion_matrix(y_test, y_pred)
    group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
    group_counts = ['{0:0.0f}'.format(value) for value in
                    cf_matrix.flatten()]
    group_percentages = ['{0:.2%}'.format(value) for value in
                         cf_matrix.flatten() / np.sum(cf_matrix)]
    labels = [f'{v1}\n{v2}\n{v3}' for v1, v2, v3 in 
              zip(group_names,group_counts,group_percentages)]
    labels = np.asarray(labels).reshape(2, 2)

    sns.heatmap(cf_matrix, annot = labels, fmt = '', cmap = 'Blues')

y_pred = clf.predict(X_test)
plot_cf_matrix(y_test, y_pred)

from sklearn.preprocessing import StandardScaler
# normalize the input data with standard scaler
sc = StandardScaler()
X = sc.fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 1 / 3)

clf2 = svm.SVC(kernel = 'rbf', C = 10, probability = True)
clf2.fit(X_train, y_train)
clf2.score(X_test, y_test)

print(classification_report(y_test, clf2.predict(X_test)))

y_pred = clf2.predict(X_test)
plot_cf_matrix(y_test, y_pred)

from sklearn.metrics import precision_recall_curve, PrecisionRecallDisplay, roc_curve, RocCurveDisplay

y_score = clf2.decision_function(X_test)

fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=clf2.classes_[1])
roc_display = RocCurveDisplay(fpr = fpr, tpr = tpr)

prec, recall, _ = precision_recall_curve(y_test, y_score,
                                         pos_label=clf2.classes_[1])
pr_display = PrecisionRecallDisplay(precision=prec, recall=recall)

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))

roc_display.plot(ax = ax1)
pr_display.plot(ax = ax2)

from sklearn.model_selection import GridSearchCV
svc = svm.SVC()

param_grid = [{'C':[0.1,1,10], 'kernel':['linear']},
             {'C':[0.1,1,10], 'gamma':[0.001,0.01],'kernel':['rbf']}]

scoring = 'accuracy'
clfs = GridSearchCV(svc, param_grid, scoring = scoring, cv=10)
clfs.fit(X_train,y_train)

print(clfs.best_estimator_)

from sklearn.metrics import classification_report

print(classification_report(y_test, clfs.best_estimator_.predict(X_test)))

y_pred = clfs.best_estimator_.predict(X_test)
plot_cf_matrix(y_test, y_pred)

def visualize_coeff(model, name, feature_col = features):
    '''
    Function to visualize the coeff importance of each model
    INPUT:
      - model: the model instance
      - name: name of the model to show on title
      - feature_col: a list of column names of all features, default value set to be a global list named features
    OUTPUT:
      None
    '''
    coeff = model.feature_importances_
    
    importance = pd.DataFrame({"feature": features, 'importance': coeff})
    importance.sort_values(by = 'importance', ascending = False, inplace = True)
    
    plt.figure(figsize = (12, 5))
    plt.xticks(rotation = 90) 

    sns.barplot(x = 'feature', y = 'importance', data = importance).set_title('The feature coeff of %s' % name)

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(max_depth = 2, random_state = 10)
rf.fit(X_train, y_train)
rf.score(X_test, y_test)

print(classification_report(y_test, rf.predict(X_test)))

visualize_coeff(rf, 'RandomForestClassifier')

y_pred = rf.predict(X_test)
plot_cf_matrix(y_test, y_pred)

from sklearn.ensemble import AdaBoostClassifier

adaboost = AdaBoostClassifier(n_estimators = 100)

adaboost.fit(X_train, y_train)
adaboost.score(X_test, y_test)

print(classification_report(y_test, adaboost.predict(X_test)))

visualize_coeff(adaboost, 'AdaBoost')

from sklearn.ensemble import GradientBoostingClassifier

dtc = GradientBoostingClassifier(n_estimators = 200, learning_rate = 0.1).fit(X_train, y_train)
dtc.score(X_test, y_test)

print(classification_report(y_test, dtc.predict(X_test)))

visualize_coeff(dtc, 'GradientBoostingClassifier')

from xgboost import XGBClassifier
xgb = XGBClassifier(colsample_bytree = 0.4603,
                    gamma            = 0.0468, 
                    learning_rate    = 0.05, 
                    max_depth        = 3, 
                    min_child_weight = 1.7817, 
                    n_estimators     = 2200,
                    reg_alpha        = 0.4640, 
                    reg_lambda       = 0.8571,
                    subsample        = 0.5213, 
                    silent           = 1,
                    random_state     = 7,
                    nthread          = -1)

xgb.fit(X_train, y_train)
xgb.score(X_test, y_test)

print(classification_report(y_test, xgb.predict(X_test)))

visualize_coeff(xgb, 'XGBoost')

y_pred = xgb.predict(X_test)
plot_cf_matrix(y_test, y_pred)