Implementación de Bagging

Les paso la versión del código implementando los clasificadores que provee scikit learn

import pandas as pd

from sklearn.neighbors import  KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import  accuracy_score

from sklearn.svm import LinearSVC
from sklearn.svm import SVC
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier

if __name__ == "__main__":
    
    path = './Bagging/data/heart.csv'
    dataset = pd.read_csv(path)

    print(dataset.head(5))
    print('')
    print(dataset['target'].describe())

    x = dataset.drop(['target'], axis=1, inplace=False)
    y = dataset['target']

    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.35, random_state=42)

    knn_class = KNeighborsClassifier().fit(x_train, y_train)
    knn_pred = knn_class.predict(x_test)

    print('')
    print('Accuracy KNeighbors:', accuracy_score(knn_pred, y_test))
    print('')

    #bag_class = BaggingClassifier(base_estimator=KNeighborsClassifier(), n_estimators=50).fit(x_train, y_train)
    #bag_pred = bag_class.predict(x_test)

    #print('')
    #print('Accuracy Bagging with KNeighbors:', accuracy_score(bag_pred, y_test))
    #print('')

    classifier = {
        'KNeighbors': KNeighborsClassifier(),
        'LinearSCV': LinearSVC(),
        'SVC': SVC(),
        'SGDC': SGDClassifier(),
        'DecisionTree': DecisionTreeClassifier()
    }

    for name, estimator in classifier.items():
        bag_class = BaggingClassifier(base_estimator=estimator, n_estimators=5).fit(x_train, y_train)
        bag_pred = bag_class.predict(x_test)

        print('Accuracy Bagging with {}:'.format(name), accuracy_score(bag_pred, y_test))
        print('')

Donde este es el output del código:

• Accuracy KNeighbors: 0.6908077994428969
• Accuracy Bagging with KNeighbors: 0.7437325905292479
• Accuracy Bagging with SVC: 0.9164345403899722
• Accuracy Bagging with SGDC: 0.5988857938718662
• Accuracy Bagging with DecisionTree: 0.9610027855153204

Hola, comparto el código con varios de los algoritmos de sklearn para clasificación:

import pandas as pd 

from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

import warnings
warnings.filterwarnings("ignore")

if __name__ == '__main__':
    dt_heart = pd.read_csv('./datasets/heart.csv')
    #print(dt_heart['target'].describe())

    x = dt_heart.drop(['target'], axis=1)
    y = dt_heart['target']

    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.35, random_state=1)

    knn_class = KNeighborsClassifier().fit(x_train, y_train)
    knn_prediction = knn_class.predict(x_test)
    print('='*64)
    print('SCORE con KNN: ', accuracy_score(knn_prediction, y_test))

    '''bag_class = BaggingClassifier(base_estimator=KNeighborsClassifier(), n_estimators=50).fit(x_train, y_train) # base_estimator pide el estimador en el que va a estar basado nuestro metodo || n_estimators nos pide cuantos de estos modelos vamos a utilizar
    bag_pred = bag_class.predict(x_test)
    print('='*64)
    print(accuracy_score(bag_pred, y_test))'''

    estimators = {
        'LogisticRegression' : LogisticRegression(),
        'SVC' : SVC(),
        'LinearSVC' : LinearSVC(),
        'SGD' : SGDClassifier(loss="hinge", penalty="l2", max_iter=5),
        'KNN' : KNeighborsClassifier(),
        'DecisionTreeClf' : DecisionTreeClassifier(),
        'RandomTreeForest' : RandomForestClassifier(random_state=0)
    }

    for name, estimator in estimators.items():
        bag_class = BaggingClassifier(base_estimator=estimator, n_estimators=50).fit(x_train, y_train)
        bag_predict = bag_class.predict(x_test)
        print('='*64)
        print('SCORE Bagging with {} : {}'.format(name, accuracy_score(bag_predict, y_test)))

Output:

Sin embargo, en la practica supongo para problemas relacionados con la salud o sector financiero, se requiere una precisión de los modelos “más alta”, por ser temáticas demasiado sensibles.

usando arboles de decisión, obtuve un accuracy de 0.9902597402597403

En algunas dió mejores resultados sin Bagging 🤔:

classifiers = {
    'KNN': KNeighborsClassifier,
    'SGD': SGDClassifier,
    'SVC': SVC,
    'LinearSVC': LinearSVC,
    'LogisticRegression': LogisticRegression,
    'DecisionTree': DecisionTreeClassifier,
    'RandomForest': RandomForestClassifier
}

for name, classifier in classifiers.items():
    model = classifier().fit(X_train, y_train)
    prediction = model.predict(X_test)
    bag_class = BaggingClassifier(base_estimator=classifier(), n_estimators=50).fit(X_train, y_train)
    bag_predict = bag_class.predict(X_test)
    print("="*64)
    print(name)
    print("Accuracy:", accuracy_score(prediction, y_test))
    print("Bagging Accuracy:", accuracy_score(bag_predict, y_test))

KNN
Accuracy: 0.7075208913649025
Bagging Accuracy: 0.754874651810585

SGD
Accuracy: 0.6768802228412256
Bagging Accuracy: 0.6880222841225627

SVC
Accuracy: 0.6963788300835655
Bagging Accuracy: 0.6880222841225627

LinearSVC
Accuracy: 0.7994428969359332
Bagging Accuracy: 0.8022284122562674

LogisticRegression
Accuracy: 0.8328690807799443
Bagging Accuracy: 0.8217270194986073

DecisionTree
Accuracy: 0.9749303621169917
Bagging Accuracy: 0.9832869080779945

RandomForest
Accuracy: 0.9832869080779945
Bagging Accuracy: 0.9777158774373259

Excelente clase, solo tengo una cosa para decir, vivan los modelos tree.

Agregue random forest y decision tree, dejo el código como guía

import pandas as pd  

from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

if __name__ == "__main__":
    dt_heart = pd.read_csv("./data/heart.csv")
    print(dt_heart.head(5))

    X = dt_heart.drop(["target"],axis=1)
    y = dt_heart["target"]

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.35, random_state=42)

    knn_class = KNeighborsClassifier().fit(X_train,y_train)
    knn_pred = knn_class.predict(X_test)

    print("="*64)
    print(accuracy_score(knn_pred, y_test))

    bag_class = BaggingClassifier(base_estimator=KNeighborsClassifier(),n_estimators=30).fit(X_train,y_train)
    bag_pred = bag_class.predict(X_test)

    print("="*64)
    print(accuracy_score(bag_pred, y_test))


    rf_class = BaggingClassifier(base_estimator=RandomForestClassifier(),n_estimators=30).fit(X_train,y_train)
    rf_pred = rf_class.predict(X_test)

    print("="*64)
    print(accuracy_score(rf_pred, y_test))

    dt_class = BaggingClassifier(base_estimator=DecisionTreeClassifier(),n_estimators=30).fit(X_train,y_train)
    dt_pred = dt_class.predict(X_test)

    print("="*64)
    print(accuracy_score(dt_pred, y_test))

Agregué el clasificador RandomForestClassifier() de Scikit-Learn y mi Accuracy Score fue de 0.9832869080779945.
Así se importa:

rom sklearn.ensemble import RandomForestClassifier

Si antes del SPLIT se normalizan los datos, los ACCURACY de los modelos aumentan.

# Normalizamos los datos
df_features = StandardScaler().fit_transform(df_features)

Implementando los clasificadores que tiene sklearne

KNeighborsClassifier
Accuracy: 0.7186629526462396

====================================================================
BaggingClassifier
Accuracy: 0.7855153203342619

====================================================================
LinearSVC
Accuracy: 0.5515320334261838
====================================================================
SVC
Accuracy: 0.6824512534818942
====================================================================
RandomForestClassifier
Accuracy: 1.0
====================================================================
GradientBoostingClassifier
Accuracy: 0.9832869080779945
====================================================================
AdaBoostClassifier
Accuracy: 0.8635097493036211
====================================================================
GaussianNB
Accuracy: 0.8328690807799443 

Para el problema de KNeighbors si primero normalzamos los datos como se vio en la clase 10 tenemos lo siguiente

Como se observa normalizando se obtiene un score mas alto que incluso usando una función de emsable, sin embargo cuando usamos el BaggingClasiffier nuestro score baja, lo cual muestra la importancia de primero normalizar los datos ( esto solo se hace en clasficaciones de acuerdo a lo mencionado en la clase 23 del curso de ML aplicado a python

Una pequeña corrección al código:
Si chequeamos en el la página de sklearn, primero debe ir la variable target con los valores reales, y luego los predichos:

sklearn.metrics.accuracy_score(y_true, y_pred, *, normalize=True, sample_weight=None)

Sin embargo en el video se lo pone al revés, con lo cual habría que cambiar las líneas 22 y 27 a:

print(accuracy_score(y_test, knn_pred))

y

print(accuracy_score(y_test, bag_pred))

respectivamente.

Utilizando el método de Bagging con 30 estimadores pero con el algoritmo de Decision Tree, alcancé un accuracy mediante cross_val_score de 0.995

Este es mi score esta 100% seguro

Use SVC con gamma=2 y C=1 con y sin bagging y me dio exactamente lo mismo, un accuracy de 0.9693

#//////////////////////SVC
    print("="*32)
    print("="*32)
    print("Using SVC")

    #SVC
    svc_class = SVC(gamma=2, C=1).fit(X_train, y_train)
    svc_predict = svc_class.predict(X_test)

    print("="*32)
    print('Score SVC:', accuracy_score(y_test, svc_predict))

    #SVC With Bagging

    bag_svc_class = BaggingClassifier(base_estimator=SVC(gamma=2, C=1), n_estimators=50).fit(X_train, y_train)
    bag_svc_predict = bag_svc_class.predict(X_test)

    print("="*32)
    print('Score SVC:', accuracy_score(y_test, bag_svc_predict))

Score SVC: 0.9610027855153204

Me base en el código de Manuel Alejandro Lopera Ospina y modifique los n_estimators de cada modelo de clasificación:

accuracy_score_bag_list = []
classifiers_bag_list = []

accuracy_score_list = []
classifiers_list = []

for name, classifier in classifiers.items():

for i in np.arange(10,100):

    if i == 10:
        model = classifier().fit(x_train, y_train)
        prediction = model.predict(x_test)
        classifiers_list.append(name)
        accuracy_score_list.append(accuracy_score(prediction, y_test))

    bag_class = BaggingClassifier(base_estimator=classifier(), n_estimators=i).fit(x_train, y_train)
    bag_predict = bag_class.predict(x_test)

    classifiers_bag_list.append(name)
    accuracy_score_bag_list.append(accuracy_score(bag_predict, y_test))

bagging_df = pd.DataFrame(np.c_[classifiers_bag_list,accuracy_score_bag_list],columns=[‘Nombre’,‘score’])
bagging_df[‘score’] = pd.to_numeric(bagging_df[‘score’]).round(3)

for ii in np.arange(len(classifiers)):
plt.figure()
plt.plot(np.arange(bagging_df[bagging_df[‘Nombre’]==list(classifiers.keys())[ii]][‘score’].shape[0]),bagging_df[bagging_df[‘Nombre’]==list(classifiers.keys())[ii]][‘score’])
aa = np.ones(bagging_df[bagging_df[‘Nombre’]==list(classifiers.keys())[ii]][‘score’].shape[0])*accuracy_score_list[ii]
plt.plot(np.arange(bagging_df[bagging_df[‘Nombre’]==list(classifiers.keys())[ii]][‘score’].shape[0]),aa,‘r–’)
plt.title(list(classifiers.keys())[ii])
plt.xlabel(‘n_estimators’)
plt.ylabel(‘accuracy_score’)
plt.legend([‘bagging_n_est’,‘classifiers’])
plt.show()

No recuerdo quien lo dijo pero utilizando el método del ensamblado con los arboles de decisión estariamos utilizando el método de clasificación de Random Fortest, esto se puede observar en las gráfcias,

Saludos y éxito para todos

Qué gran curso, cada clase aquí es una experiencia enriquecedora y gratificante para mí 💡😎🥇

Excelente clase!!!

Les comparto mi código, espero les sirva de guía.

mi compu se tarda algo por que le di n_estimator=500 y si me tardo un poco

Hola, les comparto el código con los otros clasificadores y con una pequeña modificación para poder realizar las respectivas configuraciones de cada modelo.

Excelente explicación.

Una implementación del Bagging con TreeDecision, básica, da mejores resultados en comparación con KNeighbord.

Bagging based in TreeDecisionClassifier

Bagging based in KNeighbordClassifier

import pandas as pd
import sklearn

from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

dt_heart = pd.read_csv('./heart.csv')
X = dt_heart.drop(['target'],axis=1)
y = dt_heart['target']

X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.35)

knn_class = KNeighborsClassifier().fit(X_train,y_train)
knn_pred = knn_class.predict(X_test)
print(accuracy_score(knn_pred,y_test))

bag_class = BaggingClassifier(base_estimator=KNeighborsClassifier(),n_estimators=50).fit(X_train,y_train)
bag_pred = bag_class.predict(X_test)
print(accuracy_score(bag_pred,y_test))

from sklearn.svm import LinearSVC
from sklearn.svm import SVC
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier

estimadores = {
    'SVC' : SVC(),
    'LinearSVC' : LinearSVC(),
    'SGDL' : SGDClassifier(),
    'Tree' : DecisionTreeClassifier()
              }

for name, estimador in estimadores.items():
    bag_classs = BaggingClassifier(base_estimator=estimador, n_estimators=5).fit(X_train,y_train)
    estimador.fit(X_train,y_train)
    
    bag_pred = bag_classs.predict(X_test)
    predictions = estimador.predict(X_test)
    
    print(name)
    print('Accuracy: ',accuracy_score(predictions,y_test))
    print('Accuracy_BAGG: ',accuracy_score(bag_pred,y_test))

Implemente el clasificador random forest y me dio un accuracy demasiado bueno.
Me hace dudar si lo hice bien

import pandas as pd
import numpy as np
from sklearn.neighbors import  KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier
 
from sklearn.ensemble import RandomForestClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

if __name__ == '__main__':

    dt_heart = pd.read_csv('./data/heart.csv')
    print(dt_heart['target'].describe())

    X = dt_heart.drop(['target'], axis=1)
    y = dt_heart['target']

    X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.3)

    knn_class = KNeighborsClassifier().fit(X_train, y_train)
    knn_pred = knn_class.predict(X_test)
    print('='*70)
    print(f'KNeighborsClassifier {accuracy_score(knn_pred, y_test)}')

    bag_class = BaggingClassifier(base_estimator=KNeighborsClassifier(), n_estimators=50).fit(X_train, y_train)
    bag_pred = bag_class.predict(X_test)
    print('='*70)
    print(f'BaggingClassifier {accuracy_score(bag_pred, y_test)}')
 
    forest = RandomForestClassifier(200)
    forest_class = forest.fit(X_train, y_train)
    forest_pred = forest_class.predict(X_test)
    print('='*70)
    print(f'RanfomForestClassifier: {accuracy_score(forest_pred, y_test)}')

======================================================================
KNeighborsClassifier 0.7045454545454546

BaggingClassifier 0.7597402597402597

RanfomForestClassifier: 0.9902597402597403

Agregue el siguiente codigo para probar con otros modelos

from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
estimators = {
    'KNN':KNeighborsClassifier(),
    'SVC': SVC(gamma="auto", random_state=42),
    'DecisionTree': DecisionTreeClassifier()
}

for name, estimator in estimators.items():
    bag_clf = BaggingClassifier(base_estimator=estimator, n_estimators = 50).fit(X_train, y_train)
    bag_clf.fit(X_train, y_train)
    bag_pred = bag_clf.predict(X_test)
    print('-'*64)
    print(name)
    print(accuracy_score(bag_pred,y_test))

Resultados:

----------------------------------------------------------------
KNN
0.807799442896936
----------------------------------------------------------------
SVC
0.9832869080779945
----------------------------------------------------------------
DecisionTree
1.0