Implementación de K-Folds Cross Validation

Al final me parece que quedó muy en el aire esa información, asi que quise complementar. Aquí les dejo mi codigo de la implementación del Cross-Validation con KFold:

data = dataset.drop(["country","score"],axis=1)
targets = dataset["score"]

kf = KFold(n_splits=3,shuffle=True)
    
mse_values = []
for train,test in kf.split(data):
    x_train = pd.DataFrame(columns=list(data),index=range(len(train)))
    x_test = pd.DataFrame(columns=list(data),index=range(len(test)))
    y_train = pd.DataFrame(columns=['score'],index=range(len(train)))
    y_test = pd.DataFrame(columns=['score'],index=range(len(test)))
    for i in range(len(train)):
        x_train.iloc[i] = data.iloc[train[i]]
        y_train.iloc[i] = targets.iloc[train[i]]
    for j in range(len(test)):
        x_test.iloc[j] = data.iloc[test[j]]
        y_test.iloc[j] = targets.iloc[test[j]]
    model = DecisionTreeRegressor().fit(x_train,y_train)
    predict = model.predict(x_test)
    mse_values.append(mean_squared_error(y_test,predict))
    
print("Los tres MSE fueron: ",mse_values)
print("El MSE promedio fue: ", np.mean(mse_values))

no entiendo como interpretar el resultado de score

En este caso al hacer tres splits, es decir al dividir nuestros datos en A,B,C, tenemos solo tres posibles combinaciones AB para training y C para test, luego AC con B, y por ultimo BC con A.

Dejo la implementación de los entrenamientos dentro del ciclo for de la última parte. Es decir, cómo usar los índices generados por KFold y usar una medida como la que veníamos usando antes para la pérdida (mean_squared_error):

# se importa adicional a las que ya se tienen
from sklearn.metrics import mean_squared_error

if __name__ == "__main__":

    dataset = pd.read_csv('./data/happiness.csv')

    X = dataset.drop(['country', 'score'], axis=1)
    y = dataset['score']
    
    model = DecisionTreeRegressor()

    kf = KFold(n_splits=3, shuffle=True, random_state=42)
    losses = []
    for train, test in kf.split(dataset):
        X_train = X.iloc[train]
        X_test = X.iloc[test]
        y_train = y[train]
        y_test = y[test]
        model.fit(X_train, y_train)
        predictions = model.predict(X_test)
        loss = mean_squared_error(y_test, predictions)
        losses.append(loss)

    print('Error para cada partición: ', losses)
    print('Promedio de los errores: ', np.mean(losses))

Implementación de cross validation con KFolds

X = dataset.drop(['country', 'score', 'rank'], axis=1)
y = dataset['score']
model = DecisionTreeRegressor()
kf = KFold(n_splits=10, shuffle=True, random_state=1)
results = cross_val_score(model, X, y, cv=kf, scoring='neg_mean_squared_error')
print(f'Promedio de los cross validation scores: {np.abs(np.mean(results))}')

Aquí les comparto mi implementación del KFold:

X = dataset.drop(['country','rank', 'score'], axis=1)
y = dataset[['score']]

k_fold = KFold(n_splits=3, shuffle=True, random_state=2)
dtr = DecisionTreeRegressor()
mse = []

for train_index, test_index in k_fold.split(X):

  # X_train
  X_train = pd.DataFrame(X.iloc[train_index, :], columns= list(X))
  X_train = X_train.reset_index().drop('index', axis = 1)

  # X_test
  X_test = pd.DataFrame(X.iloc[test_index, :], columns= list(X))
  X_test = X_test.reset_index().drop('index', axis = 1)

  # y_train
  y_train = pd.DataFrame(y.iloc[train_index, :], columns= list(y))
  y_train = y_train.reset_index().drop('index', axis = 1)

  # y_test
  y_test = pd.DataFrame(y.iloc[test_index, :], columns= list(y))
  y_test = y_test.reset_index().drop('index', axis = 1)

  dtr.fit(X_train, y_train)
  y_predit = dtr.predict(X_test)

  mse.append(mean_squared_error(y_test, y_predit))

print(mse)

Les dejo un código que hice bastante claro y escalable:

import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import cross_val_score, KFold
from sklearn.metrics import mean_squared_error


def train_test_split_kf(data: np.array, target: np.array, train: np.array, test: np.array) -> np.array:
    x_train = data[train]
    x_test = data[test]
    y_train = target[train]
    y_test = target[test]
    return x_train, x_test, y_train, y_test


def evaluate_model(model, metric, x_train, x_test, y_train, y_test):
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)
    score = metric(y_pred, y_test)
    return score


def main():
    dataset = pd.read_csv("../datasets/happines.csv")

    data = dataset.drop(["country", "score", "rank"], axis=1)
    target = dataset["score"]
    kf = KFold(n_splits=3, shuffle=True, random_state=42)

    models = {"DecisionTreeRegressor": DecisionTreeRegressor()}
    print('---- Easy Implementation ----')
    for name, model in models.items():
        score = cross_val_score(model, data, target, cv=3, scoring='neg_mean_squared_error')
        print("Scores:", score)
        print("Mean score: ", np.abs(np.mean(score)))
    print("=" * 64)
    scores = []
    print('---- Full Implementation ----')
    for name, model in models.items():
        print(f"I'm evaluating: {name}")
        for n_fold, (train, test) in enumerate(kf.split(data)):
            print(f"\t-I'm running fold {n_fold + 1}")
            x_train, x_test, y_train, y_test = train_test_split_kf(data.values, target.values, train, test)
            score = evaluate_model(model, mean_squared_error, x_train, x_test, y_train, y_test)
            print("\t\t-score:", score)
            scores.append(score)
        print("="*64)
        print("Scores:", scores)
        print("Mean score: ", np.mean(scores))


if __name__ == '__main__':
    main()

Una forma de validar nuestro modelo a través de K-folds es importar la función cross_validate.

results = cross_validate(LinearSVC(dual=False), X_scaled, y_bal_df, return_train_score=True, cv=50)

Donde usa K-folds determinados por analista, ingresamos el modelo a usar, nuestros features, nuestra columna Target y podemos usar el parámetro return_train_score para que nos devuelva el score de cada uno de los modelos armados con cada K-fold.

Para este ejemplo, devuelve 50 resultados y se ve de esta manera:

{'fit_time': array([0.7657094 , 0.79296184, 0.78337979, 0.76587272, 0.77635145,
        0.85032177, 0.81398034, 0.85498548, 0.83176494, 0.80038834,
        0.77539325, 0.77279139, 0.77225327, 0.7791965 , 0.78038335,
        0.80153918, 0.80040407, 0.79351783, 0.81560874, 0.80307364,
        0.81683111, 0.82906723, 0.80062199, 0.82587028, 0.82564425,
        0.82885957, 0.81278729, 0.8008554 , 0.79572654, 0.79565167,
        0.79669046, 0.80218983, 0.80508876, 0.79961634, 0.81082296,
        0.80319452, 0.77765465, 0.79044795, 0.8088398 , 0.77227616,
        0.7737639 , 0.80028009, 0.77275562, 0.80092597, 0.83133173,
        0.79332328, 0.82022905, 0.7792604 , 0.7991221 , 0.76032257]),
 'score_time': array([0.00137734, 0.00085735, 0.00090194, 0.00089025, 0.00107074,
        0.00088453, 0.00085354, 0.00089717, 0.00089884, 0.00094604,
        0.00084925, 0.00099683, 0.00086522, 0.00086045, 0.00086689,
        0.00135803, 0.00090003, 0.00088763, 0.00086021, 0.00089717,
        0.00164509, 0.00086546, 0.00089502, 0.00087857, 0.00087142,
        0.00086522, 0.00091124, 0.00086331, 0.00086379, 0.00085735,
        0.00086927, 0.00086069, 0.00087881, 0.00088525, 0.00098681,
        0.00091887, 0.00086212, 0.00087953, 0.00084734, 0.00087833,
        0.00088882, 0.00086713, 0.0008862 , 0.00087786, 0.00107956,
        0.00086331, 0.0008769 , 0.00088549, 0.00094199, 0.00087142]),
 'test_score': array([0.77079108, 0.90365112, 0.90466531, 0.90567951, 0.92697769,
        0.87423935, 0.92089249, 0.96044625, 0.96450304, 0.96653144,
        0.96450304, 0.97360406, 0.96446701, 0.97360406, 0.9715736 ,
        0.97664975, 0.98375635, 0.97969543, 0.97360406, 0.97766497,
        0.97055838, 0.97664975, 0.97766497, 0.97969543, 0.97360406,
        0.97258883, 0.97969543, 0.96649746, 0.97664975, 0.97360406,
        0.97664975, 0.97969543, 0.96649746, 0.97664975, 0.97360406,
        0.98071066, 0.97969543, 0.93299492, 0.92791878, 0.92385787,
        0.93401015, 0.93502538, 0.91269036, 0.9177665 , 0.92994924,
        0.93299492, 0.91573604, 0.91573604, 0.91472081, 0.91979695]),
 'train_score': array([0.95287416, 0.95154842, 0.95202486, 0.95127913, 0.95115484,
        0.95206629, 0.95158985, 0.95171414, 0.95113413, 0.95127913,
        0.95161056, 0.95111443, 0.95144585, 0.95115585, 0.95109371,
        0.95113514, 0.95096943, 0.95082443, 0.95101085, 0.950928  ,
        0.95103157, 0.95113514, 0.95082443, 0.95090728, 0.95109371,
        0.951363  , 0.95105228, 0.95134228, 0.95103157, 0.951073  ,
        0.95086585, 0.950928  , 0.95115585, 0.95111443, 0.95119728,
        0.95088657, 0.95096943, 0.951073  , 0.95117657, 0.95144585,
        0.95128014, 0.950928  , 0.95144585, 0.95179799, 0.95113514,
        0.95088657, 0.95148728, 0.95123871, 0.951218  , 0.95159085])}

Un poco complejo de leer, no? En realidad es todo data que ya vimos en el curso, donde podemos ver los scores de cada modelo armado.

De acá en adelante solo nos queda sacar el promedio de nuestros scores en cada etapa (train y test) y ver si debemos ajustar algo para corregir un posible overfitting o un underfitting.

test_scores = results['test_score']
train_scores = results['train_score']

print(f'Train score : {np.mean(train_scores)}')
print(f'Test score : {np.mean(test_scores)}')

Para cerrar este ejemplo, el resultado seria el siguiente:
Train score : 0.9512525761353487
Test score : 0.9486281648665067

COMPLETO

import pandas as pd
import numpy as np

from sklearn.metrics import mean_squared_error

from sklearn.tree import DecisionTreeRegressor

from sklearn.model_selection import (
cross_val_score, KFold
)

dataset = pd.read_csv(‘felicidad.csv’)
X = dataset.drop([‘country’, ‘score’], axis=1)
y = dataset[‘score’]

model = DecisionTreeRegressor()

score = cross_val_score(model, X,y, cv=5, scoring=‘neg_mean_squared_error’)
print (“MSE para cv=5, 5 valores”)
print(score)
print(np.abs(np.mean(score)))

print("")

kf = KFold(n_splits=3, shuffle=True, random_state=42)
mse_values = []

for train, test in kf.split(dataset):
X_train = X.iloc[train]
X_test = X.iloc[test]

y_train = y[train]
y_test = y[test]

model = DecisionTreeRegressor().fit(X_train,y_train)
predict = model.predict(X_test)
mse = mean_squared_error(y_test,predict)
mse_values.append(mse)

#model.fit(X_train, y_train)
#predictions = model.predict(X_test)

#.append(mean_squared_error(y_test,predict))

print('Error para cada partición: ', mse_values)
print('Promedio de los errores: ', np.mean(mse_values))

Mi implementación de K-Folds

lst = []
cv = 1
for train , test in kf.split(X,y= y): 
    # datasets 
    X_train = pd.DataFrame(X.iloc[train,:], columns=X.columns)
    y_train = pd.DataFrame(y.iloc[train,])
    X_test  = pd.DataFrame(X.iloc[test,:], columns=X.columns)
    y_test  = pd.DataFrame(y.iloc[test,])
    
    #Modelo
    dt = DecisionTreeRegressor().fit(X_train,y_train)
    y_predict = dt.predict(X_test)
    lst.append(mean_squared_error(y_predict, y_test))
    print(f'The error for the folder number {cv} =',mean_squared_error(y_predict, y_test))
    cv += 1
    
print(f'\nThe mean of ther erros is equal to {np.mean(lst)}')

Resultado

The error for the folder number 1 = 0.260119109314712
The error for the folder number 2 = 0.21976508857008575
The error for the folder number 3 = 0.23677062451690725
The error for the folder number 4 = 0.31030546626208927

The mean of ther erros is equal to 0.2567400721659486