Ttykuu

I initially posted a question on SO. I have come up with an answer for the same. Basically, given two dicts of models and parameters, user can create an object, and get the report in 5 steps.

Following is the code.

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import f1_score, roc_auc_score, recall_score, precision_score
from sklearn import datasets
from sklearn.pipeline import Pipeline
from sklearn.base import BaseEstimator
import warnings
warnings.filterwarnings('ignore')

cancer = datasets.load_breast_cancer()
df = pd.DataFrame(cancer.data, columns=cancer.feature_names)
df['target'] = cancer.target
target = df['target']
X_train, X_test, y_train, y_test = train_test_split(df.drop(columns='target', axis=1), target, test_size=0.4, random_state=13, stratify=target)

class ClfSwitcher(BaseEstimator):

 def __init__(self, model=RandomForestClassifier()):
 """
 A Custom BaseEstimator that can switch between classifiers.
 :param estimator: sklearn object - The classifier
 """ 

 self.model = model


 def fit(self, X, y=None, **kwargs):
 self.model.fit(X, y)
 return self


 def predict(self, X, y=None):
 return self.model.predict(X)


 def predict_proba(self, X):
 return self.model.predict_proba(X)

 def score(self, X, y):
 return self.estimator.score(X, y)

class report(ClfSwitcher):
 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 self.grid = None
 self.full_report = None
 self.concise_report = None
 self.scoring_metrics = 
 'precision': precision_score,
 'recall': recall_score,
 'f1': f1_score,
 'roc_auc': roc_auc_score
 


 def griddy(self, pipeLine, parameters, **kwargs):
 self.grid = GridSearchCV(pipeLine, parameters, scoring='accuracy', n_jobs=-1)


 def fit_grid(self, X_train, y_train=None, **kwargs):
 self.grid.fit(X_train, y_train)

 def make_grid_report(self):
 self.full_report = pd.DataFrame(self.grid.cv_results_)

 @staticmethod
 def get_names(col):
 return col.__class__.__name__

 @staticmethod
 def calc_score(col, metric):
 return round(metric(y_test, col.fit(X_train, y_train).predict(X_test)), 4)


 def make_concise_report(self):
 self.concise_report = pd.DataFrame(self.grid.cv_results_)
 self.concise_report['model_names'] = self.concise_report['param_cst__model'].apply(self.get_names)
 self.concise_report = self.concise_report.sort_values(['model_names', 'rank_test_score'], ascending=[True, False]) 
 .groupby(['model_names']).head(1)[['param_cst__model', 'model_names']] 
 .reset_index(drop=True)

 for metric_name, metric_func in self.scoring_metrics.items():
 self.concise_report[metric_name] = self.concise_report['param_cst__model'].apply(self.calc_score, metric=metric_func)

 self.concise_report = self.concise_report[['model_names', 'precision', 'recall', 'f1', 'roc_auc', 'param_cst__model']]

pipeline = Pipeline([
 ('cst', ClfSwitcher()),
])

parameters = [
 
 'cst__model': [RandomForestClassifier()],
 'cst__model__n_estimators': [10, 20],
 'cst__model__max_depth': [5, 10],
 'cst__model__criterion': ['gini', 'entropy']
 ,
 
 'cst__model': [SVC()],
 'cst__model__C': [10, 20],
 'cst__model__kernel': ['linear'],
 'cst__model__gamma': [0.0001, 0.001]
 ,
 
 'cst__model': [LogisticRegression()],
 'cst__model__C': [13, 17],
 'cst__model__penalty': ['l1', 'l2']
 ,
 
 'cst__model': [GradientBoostingClassifier()],
 'cst__model__n_estimators': [10, 50],
 'cst__model__max_depth': [3, 5],
 'cst__model__min_samples_leaf': [1, 2]
 
]

my_report = report()
my_report.griddy(pipeline, parameters, scoring='f1')
my_report.fit_grid(X_train, y_train)
my_report.make_concise_report()
my_report.concise_report