from scipy.stats import randint as sp_randint from sklearn.svm import SVR import matplotlib.pyplot as plt import numpy as np import csv from time import time import numpy as np import pandas as pd from math import sqrt from sklearn import tree from sklearn.linear_model import LogisticRegression,LinearRegression,Lasso,Ridge from sklearn.ensemble import RandomForestRegressor,ExtraTreesRegressor,AdaBoostRegressor,GradientBoostingRegressor from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier,AdaBoostClassifier,GradientBoostingClassifier from sklearn.metrics import confusion_matrix,precision_score,accuracy_score,recall_score,roc_curve,f1_score,auc,matthews_corrcoef from sklearn import metrics from sklearn.model_selection import RandomizedSearchCV from sklearn.metrics import mean_squared_error from sklearn.externals import joblib import plotly.offline as py import plotly.graph_objs as go import timeit class general(): def __init__(self): return def report(self,results, n_top=3): for i in range(1, n_top + 1): candidates = np.flatnonzero(results['rank_test_score'] == i) for candidate in candidates: print("Model with rank: {0}".format(i)) print("Mean validation score: {0:.3f} (std: {1:.3f})".format( results['mean_test_score'][candidate], results['std_test_score'][candidate])) print("Parameters: {0}\n".format(results['params'][candidate])) def hyperparameter_optimize(self,X,Y,param_dist,train_model,n_iter_search=20): random_search = RandomizedSearchCV(train_model, param_distributions=param_dist, n_iter=n_iter_search) start = time() random_search.fit(X, Y) print("RandomizedSearchCV took %.2f seconds for %d candidates" " parameter settings." % ((time() - start), n_iter_search)) self.report(random_search.cv_results_) return random_search def feature_importance(self,feature_df,model_name,top_rank=20,dataset=None): temp_df = feature_df.sort_values(model_name,ascending=False) y = temp_df[model_name].values[:top_rank] x = temp_df['features'].values[:top_rank] data = [go.Bar( x= x, y= y, width = 0.5, marker=dict( color = temp_df[model_name].values[:top_rank], colorscale='Portland', showscale=True, reversescale = False ), opacity=0.6 )] layout= go.Layout( autosize= True, title= 'Barplots of %s'%model_name, hovermode= 'closest', yaxis=dict( title= 'Feature Importance', ticklen= 5, gridwidth= 2 ), showlegend= False ) fig = go.Figure(data=data, layout=layout) if dataset == 'FL': py.plot(fig,filename='FL results/%s feature importance.html'%model_name) elif dataset == 'LC': py.plot(fig,filename='LC results/%s feature importance.html'%model_name) else: py.plot(fig) return class FL_models(): def __init__(self): self.model_dict = { 'RandomForest':RandomForestRegressor(), 'ExtraTrees': ExtraTreesRegressor(), 'AdaBoost':AdaBoostRegressor(), 'GradientBoost':GradientBoostingRegressor() } self.feature_dict = {} self.base_predictions_train = {} self.base_predictions_test = {} return def set_data(self,target = 'em', quantum_cal=True ,solvent=True): self.target = target self.train_set = pd.read_csv('../../Data/Fluorescence dataset/train set.csv',header = 0, low_memory=False) self.test_set = pd.read_csv('../../Data/Fluorescence dataset/test set.csv',header = 0, low_memory=False) train = self.train_set.as_matrix()[:,1:].astype(float) test = self.test_set.as_matrix()[:,1:].astype(float) #QC descriptor if quantum_cal == False: self.feature_col = list(self.train_set)[1:2145] self.X_train = train[:,:2144] self.X_test = test[:,:2144] else: self.feature_col = list(self.train_set)[1:-3] self.X_train = train[:,:-3] self.X_test = test[:,:-3] #Solvent if solvent == True: self.feature_col.append('solvent') self.X_train = np.append(self.X_train, train[:,-3].reshape((-1,1)), axis=1) self.X_test = np.append(self.X_test, test[:,-3].reshape((-1,1)), axis=1) #Absorption or emission if target == 'abs': self.y_train = train[0:,-2] self.y_test = test[0:,-2] elif target == 'em': self.y_train = train[0:,-1] self.y_test = test[0:,-1] #Save features self.feature_dict['features'] = self.feature_col def plot(self,y_cal,y_pred,model_name,target='em',save = False): print ('Model:%s'%model_name) print ('train:') print ('R2=%f'%metrics.r2_score(self.y_train,y_cal)) print ('RMSE=%f nm'%sqrt(mean_squared_error(self.y_train,y_cal))) print ('test:') print ('R2=%f'%metrics.r2_score(self.y_test,y_pred)) print ('RMSE=%f nm'%sqrt(mean_squared_error(self.y_test,y_pred))) plt.scatter(self.y_train,y_cal,color = "b",marker = '^',label='train set') plt.scatter(self.y_test,y_pred,color = "orange",label='test set') plt.axis('scaled') plt.xlim(200,850) plt.ylim(200,850) plt.xlabel("The experimental "+r'$\lambda_{%s}$ (nm)'% target,fontsize=14) plt.ylabel("The calculated "+r'$\lambda_{%s}$ (nm)'% target,fontsize=14) plt.plot([200,850],[200,850],color = "k") plt.legend(loc=0) if save == True: plt.savefig('FL results/%s_%s.pdf'% (self.target,model_name), format = 'pdf') plt.show() def plot_interactive(self,y_cal,y_pred,model_name,target='em'): trace0 = go.Scatter( x = self.y_train, y = y_cal, mode = 'markers', name = 'train set', marker = dict(size = 10), text = self.train_set['No'] ) trace1 = go.Scatter( x = self.y_test, y = y_pred, mode = 'markers', name = 'test set', marker = dict(size = 10), text = self.test_set['No'] ) trace2 = go.Scatter( x = (200,850), y = (200,850), mode = 'lines', marker = dict(color = 'black') ) data = [trace0,trace1,trace2] layout = go.Layout( xaxis=dict(range=[200, 850]), yaxis=dict(range=[200, 850]), ) fig = go.Figure(data=data, layout = layout) py.plot(fig, filename='FL results/%s_interactive.html'%model_name) def build_model(self,model_name,save_model=False,save_plot=False,interactive = False,n_iter_search=20,feature_importance = False): if model_name == 'RandomForest' or model_name == 'ExtraTrees': param = {"n_estimators":[80,90,100,110], "max_features": ["auto","sqrt"], "min_samples_split": sp_randint(5, 11), "min_samples_leaf": sp_randint(1, 8), "random_state":[0,5,10] } elif model_name == 'AdaBoost': param = {"base_estimator": [tree.DecisionTreeRegressor(max_depth=6)], "n_estimators":[80,90,100,110], "learning_rate": [0.1,0.5,1], "loss":["linear", "square", "exponential"], "random_state":[0,5,10] } elif model_name == 'GradientBoost': param = {"n_estimators":[80,90,100,110], "max_features": ["auto","sqrt"], "min_samples_split": sp_randint(5, 11), "min_samples_leaf": sp_randint(1, 8), "loss":["ls", "lad", "huber", "quantile"], "learning_rate":[0.1,0.5,1], "max_depth":[2,3,4,5], "random_state":[0,5,10] } else: print ('wrong model name') return self.model_dict[model_name] = general().hyperparameter_optimize( self.X_train,self.y_train,param,self.model_dict[model_name],n_iter_search).best_estimator_ y_cal = self.model_dict[model_name].predict(self.X_train) y_pred = self.model_dict[model_name].predict(self.X_test) self.plot(y_cal,y_pred,model_name,save=save_plot) #prepare for blending self.base_predictions_train[model_name] = y_cal self.base_predictions_test[model_name] = y_pred self.feature_dict[model_name]=self.model_dict[model_name].feature_importances_ if save_model == True: joblib.dump(self.model_dict[model_name], "FL models/%s_%s.pkl"%(self.target,model_name)) if interactive == True: self.plot_interactive(model_name) if feature_importance == True: general().feature_importance(pd.DataFrame(self.feature_dict),model_name,top_rank=20,dataset='FL') return def load_model(self,model_name,save_plot=False,interactive = False,feature_importance = False): self.model_dict[model_name] = joblib.load("FL models/%s_%s.pkl"%(self.target,model_name)) self.feature_dict[model_name]=self.model_dict[model_name].feature_importances_ y_cal = self.model_dict[model_name].predict(self.X_train) y_pred = self.model_dict[model_name].predict(self.X_test) self.plot(y_cal,y_pred,model_name,save=save_plot) #prepare for blending self.base_predictions_train[model_name] = y_cal self.base_predictions_test[model_name] = y_pred if interactive == True: self.plot_interactive(y_cal,y_pred,model_name) if feature_importance == True: general().feature_importance(pd.DataFrame(self.feature_dict),model_name,top_rank=20,dataset='FL') return def blending(self,blend_type,feature_importance = True, save_plot=False,interactive =False): base_predictions_train = pd.DataFrame(self.base_predictions_train) base_predictions_test = pd.DataFrame(self.base_predictions_test) feature_dataframe = pd.DataFrame(self.feature_dict) if blend_type == 'uniform': Y_cal = np.mean(base_predictions_train.as_matrix(),axis = 1) Y_pred = np.mean(base_predictions_test.as_matrix(),axis = 1) feature_dataframe['%s blending'%blend_type] = feature_dataframe.mean(axis= 1) elif blend_type == 'linear': lr = LinearRegression() lr.fit(base_predictions_train.as_matrix(),self.y_train) Y_cal = lr.predict(base_predictions_train.as_matrix()) Y_pred = lr.predict(base_predictions_test.as_matrix()) lr_importance = [abs(i) for i in lr.coef_] importance_rate = [i/sum(lr_importance) for i in lr_importance] feature_dataframe['%s blending'%blend_type] = (feature_dataframe[list(feature_dataframe)[:-1]]*importance_rate).sum(axis=1) elif blend_type == 'any': gbr = GradientBoostingRegressor(n_estimators=100, max_depth=8,loss= 'lad') gbr.fit(base_predictions_train.as_matrix(),self.y_train) Y_cal = gbr.predict(base_predictions_train.as_matrix()) Y_pred = gbr.predict(base_predictions_test.as_matrix()) importance_rate = gbr.feature_importances_ feature_dataframe['%s blending'%blend_type] = (feature_dataframe[list(feature_dataframe)[:-1]]*importance_rate).sum(axis=1) self.plot(Y_cal,Y_pred,'%s blending'%blend_type) if interactive == True: self.plot_interactive(Y_cal,Y_pred,'%s blending'%blend_type) if feature_importance == True: general().feature_importance(feature_dataframe,'%s blending'%blend_type,top_rank=20,dataset='FL') return class LC_models(): def __init__(self): self.model_dict = { 'RandomForest':RandomForestClassifier(), 'ExtraTrees': ExtraTreesClassifier(), 'AdaBoost':AdaBoostClassifier(), 'GradientBoost':GradientBoostingClassifier() } self.feature_dict = {} self.base_predictions_train = {} self.base_predictions_test = {} return def set_data(self): train_df = pd.read_csv('../../Data/Liquid crystal dataset/Demo data/train_set_interpretable.csv',index_col=0) test_df = pd.read_csv('../../Data/Liquid crystal dataset/Demo data/test_set.csv',index_col=0)[list(train_df)] feature_col = list(train_df)[1:] self.X_train = train_df[feature_col] self.y_train = train_df['LC'] self.X_test = test_df[feature_col] self.y_test = test_df['LC'] #Save features self.feature_dict['features'] = feature_col return def performance(self,y_cal,y_pred,model_name): print ('Model:%s'%model_name) print ('train:') train_performance = {'confusion matrix':confusion_matrix(self.y_train,y_cal), 'accuracy_score':accuracy_score(self.y_train,y_cal), 'precision':precision_score(self.y_train,y_cal), 'recall':recall_score(self.y_train,y_cal), 'F1':f1_score(self.y_train,y_cal), 'MCC':matthews_corrcoef(self.y_train,y_cal)} print (train_performance) print ('test:') test_performance = {'confusion matrix':confusion_matrix(self.y_test,y_pred), 'accuracy_score':accuracy_score(self.y_test,y_pred), 'precision':precision_score(self.y_test,y_pred), 'recall':recall_score(self.y_test,y_pred), 'F1':f1_score(self.y_test,y_pred), 'MCC':matthews_corrcoef(self.y_test,y_pred)} print (test_performance) print () return train_performance,test_performance def build_model(self,model_name,save_model=False,n_iter_search=20,feature_importance = False): if model_name == 'RandomForest' or model_name == 'ExtraTrees': param = {"n_estimators":[80,90,100,110], "max_features": ["auto","sqrt"], "min_samples_split": sp_randint(5, 11), "min_samples_leaf": sp_randint(1, 8), "random_state":[0,5,10] } elif model_name == 'AdaBoost': param = {"base_estimator": [tree.DecisionTreeClassifier(max_depth=5)], "n_estimators":[80,90,100,110], "learning_rate": [0.1,0.5,1], "algorithm":['SAMME','SAMME.R'], "random_state":[0,5,10] } elif model_name == 'GradientBoost': param = {"n_estimators":[80,90,100,110], "max_features": ["auto","sqrt"], "min_samples_split": sp_randint(5, 11), "min_samples_leaf": sp_randint(1, 8), "learning_rate":[0.1,0.5,1], "max_depth":[2,3,4,5], "random_state":[0,5,10] } else: print ('wrong model name') return self.model_dict[model_name] = general().hyperparameter_optimize( self.X_train,self.y_train,param,self.model_dict[model_name],n_iter_search).best_estimator_ y_cal = self.model_dict[model_name].predict(self.X_train) y_pred = self.model_dict[model_name].predict(self.X_test) _,_ = self.performance(y_cal,y_pred,model_name) #prepare for blending self.base_predictions_train[model_name] = self.model_dict[model_name].predict_proba(self.X_train)[:,0] self.base_predictions_test[model_name] = self.model_dict[model_name].predict_proba(self.X_test)[:,0] self.feature_dict[model_name]=self.model_dict[model_name].feature_importances_ if save_model == True: joblib.dump(self.model_dict[model_name], "LC models/%s.pkl"%model_name) if feature_importance == True: general().feature_importance(pd.DataFrame(self.feature_dict),model_name,top_rank=20,dataset='LC') return def load_model(self,model_name,feature_importance = False): self.model_dict[model_name] = joblib.load("LC models/%s.pkl"%model_name) y_cal = self.model_dict[model_name].predict(self.X_train) y_pred = self.model_dict[model_name].predict(self.X_test) _,_ = self.performance(y_cal,y_pred,model_name) #prepare for blending self.base_predictions_train[model_name] = self.model_dict[model_name].predict_proba(self.X_train)[:,0] self.base_predictions_test[model_name] = self.model_dict[model_name].predict_proba(self.X_test)[:,0] self.feature_dict[model_name]=self.model_dict[model_name].feature_importances_ if feature_importance == True: general().feature_importance(pd.DataFrame(self.feature_dict),model_name,top_rank=20,dataset='LC') return def blending(self,blend_type,feature_importance = True): base_predictions_train = pd.DataFrame(self.base_predictions_train) base_predictions_test = pd.DataFrame(self.base_predictions_test) feature_dataframe = pd.DataFrame(self.feature_dict) if blend_type == 'uniform': cal_uniform = np.mean(base_predictions_train.as_matrix(),axis = 1) y_cal = np.where(cal_uniform < 0.5, 1, 0) pred_uniform = np.mean(base_predictions_test.as_matrix(),axis = 1) y_pred = np.where(pred_uniform < 0.5, 1, 0) feature_dataframe['%s blending'%blend_type] = feature_dataframe.mean(axis= 1) elif blend_type == 'linear': lr = LogisticRegression() lr.fit(base_predictions_train.as_matrix(),self.y_train) y_cal = lr.predict(base_predictions_train.as_matrix()) y_pred = lr.predict(base_predictions_test.as_matrix()) self.lr_coef = lr.coef_ lr_importance = [abs(i) for i in lr.coef_[0]] importance_rate = [i/sum(lr_importance) for i in lr_importance] feature_dataframe['%s blending'%blend_type] = (feature_dataframe[list(feature_dataframe)[:-1]]*importance_rate).sum(axis=1) elif blend_type == 'any': gbr = GradientBoostingClassifier(n_estimators=100,max_depth=4,learning_rate = 0.2) gbr.fit(base_predictions_train.as_matrix(),self.y_train) y_cal = gbr.predict(base_predictions_train.as_matrix()) y_pred = gbr.predict(base_predictions_test.as_matrix()) importance_rate = gbr.feature_importances_ feature_dataframe['%s blending'%blend_type] = (feature_dataframe[list(feature_dataframe)[:-1]]*importance_rate).sum(axis=1) _,_ = self.performance(y_cal,y_pred,'%s blending'%blend_type) if feature_importance == True: general().feature_importance(feature_dataframe,'%s blending'%blend_type,top_rank=20,dataset='LC') return