{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF classification based on all variables used to plot the correlation matrix: 500 forests to compute average features scores and average accuracy scores (Supplementary Figure S6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_repeats = 500\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt # data visualization\n",
    "import seaborn as sns # statistical data visualization\n",
    "%matplotlib inline\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('./input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Import data specifying that first row is column names (header)\n",
    "hevonen = './input/PetteriData.csv'\n",
    "df = pd.read_csv(hevonen,sep=';', header=0)\n",
    "\n",
    "# create a classifier model with all feature variables, and estimate the \"importance\" of each variable in the final classification\n",
    "X = df.drop(['Diagnoosi','Name'], axis=1)\n",
    "y = df['Diagnoosi']\n",
    "\n",
    "# Initialize the variables that will be incremented during the runs\n",
    "features_scores=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
    "cm = [[0,0],[0,0]]\n",
    "accuracyscore = 0\n",
    "\n",
    "for k in range(1,n_repeats+1):\n",
    "    print(k)\n",
    "    # Splitting in training and testing datasets\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size = 0.5, shuffle=True, random_state = None)\n",
    "\n",
    "    # create the classifier with n_estimators = 100\n",
    "    from sklearn.ensemble import RandomForestClassifier\n",
    "    clf = RandomForestClassifier(n_estimators=300, criterion='entropy', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features='sqrt', max_leaf_nodes=None, min_impurity_decrease=0.0, bootstrap=True, oob_score=False, n_jobs=-1, random_state=None, verbose=0, warm_start=False, class_weight=None, ccp_alpha=0.0, max_samples=None)\n",
    "\n",
    "    # fit the model to the training set\n",
    "    clf.fit(X_train, y_train) \n",
    " \n",
    "    # Compute the feature scores and sum them to thtte existing scores\n",
    "    features_scores = features_scores+clf.feature_importances_\n",
    "    \n",
    "    # Predict the Test set results and compute accuracy and confusion matrix\n",
    "    y_pred = clf.predict(X_test)    \n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    from sklearn.metrics import accuracy_score\n",
    "    cm = cm + confusion_matrix(y_test, y_pred)\n",
    "    accuracyscore = accuracyscore + accuracy_score(y_test, y_pred)\n",
    "\n",
    "    \n",
    "    \n",
    "# normalize the  results to the number of repeats:\n",
    "features_scores = np.divide(features_scores,n_repeats)\n",
    "cm = np.divide(cm,n_repeats)\n",
    "accuracyscore = np.divide(accuracyscore,n_repeats)\n",
    "    \n",
    "    \n",
    "# display the feature scores and visualize them\n",
    "feature_scores = pd.Series(features_scores, index=X_train.columns).sort_values(ascending=False)\n",
    "plt.figure(figsize=(15, 15))\n",
    "sns.barplot(x=feature_scores, y=feature_scores.index)\n",
    "# Add labels to the graph\n",
    "plt.xlabel('Feature Importance Score')\n",
    "plt.ylabel('Features')\n",
    "# Add title to the graph\n",
    "plt.title(\"Visualizing Important Features\")\n",
    "# Visualize the graph\n",
    "plt.show()\n",
    "\n",
    "# print mean accuracy score and confusion matrix\n",
    "print('Model accuracy score with 100 decision-trees : {0:0.4f}'. format(accuracyscore))\n",
    "\n",
    "# confusion matrix (errorS)\n",
    "print('Confusion matrix\\n\\n', cm)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hence, a RF model trained on all variables but only half the samples predicts the status (healthy/OA) of the remaining samples with reasonably good accuracy (92%)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dimensionality reduction: RF-model built on random sampling of the 6 groups identified by the correlation matrix (Fig. 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt # data visualization\n",
    "import seaborn as sns # statistical data visualization\n",
    "%matplotlib inline\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('./input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Import data specifying that first row is column names (header)\n",
    "hevonen = './input/PetteriData.csv'\n",
    "df = pd.read_csv(hevonen,sep=';', header=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the variables in groups (according to Figure 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that to make the groups, we drop:\n",
    "#Map_TA because missing values; Age, BMI because don't want to use them as classification features;\n",
    "\n",
    "\n",
    "y0 = df['Diagnoosi']\n",
    "Xgroup1 = df[['Med_tibia_cartilage_thickness_min','Med_tibia_cartilage_thickness','Med_femur_cartilage_thickness','Chair_stand','Flexion','Med_femur_cartilage_thickness_min','CD61','PPT_ref','CCD41CD63CD9','CCD9CD63','PPT_LJC','PPT_RF','PPT_MJC','PPT_patella','PPT_CLT','PPT_CMT']]\n",
    "Xgroup2 = df[['CCD41CD81CD9','CCD63CD81','CCD63CD81CD9','CCD81CD63','CCD81CD63CD9','CCD9CD63CD81']]\n",
    "Xgroup3 = df[['Motor_threshold','VAS_pain','Fast_paced_walk','Stair_climb','CCD41','Extension','VAS_worstpain']]\n",
    "Xgroup4 = df[['CD81','CCD81CD9','CCD9CD81','CCD41CD81','CCD81']]\n",
    "Xgroup5 = df[['CD63','CD41','CD9','CCD9','CCD63CD9','CCD41CD63','CCD41CD9','CCD41CD63CD81','CCD41CD63CD81CD9']]\n",
    "Xgroup6 = df[['CCD63','TPD_ref','TPD_lat','TPD_med','QST_heatpain','QST_cold','QST_warm']]\n",
    "Xgroup5.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test of within-group information redundancy: we perform random sampling of the groups to enrich 1000-fold the dataset and then, split in training and testing datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we pick a number of sampling we will do to construct the augmented dataset\n",
    "n_aug=1000\n",
    "# then we loop n_aug times the following: first we sample each group once and append columns in \"groupXX\" attributes, and then we repeat and append in the \"lines\" direction\n",
    "column_names = [\"group1\", \"group2\", \"group3\",\"group4\", \"group5\", \"group6\"]\n",
    "X = pd.DataFrame(columns = column_names)\n",
    "column_names_y = [\"Diagnoosi\"]\n",
    "#y = pd.DataFrame(columns = column_names_y)\n",
    "y = y0\n",
    "for k in range(1,n_aug+1):\n",
    "    Xtemp=Xgroup1.sample(axis='columns')\n",
    "    Xtemp=Xtemp.join(Xgroup2.sample(axis='columns'))\n",
    "    Xtemp=Xtemp.join(Xgroup3.sample(axis='columns'))\n",
    "    Xtemp=Xtemp.join(Xgroup4.sample(axis='columns'))\n",
    "    Xtemp=Xtemp.join(Xgroup5.sample(axis='columns'))\n",
    "    Xtemp=Xtemp.join(Xgroup6.sample(axis='columns'))\n",
    "    Xtemp.columns = X.columns\n",
    "    X=X.append(Xtemp,ignore_index = True)\n",
    "    del Xtemp\n",
    "for k in range(1,n_aug):\n",
    "    y=y.append(y0,ignore_index = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, shuffle=True, random_state = None)\n",
    "# check the shape (i.e., array size) of X_train and X_test\n",
    "print(X_train.shape, X_test.shape)\n",
    "print(y_train.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RF classification on this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the classifier with n_estimators = 100\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# create an instance of the classifier\n",
    "clf = RandomForestClassifier(n_estimators=100, criterion='entropy', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=6, max_leaf_nodes=None, min_impurity_decrease=0.0, bootstrap=True, oob_score=False, n_jobs=-1, random_state=None, verbose=0, warm_start=False, class_weight=None, ccp_alpha=0.0, max_samples=None)\n",
    "\n",
    "# fit the model to the training set\n",
    "clf.fit(X_train, y_train)\n",
    "# display the feature scores and visualize them\n",
    "print(clf.feature_importances_)\n",
    "feature_scores = pd.Series(clf.feature_importances_, index=X_train.columns).sort_values(ascending=False)\n",
    "feature_scores\n",
    "plt.figure(figsize=(15, 15))\n",
    "sns.barplot(x=feature_scores, y=feature_scores.index)\n",
    "# Add labels to the graph\n",
    "plt.xlabel('Feature Importance Score')\n",
    "plt.ylabel('Features')\n",
    "# Add title to the graph\n",
    "plt.title(\"Visualizing Important Features\")\n",
    "# Visualize the graph\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assess prediction performance: accuracy and confusion matrix and classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict the Test set results\n",
    "y_pred = clf.predict(X_test)\n",
    "\n",
    "# Check accuracy score \n",
    "from sklearn.metrics import accuracy_score\n",
    "print('Model accuracy score with 100 decision-trees : {0:0.4f}'. format(accuracy_score(y_test, y_pred)))\n",
    "\n",
    "# confusion matrix (errorS)\n",
    "from sklearn.metrics import confusion_matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "print('Confusion matrix\\n\\n', cm)\n",
    "\n",
    "# classification report (another way to present data)\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Repeat the analysis above 100 times (100 forests) to compute \"average\" accuracy score, confusion matrix, and Important features (Figure 3A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "n_repeats = 100\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt # data visualization\n",
    "import seaborn as sns # statistical data visualization\n",
    "%matplotlib inline\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('./input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Import data specifying that first row is column names (header)\n",
    "hevonen = './input/PetteriData.csv'\n",
    "df = pd.read_csv(hevonen,sep=';', header=0)\n",
    "\n",
    "#y0 = df.drop(['Name','Zf140iso','Zf140','Zf141n5','Zf150iso','Zf150anteiso','Zf150','Zf160iso','Zf160','Zf161n9','Zf161n7','Zf161n5','Zf170iso','Zf170anteiso','Zf170','Zf171n8','Zf180iso','Zf180','Zf181n9','Zf181n7','Zf181n5','Zf190iso','Zf182n6','Zf190','Zf191n10','Zf191n8','Zf183n3','Zf182c9t11','Zf200','Zf201n11','Zf201n9','Zf201n7','Zf202n6','Zf203n6','Zf204n6','Zf203n3','Zf205n3','Zf220','Zf221n9','Zf225n3','Zf240','Zf226n3plusartef','Zf241n9','ZSUMSAT','ZSUMMUFA','ZSUMPUFA','ZSUMn6PUFA','ZSUMn3PUFA','ZUNSATSAT','Zn3n6','Zn3n6ilmanDHA','ZTACL','ZD9DI','ZDBI','ZProdpren6','ZProdpren3','Zdelta5n6'], axis=1)\n",
    "y0 = df['Diagnoosi']\n",
    "Xgroup1 = df[['Med_tibia_cartilage_thickness_min','Med_tibia_cartilage_thickness','Med_femur_cartilage_thickness','Chair_stand','Flexion','Med_femur_cartilage_thickness_min','CD61','PPT_ref','CCD41CD63CD9','CCD9CD63','PPT_LJC','PPT_RF','PPT_MJC','PPT_patella','PPT_CLT','PPT_CMT']]\n",
    "Xgroup2 = df[['CCD41CD81CD9','CCD63CD81','CCD63CD81CD9','CCD81CD63','CCD81CD63CD9','CCD9CD63CD81']]\n",
    "Xgroup3 = df[['Motor_threshold','VAS_pain','Fast_paced_walk','Stair_climb','CCD41','Extension','VAS_worstpain']]\n",
    "Xgroup4 = df[['CD81','CCD81CD9','CCD9CD81','CCD41CD81','CCD81']]\n",
    "Xgroup5 = df[['CD63','CD41','CD9','CCD9','CCD63CD9','CCD41CD63','CCD41CD9','CCD41CD63CD81','CCD41CD63CD81CD9']]\n",
    "Xgroup6 = df[['CCD63','TPD_ref','TPD_lat','TPD_med','QST_heatpain','QST_cold','QST_warm']]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "features_scores=[0,0,0,0,0,0]\n",
    "cm = [[0,0],[0,0]]\n",
    "accuracyscore = 0\n",
    "\n",
    "for k in range(1,n_repeats+1):\n",
    "    print(k)\n",
    "    # we pick a number of sampling we will do to construct the augmented dataset\n",
    "    n_aug=1000\n",
    "    # then we loop n_aug times the following: first we sample each group once and append columns in \"groupXX\" attributes, and then we repeat and append in the \"lines\" direction\n",
    "    column_names = [\"group1\", \"group2\", \"group3\",\"group4\", \"group5\", \"group6\"]\n",
    "    X = pd.DataFrame(columns = column_names)\n",
    "    column_names_y = [\"Diagnoosi\"]\n",
    "    #y = pd.DataFrame(columns = column_names_y)\n",
    "    y = y0\n",
    "    for k in range(1,n_aug+1):\n",
    "        Xtemp=Xgroup1.sample(axis='columns')\n",
    "        Xtemp=Xtemp.join(Xgroup2.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xgroup3.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xgroup4.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xgroup5.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xgroup6.sample(axis='columns'))\n",
    "        Xtemp.columns = X.columns\n",
    "        X=X.append(Xtemp,ignore_index = True)\n",
    "        del Xtemp\n",
    "    for k in range(1,n_aug):\n",
    "        y=y.append(y0,ignore_index = True)\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, shuffle=True, random_state = None)\n",
    "    # create the classifier with n_estimators = 100\n",
    "    from sklearn.ensemble import RandomForestClassifier\n",
    "    \n",
    "    # create an instance of the classifier\n",
    "    clf = RandomForestClassifier(n_estimators=100, criterion='entropy', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=6, max_leaf_nodes=None, min_impurity_decrease=0.0, bootstrap=True, oob_score=False, n_jobs=-1, random_state=None, verbose=0, warm_start=False, class_weight=None, ccp_alpha=0.0, max_samples=None)\n",
    "\n",
    "    # fit the model to the training set\n",
    "    clf.fit(X_train, y_train)\n",
    "    # Compute the feature scores and sum them to thtte existing scores\n",
    "    features_scores = features_scores+clf.feature_importances_\n",
    "    \n",
    "    # Predict the Test set results and compute accuracy and confusion matrix\n",
    "    y_pred = clf.predict(X_test)    \n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    from sklearn.metrics import accuracy_score\n",
    "    cm = cm + confusion_matrix(y_test, y_pred)\n",
    "    accuracyscore = accuracyscore + accuracy_score(y_test, y_pred)\n",
    "\n",
    "    \n",
    "    \n",
    "# normalize the  results to the number of repeats:\n",
    "features_scores = np.divide(features_scores,n_repeats)\n",
    "cm = np.divide(cm,n_repeats)\n",
    "accuracyscore = np.divide(accuracyscore,n_repeats)\n",
    "    \n",
    "    \n",
    "# display the feature scores and visualize them\n",
    "feature_scores = pd.Series(features_scores, index=X_train.columns).sort_values(ascending=False)\n",
    "plt.figure(figsize=(15, 15))\n",
    "sns.barplot(x=feature_scores, y=feature_scores.index)\n",
    "# Add labels to the graph\n",
    "plt.xlabel('Feature Importance Score')\n",
    "plt.ylabel('Features')\n",
    "# Add title to the graph\n",
    "plt.title(\"Visualizing Important Features\")\n",
    "# Visualize the graph\n",
    "plt.show()\n",
    "\n",
    "# print mean accuracy score and confusion matrix\n",
    "print('Model accuracy score with 100 decision-trees : {0:0.4f}'. format(accuracyscore))\n",
    "\n",
    "# confusion matrix (errorS)\n",
    "print('Confusion matrix\\n\\n', cm)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test of RF-classifier ability to predict new samples (Figure 3B): we first split the training and testing datasets (from the original 16 samples), then perform 1000-fold data enrichment within each dataset separately, and finally train and test the RF model on those (repeated 100 times to compute average scores, as above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "n_repeats = 100\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt # data visualization\n",
    "import seaborn as sns # statistical data visualization\n",
    "%matplotlib inline\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('./input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Import data specifying that first row is column names (header)\n",
    "hevonen = './input/PetteriData.csv'\n",
    "df = pd.read_csv(hevonen,sep=';', header=0)\n",
    "X = df.drop(['Diagnoosi','Name'], axis=1)\n",
    "y = df['Diagnoosi']\n",
    "\n",
    "\n",
    "features_scores=[0,0,0,0,0,0]\n",
    "cm = [[0,0],[0,0]]\n",
    "accuracyscore = 0\n",
    "\n",
    "for k in range(1,n_repeats+1):\n",
    "    print(k)\n",
    "    # we split the dataset in training and testing\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, shuffle=True, random_state = None)\n",
    "\n",
    "    # tthen we preprocess data to get it ready for data augmentation\n",
    "    ytest0 = y_test\n",
    "    Xtestgroup1 = X_test[['Med_tibia_cartilage_thickness_min','Med_tibia_cartilage_thickness','Med_femur_cartilage_thickness','Chair_stand','Flexion','Med_femur_cartilage_thickness_min','CD61','PPT_ref','CCD41CD63CD9','CCD9CD63','PPT_LJC','PPT_RF','PPT_MJC','PPT_patella','PPT_CLT','PPT_CMT']]\n",
    "    Xtestgroup2 = X_test[['CCD41CD81CD9','CCD63CD81','CCD63CD81CD9','CCD81CD63','CCD81CD63CD9','CCD9CD63CD81']]\n",
    "    Xtestgroup3 = X_test[['Motor_threshold','VAS_pain','Fast_paced_walk','Stair_climb','CCD41','Extension','VAS_worstpain']]\n",
    "    Xtestgroup4 = X_test[['CD81','CCD81CD9','CCD9CD81','CCD41CD81','CCD81']]\n",
    "    Xtestgroup5 = X_test[['CD63','CD41','CD9','CCD9','CCD63CD9','CCD41CD63','CCD41CD9','CCD41CD63CD81','CCD41CD63CD81CD9']]\n",
    "    Xtestgroup6 = X_test[['CCD63','TPD_ref','TPD_lat','TPD_med','QST_heatpain','QST_cold','QST_warm']]\n",
    "    ytrain0 = y_train\n",
    "    Xtraingroup1 = X_train[['Med_tibia_cartilage_thickness_min','Med_tibia_cartilage_thickness','Med_femur_cartilage_thickness','Chair_stand','Flexion','Med_femur_cartilage_thickness_min','CD61','PPT_ref','CCD41CD63CD9','CCD9CD63','PPT_LJC','PPT_RF','PPT_MJC','PPT_patella','PPT_CLT','PPT_CMT']]\n",
    "    Xtraingroup2 = X_train[['CCD41CD81CD9','CCD63CD81','CCD63CD81CD9','CCD81CD63','CCD81CD63CD9','CCD9CD63CD81']]\n",
    "    Xtraingroup3 = X_train[['Motor_threshold','VAS_pain','Fast_paced_walk','Stair_climb','CCD41','Extension','VAS_worstpain']]\n",
    "    Xtraingroup4 = X_train[['CD81','CCD81CD9','CCD9CD81','CCD41CD81','CCD81']]\n",
    "    Xtraingroup5 = X_train[['CD63','CD41','CD9','CCD9','CCD63CD9','CCD41CD63','CCD41CD9','CCD41CD63CD81','CCD41CD63CD81CD9']]\n",
    "    Xtraingroup6 = X_train[['CCD63','TPD_ref','TPD_lat','TPD_med','QST_heatpain','QST_cold','QST_warm']]\n",
    "\n",
    "\n",
    "    # Then we do the data augmentation\n",
    "    # we pick a number of sampling we will do to construct the augmented dataset\n",
    "    n_aug=1000\n",
    "    # then we loop n_aug times the following: first we sample each group once and append columns in \"groupXX\" attributes, and then we repeat and append in the \"lines\" direction\n",
    "    column_names = [\"group1\", \"group2\", \"group3\",\"group4\", \"group5\", \"group6\"]\n",
    "    X_train = pd.DataFrame(columns = column_names)\n",
    "    X_test = pd.DataFrame(columns = column_names)\n",
    "    y_train = ytrain0\n",
    "    y_test = ytest0\n",
    "    for k in range(1,n_aug+1):\n",
    "        Xtemp=Xtraingroup1.sample(axis='columns')\n",
    "        Xtemp=Xtemp.join(Xtraingroup2.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtraingroup3.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtraingroup4.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtraingroup5.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtraingroup6.sample(axis='columns'))\n",
    "        Xtemp.columns = X_train.columns\n",
    "        X_train=X_train.append(Xtemp,ignore_index = True)\n",
    "        del Xtemp\n",
    "        Xtemp=Xtestgroup1.sample(axis='columns')\n",
    "        Xtemp=Xtemp.join(Xtestgroup2.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtestgroup3.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtestgroup4.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtestgroup5.sample(axis='columns'))\n",
    "        Xtemp=Xtemp.join(Xtestgroup6.sample(axis='columns'))\n",
    "        Xtemp.columns = X_test.columns\n",
    "        X_test=X_test.append(Xtemp,ignore_index = True)\n",
    "        del Xtemp\n",
    "    for k in range(1,n_aug):\n",
    "        y_train=y_train.append(ytrain0,ignore_index = True)\n",
    "        y_test=y_test.append(ytest0,ignore_index = True)\n",
    "\n",
    "    # create the classifier with n_estimators = 100\n",
    "    from sklearn.ensemble import RandomForestClassifier\n",
    "    \n",
    "    # create an instance of the classifier\n",
    "    clf = RandomForestClassifier(n_estimators=300, criterion='entropy', max_depth=10, min_samples_split=4, min_samples_leaf=2, min_weight_fraction_leaf=0.0, max_features=6, max_leaf_nodes=None, min_impurity_decrease=0.0, bootstrap=True, oob_score=False, n_jobs=-1, random_state=None, verbose=0, warm_start=False, class_weight=None, ccp_alpha=0.0, max_samples=None)\n",
    "\n",
    "    # fit the model to the training set\n",
    "    clf.fit(X_train, y_train)\n",
    "    # Compute the feature scores and sum them to thtte existing scores\n",
    "    features_scores = features_scores+clf.feature_importances_\n",
    "    \n",
    "    # Predict the Test set results and compute accuracy and confusion matrix\n",
    "    y_pred = clf.predict(X_test)    \n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    from sklearn.metrics import accuracy_score\n",
    "    cm = cm + confusion_matrix(y_test, y_pred)\n",
    "    accuracyscore = accuracyscore + accuracy_score(y_test, y_pred)\n",
    "\n",
    "    \n",
    "    \n",
    "# normalize the  results to the number of repeats:\n",
    "features_scores = np.divide(features_scores,n_repeats)\n",
    "cm = np.divide(cm,n_repeats)\n",
    "accuracyscore = np.divide(accuracyscore,n_repeats)\n",
    "    \n",
    "    \n",
    "# display the feature scores and visualize them\n",
    "feature_scores = pd.Series(features_scores, index=X_train.columns).sort_values(ascending=False)\n",
    "plt.figure(figsize=(15, 15))\n",
    "sns.barplot(x=feature_scores, y=feature_scores.index)\n",
    "# Add labels to the graph\n",
    "plt.xlabel('Feature Importance Score')\n",
    "plt.ylabel('Features')\n",
    "# Add title to the graph\n",
    "plt.title(\"Visualizing Important Features\")\n",
    "# Visualize the graph\n",
    "plt.show()\n",
    "\n",
    "# print mean accuracy score and confusion matrix\n",
    "print('Model accuracy score with 100 decision-trees : {0:0.4f}'. format(accuracyscore))\n",
    "\n",
    "# confusion matrix (errorS)\n",
    "print('Confusion matrix\\n\\n', cm)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}