{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LBt8ma8Rbj_O"
   },
   "outputs": [],
   "source": [
    "## source: https://towardsdatascience.com/word-embeddings-for-sentiment-analysis-65f42ea5d26e\n",
    "## source: https://towardsdatascience.com/twitter-topic-modeling-e0e3315b12e2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dQn4V5H-ibIo",
    "outputId": "798c38f4-e8ec-48bf-fc9f-60b98530f08a"
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import re\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n",
    "import csv\n",
    "import seaborn as sns\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "import keras\n",
    "from keras import layers, regularizers, Sequential\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.layers import *\n",
    "from keras.models import Model\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.keras.optimizers import RMSprop\n",
    "from keras.losses import CategoricalCrossentropy\n",
    "from keras.metrics import Accuracy\n",
    "\n",
    "import sklearn\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer \n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import multilabel_confusion_matrix, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xxl2v7heFySU"
   },
   "outputs": [],
   "source": [
    "def custom_tokenizer(df, tk):\n",
    "    \"\"\"\n",
    "    Tokenizes tweets using Keras' Tokenizer after stop word removal\n",
    "    \n",
    "    Args:     df, Pandas DataFrame: Contains all tweets and labels\n",
    "              tk, Keras Tokenizer Object:  Creates tokens via fit_on_texts fn\n",
    "    \n",
    "    Returns:  df, Pandas DataFrame:  Updated DataFrame with tokenized tweets in new column\n",
    "    \"\"\"\n",
    "    \n",
    "    tokenizeList = []\n",
    "    \n",
    "    # remove stopwords\n",
    "    df.tweet = df.tweet.apply(remove_stopwords)\n",
    "    \n",
    "    tweetList = df['tweet'].tolist()\n",
    "    tk.fit_on_texts(df['tweet'])\n",
    "    inv_map = {v: k for k, v in tk.word_index.items()}\n",
    "    \n",
    "    for sentence in tweetList:\n",
    "        tweet = re.split('\\s+', sentence)\n",
    "        processed_seq = tk.texts_to_sequences(tweet)\n",
    "        tokens = [inv_map[tok] for seq in processed_seq for tok in seq]\n",
    "        tokenizeList.append(tokens)\n",
    "    \n",
    "    df['tokens'] = tokenizeList\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yPYHwmvj2DXW"
   },
   "outputs": [],
   "source": [
    "def remove_stopwords(input_text):\n",
    "    \"\"\"\n",
    "    Removes stopwords from tweets based on Indonesian stop word list\n",
    "\n",
    "    Args:     input_text, string:  tweet\n",
    "\n",
    "    Returns:  string: cleaned tweet\n",
    "    \"\"\"\n",
    "\n",
    "    stopwords_list = stopwords.words('indonesian')\n",
    "        \n",
    "    words = input_text.split()\n",
    "    clean_words = [word for word in words if word not in stopwords_list]\n",
    "    \n",
    "    return \" \".join(clean_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "K1QMmLpTibIq"
   },
   "outputs": [],
   "source": [
    "def embedding_to_matrix(token_dict, embeddings, dimensionality, NB_WORDS):\n",
    "    \"\"\" \n",
    "    Converts Pandas df of pre-trained embeddings to a NxN matrix cross-referenced with \n",
    "    all tokens in corpus.  \n",
    "    \n",
    "    Args:     token_dict, dict: dictionary of tokens representing all tweets\n",
    "              Embeddings, Pandas DataFrame: pre-trained Embeddings\n",
    "              Dimensionality, int: dimension of embeddings (i.e. 400-D for Word2Vec or 100-D for FT)\n",
    "              NB_WORDS, int: Parameter indicating the number of words we'll put in the embedding dictionary\n",
    "    \n",
    "    Returns:  emb_matrix, NumPy Array:  NxN matrix cross-referenced with all tokens in corpus\n",
    "    \"\"\"\n",
    "\n",
    "    emb_dict = {}\n",
    "    \n",
    "    for line in embeddings:\n",
    "        word = line[0]\n",
    "        vector = np.asarray(line[1:], dtype='float32')\n",
    "        emb_dict[word] = vector\n",
    "    \n",
    "    emb_matrix = np.zeros((NB_WORDS, dimensionality))\n",
    "\n",
    "    for w, i in token_dict:\n",
    "        # The word_index contains a token for all words of the training data so we need to limit that\n",
    "        if i < NB_WORDS:\n",
    "            vect = emb_dict.get(w)\n",
    "            # Check if the word from the training data occurs in the pre-trained word embeddings\n",
    "            # Otherwise the vector is kept with only zeros\n",
    "            if vect is not None:\n",
    "                emb_matrix[i] = vect\n",
    "        else:\n",
    "            break\n",
    "            \n",
    "    return emb_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "x78rDylaibIr"
   },
   "outputs": [],
   "source": [
    "def dict_of_tokens(df, tk):\n",
    "    \"\"\"\n",
    "    Creates a dictionary of tokens found in the tweets.\n",
    "    \n",
    "    Args:     df, Pandas Dataframe: Contains all tweets and labels\n",
    "              tk, Keras Tokenizer Object:  Creates dict of tokens via fit_on_texts fn\n",
    "          \n",
    "    Returns:  dictionary of tokens\n",
    "    \"\"\"\n",
    "    \n",
    "    tk.fit_on_texts(df['tweet'])\n",
    "    \n",
    "    return tk.word_index.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WTFihg3bibIs"
   },
   "outputs": [],
   "source": [
    "def convert_text_to_sequences(df, tk):\n",
    "    \"\"\"\n",
    "    Converts tokenized tweets to sequences.\n",
    "    \n",
    "    Args:       df, Pandas Dataframe: Contains all tweets, tokenized tweets and labels\n",
    "                tk, Keras Tokenizer Object:  Converts tokenized tweets to sequences via texts_to_sequences fn\n",
    "\n",
    "    Returns:    NumPy array:  List of all tokenized tweets represented as a sequence of numbers\n",
    "    \"\"\"\n",
    "\n",
    "    tokens = df['tokens']\n",
    "    \n",
    "    return tk.texts_to_sequences(tokens)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rTrPwRy9ibIt"
   },
   "outputs": [],
   "source": [
    "def import_embedding(filepath, Dimensionality):\n",
    "    \"\"\"\n",
    "    Imports a pre-trained embedding file and converts to NumPy array.\n",
    "    \n",
    "    Args:       filepath, string:  file path for embedding data\n",
    "                Dimensionality, int: dimension of embeddings (i.e. 400-D for Word2Vec or 100-D for FT)  \n",
    "    \n",
    "    Returns:    NumPy array:  pre-trained embeddings \n",
    "    \"\"\"\n",
    "    \n",
    "    header_list = list(range(0, Dimensionality+1))\n",
    "    \n",
    "    embeddings = pd.read_csv(filepath, delimiter=' ', skiprows=1, index_col=False, names=header_list)\n",
    "    \n",
    "    return embeddings.to_numpy()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PiG8Fl1GibIu"
   },
   "outputs": [],
   "source": [
    "def sequence_pad(X_seq):\n",
    "    \"\"\"\n",
    "    Pads the sequences to some length of max tweet in tokens.\n",
    "    \n",
    "    Args:       X_seq, NumPy array:  List of all tokenized tweets represented as a sequence of numbers\n",
    "    \n",
    "    Returns:    X_seq_trunc, NumPy array:  List of all tokenized tweets appended to value of max tweet length\n",
    "                MAX_LEN, int:   Maximum tweet length in tokens\n",
    "    \"\"\"\n",
    "    \n",
    "    lengths = []\n",
    "\n",
    "    for i in X_seq:\n",
    "        lengths.append(len(i))\n",
    "\n",
    "    MAX_LEN = max(lengths)\n",
    "    \n",
    "    X_seq_trunc = pad_sequences(X_seq, maxlen=MAX_LEN)\n",
    "    \n",
    "    return X_seq_trunc, MAX_LEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9r6ZXoHfibIu"
   },
   "outputs": [],
   "source": [
    "def encode_labels(df):\n",
    "    \"\"\"\n",
    "    One-hot encodes the emotion labels.\n",
    "\n",
    "    Args:       df, Pandas DataFrame:  Contains all tweets and labels\n",
    "    \n",
    "    Returns:    y_oh, NumPy array:  one-hot encoded emotion labels\n",
    "    \"\"\"\n",
    "    \n",
    "    le = LabelEncoder()\n",
    "    \n",
    "    y = le.fit_transform(df['label'])\n",
    "    \n",
    "    return to_categorical(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "27wOywUpibIv"
   },
   "outputs": [],
   "source": [
    "def deep_model(model, X_train, y_train, X_valid, y_valid, checkpoint_filepath):\n",
    "    \"\"\"\n",
    "    Function to used for individual model exploration.  \n",
    "    Saves file path at epoch with max validation accuracy.\n",
    "    \n",
    "    Args:       model, Keras object: model with the chosen architecture\n",
    "                X_train, NumPy array: training features\n",
    "                y_train, NumPy array: training target\n",
    "                X_valid, NumPy array: validation features\n",
    "                Y_valid, NumPy array: validation target\n",
    "                checkpoint_filepath:  file path to save epoch with max validation accuracy\n",
    "\n",
    "    Returns:    model training history, Keras object: contains the output of the Keras model.fit fn\n",
    "    \"\"\"\n",
    "\n",
    "    model_checkpoint_callback = keras.callbacks.ModelCheckpoint(\n",
    "                            filepath=checkpoint_filepath,\n",
    "                            save_weights_only=False,\n",
    "                            monitor='val_accuracy',\n",
    "                            mode='max',\n",
    "                            save_best_only=True)\n",
    "    \n",
    "    model.compile(optimizer='rmsprop'\n",
    "                  , loss='categorical_crossentropy'\n",
    "                  , metrics=['accuracy'])\n",
    "    \n",
    "    history = model.fit(X_train\n",
    "                       , y_train\n",
    "                       , epochs=NB_START_EPOCHS\n",
    "                       , batch_size=BATCH_SIZE\n",
    "                       , validation_data=(X_valid, y_valid)\n",
    "                       , verbose=2\n",
    "                       , callbacks=[model_checkpoint_callback]\n",
    "                       , shuffle=True )\n",
    "    return history\n",
    "\n",
    "\n",
    "def eval_metric(history, metric_name):\n",
    "    '''\n",
    "    Function to evaluate a trained model on a chosen metric. \n",
    "    Training and validation metric are plotted in a\n",
    "    line chart for each epoch.\n",
    "    \n",
    "    Args:     history, Keras object: model training history\n",
    "              metric_name, string: loss or accuracy\n",
    "\n",
    "    Returns:  matplotlib chart:  line chart with epochs of x-axis and metric on y-axis\n",
    "    '''\n",
    "    metric = history.history[metric_name]\n",
    "    val_metric = history.history['val_' + metric_name]\n",
    "\n",
    "    e = range(1, NB_START_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, metric, label='Train ' + metric_name)\n",
    "    plt.plot(e, val_metric, label='Validation ' + metric_name)\n",
    "    plt.title('Bi-LSTM ' + metric_name)\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel(metric_name)\n",
    "    plt.xticks(np.arange(min(e)+1, max(e)+1, 2.0))\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6EbRlVt4yVqY"
   },
   "outputs": [],
   "source": [
    "def make_classification_report(X_valid, y_valid, checkpoint_filepath):\n",
    "    \"\"\"\n",
    "    Makes a classification report.\n",
    "\n",
    "    Args:     X_valid, NumPy array: validation features\n",
    "              Y_valid, NumPy array: validation target\n",
    "              checkpoint_filepath:  file path to save epoch with max validation accuracy\n",
    "\n",
    "    Returns:  classification report\n",
    "    \"\"\"\n",
    "\n",
    "    model.load_weights(checkpoint_filepath)\n",
    "\n",
    "    label_names = [\"anger\", \"fear\", \"joy\", \"love\", \"sadness\"]\n",
    "\n",
    "    y_prob = model.predict(X_valid)\n",
    "    prediction_ints = np.zeros_like(y_prob)\n",
    "    prediction_ints[np.arange(len(y_prob)), y_prob.argmax(1)] = 1\n",
    "    prediction = np.where(prediction_ints==1)[1]\n",
    "    \n",
    "    return print(classification_report(y_valid, prediction_ints, target_names=label_names, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AO_fsE7H0ETF"
   },
   "outputs": [],
   "source": [
    "def make_confusion_matrix(X_valid, y_valid, checkpoint_filepath):\n",
    "    \"\"\"\n",
    "    Makes a confusion matrix.\n",
    "\n",
    "    Args:     X_valid, NumPy array: validation features\n",
    "              Y_valid, NumPy array: validation target\n",
    "              checkpoint_filepath:  file path to save epoch with max validation accuracy\n",
    "\n",
    "    Returns:  confusion matrix\n",
    "    \"\"\"\n",
    "\n",
    "    model.load_weights(checkpoint_filepath)\n",
    "\n",
    "    label_names = [\"anger\", \"fear\", \"joy\", \"love\", \"sadness\"]\n",
    "\n",
    "    y_prob = model.predict(X_valid)\n",
    "    prediction_ints = np.zeros_like(y_prob)\n",
    "    prediction_ints[np.arange(len(y_prob)), y_prob.argmax(1)] = 1\n",
    "    prediction = np.where(prediction_ints==1)[1]\n",
    "\n",
    "    y_cat_valid_emb = np.where(y_valid==1)[1]\n",
    "\n",
    "    cf_matrix = confusion_matrix(prediction, y_cat_valid_emb)\n",
    "\n",
    "    cf_matrix_norm = cf_matrix / cf_matrix.astype(np.float).sum(axis=1, keepdims=True)\n",
    "\n",
    "    cf_matrix_norm_round = np.around(cf_matrix_norm, decimals=2)\n",
    "\n",
    "    df_cm = pd.DataFrame(cf_matrix_norm_round, columns=label_names, index=label_names)\n",
    "\n",
    "    df_cm.index.name = 'Actual'\n",
    "    df_cm.columns.name = 'Predicted'\n",
    "\n",
    "    sns.heatmap(df_cm, cmap='Blues', annot=True)\n",
    "    plt.yticks(rotation=0)\n",
    "    plt.title(\"Confusion Matrix\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oc1lMkp1ibIw"
   },
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xUK77OJeibIw"
   },
   "source": [
    "### Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lOBm5ayvibIw"
   },
   "outputs": [],
   "source": [
    "NB_WORDS = 20000  # Parameter indicating the number of words to be put in the dictionary\n",
    "NB_START_EPOCHS = 20  # Number of epochs used for training\n",
    "BATCH_SIZE = 64  # Size of the batches used in the mini-batch gradient descent\n",
    "MAX_LEN = 64  # Maximum number of words in a sequence\n",
    "WORD2VEC_DIM = 400  # Number of dimensions of the pre-trained word2vec embeddings\n",
    "FASTEXT_DIM = 100 # Number of dimensions of the pre-trained fasttext embeddings\n",
    "NUM_FOLDS = 10 # Number of folds in K-Fold Cross Validation experiment\n",
    "\n",
    "checkpoint_filepath = '~\\\\insert_filepath_here'\n",
    "tweets_filepath = '~\\\\insert_filepath_here\\\\Twitter_Emotion_Dataset.csv'\n",
    "word2vec_filepath = '~\\\\insert_filepath_here\\\\Word2Vec_400dim.txt'\n",
    "fastext_filepath = '~\\\\insert_filepath_here\\\\ft_model.vec'\n",
    "\n",
    "TRAIN_SIZE = .9\n",
    "VALID_SIZE = .1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VO8ZpJURibIx"
   },
   "source": [
    "## Data Handling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1puI5jgAibIx"
   },
   "source": [
    "### File Read-In"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DUw0OnyMibIx"
   },
   "outputs": [],
   "source": [
    "# import tweets\n",
    "df = pd.read_csv(tweets_filepath)\n",
    "\n",
    "# import word2vec pre-trained embeddings\n",
    "w2v_embeddings = import_embedding(word2vec_filepath, WORD2VEC_DIM)\n",
    "\n",
    "# import fasttext pre-trained embeddings\n",
    "ft_embeddings = import_embedding(fastext_filepath, FASTEXT_DIM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RwrmZbjFibIy"
   },
   "source": [
    "### Pre-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "16yh9zsOFlcP"
   },
   "outputs": [],
   "source": [
    "# initialize the tokenizer\n",
    "\n",
    "tk = Tokenizer(num_words=NB_WORDS,\n",
    "               filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
    "               lower=True,\n",
    "               split=\" \")\n",
    "\n",
    "# tokenize tweets and remove stopwords\n",
    "df = custom_tokenizer(df, tk)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pK_tykU8ibIz"
   },
   "source": [
    "### Label Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ucWr1GiCibIz"
   },
   "outputs": [],
   "source": [
    "y_oh = encode_labels(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Uo4k23aRibI0"
   },
   "source": [
    "### Create Token Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JyxR4mZ5ibI0"
   },
   "outputs": [],
   "source": [
    "token_dict = dict_of_tokens(df, tk)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YUKvwjP4ibI0"
   },
   "source": [
    "### Convert data to numerical sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uLqHz6i-ibI0"
   },
   "outputs": [],
   "source": [
    "X_seq = convert_text_to_sequences(df, tk)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jbaPX9x0ibI0"
   },
   "source": [
    "### Create word sequences of equal length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IRCOs8GSibI0"
   },
   "outputs": [],
   "source": [
    "X_seq_trunc, MAX_LEN = sequence_pad(X_seq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "alFtUKa6ibI1"
   },
   "source": [
    "## Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HBA6uDG_ibI1"
   },
   "source": [
    "### Embeddings (Word2Vec and FasText)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_Ei8vvdHibI1"
   },
   "outputs": [],
   "source": [
    "w2v_emb_matrix = embedding_to_matrix(token_dict, w2v_embeddings, WORD2VEC_DIM, NB_WORDS)\n",
    "\n",
    "ft_emb_matrix = embedding_to_matrix(token_dict, ft_embeddings, FASTEXT_DIM, NB_WORDS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5fT1ss0RibI1"
   },
   "source": [
    "### Train / Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aGws7r24vnC4"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_seq_trunc, y_oh, test_size=VALID_SIZE, random_state=12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IzC1sB1dibI2"
   },
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Xf_sSMUcwyuf"
   },
   "source": [
    "## Model Examination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hPGB3oF6w3uH",
    "outputId": "69578ea9-357e-431b-f264-b791c98762a6"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(NB_WORDS, FASTEXT_DIM, input_length=MAX_LEN))\n",
    "model.add(SpatialDropout1D(0.2))\n",
    "model.add(Bidirectional(LSTM(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)))\n",
    "model.add(GlobalMaxPool1D())\n",
    "model.add(Dense(512, activation=\"relu\"))\n",
    "model.add(layers.Dropout(rate=0.5))\n",
    "model.add(Dense(256, activation=\"relu\"))\n",
    "model.add(layers.Dropout(rate=0.5))\n",
    "model.add(layers.Dense(5, activation='softmax'))\n",
    "model.layers[0].set_weights([ft_emb_matrix])\n",
    "model.layers[0].trainable = True\n",
    "model_history = deep_model(model, X_train, y_train, X_test, y_test, checkpoint_filepath)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SxtO6wGu1q3P"
   },
   "source": [
    "### Model Accuracy Over Epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "id": "jscAJ6Jw4fR0",
    "outputId": "de44e517-500d-4695-c7d1-cc3db8de9006"
   },
   "outputs": [],
   "source": [
    "eval_metric(model_history, 'accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I0mBkIHW1ve1"
   },
   "source": [
    "### Model Loss Over Epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "id": "l91pkp854fZq",
    "outputId": "429ef03b-a112-4a8c-abb6-21e7d363e933"
   },
   "outputs": [],
   "source": [
    "eval_metric(model_history, 'loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "swOt1Pg-1z8D"
   },
   "source": [
    "### Model Classification Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "A4Ojovpr4fej",
    "outputId": "8c7f9fcd-f634-46cb-b8a9-d8fae5771b50"
   },
   "outputs": [],
   "source": [
    "make_classification_report(X_test, y_test, checkpoint_filepath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "s0VfLBbL13Fr"
   },
   "source": [
    "### Model Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "id": "bnXL5Fbeyro5",
    "outputId": "5345ca1b-a4b6-4a05-eefe-308e03aad4ed"
   },
   "outputs": [],
   "source": [
    "make_confusion_matrix(X_test, y_test, checkpoint_filepath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "x_LLgkeErol3"
   },
   "source": [
    "## K-Fold Experiment\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "i7Ilh3xG9ZEU",
    "outputId": "643e1d5e-db2c-4ad2-bcb3-a6e9957e1bcf"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "kf = KFold(n_splits=NUM_FOLDS)\n",
    "\n",
    "i=0\n",
    "\n",
    "for train_index, test_index in kf.split(X_seq_trunc):\n",
    "    i+=1\n",
    "    print(\"FOLD\", i)\n",
    "    X_train, X_test = X_seq_trunc[train_index], X_seq_trunc[test_index]\n",
    "    y_train, y_test = y_oh[train_index], y_oh[test_index]\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(NB_WORDS, FASTEXT_DIM, input_length=MAX_LEN))\n",
    "    model.add(SpatialDropout1D(0.2))\n",
    "    model.add(Bidirectional(LSTM(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)))\n",
    "    #model.add(Bidirectional(LSTM(256, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)))\n",
    "    #model.add(LSTM(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))\n",
    "    #model.add(GRU(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))\n",
    "    model.add(GlobalMaxPool1D())\n",
    "    model.add(Dense(512, activation=\"relu\"))\n",
    "    model.add(layers.Dropout(rate=0.5))\n",
    "    model.add(Dense(256, activation=\"relu\"))\n",
    "    model.add(layers.Dropout(rate=0.5))\n",
    "    model.add(layers.Dense(5, activation='softmax'))\n",
    "    model.layers[0].set_weights([ft_emb_matrix])\n",
    "    model.layers[0].trainable = True\n",
    "    print(model.summary())\n",
    "\n",
    "    model.compile(loss=\"categorical_crossentropy\",\n",
    "                optimizer=\"rmsprop\", metrics=[\"acc\"])\n",
    "    \n",
    "    model.fit(X_train, y_train, \n",
    "            validation_data=(X_test, y_test), \n",
    "            epochs=NB_START_EPOCHS, batch_size=BATCH_SIZE, verbose=2)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WXBlIq0jr30o"
   },
   "source": [
    "## Bootstrap Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 623
    },
    "id": "_x47Zdajr3-5",
    "outputId": "a0b7786c-459c-4781-fe48-0fde6cfb62e8"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "bootdf = pd.DataFrame(X_seq_trunc)\n",
    "\n",
    "bootdf['label'] = df['label']\n",
    "\n",
    "for i in range(NUM_FOLDS):\n",
    "\n",
    "    print(\"FOLD\", i)\n",
    "    train = resample(bootdf, replace=True, n_samples=3960, random_state=i)\n",
    "    test = bootdf[~bootdf.index.isin(train.index)]\n",
    "\n",
    "    X_train = train.iloc[:,:-1]\n",
    "    X_test = test.iloc[:,:-1]\n",
    "\n",
    "    y_train = train.iloc[:, -1]\n",
    "    y_test = test.iloc[:, -1]\n",
    "\n",
    "    le = LabelEncoder()\n",
    "    y = le.fit_transform(y_train)\n",
    "    y_train = to_categorical(y)\n",
    "\n",
    "    y2 = le.fit_transform(y_test)\n",
    "    y_test = to_categorical(y2)\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(NB_WORDS, FASTEXT_DIM, input_length=MAX_LEN))\n",
    "    model.add(SpatialDropout1D(0.2))\n",
    "    model.add(Bidirectional(LSTM(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)))\n",
    "    #model.add(Bidirectional(LSTM(256, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)))\n",
    "    #model.add(LSTM(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))\n",
    "    #model.add(GRU(512, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))\n",
    "    model.add(GlobalMaxPool1D())\n",
    "    model.add(Dense(512, activation=\"relu\"))\n",
    "    model.add(layers.Dropout(rate=0.5))\n",
    "    model.add(Dense(256, activation=\"relu\"))\n",
    "    model.add(layers.Dropout(rate=0.5))\n",
    "    model.add(layers.Dense(5, activation='softmax'))\n",
    "    model.layers[0].set_weights([ft_emb_matrix])\n",
    "    model.layers[0].trainable = True\n",
    "    print(model.summary())\n",
    "\n",
    "    model.compile(loss=\"categorical_crossentropy\",\n",
    "                optimizer=\"rmsprop\", metrics=[\"acc\"])\n",
    "    \n",
    "    model.fit(X_train, y_train, \n",
    "            validation_data=(X_test, y_test), \n",
    "            epochs=NB_START_EPOCHS, batch_size=BATCH_SIZE, verbose=2)\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Thesis_Master.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}