{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BSD 3-Clause License\n",
    "\n",
    "Copyright (c) 2019, The Broad Institute, Inc. and The General Hospital Corporation.\n",
    "All rights reserved.\n",
    "\n",
    "Redistribution and use in source and binary forms, with or without\n",
    "modification, are permitted provided that the following conditions are met:\n",
    "\n",
    "* Redistributions of source code must retain the above copyright notice, this\n",
    "  list of conditions and the following disclaimer.\n",
    "\n",
    "* Redistributions in binary form must reproduce the above copyright notice,\n",
    "  this list of conditions and the following disclaimer in the documentation\n",
    "  and/or other materials provided with the distribution.\n",
    "\n",
    "* Neither the name of the copyright holder nor the names of its\n",
    "  contributors may be used to endorse or promote products derived from\n",
    "  this software without specific prior written permission.\n",
    "\n",
    "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n",
    "AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n",
    "IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n",
    "DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\n",
    "FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n",
    "DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n",
    "SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n",
    "CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n",
    "OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n",
    "OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blas_mkl_info:\n",
      "  NOT AVAILABLE\n",
      "blis_info:\n",
      "  NOT AVAILABLE\n",
      "openblas_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n",
      "blas_opt_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n",
      "lapack_mkl_info:\n",
      "  NOT AVAILABLE\n",
      "openblas_lapack_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n",
      "lapack_opt_info:\n",
      "    libraries = ['openblas', 'openblas']\n",
      "    library_dirs = ['/usr/local/lib']\n",
      "    language = c\n",
      "    define_macros = [('HAVE_CBLAS', None)]\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "from scipy.stats import median_absolute_deviation as mad\n",
    "\n",
    "os.environ.putenv('OPENBLAS_NUM_THREADS', '1')\n",
    "np.__config__.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 17/17\n",
      "Iteration 17/17\n",
      "Iteration 17/17\n"
     ]
    }
   ],
   "source": [
    "num_exp = 17\n",
    "n_range = [100, 1000, 10000]\n",
    "\n",
    "np.random.seed(0)\n",
    "numpy_df = {}\n",
    "for n in n_range:\n",
    "    t = []\n",
    "    for i in range(num_exp):\n",
    "        print('\\rIteration {}/{}'.format(i+1, num_exp), end='')\n",
    "        a = np.random.randn(n,n).astype(np.float32)\n",
    "        b = np.random.randn(n,n).astype(np.float32)\n",
    "        start_time = time.time()\n",
    "        c = np.dot(a,b)\n",
    "        end_time = time.time()\n",
    "        t.append((end_time - start_time))\n",
    "        del a,b,c\n",
    "    print()\n",
    "    numpy_df[n] = t[2:]  # skip initial iterations\n",
    "numpy_df = pd.DataFrame(numpy_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 17/17\n",
      "Iteration 17/17\n",
      "Iteration 17/17\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(0)\n",
    "cpu_df = {}\n",
    "for n in n_range:\n",
    "    t = []\n",
    "    for i in range(num_exp):\n",
    "        print('\\rIteration {}/{}'.format(i+1, num_exp), end='')\n",
    "        a = torch.rand(n,n, device='cpu')\n",
    "        b = torch.rand(n,n, device='cpu')\n",
    "        start_time = time.time()\n",
    "        c = torch.mm(a,b)\n",
    "        end_time = time.time()\n",
    "        t.append((end_time - start_time))\n",
    "        del a,b,c\n",
    "        torch.cuda.empty_cache()\n",
    "    print()\n",
    "    cpu_df[n] = t[2:]\n",
    "cpu_df = pd.DataFrame(cpu_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 17/17\n",
      "Iteration 17/17\n",
      "Iteration 17/17\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(0)\n",
    "gpu_df = {}\n",
    "for n in n_range:\n",
    "    t = []\n",
    "    for i in range(num_exp):\n",
    "        print('\\rIteration {}/{}'.format(i+1, num_exp), end='')\n",
    "        a = torch.rand(n,n, device='cuda')\n",
    "        b = torch.rand(n,n, device='cuda')\n",
    "        start_time = time.time()\n",
    "        c = torch.mm(a,b)\n",
    "        end_time = time.time()\n",
    "        t.append((end_time - start_time))\n",
    "        del a,b,c\n",
    "        torch.cuda.empty_cache()\n",
    "    print()\n",
    "    gpu_df[n] = t[2:]\n",
    "gpu_df = pd.DataFrame(gpu_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 17/17\n",
      "Iteration 17/17\n",
      "Iteration 17/17\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(0)\n",
    "gpu_copy_df = {}\n",
    "for n in n_range:\n",
    "    t = []\n",
    "    for i in range(num_exp):\n",
    "        print('\\rIteration {}/{}'.format(i+1, num_exp), end='')\n",
    "        a = torch.rand(n,n, device='cpu')\n",
    "        b = torch.rand(n,n, device='cpu')\n",
    "        start_time = time.time()\n",
    "        c = torch.mm(a.cuda(),b.cuda())\n",
    "        end_time = time.time()\n",
    "        t.append((end_time - start_time))\n",
    "        del a,b,c\n",
    "        torch.cuda.empty_cache()\n",
    "    print()\n",
    "    gpu_copy_df[n] = t[2:]\n",
    "gpu_copy_df = pd.DataFrame(gpu_copy_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Runtime (s)')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAENCAYAAAAYIIIKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhU9fn38fcdIICAVhbFijRQ/CFKADHqQwEN+oB4FUTUHxW0lgLys4+2BfVntdiWLm5t1apYLSqC1opLrUrFqqgp4gYBSdmqIsUSF6BYtSoiyP38cSZxCFnmZM5kzkw+r+uai5zv2e6ZM+TO9yz319wdERGRVBVkOwAREcktShwiIhKKEoeIiISixCEiIqEocYiISCgtsx1AU+jcubMXFRVlOwwRkZyyfPnyf7l7l5rtzSJxFBUVUV5enu0wRERyipm9WVu7TlWJiEgoShwiIhJKziUOM+tpZneY2YPZjkVEpDmKxTUOM5sDjAK2uHvfpPaRwA1AC+B2d7/a3TcAk9NNHDt37qSyspJPP/00nc1IM9GmTRu6detGq1atsh2KSNbFInEAc4FZwF1VDWbWArgZGA5UAsvM7FF3XxvFDisrK+nQoQNFRUWYWRSblDzl7mzbto3Kykp69OiR7XBEsi4Wp6rcfTHwXo3mY4D17r7B3T8D5gNjUt2mmU01s3IzK9+6dete8z/99FM6deqkpCENMjM6deqk3qlIQiwSRx0OBjYlTVcCB5tZJzO7FTjSzC6ra2V3n+3uJe5e0qXLXrchAyhpSMr0XRH5QlxOVaXM3bcB56WyrJmNBkb36tUrs0GJiDQjcU4cbwGHJE13S7RlRNGlj0W6vY1Xf73BZcyMCy+8kGuvvRaAX//613z00UfMnDkz7f3PnDmT2267jS5durBr1y6uvPJKTjnllLS3K5KLSktLASgrK8tqHPkizoljGXComfUgSBhnAhPCbMDdFwALSkpKzs1AfGlr3bo1Dz30EJdddhmdO3eOfPvTp0/n4osvZt26dQwdOpQtW7ZQUBDns5Mi6anrD8B3N2yrd34qf+jJF2LxW8TM7gVeBHqbWaWZTXb3XcAFwBPAOuB+d18TcrujzWz2Bx98EH3QEWjZsiVTp07l+uuv32vexIkTefDBL+44bt++PRD8xXT88cczZswYevbsyaWXXso999zDMcccQ3FxMW+88cZe2+rTpw8tW7Zk06ZN9OjRg507dwLw4Ycf7jEtkq+6TriarhOuDr1eaWlpdW9FvhCLHoe7j6+jfSGwMI3txrrHAXD++efTr18/LrnkkpTXqaioYN26dXTs2JGePXsyZcoUli5dyg033MBNN93Eb37zmz2Wf/nllykoKKB79+6Ulpby2GOPceqppzJ//nxOO+00PZsgMnO/WpvLSuufz8x4/lGaabHocTRn++67L+eccw433nhjyuscffTRHHTQQbRu3ZqvfvWrjBgxAoDi4mI2btxYvdz111/PgAEDuPjii7nvvvswM6ZMmcKdd94JwJ133sm3v/3tSN+PiOS/WPQ4MiVX7qqaNm0aAwcO3OOXeMuWLdm9ezcAu3fv5rPPPque17p16+qfCwoKqqcLCgrYtWtX9byqaxzJBg8ezMaNGykrK+Pzzz+nb9++iIiEkdc9Dndf4O5T99uvjm5mTHTs2JFx48Zxxx13VLcVFRWxfPlyAB599NFIr0Occ845TJgwQb0NEWkU9TgSsn1XxUUXXcSsWbOqp88991zGjBlD//79GTlyJO3atYtsX2eddRaXX34548fXemlJRKRe5u7ZjiHjSkpKvOZATuvWraNPnz5Ziii7HnzwQR555BHuvvvubIeSU5rzdyZXNPZ5rI1tQt3p/4U8vzhuZsvdvaRme173OGRv3/3ud3n88cdZuLDRN6uJSDOnxNHM3HTTTdkOQURyXF5fHI/7A4AiIrkorxNHrtxVJSKSS/I6cYiISPSUOEREJJS8vjge6snxumrRNFYKt+m1aNGC4uJidu3aRZ8+fZg3bx777LPPXsutWrWKb37zmwD885//ZL/99mO//fajc+fOLFq0KK0wzz77bM444wxOPfXUepd7++23mTZtGitWrGC//faja9eu3HDDDUBQ6qR379589tlnDBs2jFmzZvH0008za9YsHn744dD7EpF4y+seR9yvcbRt25aVK1eyevVqCgsLufXWW2tdrri4mJUrV7Jy5UpOOeUUfvWrX7Fy5cqUk0ZyGZLGcHdOPfVURowYwfr161m+fDm/+MUv2Lx5MwC9e/dm5cqVVFRUsHLlShYsWJDW/kQk3vI6ceSSoUOHsn79en784x/vUd12xowZ1X/Z12b37t1ceOGF9O3bl+Li4upS7IsWLaK0tJRRo0ZRXFwMBEUN+/XrR//+/fcoN/Lss8/yta99jZ49e/KnP/1pr3089dRTtG/fnilTplS3HXnkkQwePHiP5Vq1asWgQYNYv3594z4EEckJeX2qKlfs2rWLxx9/nJEjR3LyySdz2mmnMW3aNHbv3s38+fNZunRpnes+8MADrFu3joqKCrZu3crRRx/NcccdB0B5eTlr166le/fuVFRUcM011/DCCy/QsWNH3nvvveptbNmyheeff55Vq1Yxbtw4xo4du8c+Vq9ezVFHHdXg+/j444955plnuOaaaxr5SYhILlDiyKLt27czYMAAIOhxTJ48mcLCQjp16sQrr7zC5s2bOfLII+nUqVOd21iyZAnjx4+nRYsWdO3alSFDhlBeXk5hYSGDBg2ie/fuADzzzDN84xvfoGPHjgDV/wKceuqpmBn9+vXjrbfCj8776quvMmDAAAoKChg7dizDhw/n6aefrnVZMwu9fRGJFyWOLKq6xlHTlClTmDt3Lu+++y6TJk1q9PZTLYyYXKa9ttplRxxxBH/+85/rXL/qGkeyTp068e9//3uPtvfeey8jQ+SKSNPK62scufrk+NixY/nLX/7CsmXLOOmkk+pddujQocyfP5/du3ezefNmnn/+eUpK9qpJxgknnMB9991XfYoq+VRVQ0aMGMGHH37InDlzqtsqKip4/vnn61znsMMO48033+S1114D4B//+Adr1qyhX79+Ke9XROIpr3scoYaOjVGVy8LCQoYNG8aXvvQlWrRoUe+yZ5xxBi+99BL9+vXDzLjuuus44IAD9lquf//+XHLJJRx33HG0bNmSo446ao/xP+pjZjzyyCN8//vf54orrqBNmzb07NmT3/zmN7X2UADatGnDXXfdxTe/+U127NhBYWEhc+bMoUOHDintU0TiS2XVY2j37t0MHDiQBx54gEMPPTTb4UhCnL8zElBZ9WjVVVY9r09V5aK1a9fSq1cvTjzxRCUNEYmlvD5VlYsOP/xwNmzYkO0wRETqpB6HiIiEosQhIiKh5NypKjNrB/wW+Awoc/d7shySiEizEoseh5nNMbMtZra6RvtIM3vVzNab2aWJ5tOAB939XOCUJg9WRKSZi0uPYy4wC7irqsHMWgA3A8OBSmCZmT0KdANWJRb7PKoAiucVR7UpAFZ9a1WDy+RSWfV33nmH6dOns3TpUvbff39at27NpZdeyimnnMKiRYs4/fTT6dGjBzt27ODss89mxowZ3H777axevXqPoo1Dhgxh1qxZ1aVWRCT3xKLH4e6LgZqPMh8DrHf3De7+GTAfGEOQRLollqkzfjObamblZla+devWTISdtlwqqz5mzBhOPPFENmzYwPLly7nnnnvYtGlT9TLDhg1j5cqVLFu2jDvuuIOKioq09iki8RWLxFGHg4FNSdOVibaHgNPN7BagzoEf3H22u5e4e0mXLl0yG2kE4lxW/cknn6RDhw6ce+4XD+D36NGD888/f69l27dvz8CBA3njjTfCfwgikhPicqoqZe7+MfDtBhck5AiAWRT3supr1qxh4MCBKb2XrVu3snTpUq644gqee+65RnwaIhJ3cU4cbwGHJE13S7SlLFStqizI1bLq5513Hi+88ALt2rXjxRdfBIJey5FHHklBQQE/+tGP6N27N0uWLKl1fZVWF8ltcU4cy4BDzawHQcI4EwhVUCbuPY5cKqv+2GNf1AC69dZbeffddxkyZEh127Bhw/YYXxxUWl0kX8XiGoeZ3Qu8CPQ2s0ozm+zuu4ALgCeAdcD97r4mzHbjPuZ4XeJYVv2DDz7gtttuq2775JNPGlzv2GOPZfHixWzZsgWAl19+GXfny1/+csr7FpH4iUWPw93H19G+EFjY2O2G6XGkcvtsU4lrWfXp06dz5ZVXcsABB7DPPvtw9dVX17veQQcdxLXXXstJJ52Eu9OhQwfuvfdenaoSyXEqqx5DKqseT3H+zkhAZdWj1SzLqufiCIAqqy4icReLU1WZEve7qmqjsuoiEnd53eMQEZHo5XXiyMVTVSIicZfXiSNXb8cVEYmzvE4cIiISvby+OB7mOY51h0V7m2Wfv69rcBmVVQ9nyZIl3HPPPdxyyy2h1xWR6OR1jyPup6pUVj2cqkKQIpJdeZ04cklzLqv+4osvMm7cOAD++Mc/0q5dO3bu3MnHH39Mcm/xmWee4cQTT9xr/SuvvJLi4mL69+/PjBkzAFixYgXHHnss/fr14/TTT6fqBokhQ4Ywbdo0BgwYQHFxMeXl5ezevZtevXpVl2H5/PPP6dmzZ6iyLCLNiRJHDFSVVS8uLmbSpEncdVcwEGJVWfWzzz67znWTy6o/9dRTTJ8+vbo2VHl5Ob/97W+r519zzTWUlZVRUVHBtddeW72NqrLqDz/8MJdddtle+2hMWfUjjjgi5fd/1FFHsXz5cgCee+45Dj/8cFasWMFLL73EoEGDANi8eTPt2rWjffv2e6y7YMECHn/8cZYuXUpFRQUXXXQREJyCu+666/jb3/5G7969+fnPf169zo4dO1i5ciU33HADU6ZMoaCggPHjx/OHP/wBgCeeeIKjjz56jwrCIvKFvE4ccb8dt6qseklJCd27d2fy5MkUFRVVl1V/8sknG11WHchoWfV+/fpV/1KHL8qqjxw5srqsel01qWq2FxYW0r17d15//XXKy8uZNm0aixcv5rnnnmPo0KFA8Mu8toKPixYtYtKkSbRt27b6fW3bto1PP/2UwYMHA/Ctb32LxYsXV68zfnxQGu2EE05gy5YtfPTRR0yePJl58+YBMGfOnD16ZCKyp7y+OB73J8dVVv0Lxx13HI899hht27blxBNPZOrUqXzyySfcdNNNQHB944c//GFK76chNROXmVFUVMT+++/Ps88+yyuvvMKIESMi2ZdIPsrrHkeuao5l1YcOHcp1113H4MGD6dq1K++++y5vvPEGffr0Yffu3axbt676Wk2y4cOHM2fOHLZv3179vjp16kTbtm154YUXALj77rs5/vjjq9e57777ACgrK+PAAw+sTrCTJ0/mrLPO4swzz6SgQP81ROqS1z2OMFK5fbapNMey6oMGDeKdd96pHva2b9++vP/++wAsXbq01mQIMGrUKCoqKigpKaFVq1aMHj2an//859x999185zvfYfv27fTq1Ys777yzep1WrVoxYMAAPv/88z3ax44dy6RJk5g4cWJKn4tIc6Wy6jGksup7mjlzJn379uWMM85Ie1v1PUfy0ksvcdlll/Hss8/Wum6cvzMSUFn1aNVVVl09jphZu3Yto0aNYuzYsUoaCTNnzsz4Pq644gpmz57N/PnzM74vkbqUlpYCwWnUOMvrxBH3Mcdro7LqmbVkyZJa22fMmFH9DIiI1C+vrwA29OR4czhNJ9HQd0XkC3mdOOrTpk0btm3bpl8I0iB3Z9u2bbRp0ybboYjEQl6fqqpPt27dqKysZOvWrdkORXJAmzZt6NatW7bDkDxRV1HVT/75Zr3z43L3Z7NNHK1ataJHjx7ZDkNEJOc021NVIiLSOCn1OMysDzAeOB4oAtoCW4EVwOPAH919R4ZiFBGRGKm3x2FmA81sEfAKMBh4Afg18ENgHuDAFcDbZvYDM2td58YiYmY9zewOM3sw0/sSEZG9NdTj+BPwS+C/3f3fdS1kZoOA6cDFBImkruXmAKOALe7eN6l9JHAD0AK43d3rrGXh7huAyUocIiLZ0VDiONTdP2toI+7+IvCimRU2sOhcYBZwV1WDmbUAbgaGA5XAMjN7lCCJXFVj/UnuvqWheEREJHPqTRwNJQ0za+XuO1Nd3t0Xm1lRjeZjgPWJngRmNh8Y4+5XEfROGsXMpgJTgeoxKURE4mxe969kO4SUpHxXlZl9z8xOT5q+A9huZq+aWe80YjgY2JQ0XZloqyuOTmZ2K3Ckme09XF2Cu8929xJ3L+nSpUsa4YmISLIwt+N+j+BOKszsOGAcMAFYCVxbz3qRcvdt7n6eu3810SupU9xHABQRyUVhEsfBwD8SP48GHnD3+4GZwP9JI4a3gEOSprsl2tLWUK0qEREJL0zi+BCoGiFoOPB04uedQDpFfJYBh5pZj8TF9TOBR9PYXjX1OEREohcmcTwJ3GZmtwO9CB78AziCL3oi9TKze4EXgd5mVmlmk919F3AB8ASwDrjf3deEiKtO6nGIiEQvTK2q8wme0egOnOHuVYNWDwTuTWUD7j6+jvaFwMIQsaQkF8fjEBGJu5QTh7t/CHy3lvafRBpRhNx9AbCgpKTk3GzHIiKSLxoqOdIhzMbCLp9pusYhIhK9hq5xvG5ml5tZnQMRmFmBmZ1sZk8RnM6KDV3jEBGJXkOnqoYSXNfYYGargHLgbeBTYH/gcIJbcbcDVwK3ZS5UERGJg4ZKjrwOjDOzQwge+BtKUCKkLfAvgqq5s4GF7r47w7GGpovjIiLRS+niuLtvIng6vMmeEI+CLo6LiERPIwCKiEgoeZ04dFeViEj08jpx6K4qEZHo5XXiEBGR6ClxiIhIKKESh5kdaGYXm9ktZtY50TbYzHpkJrz06BqHiEj0wowAeBTwKnAWMBnYNzFrOMFDgrGjaxwiItEL0+P4NXCDux8J7EhqfwIYHGlUIiISW2ESx1HAvFra3wEOjCYcERGJuzCJYztBfaqaDgO2RBOOiIjEXZjE8QjwEzNrnZh2MysCrgH+GHFcIiISU2ESx8VAR2ArsA+wBFgPvA9cHn1o6dNdVSIi0Qs7AuAQMzuBYLjYAmCFuy/KVHDpUpFDEZHohRlzHAB3fwZ4JgOxiIhIDgiVOMzsSGAYcAA1TnO5+yURxiUiIjGVcuIws0uAq4E3gc2AJ832WlcSEZG8E6bHMR34jrv/LlPBiIhI/IW5q6oAeDpTgYiISG4IkzhuAb6dqUDCMLNTzew2M7vPzEZkOx4RkeYkzKmqnwILzewVYDWwM3mmu09KZSNmNgcYBWxx975J7SOBG4AWwO3ufnVd23D3h4GHzWx/ghpaT4Z4HyIikoYwieMKYASwgqD0SGMviM8FZgF3VTWYWQvgZoJKu5XAMjN7lCCJXFVj/UnuXlXi5PLEeiIi0kTCJI7/B0xw9/vS2aG7L06UKkl2DLDe3TcAmNl8YIy7X0XQO9mDmRnBHV6Pu/uKdOIREZFwwhY5fCVDcRwMbEqarky01eW7wP8FzjCz82pbwMymmlm5mZVv3bo1ukhFRJq5MD2O64FpZna+u2f1uQ13vxG4sYFlZpvZO8DowsLCo5omMpE9lZaWAlBWVpbVOESiFCZxDAWOA75uZmvZ++L4KWnE8RZwSNJ0t0RbWlSrSqKiBCDyhTCJ41/AQxmKYxlwaGLs8reAM4EJ6W7UzEYDo3v16pXupkTqN7OO4Yk3flz//Jmq3Cy5J0x13Eie4TCze4FSoLOZVQI/cfc7zOwCgmFoWwBz3H1NuvtSj0NEJHqhq+Omy93H19G+EFgY5b7U45BsK5vYLtshiESu3sRhZn8Djnf3f5vZKup5dsPd+0UdXLrU45Cwii59rNb2dzdsq3f+xjYZC0kkdhrqcfwR2JH0c05VwVWPQ0QkevUmDnf/adLPMzMeTcTU4xCRTCqeV9yo9e6POI6mlvIDgGb2jJl9qZb2fc1MIwKKiDQTYS6OlwKFtbS3IXjGI3Z0qkqi0nVCnTU3RZqdBhOHmQ1MmuxnZu8lTbcATiKCh/UyQaeqRESil0qPo5zgorhTe/ny7QS1o0REpBlIJXH0AAzYQFDFNrli4GcE42p8noHY0qZTVSIi0Wswcbj7m4kfw1TSjQWdqhIRiV6oJ8fNrBtBocMDqJFI3P26COMSEZGYSjlxmNlZwBxgF8HpquSHAR1Q4hARaQbC9Dh+BlwL/Ciu1zRERCTzwly3OBC4PZeShpmNNrPZH3yg0tUiIlEJkzgWAsdmKpBMcPcF7j51v/3qGAtBRERCC3Oq6ingGjM7AljF3iMAZmqQJxERiZEwieN3iX9/WMs8J3iKXERE8lyYEQBz7jkOERGJnpKBiIiEEuY5jgvrmx/HBwBVckSam9LSUgDKysqyGofktzDXOGoWMmwFHERQ5HALMXwAUCVHRESiF+YaR4+abWZ2IHAncFuUQYlI/dYd1qfW9k/++Wa98/v8fV3GYpLmI61rHO6+GZgB/DKacEREJO6iuDheQPBUuYiINANhLo6fVrOJ4BrH+cBzUQYlIo0zr/tXsh2CNANhLo4/WGPaCarkPgNcFFlEIiISazn3AKCZ9QG+D3QGnnb3W7IckohIsxJJMjCzQ1Jcbo6ZbTGz1TXaR5rZq2a23swurW8b7r7O3c8DxgGDGx+1iIg0RlqJw8y6mtnNwGsprjIXGFljGy2Am4GTgcOB8WZ2uJkVm9mfa7wOSKxzCvAYQcVeERFpQg0mDjP7kpndY2ZbzextM/ueBX4CbCAotT4plZ25+2LgvRrNxwDr3X2Du38GzAfGuPsqdx9V47UlsZ1H3f1k4Kx64p5qZuVmVr5169ZUwhMRkRSkco3jSoJxxucR9BauB4YD7YCT3f2vacZwMLApabqSesb9MLNS4DSgNfX0ONx9tpm9A4wuLCw8Ks0YRUQkIZXE8XXg2+6+yMx+C6wH3nD3aZkNrXbuXgaUpbisSo6IiEQslWscXwbWArj7BuBToi0x8haQfHG9W6ItbRo6VkQkeqkkjgL2HO3vc+CTCGNYBhxqZj3MrBA4E3g0ig1r6FgRkeilcqrKgN+b2Y7EdBvgNjPbI3m4+ykNbsjsXqAU6GxmlcBP3P0OM7sAeIJgFME57r4mxHuob38qqy4iErFUEse8GtO/b+zO3H18He0LycCttbrGISISvQYTh7t/uykCyQT1OEREoheLMiKZomscIiLRy+vEISIi0cvrxKHbcUVEopfXiUOnqkREopfXiUNERKKX14lDp6pERKKX14lDp6pERKKX14lDRESip8QhIiKhKHGIiEgoeZ04dHFcRCR6eZ04dHFcRCR6eZ04REQkekocIiISihKHiIiEosQhIiKh5HXi0F1VIiLRy+vEobuqRESil9eJQ0REoqfEISIioShxiIhIKEocIiISihKHiIiEkpOJw8zamVm5mY3KdiwiIs1NkyYOM5tjZlvMbHWN9pFm9qqZrTezS1PY1A+A+zMTpYiI1KdlE+9vLjALuKuqwcxaADcDw4FKYJmZPQq0AK6qsf4koD+wFmjTBPGKiEgNTZo43H2xmRXVaD4GWO/uGwDMbD4wxt2vAvY6FWVmpUA74HBgu5ktdPfdtSw3FZgK0L179wjfRf4rLS0FoKysLKtxiEg8NXWPozYHA5uSpiuBY+ta2N1nAJjZROBftSWNxHKzgdkAJSUlHlWwIiLNXRwSR6O4+9yGljGz0cDoXr16ZT6gFOgveRHJB3G4q+ot4JCk6W6JNhERiaE49DiWAYeaWQ+ChHEmMCGKDbv7AmBBSUnJuVFsL9+sO6xPre2f/PPNeuf3+fu6jMUkIvHXpInDzO4FSoHOZlYJ/MTd7zCzC4AnCO6kmuPuayLaX1ZOVRVd+lit7e9u2Fbv/I1Xf73Wdp3iEpE4aeq7qsbX0b4QWJiB/anHISISsTicqpIqM+sYN2Tjx/XPn6mBqkSk6eR14ojbXVW5Yl73r2Q7BBGJsbxOHHE7VdV1wtWNWq9sYruIIxERabw43I6bMRpzXEQkenmdODTmuIhI9PI6cYiISPSUOEREJJS8Thy6xiEiEr28Thy6xiEiEr28Thwiuaq0tLS61IxI3OT1cxwicVc8r7jW9g3vbqh3vsZNlmzK68ShJ8clV/W8rGe2QxCpU16fqtI1DhGR6OV14hARkegpcYiISChKHCIiEooSh4iIhJLXiUNPjouIRC+vE4fuqhIRiV5eJ47mTk8fi0gmKHGIiEgoef3keHOhshUi0pTU4xARkVDU48hjqnckIpmQcz0OMys1s+fM7FYzK812PCIizU2TJg4zm2NmW8xsdY32kWb2qpmtN7NLG9iMAx8BbYDKTMUqIiK1a+pTVXOBWcBdVQ1m1gK4GRhOkAiWmdmjQAvgqhrrTwKec/e/mtmBwHXAWU0Qt4iIJDRp4nD3xWZWVKP5GGC9u28AMLP5wBh3vwoYVc/m/g20rmummU0FpgJ07949jahFRCRZHK5xHAxsSpquTLTVysxOM7PfAXcT9F5q5e6z3b3E3Uu6dOkSWbAiIs1dzt1V5e4PAQ+lsqxGABQRiV4cehxvAYckTXdLtKVNtapERKIXh8SxDDjUzHqYWSFwJvBoFBtWdVwRkeg19e249wIvAr3NrNLMJrv7LuAC4AlgHXC/u6+JYn/qcYiIRM/cPdsxZEzVNQ7gG8DrWQ4nkzoD/8p2ENIoOna5Ld+P31fcfa+7i/I6cTQXZlbu7iXZjkPC07HLbc31+MXhGoeIiOQQJQ4REQlFiSM/zM52ANJoOna5rVkeP13jEBGRUNTjEBGRUJQ4REQkFCUOEREJRYkjQ8zsAjMrN7MdZja3xryOZvYnM/vYzN40swlh5qe7/6aKIc4ydXyi+tzSOX75cGyzdXya4vilG0NTfP8aknPVcXPI28AvgJOAtjXm3Qx8BhwIDAAeM7OKpFIrdc43MwMGuPsryRs0s/7Aanf/PIX9pxVDIz6LOMrI8Wlo3SY6fvlwbLNyfBqaH9HxS/f4pPP+ouHuemXwlfjyzE2abpc4sP+V1HY3cHWK83sAW4GRSfOHEsmmb4QAAAeYSURBVJQ96NvQ/qOIIZ9eUR6fVD63TB+/fDu2TXl8muL4pRtDut+/qF46VdX0/gvY5e6vJbVVAEekMt/d/wGcDvzezIaZ2TEE45Oc7e57jOWeqRjyXDqfTYOfWxMcv3w/tpk8Pk1x/NKNIa3vX1R0qqrptQc+rNH2AdAhxfl4MATvWcCDwOfAue7+l6aMIY+l89mk9Lll+Pjl+7HN5PFpiuOXbgxpf/+ioB5H0/sI2LdG277Af1KcX+WfwC7AgDezFEM+SuezCfO5Zer45fuxzeTxaYrjl24MUX3/0qLE0fReA1qa2aFJbf2BNSnOx8y+CjwFXAKcByw0szDd0bRjyGPpfDYpfW4ZPn75fmwzeXya4vilG0Pa379IZPviV76+CE4DtgGuIrhA1QZomZg3H7iX4GLWYILu5BFJ69Y5H/gysBH4TtLyZxEMt9szlf2nG0M+vDJ4fBpaN+PHLx+ObRaPT8aPXwQxNPr9RXZ8sv0FydcXMBPwGq+ZiXkdgYeBjwm6vBNqrFvnfKA1cHot+xsDtE9l/+nGkA+vDB6fhtbN+PHLh2ObxeOT8eMXQQyNfn9RvVTkUEREQtE1DhERCUWJQ0REQlHiEBGRUJQ4REQkFCUOEREJRYlDRERCUeKQZs/M5prZnzOw3Y/MbGLStJvZGVHvJ2pmNtPMUi24KM2QEodkXeIXt5vZHbXMuyYxL9Qv9pC/pL8PnB1m+410ELCgCfaTrl8Dx2c7CIkvJQ6Ji03AODNrV9VgZi2BcwiegI2cmbU0M3P3D9z9/UzsI5m7v+vuOzK9n3S5+0fuvi3bcUh8KXFIXPwNeB0Yl9T2deBToCx5QTM72syeNLN/mdmHZrbEzAYlzd+Y+PGBRM9jY6J9ppmtNrOJZvYGsANol3yqysy6mNk7ZvaTpO31M7NPzey/6wrezHqZWVliuVfNbFQty1T3gsysKDF9ppn91cy2m9kriX31NbMXEsN/LjGzHjW2M9rMlif29Q8zu8LMCpPfv5ldbma/S3w+lWb2vzW28T9m9lpiG/8ysycSiXqvU1VmVmBmPzKzTRYMhbrKzMYkza96L6eb2VNm9omZrTWz4UnLtDKzG83s7cQ2NpnZ1XV9nhJvShwSJ3cAk5KmJwF3EtT5SdaBoHDcUOAYYCVBhdJOiflHJ/49l+D00NFJ6/YAJgD/TVA59NPkDbv7VmAiMMPMBplZW4Kicfe6+wO1BW1mBcCfCP4/DUrEPZOgrlFDfgpcAxwJvJ/Y103AjMR7awPcmLSvk4B7gFkEA/RMAs4Arqyx3enAKmBgYvu/rEquZlZCMMToT4HewIlAfeNJfB/4X+AHQHHivT5kZgNqLHdFItb+wDJgvpm1T8z7HjAWOBM4FPgG8Gp9H4zEWLaLmemlFzAX+DOwP7Cd4BdLV4IeQfeq+fWsb8A7BKOwVbU5cEaN5WYCO4EDa9t/jbbfABsIEtd6kgrY1bL/EQQD+nRPahuSiGFibTEBRYnp/0maPyrRdlpS20Tgo6TpxcCPauz/VIKxGKpqz20kSHTJy7wOXJ74+TQSA/zU8X5mEoyfXTX9FvDjGsuUAb+v570cnGgbkpi+EXi6Kka9cvulHofEhrv/m+Cv2UnAt4Ayd9/r+oaZHZA4DfOamX1AMFDNAQRJpiGV7r45heV+QDB+8znAWe7+UT3L9gHeqhHry8DuFPbzt6Sfq+JaVaOtnZntk5g+iqA39FHVC/gDQRntrnVsF+Btgs8IgrEk3gT+YWb3mNm3zKzWUeLMbF+CUuLP15i1BDi8nvfyduLfqn3OBQYAr5nZzWb29URPTXKQDpzEzRyCX9aTEj/XZh7B6afpwNcIfiFVAoV1LJ/s4xTjKAIOIfiruWeK6zTGzqSfvZ62gqR/f0rwnqte/Qh6aVvr2G7VdgoA3P0/BKewxhHceHAZ8Hcz+3LI2GueQqzepye6GUn7XEHwmV6WaJsHPKXkkZt00CRunib4S78zwbgCtRkC3OTuj7n7GoIex0E1ltkJtGhMAGbWiuCv+EeBi4Hfmll9vZl1wMFmdkhS2zFk5v/XCuAwd19fy2tXqhtx913u/oy7X0aQeNoRnCqrudyHBL2HwTVmDQHWhgnc3f/j7g+6+3cIbnw4AegVZhsSDy2zHYBIMnd3M+tHcC68rltXXwPONrOXCX7h/ZIg2STbCJxoZn8FdiROg6Xq50AXgovGHwAjgbvM7AR3r+300yLg74llpgNtgesJxqSO2s+AP5vZm8D9iX30BY5x90tS2UDijq+vElwveQ8YRnDDwbo6VvkV8DMzex1YTvDMy1CCXktKzOxCgutQKwmS+gTgQ4KeouQY9TgkdhJ/mX5YzyKTgPYEv8TmE5zS2lhjmYsIfiFuAl5Jdd9mdnxi3XPc/f3EKZeJBOfzf1BHvLsJ7hgqILi2cRfwC4KL+5Fy9ycI/lofBixNvC4l3LMu7xNcUK9KeBcDU9z9uTqWv5EgefwSWE3wXk9394oQ+/wPwZ1ZSwl6TQOAk939kxDbkJjQCIAiIhKKehwiIhKKEoeIiISixCEiIqEocYiISChKHCIiEooSh4iIhKLEISIioShxiIhIKP8fMq+v8PM9Q7wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cols = ['NumPy', 'PyTorch CPU', 'PyTorch GPU', 'PyTorch GPU w/ copy']\n",
    "median_df = pd.concat([\n",
    "    numpy_df.median(),\n",
    "    cpu_df.median(),\n",
    "    gpu_df.median(),\n",
    "    gpu_copy_df.median()\n",
    "], axis=1, keys=cols)\n",
    "mad_df = pd.DataFrame(np.c_[\n",
    "    mad(numpy_df),\n",
    "    mad(cpu_df),\n",
    "    mad(gpu_df),\n",
    "    mad(gpu_copy_df)\n",
    "], columns=cols, index=n_range)\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.set_yscale('log')\n",
    "median_df.plot(ax=ax, yerr=mad_df*1.96, kind='bar')\n",
    "ax.set_xticklabels([r'{0}$\\times${0}'.format(i) for i in n_range],\n",
    "                   rotation=0, fontsize=12)\n",
    "ax.set_xlabel('Matrix dimensions', fontsize=14)\n",
    "ax.set_ylabel('Runtime (s)', fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numpy_df.to_csv('matrix_mul_benchmark.numpy.txt', sep='\\t', float_format='%.6g')\n",
    "cpu_df.to_csv('matrix_mul_benchmark.pytorch_cpu.txt', sep='\\t', float_format='%.6g')\n",
    "gpu_df.to_csv('matrix_mul_benchmark.pytorch_gpu.txt', sep='\\t', float_format='%.6g')\n",
    "gpu_copy_df.to_csv('matrix_mul_benchmark.pytorch_gpu_copy.txt', sep='\\t', float_format='%.6g')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}