{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14:00:33 [I] klustakwik KlustaKwik2 version 0.2.6\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import expipe\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import spatial_maps.stats as stats\n",
    "import septum_mec\n",
    "import septum_mec.analysis.data_processing as dp\n",
    "import septum_mec.analysis.registration\n",
    "import head_direction.head as head\n",
    "import spatial_maps as sp\n",
    "import speed_cells.speed as spd\n",
    "import re\n",
    "import joblib\n",
    "import multiprocessing\n",
    "import shutil\n",
    "import psutil\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from distutils.dir_util import copy_tree\n",
    "from neo import SpikeTrain\n",
    "import scipy\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from tqdm._tqdm_notebook import tqdm_notebook\n",
    "tqdm_notebook.pandas()\n",
    "\n",
    "from spike_statistics.core import permutation_resampling\n",
    "\n",
    "from spikewaveform.core import calculate_waveform_features_from_template, cluster_waveform_features\n",
    "\n",
    "from septum_mec.analysis.plotting import violinplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "color_control = '#4393c3'\n",
    "color_stimulated = '#d6604d'\n",
    "\n",
    "color_bs = '#5aae61'\n",
    "color_ns = '#9970ab'\n",
    "\n",
    "figsize_violin = (1.7, 3)\n",
    "figsize_gen = (4, 3)\n",
    "\n",
    "output_path = pathlib.Path(\"output\") / \"waveform-analysis\"\n",
    "(output_path / \"statistics\").mkdir(exist_ok=True, parents=True)\n",
    "(output_path / \"figures\").mkdir(exist_ok=True, parents=True)\n",
    "output_path.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader = dp.Data()\n",
    "actions = data_loader.actions\n",
    "project = data_loader.project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N cells: 1298\n"
     ]
    }
   ],
   "source": [
    "identify_neurons = actions['identify-neurons']\n",
    "sessions = pd.read_csv(identify_neurons.data_path('sessions'))\n",
    "units = pd.read_csv(identify_neurons.data_path('all_non_identified_units'))\n",
    "# units = pd.read_csv(identify_neurons.data_path('units'))\n",
    "session_units = pd.merge(sessions, units, on='action')\n",
    "#########################3\n",
    "# session_units = session_units.drop_duplicates('unit_id')\n",
    "#################################\n",
    "print('N cells:',session_units.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>action</th>\n",
       "      <th>baseline</th>\n",
       "      <th>entity</th>\n",
       "      <th>frequency</th>\n",
       "      <th>i</th>\n",
       "      <th>ii</th>\n",
       "      <th>session</th>\n",
       "      <th>stim_location</th>\n",
       "      <th>stimulated</th>\n",
       "      <th>tag</th>\n",
       "      <th>channel_group</th>\n",
       "      <th>unit_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1849-060319-3</td>\n",
       "      <td>True</td>\n",
       "      <td>1849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>baseline ii</td>\n",
       "      <td>1</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1849-060319-3</td>\n",
       "      <td>True</td>\n",
       "      <td>1849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>baseline ii</td>\n",
       "      <td>1</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1849-060319-3</td>\n",
       "      <td>True</td>\n",
       "      <td>1849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>baseline ii</td>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1849-060319-3</td>\n",
       "      <td>True</td>\n",
       "      <td>1849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>baseline ii</td>\n",
       "      <td>1</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1849-060319-3</td>\n",
       "      <td>True</td>\n",
       "      <td>1849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>baseline ii</td>\n",
       "      <td>1</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          action  baseline  entity  frequency      i    ii  session  \\\n",
       "0  1849-060319-3      True    1849        NaN  False  True        3   \n",
       "1  1849-060319-3      True    1849        NaN  False  True        3   \n",
       "2  1849-060319-3      True    1849        NaN  False  True        3   \n",
       "3  1849-060319-3      True    1849        NaN  False  True        3   \n",
       "4  1849-060319-3      True    1849        NaN  False  True        3   \n",
       "\n",
       "  stim_location  stimulated          tag  channel_group  unit_name  \n",
       "0           NaN       False  baseline ii              1        104  \n",
       "1           NaN       False  baseline ii              1        108  \n",
       "2           NaN       False  baseline ii              1         85  \n",
       "3           NaN       False  baseline ii              1         94  \n",
       "4           NaN       False  baseline ii              1         98  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session_units.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "action\n",
       "1849-110319-2    13\n",
       "1849-010319-4    13\n",
       "1849-220319-4    13\n",
       "1834-220319-3    13\n",
       "1833-200619-1    13\n",
       "1849-220319-3    13\n",
       "1849-060319-3    13\n",
       "1834-110319-1    13\n",
       "1833-050619-3    13\n",
       "1833-120619-3    13\n",
       "1833-050619-4    13\n",
       "1833-060619-1    13\n",
       "1834-120319-3    12\n",
       "1834-120319-2    12\n",
       "1839-120619-3    12\n",
       "1849-220319-5    12\n",
       "1834-150319-1    12\n",
       "1839-290519-1    12\n",
       "1834-010319-5    11\n",
       "1849-060319-4    11\n",
       "1849-060319-2    11\n",
       "1834-120319-4    10\n",
       "1839-120619-2    10\n",
       "1839-060619-5     9\n",
       "1833-050619-1     9\n",
       "1834-110319-5     9\n",
       "1834-110319-3     9\n",
       "1833-120619-4     9\n",
       "1834-120319-1     8\n",
       "1833-120619-1     7\n",
       "1839-290519-3     6\n",
       "1839-200619-1     5\n",
       "1834-110319-6     5\n",
       "1849-110319-3     4\n",
       "1839-060619-4     4\n",
       "1839-060619-3     4\n",
       "1839-120619-1     3\n",
       "1839-060619-1     2\n",
       "Name: unit_name, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session_units.groupby('action').count().unit_name.hist()\n",
    "# session_units.groupby('action').count().sort_values('unit_name', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process all data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def features(row):\n",
    "    action_id = row['action']\n",
    "    channel_id = row['channel_group']\n",
    "    unit = row['unit_name']\n",
    "    template = data_loader.template(action_id, channel_id, unit)\n",
    "    spike_times = data_loader.spike_train(action_id, channel_id, unit)\n",
    "    half_widths, peak_to_troughs = calculate_waveform_features_from_template(\n",
    "        template.data, template.sampling_rate)\n",
    "    peak_amps = template.data.min(axis=1)\n",
    "    half_widths = half_widths * 1000 # to ms\n",
    "    peak_to_troughs = peak_to_troughs * 1000 # to ms\n",
    "    idxs = np.argsort(peak_amps)\n",
    "    peak_to_trough = np.nan\n",
    "    for p2t in peak_to_troughs[idxs]:\n",
    "        if np.isfinite(p2t) and p2t > .1:\n",
    "            peak_to_trough = p2t\n",
    "            break\n",
    "    half_width = np.nan\n",
    "    for hw in half_widths[idxs]:\n",
    "        if np.isfinite(hw):\n",
    "            half_width = hw\n",
    "            break\n",
    "    \n",
    "    return pd.Series({\n",
    "        'half_width': half_width,\n",
    "        'peak_to_trough': peak_to_trough,\n",
    "        'average_firing_rate': float(len(spike_times) / spike_times.t_stop),\n",
    "        'template': template.data[idxs[0]]\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb8ac000ef5b473a833b96cbfd854d1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1298), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = session_units.merge(\n",
    "    session_units.progress_apply(features, axis=1), \n",
    "    left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%debug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = results.loc[:, ['half_width', 'peak_to_trough']].dropna()\n",
    "\n",
    "idxs_df = cluster_waveform_features(df.half_width, df.peak_to_trough)\n",
    "\n",
    "results.loc[df.index, 'bs'] = idxs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize_gen)\n",
    "size = 5\n",
    "mew = .5\n",
    "marker_bs = '.'\n",
    "marker_ns = '+'\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs==0')['half_width'], \n",
    "    results.query('bs==0')['peak_to_trough'], \n",
    "    c=color_ns, s=size, marker=marker_ns, linewidth=mew, label='NS')\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs==1')['half_width'], \n",
    "    results.query('bs==1')['peak_to_trough'], \n",
    "    c=color_bs, s=size, marker=marker_bs, linewidth=mew, label='BS')\n",
    "\n",
    "plt.xlabel('half width')\n",
    "plt.ylabel('peak to through')\n",
    "\n",
    "plt.legend(bbox_to_anchor=(0,1.02,1,0.2), loc=\"lower left\",\n",
    "                mode=\"expand\", borderaxespad=0, ncol=2)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"clusters.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"clusters.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize_gen)\n",
    "\n",
    "size = 5\n",
    "mew = .5\n",
    "marker_bs = '.'\n",
    "marker_ns = '+'\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs==0')['half_width'], \n",
    "    results.query('bs==0')['peak_to_trough'], \n",
    "    c=color_ns, s=size, marker=marker_ns, linewidth=mew, label='NS')\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs==1')['half_width'], \n",
    "    results.query('bs==1')['peak_to_trough'], \n",
    "    c=color_bs, s=size, marker=marker_bs, linewidth=mew, label='BS')\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs==1 and average_firing_rate > 10')['half_width'], \n",
    "    results.query('bs==1 and average_firing_rate > 10')['peak_to_trough'], \n",
    "    c='red', s=size, marker=marker_bs, linewidth=mew, label='BS rate > 10 Hz')\n",
    "\n",
    "plt.xlabel('half width')\n",
    "plt.ylabel('peak to through')\n",
    "\n",
    "plt.legend(bbox_to_anchor=(0,1.02,1,0.2), loc=\"lower left\",\n",
    "                mode=\"expand\", borderaxespad=0, ncol=2)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"clusters_and_rate.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"clusters_and_rate.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stim = results.query('stimulated').loc[:, ['half_width', 'peak_to_trough']].dropna()\n",
    "\n",
    "idxs_stim = cluster_waveform_features(stim.half_width, stim.peak_to_trough)\n",
    "\n",
    "results.loc[stim.index, 'bs_stim'] = idxs_stim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "control = results.query('not stimulated').loc[:, ['half_width', 'peak_to_trough']].dropna()\n",
    "\n",
    "idxs_control = cluster_waveform_features(control.half_width, control.peak_to_trough)\n",
    "\n",
    "results.loc[control.index, 'bs_ctrl'] = idxs_control"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize_gen)\n",
    "size = 5\n",
    "mew = .5\n",
    "marker_bs = '.'\n",
    "marker_ns = '+'\n",
    "plt.scatter(\n",
    "    results.query('bs_stim==0')['half_width'], \n",
    "    results.query('bs_stim==0')['peak_to_trough'], \n",
    "    c=color_stimulated, s=size, marker=marker_ns, \n",
    "    linewidth=mew, label='Stimulated NS', alpha=.5)\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs_stim==1')['half_width'], \n",
    "    results.query('bs_stim==1')['peak_to_trough'], \n",
    "    c=color_stimulated, s=size, marker=marker_bs, \n",
    "    linewidth=mew, label='Stimulated BS', alpha=.5)\n",
    "\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs_ctrl==0')['half_width'], \n",
    "    results.query('bs_ctrl==0')['peak_to_trough'], \n",
    "    c=color_control, s=size, marker=marker_ns, \n",
    "    linewidth=mew, label='Control NS', alpha=.5)\n",
    "\n",
    "plt.scatter(\n",
    "    results.query('bs_ctrl==1')['half_width'], \n",
    "    results.query('bs_ctrl==1')['peak_to_trough'], \n",
    "    c=color_control, s=size, marker=marker_bs, \n",
    "    linewidth=mew, label='Control BS', alpha=.5)\n",
    "\n",
    "plt.xlabel('half width')\n",
    "plt.ylabel('peak to through')\n",
    "plt.legend(bbox_to_anchor=(0,1.02,1,0.2), loc=\"lower left\",\n",
    "                mode=\"expand\", borderaxespad=0, ncol=2)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"compare-clusters.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"compare-clusters.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.average_firing_rate = results.apply(lambda x: float(x.average_firing_rate), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.frequency = results.apply(\n",
    "    lambda x: \n",
    "        float(x.frequency.replace('Hz', '')) if isinstance(x.frequency, str) \n",
    "        else float(x.frequency), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins=100\n",
    "density=True\n",
    "cumulative=True\n",
    "histtype='step'\n",
    "lw = 2\n",
    "\n",
    "plt.figure(figsize=figsize_gen)\n",
    "plt.title('Narrow spiking')\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_ctrl==0')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_control, lw=lw, label='Control');\n",
    "\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_stim==0')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_stimulated, lw=lw, label='Stimulated');\n",
    "\n",
    "plt.xlim(-.5, 56)\n",
    "# plt.legend(bbox_to_anchor=(0,1.02,1,0.2), loc=\"lower left\",\n",
    "#                 mode=\"expand\", borderaxespad=0, ncol=2)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_ns.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_ns.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins=100\n",
    "density=True\n",
    "cumulative=True\n",
    "histtype='step'\n",
    "lw = 2\n",
    "\n",
    "plt.figure(figsize=figsize_gen)\n",
    "plt.title('Narrow spiking')\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_ctrl==0')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_control, lw=lw, label='Control');\n",
    "\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_stim==0 and frequency==30')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_stimulated, lw=lw, label='Stimulated');\n",
    "\n",
    "plt.xlim(-.5, 56)\n",
    "# plt.legend(bbox_to_anchor=(0,1.02,1,0.2), loc=\"lower left\",\n",
    "#                 mode=\"expand\", borderaxespad=0, ncol=2)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_ns_30.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_ns_30.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins=100\n",
    "density=True\n",
    "cumulative=True\n",
    "histtype='step'\n",
    "lw = 2\n",
    "\n",
    "plt.figure(figsize=figsize_gen)\n",
    "plt.title('Narrow spiking')\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_ctrl==0')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_control, lw=lw, label='Control');\n",
    "\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_stim==0 and frequency==11')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_stimulated, lw=lw, label='Stimulated');\n",
    "\n",
    "plt.xlim(-.5, 56)\n",
    "# plt.legend(bbox_to_anchor=(0,1.02,1,0.2), loc=\"lower left\",\n",
    "#                 mode=\"expand\", borderaxespad=0, ncol=2)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_ns_11.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_ns_11.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = 100\n",
    "\n",
    "plt.figure(figsize=figsize_gen)\n",
    "plt.title('Broad spiking')\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_ctrl==1')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_control, lw=lw);\n",
    "\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_stim==1')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_stimulated, lw=lw);\n",
    "\n",
    "plt.xlim(-.5, 44)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_bs.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_bs.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = 100\n",
    "\n",
    "plt.figure(figsize=figsize_gen)\n",
    "plt.title('Broad spiking')\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_ctrl==1')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_control, lw=lw);\n",
    "\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_stim==1 and frequency==11')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_stimulated, lw=lw);\n",
    "\n",
    "plt.xlim(-.5, 44)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_bs_11.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_bs_11.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = 100\n",
    "\n",
    "plt.figure(figsize=figsize_gen)\n",
    "plt.title('Broad spiking')\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_ctrl==1')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_control, lw=lw);\n",
    "\n",
    "_, bins, _ = plt.hist(\n",
    "    results.query('bs_stim==1 and frequency==30')['average_firing_rate'], \n",
    "    bins=bins, density=density, cumulative=cumulative, \n",
    "    histtype=histtype, color=color_stimulated, lw=lw);\n",
    "\n",
    "plt.xlim(-.5, 44)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_bs_30.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"cumulative_bs_30.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize_violin)\n",
    "# test = 'permutation_resampling'\n",
    "test = 'mann_whitney'\n",
    "\n",
    "plt.title('Broad spiking')\n",
    "violinplot(\n",
    "    results.query('bs_ctrl==1')['average_firing_rate'].to_numpy(), \n",
    "    results.query('bs_stim==1')['average_firing_rate'].to_numpy(), \n",
    "    test=test)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"rates_bs.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"rates_bs.png\", dpi=600, bbox_inches=\"tight\")\n",
    "\n",
    "plt.figure(figsize=figsize_violin)\n",
    "plt.title('Narrow spiking')\n",
    "violinplot(\n",
    "    results.query('bs_ctrl==0')['average_firing_rate'].to_numpy(), \n",
    "    results.query('bs_stim==0')['average_firing_rate'].to_numpy(), \n",
    "    test=test)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"rates_ns.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"rates_ns.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize_violin)\n",
    "# test = 'permutation_resampling'\n",
    "test = 'mann_whitney'\n",
    "\n",
    "plt.title('Broad spiking')\n",
    "violinplot(\n",
    "    results.query('bs_ctrl==1')['average_firing_rate'].to_numpy(), \n",
    "    results.query('bs_stim==1 and frequency==11')['average_firing_rate'].to_numpy(), \n",
    "    test=test)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"rates_bs_11.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"rates_bs_11.png\", dpi=600, bbox_inches=\"tight\")\n",
    "\n",
    "plt.figure(figsize=figsize_violin)\n",
    "plt.title('Narrow spiking')\n",
    "violinplot(\n",
    "    results.query('bs_ctrl==0')['average_firing_rate'].to_numpy(), \n",
    "    results.query('bs_stim==0 and frequency==11')['average_firing_rate'].to_numpy(), \n",
    "    test=test)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"rates_ns_11.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"rates_ns_11.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize_violin)\n",
    "# test = 'permutation_resampling'\n",
    "test = 'mann_whitney'\n",
    "\n",
    "plt.title('Broad spiking')\n",
    "violinplot(\n",
    "    results.query('bs_ctrl==1')['average_firing_rate'].to_numpy(), \n",
    "    results.query('bs_stim==1 and frequency==30')['average_firing_rate'].to_numpy(), \n",
    "    test=test)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"rates_bs.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"rates_bs.png\", dpi=600, bbox_inches=\"tight\")\n",
    "\n",
    "plt.figure(figsize=figsize_violin)\n",
    "plt.title('Narrow spiking')\n",
    "violinplot(\n",
    "    results.query('bs_ctrl==0')['average_firing_rate'].to_numpy(), \n",
    "    results.query('bs_stim==0 and frequency==30')['average_firing_rate'].to_numpy(), \n",
    "    test=test)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"rates_ns_30.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"rates_ns_30.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\n",
    "    'average_firing_rate',\n",
    "    'half_width',\n",
    "    'peak_to_trough'\n",
    "]\n",
    "\n",
    "\n",
    "def summarize(data):\n",
    "    return \"{:.2f} ± {:.2f} ({})\".format(data.mean(), data.sem(), sum(~np.isnan(data)))\n",
    "\n",
    "\n",
    "bs = pd.DataFrame()\n",
    "\n",
    "bs['Control'] = results.query('bs_ctrl==1')[columns].agg(summarize)\n",
    "bs['Stimulated'] = results.query('bs_stim==1')[columns].agg(summarize)\n",
    "\n",
    "ns = pd.DataFrame()\n",
    "\n",
    "ns['Control'] = results.query('bs_ctrl==0')[columns].agg(summarize)\n",
    "ns['Stimulated'] = results.query('bs_stim==0')[columns].agg(summarize)\n",
    "\n",
    "\n",
    "def MWU(column, df, cluster, extra):\n",
    "    '''\n",
    "    Mann Whitney U\n",
    "    '''\n",
    "    Uvalue, pvalue = scipy.stats.mannwhitneyu(\n",
    "        df.query('bs_ctrl=={} {}'.format(cluster, extra))[column].dropna(), \n",
    "        df.query('bs_stim=={} {}'.format(cluster, extra))[column].dropna(),\n",
    "        alternative='two-sided')\n",
    "\n",
    "    return \"{:.2f}, {:.3f}\".format(Uvalue, pvalue)\n",
    "\n",
    "\n",
    "def PRS(column, df, cluster, extra):\n",
    "    '''\n",
    "    Permutation ReSampling\n",
    "    '''\n",
    "    pvalue, observed_diff, diffs = permutation_resampling(\n",
    "        df.query('bs_ctrl=={} {}'.format(cluster, extra))[column].dropna(), \n",
    "        df.query('bs_stim=={} {}'.format(cluster, extra))[column].dropna())\n",
    "\n",
    "    return \"{:.2f}, {:.3f}\".format(observed_diff, pvalue)\n",
    "\n",
    "\n",
    "bs['MWU'] = list(map(lambda x: MWU(x, results, 1, ''), columns))\n",
    "bs['PRS'] = list(map(lambda x: PRS(x, results, 1, ''), columns))\n",
    "\n",
    "ns['MWU'] = list(map(lambda x: MWU(x, results, 0, ''), columns))\n",
    "ns['PRS'] = list(map(lambda x: PRS(x, results, 0, ''), columns))\n",
    "\n",
    "bs.to_latex(output_path / \"statistics\" / \"broad_spiking.tex\")\n",
    "bs.to_csv(output_path / \"statistics\" / \"broad_spiking.csv\")\n",
    "\n",
    "ns.to_latex(output_path / \"statistics\" / \"narrow_spiking.tex\")\n",
    "ns.to_csv(output_path / \"statistics\" / \"narrow_spiking.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\n",
    "    'average_firing_rate',\n",
    "    'half_width',\n",
    "    'peak_to_trough'\n",
    "]\n",
    "\n",
    "\n",
    "def summarize(data):\n",
    "    return \"{:.2f} ± {:.2f} ({})\".format(data.mean(), data.sem(), sum(~np.isnan(data)))\n",
    "\n",
    "\n",
    "bs = pd.DataFrame()\n",
    "\n",
    "bs['Control'] = results.query('bs_ctrl==1')[columns].agg(summarize)\n",
    "bs['Stimulated'] = results.query('bs_stim==1 and frequency==11')[columns].agg(summarize)\n",
    "\n",
    "ns = pd.DataFrame()\n",
    "\n",
    "ns['Control'] = results.query('bs_ctrl==0')[columns].agg(summarize)\n",
    "ns['Stimulated'] = results.query('bs_stim==0 and frequency==11')[columns].agg(summarize)\n",
    "\n",
    "\n",
    "def MWU(column, df, cluster, extra):\n",
    "    '''\n",
    "    Mann Whitney U\n",
    "    '''\n",
    "    Uvalue, pvalue = scipy.stats.mannwhitneyu(\n",
    "        df.query('bs_ctrl=={} {}'.format(cluster, extra))[column].dropna(), \n",
    "        df.query('bs_stim=={} {} and frequency==11'.format(cluster, extra))[column].dropna(),\n",
    "        alternative='two-sided')\n",
    "\n",
    "    return \"{:.2f}, {:.3f}\".format(Uvalue, pvalue)\n",
    "\n",
    "\n",
    "def PRS(column, df, cluster, extra):\n",
    "    '''\n",
    "    Permutation ReSampling\n",
    "    '''\n",
    "    pvalue, observed_diff, diffs = permutation_resampling(\n",
    "        df.query('bs_ctrl=={} {}'.format(cluster, extra))[column].dropna(), \n",
    "        df.query('bs_stim=={} {} and frequency==11'.format(cluster, extra))[column].dropna())\n",
    "\n",
    "    return \"{:.2f}, {:.3f}\".format(observed_diff, pvalue)\n",
    "\n",
    "\n",
    "bs['MWU'] = list(map(lambda x: MWU(x, results, 1, ''), columns))\n",
    "bs['PRS'] = list(map(lambda x: PRS(x, results, 1, ''), columns))\n",
    "\n",
    "ns['MWU'] = list(map(lambda x: MWU(x, results, 0, ''), columns))\n",
    "ns['PRS'] = list(map(lambda x: PRS(x, results, 0, ''), columns))\n",
    "\n",
    "bs.to_latex(output_path / \"statistics\" / \"broad_spiking_11.tex\")\n",
    "bs.to_csv(output_path / \"statistics\" / \"broad_spiking_11.csv\")\n",
    "\n",
    "ns.to_latex(output_path / \"statistics\" / \"narrow_spiking_11.tex\")\n",
    "ns.to_csv(output_path / \"statistics\" / \"narrow_spiking_11.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\n",
    "    'average_firing_rate',\n",
    "    'half_width',\n",
    "    'peak_to_trough'\n",
    "]\n",
    "\n",
    "\n",
    "def summarize(data):\n",
    "    return \"{:.2f} ± {:.2f} ({})\".format(data.mean(), data.sem(), sum(~np.isnan(data)))\n",
    "\n",
    "\n",
    "bs = pd.DataFrame()\n",
    "\n",
    "bs['Control'] = results.query('bs_ctrl==1')[columns].agg(summarize)\n",
    "bs['Stimulated'] = results.query('bs_stim==1 and frequency==30')[columns].agg(summarize)\n",
    "\n",
    "ns = pd.DataFrame()\n",
    "\n",
    "ns['Control'] = results.query('bs_ctrl==0')[columns].agg(summarize)\n",
    "ns['Stimulated'] = results.query('bs_stim==0 and frequency==30')[columns].agg(summarize)\n",
    "\n",
    "\n",
    "def MWU(column, df, cluster, extra):\n",
    "    '''\n",
    "    Mann Whitney U\n",
    "    '''\n",
    "    Uvalue, pvalue = scipy.stats.mannwhitneyu(\n",
    "        df.query('bs_ctrl=={} {}'.format(cluster, extra))[column].dropna(), \n",
    "        df.query('bs_stim=={} {} and frequency==30'.format(cluster, extra))[column].dropna(),\n",
    "        alternative='two-sided')\n",
    "\n",
    "    return \"{:.2f}, {:.3f}\".format(Uvalue, pvalue)\n",
    "\n",
    "\n",
    "def PRS(column, df, cluster, extra):\n",
    "    '''\n",
    "    Permutation ReSampling\n",
    "    '''\n",
    "    pvalue, observed_diff, diffs = permutation_resampling(\n",
    "        df.query('bs_ctrl=={} {}'.format(cluster, extra))[column].dropna(), \n",
    "        df.query('bs_stim=={} {} and frequency==30'.format(cluster, extra))[column].dropna())\n",
    "\n",
    "    return \"{:.2f}, {:.3f}\".format(observed_diff, pvalue)\n",
    "\n",
    "\n",
    "bs['MWU'] = list(map(lambda x: MWU(x, results, 1, ''), columns))\n",
    "bs['PRS'] = list(map(lambda x: PRS(x, results, 1, ''), columns))\n",
    "\n",
    "ns['MWU'] = list(map(lambda x: MWU(x, results, 0, ''), columns))\n",
    "ns['PRS'] = list(map(lambda x: PRS(x, results, 0, ''), columns))\n",
    "\n",
    "bs.to_latex(output_path / \"statistics\" / \"broad_spiking_30.tex\")\n",
    "bs.to_csv(output_path / \"statistics\" / \"broad_spiking_30.csv\")\n",
    "\n",
    "ns.to_latex(output_path / \"statistics\" / \"narrow_spiking_30.tex\")\n",
    "ns.to_csv(output_path / \"statistics\" / \"narrow_spiking_30.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example waveforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(a):\n",
    "    t = a - a.min()\n",
    "    return t / t.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'half_width','peak_to_trough'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize_gen)\n",
    "\n",
    "\n",
    "lw = 3\n",
    "\n",
    "row = results.query('bs==1').sort_values('half_width', ascending=False).iloc[50]\n",
    "template = data_loader.template(\n",
    "    row.action, row.channel_group, row.unit_name)\n",
    "\n",
    "mean_wf = template.data\n",
    "peak_wf = mean_wf[np.argmin(mean_wf.min(1))]\n",
    "plt.plot(normalize(peak_wf.T), color=color_bs, lw=lw)\n",
    "\n",
    "\n",
    "row = results.query('bs==0').sort_values('half_width').iloc[10]\n",
    "template = data_loader.template(\n",
    "    row.action, row.channel_group, row.unit_name)\n",
    "\n",
    "mean_wf = template.data\n",
    "peak_wf = mean_wf[np.argmin(mean_wf.min(1))]\n",
    "plt.plot(normalize(peak_wf.T), color=color_ns, lw=lw)\n",
    "\n",
    "plt.savefig(output_path / \"figures\" / \"example_waveforms.svg\", bbox_inches=\"tight\")\n",
    "plt.savefig(output_path / \"figures\" / \"example_waveforms.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store results in Expipe action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "action = project.require_action(\"waveform-analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "action.data['results'] = 'results.csv'\n",
    "results.to_csv(action.data_path('results'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stuff = {\n",
    "    \"figures\": \"figures\",\n",
    "    \"statistics\": \"statistics\"\n",
    "}\n",
    "\n",
    "for key, value in stuff.items():\n",
    "    action.data[key] = value\n",
    "    data_path = action.data_path(key)\n",
    "    data_path.parent.mkdir(exist_ok=True, parents=True)\n",
    "    source = output_path / value\n",
    "    if source.is_file():\n",
    "        shutil.copy(source, data_path)\n",
    "    else:\n",
    "        copy_tree(str(source), str(data_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "septum_mec.analysis.registration.store_notebook(action, \"10_waveform_analysis.ipynb\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}