2019-10-08 18:52:35 +00:00
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
2019-10-09 13:38:38 +00:00
"execution_count": 2,
2019-10-08 18:52:35 +00:00
"metadata": {},
2019-10-09 13:38:38 +00:00
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
2019-10-17 11:52:49 +00:00
"12:42:00 [I] klustakwik KlustaKwik2 version 0.2.6\n",
2019-10-09 13:38:38 +00:00
"/home/mikkel/.virtualenvs/expipe/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
" return f(*args, **kwds)\n",
"/home/mikkel/.virtualenvs/expipe/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
" return f(*args, **kwds)\n"
]
}
],
2019-10-08 18:52:35 +00:00
"source": [
"import os\n",
"import expipe\n",
"import pathlib\n",
"import numpy as np\n",
"import septum_mec.analysis.data_processing as dp\n",
"from septum_mec.analysis.registration import store_notebook\n",
"import re\n",
"import joblib\n",
"import multiprocessing\n",
"import shutil\n",
"import psutil\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import quantities as pq\n",
"import exdir\n",
"from distutils.dir_util import copy_tree\n",
"from datetime import timedelta\n",
"from tqdm import tqdm_notebook as tqdm\n",
"from septum_mec.analysis.trackunitmulticomparison import TrackMultipleSessions\n",
"import networkx as nx\n",
"from nxpd import draw\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"project_path = dp.project_path()\n",
"\n",
"project = expipe.get_project(project_path)\n",
"actions = project.actions"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
2019-10-09 13:38:38 +00:00
"output = pathlib.Path('output/identify_neurons')\n",
"output.mkdir(parents=True, exist_ok=True)"
2019-10-08 18:52:35 +00:00
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"identify_neurons = project.require_action('identify-neurons')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'users': ['Mikkel Lepperød'],\n",
" 'tags': ['11hz', 'stim-ms', 'stim i', 'septum', 'open-ephys'],\n",
" 'datetime': '2019-07-01T12:54:49',\n",
" 'type': 'Recording',\n",
" 'registered': '2019-07-02T14:53:28',\n",
" 'data': {'main': 'main.exdir'},\n",
" 'location': 'IMB',\n",
" 'entities': ['1833']}"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"actions['1833-010719-2'].attributes"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"data_loader = dp.Data()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"skip_actions = [\n",
" '1849-270219-1', \n",
" '1849-260219-2', \n",
" '1834-250219-1',\n",
" '1834-230219-1'\n",
2019-10-17 11:52:49 +00:00
" '1833-050619-4'\n",
2019-10-08 18:52:35 +00:00
"]"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 10,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"sessions = []\n",
"for action in actions.values():\n",
" if action.id in skip_actions:\n",
" continue\n",
" if action.type != 'Recording':\n",
" continue\n",
" action_data_path = pathlib.Path(action.data_path('main'))\n",
" processing = exdir.File(action_data_path)['processing']\n",
"\n",
" if not 'electrophysiology' in processing:\n",
" continue\n",
" elphys = processing['electrophysiology']\n",
" if 'spikesorting' not in elphys:\n",
" continue\n",
" tags = [t.lower() for t in action.tags]\n",
" \n",
" freq = np.nan\n",
" stimulated = False\n",
" baseline = False\n",
" is_i = False\n",
" is_ii = False\n",
" tag = None\n",
" stim_location = None\n",
" \n",
" stim_times = data_loader.stim_times(action.id)\n",
" if stim_times is not None:\n",
" stimulated = True\n",
" freq = round(1 / np.median(np.diff(stim_times)))\n",
" \n",
" \n",
" \n",
" tag_i = [i for i, t in enumerate(tags) if 'baseline ' in t or 'stim ' in t]\n",
" if len(tag_i) == 1:\n",
" tag = tags[tag_i[0]]\n",
" what, how = tag.split(' ')\n",
" if what == 'stim':\n",
" where = [t for t in tags if 'stim-' in t]\n",
" assert len(where) == 1\n",
" stim_location = where[0].split('-')[-1]\n",
" assert stimulated\n",
" elif what == 'baseline':\n",
" baseline = True\n",
" assert not stimulated\n",
" else:\n",
" raise Exception(f'Found {tag}, what to do?')\n",
" if how == 'i':\n",
" is_i = True\n",
" elif how == 'ii':\n",
" is_ii = True\n",
" else:\n",
" raise Exception(f'Found {tag}, what to do?')\n",
" elif len(tag_i) > 1:\n",
" print(action.id, [tags[i] for i in tag_i])\n",
" \n",
" \n",
"\n",
" sessions.append({\n",
" 'tag': tag,\n",
" 'action': action.id,\n",
" 'stimulated': stimulated,\n",
" 'baseline': baseline,\n",
" 'i': is_i,\n",
" 'ii': is_ii,\n",
" 'frequency': float(freq),\n",
" 'session': int(action.id.split('-')[-1]),\n",
" 'stim_location': stim_location,\n",
" 'entity': int(action.entities[0]),\n",
"\n",
" })\n",
"sessions = pd.DataFrame(sessions)"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 10,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>action</th>\n",
" <th>baseline</th>\n",
" <th>entity</th>\n",
" <th>frequency</th>\n",
" <th>i</th>\n",
" <th>ii</th>\n",
" <th>session</th>\n",
" <th>stim_location</th>\n",
" <th>stimulated</th>\n",
" <th>tag</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1839-060619-3</td>\n",
" <td>False</td>\n",
" <td>1839</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>3</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>1849-110319-2</td>\n",
" <td>False</td>\n",
" <td>1849</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>1834-220319-2</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>1834-120319-2</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>1833-260619-2</td>\n",
" <td>False</td>\n",
" <td>1833</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>1849-010319-4</td>\n",
" <td>False</td>\n",
" <td>1849</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>4</td>\n",
" <td>mecr</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>1849-220319-3</td>\n",
" <td>False</td>\n",
" <td>1849</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>3</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>1834-150319-2</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30</th>\n",
" <td>1839-290519-2</td>\n",
" <td>False</td>\n",
" <td>1839</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>37</th>\n",
" <td>1849-150319-2</td>\n",
" <td>False</td>\n",
" <td>1849</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>38</th>\n",
" <td>1849-280219-2</td>\n",
" <td>False</td>\n",
" <td>1849</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>44</th>\n",
" <td>1834-010319-3</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>3</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45</th>\n",
" <td>1833-050619-2</td>\n",
" <td>False</td>\n",
" <td>1833</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>47</th>\n",
" <td>1833-120619-2</td>\n",
" <td>False</td>\n",
" <td>1833</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50</th>\n",
" <td>1849-010319-5</td>\n",
" <td>False</td>\n",
" <td>1849</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>5</td>\n",
" <td>mecl</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53</th>\n",
" <td>1834-060319-2</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>55</th>\n",
" <td>1834-110319-6</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>6</td>\n",
" <td>mecr</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>57</th>\n",
" <td>1849-060319-2</td>\n",
" <td>False</td>\n",
" <td>1849</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>61</th>\n",
" <td>1834-110319-2</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>62</th>\n",
" <td>1833-020719-2</td>\n",
" <td>False</td>\n",
" <td>1833</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>66</th>\n",
" <td>1833-200619-2</td>\n",
" <td>False</td>\n",
" <td>1833</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75</th>\n",
" <td>1833-290519-2</td>\n",
" <td>False</td>\n",
" <td>1833</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>77</th>\n",
" <td>1839-120619-2</td>\n",
" <td>False</td>\n",
" <td>1839</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>82</th>\n",
" <td>1834-110319-5</td>\n",
" <td>False</td>\n",
" <td>1834</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>5</td>\n",
" <td>mecl</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>83</th>\n",
" <td>1839-200619-2</td>\n",
" <td>False</td>\n",
" <td>1839</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" <tr>\n",
" <th>87</th>\n",
" <td>1833-010719-2</td>\n",
" <td>False</td>\n",
" <td>1833</td>\n",
" <td>11.0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>ms</td>\n",
" <td>True</td>\n",
" <td>stim i</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" action baseline entity frequency i ii session \\\n",
"3 1839-060619-3 False 1839 11.0 True False 3 \n",
"7 1849-110319-2 False 1849 11.0 True False 2 \n",
"9 1834-220319-2 False 1834 11.0 True False 2 \n",
"12 1834-120319-2 False 1834 11.0 True False 2 \n",
"19 1833-260619-2 False 1833 11.0 True False 2 \n",
"20 1849-010319-4 False 1849 11.0 True False 4 \n",
"23 1849-220319-3 False 1849 11.0 True False 3 \n",
"28 1834-150319-2 False 1834 11.0 True False 2 \n",
"30 1839-290519-2 False 1839 11.0 True False 2 \n",
"37 1849-150319-2 False 1849 11.0 True False 2 \n",
"38 1849-280219-2 False 1849 11.0 True False 2 \n",
"44 1834-010319-3 False 1834 11.0 True False 3 \n",
"45 1833-050619-2 False 1833 11.0 True False 2 \n",
"47 1833-120619-2 False 1833 11.0 True False 2 \n",
"50 1849-010319-5 False 1849 11.0 True False 5 \n",
"53 1834-060319-2 False 1834 11.0 True False 2 \n",
"55 1834-110319-6 False 1834 11.0 True False 6 \n",
"57 1849-060319-2 False 1849 11.0 True False 2 \n",
"61 1834-110319-2 False 1834 11.0 True False 2 \n",
"62 1833-020719-2 False 1833 11.0 True False 2 \n",
"66 1833-200619-2 False 1833 11.0 True False 2 \n",
"75 1833-290519-2 False 1833 11.0 True False 2 \n",
"77 1839-120619-2 False 1839 11.0 True False 2 \n",
"82 1834-110319-5 False 1834 11.0 True False 5 \n",
"83 1839-200619-2 False 1839 11.0 True False 2 \n",
"87 1833-010719-2 False 1833 11.0 True False 2 \n",
"\n",
" stim_location stimulated tag \n",
"3 ms True stim i \n",
"7 ms True stim i \n",
"9 ms True stim i \n",
"12 ms True stim i \n",
"19 ms True stim i \n",
"20 mecr True stim i \n",
"23 ms True stim i \n",
"28 ms True stim i \n",
"30 ms True stim i \n",
"37 ms True stim i \n",
"38 ms True stim i \n",
"44 ms True stim i \n",
"45 ms True stim i \n",
"47 ms True stim i \n",
"50 mecl True stim i \n",
"53 ms True stim i \n",
"55 mecr True stim i \n",
"57 ms True stim i \n",
"61 ms True stim i \n",
"62 ms True stim i \n",
"66 ms True stim i \n",
"75 ms True stim i \n",
"77 ms True stim i \n",
"82 mecl True stim i \n",
"83 ms True stim i \n",
"87 ms True stim i "
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sessions.query('stimulated and frequency!=30')"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 11,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"sessions.to_csv(output / 'sessions.csv', index=False)"
]
},
2019-10-09 13:38:38 +00:00
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"all_non_identified_units = []\n",
"for action in sessions.action.values:\n",
" for ch in range(8):\n",
" for unit_name in data_loader.unit_names(action, ch):\n",
" all_non_identified_units.append({\n",
" 'unit_name': unit_name, \n",
" 'action': action, \n",
" 'channel_group': ch\n",
" })\n",
"all_non_identified_units = pd.DataFrame(all_non_identified_units)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"all_non_identified_units.to_csv(output / 'all_non_identified_units.csv', index=False)"
]
},
2019-10-08 18:52:35 +00:00
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Identify unique neurons"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 12,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
2019-10-09 13:38:38 +00:00
"Processing 1833\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "93a0e7d7971a4e2d81e87adee47d91e5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=378), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Processing 1834\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "da0225d67b7948e18d4b1bf5447156e4",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=300), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Processing 1839\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "963db44d08da4a8a990e631e9756aca0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=78), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
2019-10-08 18:52:35 +00:00
"Processing 1849\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
2019-10-09 13:38:38 +00:00
"model_id": "0868adb93b4b4c92b75638b7bc2e6a32",
2019-10-08 18:52:35 +00:00
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=231), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"# save graphs\n",
"for entity, values in sessions.groupby('entity'):\n",
" data_path = output / f'{entity}-graphs'\n",
" if data_path.exists():\n",
" continue\n",
" print('Processing', entity)\n",
" unit_matching = TrackMultipleSessions(\n",
" actions, values.action.values.tolist(), \n",
" progress_bar=tqdm, verbose=False, data_path=data_path\n",
" )\n",
" unit_matching.do_matching()\n",
" unit_matching.make_graphs_from_matches()\n",
" unit_matching.compute_time_delta_edges()\n",
" unit_matching.compute_depth_delta_edges()\n",
" # save graph with all dissimilarities for later use\n",
" unit_matching.save_graphs()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Plot comparisons"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 13,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"unit_comp = TrackMultipleSessions(actions, data_path=f'output/identify_neurons/1833-graphs')"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 14,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"unit_comp.load_graphs()"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 15,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"max_dissimilarity = .05\n",
"max_depth_delta = 100\n",
"\n",
"unit_comp.remove_edges_above_threshold('weight', max_dissimilarity)\n",
"unit_comp.remove_edges_above_threshold('depth_delta', max_depth_delta)"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 16,
2019-10-08 18:52:35 +00:00
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
2019-10-09 13:38:38 +00:00
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAokCAYAAAAwwpPoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd4VEXbwOHf2V6y6Y0SWoClFxEIRQHFAlIUUBQLKqCoiAU+u9heRRRerAgIvKCggjQbivQOUkKHJfQa0kiyJZtt5/vjBERpCaSQzdzXlYvs5pydWbI5++zMM89IsiwjCIIgCIIgCIJCVdYdEARBEARBEITriQiQBUEQBEEQBOE8IkAWBEEQBEEQhPOIAFkQBEEQBEEQziMCZEEQBEEQBEE4jwiQBUEQBEEQBOE8IkAWBEEQBEEQhPOIAFkQBEEQBEEQziMCZEEQBEEQBEE4j6asO3AxrVu3lqtUqVLW3RCuwa5duzJsNltMcT+ueG2UfyX12gDx+ggG4tohXEpJXjsE4d+uywC5SpUqzJ07t6y7IVwDq9V6pCQeV7w2yr+Sem2AeH0EA3HtEC6lJK8dgvBvIsVCEARBEARBEM4jAmRBEARBEARBOI8IkAVBEARBEAThPCWSg2y1WlsDo2w2W0er1VobmArIwE7gGZvNFiiJdgVBEARBEAThWhX7CLLVan0JmAQYCu76L/CGzWa7CZCAnsXdpiAIgiAIgiAUl5JIsTgA9DrvdgtgRcH3vwOdS6BNQRAEQSgfTu8Ge+pVnx4IyBzOcBIIyMXYKUEQzlfsAbLNZpsDeM+7S7LZbGf/iu1AWHG3KQiCIAjlQiAAU++CCTdDuq3Ip689kMHtn6yk4+jl3DZ2BUv3ni6BTgqCUBqL9M7PN7YA2aXQpiAIQrkmyzJnnB5yXN4rHyyUH5kpkJcFznT49h4I+At9aq7by9Dvk/H4Arx8Zz3UKomB0zYxe/PxEuywIFRMpREgJ1ut1o4F33cBVpVCm4IgCOVWrtvLfRPW0fy9RTR770+Gfp/M0UxXWXdLKA7HNyr/tn8Bck9A2u5Cn/rp4hQynR6+vK8+T9U8zbxBN9A2MZrhP25j1qZjJdRhQaiYSiNAHga8Y7Va1wE6YHYptCkIglAueXwBHpn8F1uPZfPibXUZdFMtFu0+ze2frGD8igO4vYUfcRSuQ8c34ZLMPJpsVW4f21Co0zIc+Uxbe5h36h6m8eyb4X9dMH/ZhP812c1NdaJ5ec52pq09jCyLvGRBKA4lUubNZrMdBpIKvt8HdCiJdgRBEILNz9tOsvVYNp/e34yeTSuDJPFYuxq8OX8nH/6+l0mrDtI2MZpIs45sl4cjWS7OOD2oJIlqUSa6NqpE96aVMerUZf1UhIvwH9vIJl8tkh3hpMnhmA+sxdxy4BXP+33HKcID2Tx46kOIqAad34at36Fd8DxTbn2PpzRteOvnXaw/mMnTHWvTsHIoKpVU4s9HEIJViQTIgiAIQtHJsqwEwLFeeiQPhJ83QkQNKnV6jUmP3MP6Q1lMW3uYjYezcLh9hBq1VI8yUbVqOP5AgD2n7Lw0ZzufLknhg16N6VA3pqyfknA+jxNV+m6S5Z4817kumxfWof3Rwo0g/7LtFCMts1D78uDeqRBdBxr1gbmD0C55k4md3+XLql0Zv+IAv+9MxaLXkBBpolKYgTpxFu5sFE/TqmFIkgiaBaEwRIAsCIJwnViVksGp1FPMDX8T6aQdWg6AI2th9mOwYzZJ3T8l6aEWlzxflmXWHshkxE876T/lL17oXJeht9YWQdH14tQ2JDnA1kBtxjSvwvRlDeji2giONAiJvfRpOXmcPLKX2/TLoN1zSnAMoNFB78mgUqNaPIJnb7bzyCsvsXDXaXacyOFkdh4nsvNYmZLO+BUHaFUzko96N6FGtLmUnrAglF8iQBYEQShuAT+c3gmxDUCtLfRp87ee4FnD7xjdaTBwMVS9Efw+WD8Olr4H41pDp9ehWT/QGi84X5Ik2tWO5rehN/HavB2MXbyPE9kuRvZqglpMt5e9HKXahNtSnUizDnX1JDg4Df+R9agb9rjkaX/sTOVu1Rrlxr/TMdQauGciaE2w8mPCTu/mvi6juK9lo3OH5Lq9zNtygtF/2ujy6SrGPXQDnayXDsgFQRAB8nXN5w9gO23nYLoTt9ePUaemUpiR+pUsmHTiVycI161d82DOADBGQJ8pkHjLFU8JBGS27d3PSOl3pIa9lOAYlACo3VCo3RkWDIffXoQ/34QqN4AxHHz5ypdGD1G1oUFPDAmtGXNvU6qGG/ls6X58AZnRfZqKnNSy5lBqFsdXrgZAzcZJ+A6oSN+3gUqXCZBX2tJ4R7cWqrWD8GoXHqDWQI/PIbY+LHkPPm8B1i5Q906o3obQ8Or0b1uDOxrGM/CbjQyatolP72/OXU0qlcjTFIRgIKKs60wgILMiJZ05m4+zwpaOPd93wTEqCZomhHNnw3j6tkwg3KQrg54KgnBJx/5SRvQ0BtgwoVAB8rbj2XTLX4BOmw8dXrnwgLgG8OhvcGgF7JoPaXsgI0VpQ2MAVxYcWqWMNickIfX4nBdvt6JRq/jvon3o1Co+uKexCJLLUH52KrKspXZCZQDa1K3KAbky+uPbLn2Oz4/90CaqqU9Ak5cu/eCSBG2egfrdYc1nsGsu7J6v/CysGjTuTXzLgXw/KInHp27k+ZnJhBm1tK8TXZxPURCChgiQryOLd5/m44U2bKftRJi03NWkEm0So7DGWzDrNDg9Po5l5bH9eDYr9qUz8ve9fLI4hUE31+LpjokYtGLVuiBcF1K3Q3xjJcVix2wlTUJ9+cvtsr1pdFOvx5fQFm1M3YsfJElQq6PydTH5Dtj+gzKKOOFm6P01Q2/tjtcf4POl+wFEkFyGcjNO4JbDaVw1HIAIs45NutrcmL3zkudsOnyGW+T1BCQNqgY9r9xIeDW4azR0GQXpe5Uc9v2LYc2nsGECllvfYtIjj9N34nqemr6Z+UPakRgTUlxPURCChgiQrwPp9nxem7eDRbtPkxhj5pO+zejauBI6zYVlquvFh3JbgziG3W5lb2ouXyzdz2dLUvh56wk+vb85TRPCy+AZCNfCH1Dqlooc0SARCEDqDmj6ANRoB5v/Bye3QEKry56WsmszdVUnoNHzV9+2PkTJUa3XDX54EGY+DF1G8eJtTwDw+dL9ePwBPu7TVLzeykDAfpp0wkiINJ27zxXVgIjTy5Ed6UghF1YdWZmSzj3qZOSEJCVlp7BUaohrqHy1GgRnjigpOn+8TNipbUx++EN6jPuLQd9s4qdn2mExFD5XXhAqAhEgl7HVKRk890My9nwfr3apx+Pta6JVF27/lnrxoXzR7wb6tcpg+I/b6P3VWt6/pxF9W14kR0247uxPc/Dm/J0kHzuDSpJoUjWM2xvE09EaQ81o87nKA2ecHg5nOknNcePxBwgzarHGW6gUduEiLeE6cOYQeBxQqQnUuFm579CKywbIOXleamcuVa7I9btfex8s8fDorzBnIPz+EpIrk2G3vYq2IN3C4wswtm+zQl9rhOKhcaWRIYdRJ+TvtDhTtWZwGtL3byK2WZcLztmzdzevSsfA+sS1NR5RHfrNghWjYPlIquTn8uUD/+XBKZv5vx+389VDN4hqJ4JwHhEglxFZlpm8+hAfLNhD7dgQvn8iibpxlr8PCAQgwwand4E7ByQV6MxgjFRyEUMrnzu0be1oFjx3E89+n8zLc3Zw/EweL95WV1zsStrid8DvgVtHKAukimDLwVOs/uZtxrEAyWjkcMgNfG7vybu/ZvHurxCi12AxaMjN8+L0XHzntFrRZro3rcyDSdWItRiK4xkJxeGUkk/qj2vMryluusY0QntwBdz8f5c8ZdPhLO5QbSQ3pgWhocW0cEprhHunwS/PKUGRM4OhXT/GoFXxwYK9eHwBPu/XHL1GpGaVFkN+JllSDUL0f7/1Vq3XGjZCesqFAXKa3U1CxmrQAnXvuPYOSBJ0fAUM4fDHyyTpzLx650v8Z4GNCSsPMrhD4rW3IQhBQgTIZcD
2019-10-08 18:52:35 +00:00
"text/plain": [
"<Figure size 720x11448 with 48 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"unit_comp.remove_edges_with_duplicate_actions()\n",
"unit_comp.identify_units()\n",
"unit_comp.plot_matches('template', chan_group=6, step_color=False)\n",
"plt.tight_layout()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Store uniqe unit ids to csv"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 19,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"max_dissimilarity = .05\n",
"max_depth_delta = 100\n",
2019-10-17 11:52:49 +00:00
"id_nums = {}\n",
"num = 0\n",
"for entity, values in sessions.groupby('entity'):\n",
" data_path = output / f'{entity}-graphs'\n",
2019-10-08 18:52:35 +00:00
" unit_matching = TrackMultipleSessions(\n",
2019-10-17 11:52:49 +00:00
" actions, values.action.values.tolist(), \n",
" progress_bar=tqdm, verbose=False, data_path=data_path\n",
2019-10-08 18:52:35 +00:00
" )\n",
" unit_matching.load_graphs()\n",
" # cutoff large dissimilarities\n",
" unit_matching.remove_edges_above_threshold('weight', max_dissimilarity)\n",
" unit_matching.remove_edges_above_threshold('depth_delta', max_depth_delta)\n",
" unit_matching.remove_edges_with_duplicate_actions()\n",
" unit_matching.identify_units()\n",
" units = []\n",
" for ch, group in unit_matching.identified_units.items():\n",
" for unit_id, val in group.items():\n",
" for action_id, orig_unit_ids in val['original_unit_ids'].items():\n",
2019-10-17 11:52:49 +00:00
" if action_id in skip_actions:\n",
" continue\n",
" if unit_id not in id_nums:\n",
" id_nums[unit_id] = num\n",
" num += 1\n",
2019-10-08 18:52:35 +00:00
" units.extend([\n",
" {\n",
" 'unit_name': name, \n",
2019-10-17 11:52:49 +00:00
" 'unit_idnum': id_nums[unit_id],\n",
2019-10-08 18:52:35 +00:00
" 'unit_id': unit_id, \n",
" 'action': action_id,\n",
" 'channel_group': ch,\n",
" 'max_dissimilarity': max_dissimilarity,\n",
" 'max_depth_delta': max_depth_delta\n",
" } \n",
" for name in orig_unit_ids])\n",
"\n",
" pd.DataFrame(units).to_csv(output / f'{entity}-units.csv', index=False)"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 20,
2019-10-08 18:52:35 +00:00
"metadata": {},
2019-10-09 13:38:38 +00:00
"outputs": [],
2019-10-08 18:52:35 +00:00
"source": [
"unique_units = pd.concat([\n",
" pd.read_csv(p) \n",
" for p in output.iterdir() \n",
2019-10-17 11:52:49 +00:00
" if p.name.endswith('-units.csv')])"
2019-10-08 18:52:35 +00:00
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 21,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"unique_units.to_csv(output / 'units.csv', index=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Store results in Expipe action"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 22,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"identify_neurons.data['sessions'] = 'sessions.csv'\n",
"identify_neurons.data['units'] = 'units.csv'"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 23,
2019-10-08 18:52:35 +00:00
"metadata": {},
2019-10-17 11:52:49 +00:00
"outputs": [
{
"data": {
"text/plain": [
"['/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-units.csv',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-7.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-0.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-6.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-1.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-2.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-5.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-3.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1834-graphs/graph-group-4.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-units.csv',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/sessions.csv',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-units.csv',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-7.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-0.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-6.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-1.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-2.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-5.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-3.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1849-graphs/graph-group-4.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/units.csv',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-7.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-0.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-6.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-1.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-2.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-5.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-3.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-graphs/graph-group-4.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-7.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-0.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-6.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-1.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-2.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-5.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-3.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1833-graphs/graph-group-4.yaml',\n",
" '/media/storage/expipe/septum-mec/actions/identify-neurons/data/1839-units.csv']"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
2019-10-08 18:52:35 +00:00
"source": [
"copy_tree(output, str(identify_neurons.data_path()))"
]
},
{
"cell_type": "code",
2019-10-17 11:52:49 +00:00
"execution_count": 23,
2019-10-08 18:52:35 +00:00
"metadata": {},
"outputs": [],
"source": [
"store_notebook(\n",
" identify_neurons, \"00-identify-neurons.ipynb\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
