septum-mec/actions/calculate-statistics-extra/data/10_calculate_statistics_ext...

459 lines
17 KiB
Plaintext
Raw Permalink Normal View History

2021-03-10 12:57:09 +00:00
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/mikkel/.virtualenvs/expipe/lib/python3.6/site-packages/ipykernel_launcher.py:25: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
"Please use `tqdm.notebook.*` instead of `tqdm._tqdm_notebook.*`\n"
]
}
],
"source": [
"import os\n",
"import expipe\n",
"import pathlib\n",
"import numpy as np\n",
"import spatial_maps.stats as stats\n",
"import septum_mec.analysis.data_processing as dp\n",
"import head_direction.head as head\n",
"import spatial_maps as sp\n",
"import septum_mec.analysis.registration\n",
"import speed_cells.speed as spd\n",
"import septum_mec.analysis.spikes as spikes\n",
"import re\n",
"import joblib\n",
"import multiprocessing\n",
"import shutil\n",
"import psutil\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import septum_mec\n",
"import scipy.ndimage.measurements\n",
"from distutils.dir_util import copy_tree\n",
"from spike_statistics.core import theta_mod_idx\n",
"\n",
"from tqdm import tqdm_notebook as tqdm\n",
"from tqdm._tqdm_notebook import tqdm_notebook\n",
"tqdm_notebook.pandas()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"max_speed = 1, # m/s only used for speed score\n",
"min_speed = 0.02, # m/s only used for speed score\n",
"position_sampling_rate = 100 # for interpolation\n",
"position_low_pass_frequency = 6 # for low pass filtering of position\n",
"\n",
"box_size = [1.0, 1.0]\n",
"bin_size = 0.02\n",
"smoothing_low = 0.03\n",
"smoothing_high = 0.06\n",
"\n",
"stim_mask = True\n",
"baseline_duration = 600"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"project_path = dp.project_path()\n",
"\n",
"project = expipe.get_project(project_path)\n",
"actions = project.actions"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>action</th>\n",
" <th>channel_group</th>\n",
" <th>max_depth_delta</th>\n",
" <th>max_dissimilarity</th>\n",
" <th>unit_id</th>\n",
" <th>unit_idnum</th>\n",
" <th>unit_name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1834-010319-1</td>\n",
" <td>0</td>\n",
" <td>100</td>\n",
" <td>0.05</td>\n",
" <td>ae0353a9-a406-409e-8ff7-2e940b8af03f</td>\n",
" <td>327</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1834-010319-1</td>\n",
" <td>0</td>\n",
" <td>100</td>\n",
" <td>0.05</td>\n",
" <td>7f514d43-17ba-4d88-a390-20eec8bc1378</td>\n",
" <td>328</td>\n",
" <td>39</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1834-010319-3</td>\n",
" <td>0</td>\n",
" <td>100</td>\n",
" <td>0.05</td>\n",
" <td>c977aa51-06cc-4d54-9430-a94ad422a03b</td>\n",
" <td>329</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1834-010319-3</td>\n",
" <td>0</td>\n",
" <td>100</td>\n",
" <td>0.05</td>\n",
" <td>bd96a67d-ee7d-4cb6-90ab-a5fa751891b9</td>\n",
" <td>330</td>\n",
" <td>12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1834-010319-4</td>\n",
" <td>0</td>\n",
" <td>100</td>\n",
" <td>0.05</td>\n",
" <td>abc01041-2971-4f62-bf06-5132cf356737</td>\n",
" <td>332</td>\n",
" <td>7</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" action channel_group max_depth_delta max_dissimilarity \\\n",
"0 1834-010319-1 0 100 0.05 \n",
"1 1834-010319-1 0 100 0.05 \n",
"2 1834-010319-3 0 100 0.05 \n",
"3 1834-010319-3 0 100 0.05 \n",
"4 1834-010319-4 0 100 0.05 \n",
"\n",
" unit_id unit_idnum unit_name \n",
"0 ae0353a9-a406-409e-8ff7-2e940b8af03f 327 2 \n",
"1 7f514d43-17ba-4d88-a390-20eec8bc1378 328 39 \n",
"2 c977aa51-06cc-4d54-9430-a94ad422a03b 329 1 \n",
"3 bd96a67d-ee7d-4cb6-90ab-a5fa751891b9 330 12 \n",
"4 abc01041-2971-4f62-bf06-5132cf356737 332 7 "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"identify_neurons = actions['identify-neurons']\n",
"units = pd.read_csv(identify_neurons.data_path('units'))\n",
"units.head()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x7f9234454cf8>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAD3CAYAAADSftWOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAL60lEQVR4nO3dbYhld33A8e9kszejsgYLIrqog9j+uK9C2OKmWOuCkRoXieRlMRSDSCUvUmqN23SDUBQ2JdmCKVrIQ7f4AGIefOGyNS/a5qFFpEkEEy+/1dRVXyi0gWA03M7sZnwxd9ub2Xvn3j1z7z3zm/l+IHAfZub8/nvmfnP27D0zS+vr60iSarmi7QEkSZfPeEtSQcZbkgoy3pJUkPGWpIKuXMRGDh8+vH7w4MFFbGpmVldX6XQ6bY+xLa5hZ3ANO0PFNTz//PP/k5lvHvXcQuJ98OBBHnnkkUVsamZ6vR7dbrftMbbFNewMrmFnqLiGiPjpuOc8bSJJBRlvSSrIeEtSQcZbkgoy3pJUkPGWpIKMtyQVZLwlqSDjLUkFGW+9Rn/twp7arlTVQi6PVx3L+/excuz0wrd77sTRhW9Tqswjb0kqyHhLUkHGW5IKMt6SVJDxlqSCjLckFWS8Jakg4y1JBRlvSSrIeEtSQcZbkgoy3pJUkPGWpIKMtyQVtOWPhI2I/cCDwApwFfB54IfAKWAdeA64NTNfneuUkqTXmHTk/THgxcx8H/Ah4O+Bk8DxwWNLwI3zHVGStNmkeH8TuHNwewk4DxwCHh88dga4fj6jSZLG2fK0SWb+GiAiDgAPAceBuzNzffAhLwNXT9rI6uoqvV5vm6MuVr/fLzfzZk3W0O125zTNZKNm3av7YadxDTvPxF+DFhFvBx4FvpSZX4+Ivx16+gDw0qSv0el0Wo1CE71er9zMm1Vbw6hZq61hFNewM+yGNQzb8rRJRLwFeAz4bGY+OHj42Yg4Mrh9A/Dk/MaTJI0y6cj7DuBNwJ0RcfHc923AFyOiA/TYOJ0iSVqgSee8b2Mj1pu9fz7jSJKm4UU6klSQ8Zakgoy3JBVkvCWpIOOtHaG/dmHk44t4X+64bUs72cSLdKRFWN6/j5Vjp1vZ9rkTR1vZrrQdHnlLUkHGW5IKMt6SVJDxlqSCjLckFWS8Jakg4y1JBRlvSSrIeEtSQcZbkgoy3pJUkPGWpIKMtyQVZLwlqSDjLUkFGW9JKsh4S1JBxluSCjLeklSQ8Zakgoy3JBVkvCWpIOMtSQUZb0kqyHhLUkHGW5IKMt7a8/prF+b69bvdbivb1e52ZdsDSG1b3r+PlWOnF77dcyeOLnyb2j088pakgoy3JBVkvCWpIOMtSQUZb0kqaKp3m0TEYeCuzDwSEdcC3wZ+NHj6y5n5jXkNKEm61MR4R8TtwM3AbwYPHQJOZuY98xxMkjTeNKdNXgBuGrp/CDgaEU9ExAMRcWA+o0mSxpl45J2ZD0fEytBD3wPuz8ynI+Kvgc8Bf7nV11hdXaXX621r0EXr9/vlZt6syRrGXQ2o+ajyPbZXXw87WZMrLB/NzJcu3gbunfQJnU6nXBR6vV65mTfbDWvY7arsn93wvbQb1jCsybtNvhMR7xnc/gDw9AznkSRNocmR96eAeyNiDfgl8MnZjiRJmmSqeGfmOeC6we1ngPfOcSZJ0gRepCNJBRlvSSrIeEtSQcZbkgoy3pJUkPGWpIKMtyQVZLwlqSDjLUkFGW9JKsh4S1JBxluSCjLeklSQ8Zakgoy3JBVkvCWpIOMtSQUZb0kqyHhLUkHGW5IKMt6SVJDxlqSCjLckFWS8Jakg4y1JBRlvSSrIeEtSQcZbkgoy3pJUkPGWpIKMtyQVZLwlqSDjLUkFGW9JKsh4S1JBxluSCjLeklSQ8Zakgoy3JBV05TQfFBGHgbsy80hEvBs4BawDzwG3Zuar8xtRkrTZxCPviLgduB9YHjx0Ejieme8DloAb5zeeJGmUaU6bvADcNHT/EPD44PYZ4PpZDyVJ2trE0yaZ+XBErAw9tJSZ64PbLwNXT/oaq6ur9Hq9ZhO2pN/vtzbzO1bexRted9W2v063253BNJqnKq+LNl8Ps7Ib1jBsqnPemwyf3z4AvDTpEzqdTrmQ9Hq9VmdeOXa6le2eO3G0le3uVVVeF22/HmZhN6xhWJN3mzwbEUcGt28AnpzdOJKkaTQ58v40cF9EdIAe8NBsR5IkTTJVvDPzHHDd4PZZ4P1znEmSNIEX6UhSQcZbkgoy3pJUkPGWpIKMtyQVZLwlqSDjLUkFGW9JKsh4S1JBxluSCjLeklSQ8Zakgoy3JBVkvCWpIOMtSQUZb0kqyHhLUkHGW5IKMt6SVJDxlqSCjLckFWS8Jakg4y1JBZWId3/twsK32e12W9muJE3jyrYHmMby/n2sHDu98O2eO3F04duUpGmUOPKWJL2W8Zakgoy3JBVkvCWpIOMtSQUZb0kqyHhLUkHGW5IKMt6SVJDxlqSCjLckFWS8Jakg4y1JBRlvSSqo8Y+EjYhngF8N7v4kMz8+m5EkSZM0indELANLmXlktuNIkqbR9Mj7GuD1EfHY4GvckZnfnd1YkqStNI33K8DdwP3A7wJnIiIy8/yoD15dXaXX6zXc1MavJGvLduZuqs31arHa+P5qot/vl5l1nN2whmFN430W+HFmrgNnI+JF4K3Az0d9cKfTKRukqnOrhirfX71er8ys4+yGNQxr+m6TW4B7ACLibcAbgV/MaihJ0taaHnk/AJyKiKeAdeCWcadMJEmz1yjembkK/MmMZ5EkTcmLdCSpIOMtSQUZb0kqyHhLUkHGW2pJf+3Cnty2ZqPxD6aStD3L+/excux0K9s+d+JoK9vV7HjkLUkFGW9JKsh4S1JBxluSCjLeklSQ8Zakgoy3JBVkvCWpIOO9Ba9Ck2arrdfUbnwte4XlFtq6As6r37Rb+ZqaHY+8Jakg4y1JBRlvSSrIeEtSQcZbkgoy3pJUkPGWpIKMt7QHXe5FK91ud06TqCkv0pH2IC+Wqc8jb0kqyHhLUkHGW5IKMt6SVJDxlqSCjLckFWS8Jakg4y1p1+uvXWjtQqN5/RYfL9KRtOu1dVESzO/CJI+8Jakg4y1JBRlvSSrIeEtSQcZbkgpq9G6TiLgC+BJwDfC/wCcy88ezHEySNF7TI++PAsuZ+QfAMeCe2Y0kSZqkabz/EPhngMz8LvD7M5tIkjTR0vr6+mV/UkTcDzycmWcG938GvCszz4/5+P8GfrqdQSVpD3pnZr551BNNr7D8FXBg6P4V48INMG7jkqRmmp42+XfgwwARcR3wg5lNJEmaqOmR96PAByPiP4Al4OOzG0mSNEmjc96SpHZ5kY4kFWS8Jakg4y1JBfnLGEaIiGfYeDskwE8ys8w/yEbEYeCuzDwSEe8GTgHrwHPArZn5apvzTWPTGq4Fvg38aPD0lzPzG+1Nt7WI2A88CKwAVwGfB35Iof0wZg0/p9Z+2AfcBwQbf+5/BvQptB8mMd6bRMQysJSZR9qe5XJFxO3AzcBvBg+dBI5n5r9FxD8AN7LxTqEda8QaDgEnM7PKj2D4GPBiZt4cEb8DfH/wX6X9MGoNf0Ot/fARgMx8b0QcAb7AxjvjKu2HLXna5FLXAK+PiMci4l8G72Ov4gXgpqH7h4DHB7fPANcvfKLLN2oNRyPiiYh4ICIOjPm8neKbwJ2D20vAeerth3FrKLMfMvNbwCcHd98JvES9/bAl432pV4C7gT9m469aX4uIEn9DycyHgbWhh5Yy8+J7QV8Grl78VJdnxBq+B3wmM/8I+C/gc60MNqXM/HVmvjyI20PAcYrthzFrKLUfADLzfET8E3Av8DWK7YdJjPelzgJfzcz1zDwLvAi8teWZmho+n3eAjaOPah7NzKcv3gaubXOYaUTE24F/Bb6SmV+n4H4YsYZy+wEgM/8U+D02zn+/buipEvthK8b7Urcw+BG3EfE24I3AL1qdqLlnB+f7AG4Anmxxlqa+ExHvGdz+APD0Vh/ctoh4C/AY8NnMfHDwcKn9MGYN1fbDzRHxV4O7r7D
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"%matplotlib inline\n",
"units.groupby('action').count().unit_name.hist()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"data_loader = dp.Data(\n",
" position_sampling_rate=position_sampling_rate, \n",
" position_low_pass_frequency=position_low_pass_frequency,\n",
" box_size=box_size, bin_size=bin_size, stim_mask=stim_mask, baseline_duration=baseline_duration\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"first_row = units[units['action'] == '1849-060319-3'].iloc[0]\n",
"#first_row = sessions.iloc[50]"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"spatial_average_rate 3.181077\n",
"dtype: float64"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def process(row):\n",
" action_id = row['action']\n",
" channel_id = row['channel_group']\n",
" unit_id = row['unit_name']\n",
" \n",
" # common values for all units == faster calculations\n",
" x, y, t, speed = map(data_loader.tracking(action_id).get, ['x', 'y', 't', 'v'])\n",
"# ang, ang_t = map(data_loader.head_direction(action_id).get, ['a', 't'])\n",
" \n",
" occupancy_map = data_loader.occupancy(action_id)\n",
" xbins, ybins = data_loader.spatial_bins\n",
" box_size_, bin_size_ = data_loader.box_size_, data_loader.bin_size_\n",
" prob_dist = data_loader.prob_dist(action_id)\n",
" \n",
"# smooth_low_occupancy_map = sp.maps.smooth_map(\n",
"# occupancy_map, bin_size=bin_size_, smoothing=smoothing_low)\n",
" smooth_high_occupancy_map = sp.maps.smooth_map(\n",
" occupancy_map, bin_size=bin_size_, smoothing=smoothing_high)\n",
" \n",
" spike_times = data_loader.spike_train(action_id, channel_id, unit_id)\n",
" if len(spike_times) == 0:\n",
" result = pd.Series({\n",
" 'spatial_average_rate': np.nan\n",
" })\n",
" return result\n",
"\n",
" # common\n",
" spike_map = sp.maps._spike_map(x, y, t, spike_times, xbins, ybins)\n",
"\n",
"# smooth_low_spike_map = sp.maps.smooth_map(spike_map, bin_size=bin_size_, smoothing=smoothing_low)\n",
" smooth_high_spike_map = sp.maps.smooth_map(spike_map, bin_size=bin_size_, smoothing=smoothing_high)\n",
"\n",
"# smooth_low_rate_map = smooth_low_spike_map / smooth_low_occupancy_map\n",
" smooth_high_rate_map = smooth_high_spike_map / smooth_high_occupancy_map\n",
"\n",
" \n",
"\n",
" tmp_rate_map = smooth_high_rate_map.copy()\n",
" tmp_rate_map[np.isnan(tmp_rate_map)] = 0\n",
" avg_rate = np.sum(np.ravel(tmp_rate_map * prob_dist))\n",
"\n",
"\n",
" result = pd.Series({\n",
" 'spatial_average_rate': avg_rate\n",
" })\n",
" return result\n",
" \n",
"process(first_row)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8baa6ab9462541b6bcdf6c14c208eeb1",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=1284), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"results = units.merge(\n",
" units.progress_apply(process, axis=1), \n",
" left_index=True, right_index=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"output_path = pathlib.Path(\"output\") / \"calculate-statistics-extra\"\n",
"output_path.mkdir(exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"results.to_csv(output_path / \"results.csv\", index=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Store results in Expipe action"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"statistics_action = project.require_action(\"calculate-statistics-extra\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"statistics_action.data[\"results\"] = \"results.csv\"\n",
"copy_tree(output_path, str(statistics_action.data_path()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"statistics_action.modules['parameters'] = {\n",
" 'max_speed': max_speed,\n",
" 'min_speed': min_speed,\n",
" 'position_sampling_rate': position_sampling_rate,\n",
" 'position_low_pass_frequency': position_low_pass_frequency,\n",
" 'box_size': box_size,\n",
" 'bin_size': bin_size,\n",
" 'smoothing_low': smoothing_low,\n",
" 'smoothing_high': smoothing_high,\n",
" 'stim_mask': stim_mask,\n",
" 'baseline_duration': baseline_duration\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"septum_mec.analysis.registration.store_notebook(statistics_action, \"10_calculate_statistics_extra.ipynb\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 4
}