%load_ext autoreload
%autoreload 2
import os
import pathlib
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import re
import shutil
import pandas as pd
import scipy.stats
from functools import reduce
import statsmodels
import seaborn as sns
import exdir
import expipe
from distutils.dir_util import copy_tree
import septum_mec
import spatial_maps as sp
import head_direction.head as head
import septum_mec.analysis.data_processing as dp
import septum_mec.analysis.registration
from septum_mec.analysis.plotting import violinplot, savefig, despine
from septum_mec.analysis.statistics import load_data_frames, make_paired_tables, make_statistics_table
project_path = dp.project_path()
project = expipe.get_project(project_path)
actions = project.actions
output_path = pathlib.Path("output") / "comparisons-allcells"
(output_path / "statistics").mkdir(exist_ok=True, parents=True)
(output_path / "figures").mkdir(exist_ok=True, parents=True)
data, labels, colors, queries = load_data_frames()
data.keys()
columns = [
'average_rate',
# 'spatial_average_rate',
'gridness',
'sparsity',
'selectivity',
'information_rate',
'information_specificity',
'max_rate',
'interspike_interval_cv',
'burst_event_ratio',
'in_field_mean_rate',
'out_field_mean_rate',
'max_field_mean_rate',
'specificity',
'speed_score',
'spacing',
'field_area',
'head_mean_vec_len',
'border_score'
]
results, labels = make_paired_tables(data, columns)
results['gridcell']['average_rate']
# TODO: control for information rate
stat = {}
stat_values = {}
for cell_type, result in results.items():
stat[cell_type], stat_values[cell_type] = make_statistics_table(result, labels)
stat['gridcell'].T
for cell_type, sta in stat.items():
sta.to_latex(output_path / "statistics" / f"statistics_{cell_type}.tex")
sta.to_csv(output_path / "statistics" / f"statistics_{cell_type}.csv")
for cell_type, cell_results in results.items():
for key, result in cell_results.items():
result.to_latex(output_path / "statistics" / f"values_{cell_type}_{key}.tex")
result.to_csv(output_path / "statistics" / f"values_{cell_type}_{key}.csv")
title_xlabel = {
'information_rate': ('Spatial information','bits/s'),
'information_specificity': ("Spatial information specificity","bits/spike"),
'specificity': ("Spatial specificity", ""),
'average_rate': ("Average rate", "spikes/s"),
# 'spatial_average_rate': ("Spatial average rate", "spikes/s"),
'max_rate': ("Max rate", "spikes/s"),
'interspike_interval_cv': ("ISI CV", "Coefficient of variation"),
'in_field_mean_rate': ("In-field rate", "spikes/s"),
'out_field_mean_rate': ("Out-of-field rate", "spikes/s"),
'burst_event_ratio': ("Bursting ratio", ""),
'gridness': ("Gridness", "Gridness"),
'speed_score': ("Speed score", "Speed score"),
'spacing': ('Field spacing', 'cm'),
'field_area': ('Field area', 'cm$^2$'),
'border_score': ('Border Score', ''),
'head_mean_vec_len': ('Head direction score', ''),
'max_field_mean_rate': ('Max field mean rate', 'spike/s'),
'selectivity': ('Selectivity', ''),
'sparsity': ('Sparsity', '')
}
set(columns) - set(title_xlabel.keys()).intersection(set(columns))
plt.rc('axes', titlesize=12)
plt.rcParams.update({
'font.size': 12,
'figure.figsize': (3.7, 2.2),
'figure.dpi': 150
})
for cell_type, cell_results in results.items():
for key, result in cell_results.items():
if key not in title_xlabel:
continue
fig = plt.figure()
plt.suptitle(cell_type + ' ' + title_xlabel[key][0])
legend_lines = []
for color, label in zip(colors, labels):
legend_lines.append(matplotlib.lines.Line2D([0], [0], color=color, label=label))
sns.kdeplot(data=result.loc[:,labels], cumulative=True, legend=False, palette=colors, common_norm=False)
plt.xlabel(title_xlabel[key][1])
plt.legend(
handles=legend_lines,
bbox_to_anchor=(1.04,1), borderaxespad=0, frameon=False)
plt.tight_layout()
plt.grid(False)
despine()
savefig(output_path / "figures" / f'{cell_type}_{key}')
%matplotlib inline
plt.rc('axes', titlesize=12)
plt.rcParams.update({
'font.size': 12,
'figure.figsize': (1.7*3, 3),
'figure.dpi': 150
})
def violinplot(df, labels, colors, statistics=None):
data = np.array([df.loc[:,label].dropna().to_numpy() for label in labels])
pos = np.array([i * 0.6 for i in range(len(data))])
labels = np.array(labels)
# print(pos)
violins = plt.violinplot(data, pos, showmedians=True, showextrema=False)
for i, b in enumerate(violins['bodies']):
b.set_color(colors[i])
b.set_alpha (0.8)
# for i, body in enumerate(violins['cbars']):
# body.set_color('C{}'.format(i))
for category in ['cbars', 'cmins', 'cmaxes', 'cmedians']:
if category in violins:
violins[category].set_color(['k', 'k'])
violins[category].set_linewidth(2.0)
plt.xticks(pos, labels, rotation=45)
plt.gca().spines['top'].set_visible(False)
plt.gca().spines['right'].set_visible(False)
if statistics is not None:
tests = [[0,1], [2,3], [0,2]]
ds = [0,0,1]
for test, d in zip(tests, ds):
pvalue = statistics.loc[' - '.join(labels[test])]
# significance
if pvalue < 0.0001:
significance = "****"
elif pvalue < 0.001:
significance = "***"
elif pvalue < 0.01:
significance = "**"
elif pvalue < 0.05:
significance = "*"
else:
significance = "ns"
x1, x2 = pos[test]
data_max = np.max([a.max() for a in data[test]])
data_min = np.min([a.min() for a in data[test]])
y = (data_max * 1.05)
h = 0.025 * (data_max - data_min)
d_ = d * 0.15 * (data_max - data_min)
plt.plot([x1, x1, x2, x2], np.array([y - h, y, y, y - h]) + d_, c='k')
plt.text((x1 + x2) / 2, y + h + d_, significance, ha='center', va='bottom')
# plt.gca().spines['top'].set_visible(False)
# plt.gca().spines['right'].set_visible(False)
for cell_type, cell_results in results.items():
for key, result in cell_results.items():
if key not in title_xlabel:
continue
fig = plt.figure()
plt.suptitle(cell_type + ' ' + title_xlabel[key][0])
violinplot(result, labels=labels, colors=colors, statistics=stat_values[cell_type][key])
plt.ylabel(title_xlabel[key][1])
# plt.tight_layout()
plt.grid(False)
despine()
savefig(output_path / "figures" / f'{cell_type}_violin_{key}')
action = project.require_action("comparisons-allcells")
copy_tree(output_path, str(action.data_path()))
septum_mec.analysis.registration.store_notebook(action, "20_comparisons_allcells.ipynb")