#!/usr/bin/env python
# coding: utf-8

# ## Apply an image-based profiling pipeline using pycytominer
# 
# As described fully in [Caicedo et al. 2017](https://doi.org/10.1038/nmeth.4397), an image-based profiling pipeline consists of three core steps:
# 
# 1. Aggregation
# 2. Normalization
# 3. Feature selection
# 
# [Pycytominer](https://github.com/cytomining/pycytominer) is a python package, built on top of pandas, that facilitates all of these steps and more.
# 
# ### Data levels
# 
# The concept of "data levels" is important to understand when implementing an image-based profiling pipeline.
# 
# | Data | Level |
# | :---- | :---- |
# | Images | Level 1 |
# | Single cell profiles (SQLite) | Level 2 |
# | Aggregated profiles with metadata information | Level 3 |
# | Normalized aggregated profiles | Level 4a |
# | Normalized and feature selected profiles | Level 4b |
# | Consensus profiles | Level 5 |

# In[42]:


import pathlib
import pandas as pd
import numpy as np

from pycytominer import aggregate, annotate, normalize, feature_select, consensus


# ### Step 0 - Initialize data

# In[43]:


# Load all .csv files
platemap = "./Conditions.csv"
cells_csv_path = "./Cells.csv"
nuclei_csv_path = "./Nucleus.csv"
cytoplasm_csv_path = "./Cytoplasm.csv"

platemap_df = pd.read_csv(platemap)
cells_ori = pd.read_csv(cells_csv_path)
nuclei_ori = pd.read_csv(nuclei_csv_path)
cytoplasm_ori = pd.read_csv(cytoplasm_csv_path)

print("Original Cells.csv size (row,col): ", cells_ori.shape)
print("Original Nuclei.csv size (row,col): ", nuclei_ori.shape)
print("Original Cytoplasm.csv size (row,col): ", cytoplasm_ori.shape)

# In[44]:


# Delete duplicate columns (by column name)
cells = cells_ori.loc[:,cells_ori.columns.str.find('.') < 0]
nuclei = nuclei_ori.loc[:,nuclei_ori.columns.str.find('.') < 0]
cytoplasm = cytoplasm_ori.loc[:,cytoplasm_ori.columns.str.find('.') < 0]

print("Cells.csv size after removing duplicate cols (row,col): ", cells.shape)
print("Nuclei.csv size after removing duplicate cols (row,col): ", nuclei.shape)
print("Cytoplasm.csv size after removing duplicate cols (row,col): ", cytoplasm.shape)


# In[45]:


# Check to see how Cell data looks
print("Cell data (rows, columns):", cells.shape)
cells.head(5)


# In[46]:


# Check to see how platemap/conditions file looks
print("Conditions data (rows, columns):", platemap_df.shape)
platemap_df.head(6)


# In[47]:


# Scan through column headers in .csvs until you reach "AreaShape_Area", then append prefix to every column thereafter
x = cells.columns.get_loc("AreaShape_Area")
new_cell_names = [(i,'Cells_'+i) for i in cells.iloc[:, x:].columns.values]
cells.rename(columns = dict(new_cell_names), inplace=True)

x = nuclei.columns.get_loc("AreaShape_Area")
new_nuclei_names = [(i,'Nuclei_'+i) for i in nuclei.iloc[:, x:].columns.values]
nuclei.rename(columns = dict(new_nuclei_names), inplace=True)

x = cytoplasm.columns.get_loc("AreaShape_Area")
new_cytoplasm_names = [(i,'Cytoplasm_'+i) for i in cytoplasm.iloc[:, x:].columns.values]
cytoplasm.rename(columns = dict(new_cytoplasm_names), inplace=True)

cells.head(5)


# In[48]:


# Store information regarding cell number from single-cell data before calculating medians. Append this to full df later
N = cells.groupby(['Metadata_Plate','Metadata_Well'],as_index=False).size()
N.drop(['Metadata_Plate','Metadata_Well'],axis=1,inplace=True)
N.head(12)


# In[49]:


# Concatenate Cells, Nuclei, Cytoplasm data together, delete duplicated 'Metadata' columns
full_data = pd.concat([cells,nuclei,cytoplasm], axis=1, sort=False)
full_data = full_data.loc[:,~full_data.columns.duplicated()]

print("Full data (rows,cols): ", full_data.shape)
full_data.head(24)


# In[50]:


# Feature Engineering - Addition of new data columns based on calculation from existing data

full_data['Cells_NC_Ratio'] = full_data['Nuclei_AreaShape_Area']/full_data['Cells_AreaShape_Area']
full_data.info()
print("Full data (rows, cols) after adding NC Ratio: ", full_data.shape)


# In[51]:


full_data['Cells_Aspect_Ratio_Cell'] = full_data['Cells_AreaShape_MajorAxisLength']/full_data['Cells_AreaShape_MinorAxisLength']


# In[52]:


full_data['Cells_Aspect_Ratio_Nucleus'] = full_data['Nuclei_AreaShape_MajorAxisLength']/full_data['Nuclei_AreaShape_MinorAxisLength']


# In[53]:


full_data['Cells_PerimeterArea_Ratio_Cell'] = full_data['Cells_AreaShape_Perimeter']/full_data['Cells_AreaShape_Area']


# In[54]:


full_data['Cells_PerimeterArea_Ratio_Nucleus'] = full_data['Nuclei_AreaShape_Perimeter']/full_data['Nuclei_AreaShape_Area']
full_data.info()
full_data.head(10)
print("Full data (rows,cols) after adding all additional features: ", full_data.shape)


# In[55]:


# Filter data based on NC Ratio, Nucleus Solidity, & Cell Solidity

#Find outliers based on NC Ratio >= 0.75
full_data.groupby(pd.cut(full_data['Cells_NC_Ratio'], np.arange(0, 1.1, 0.1)))['Metadata_Well'].count()
outlier_objects = full_data[full_data['Cells_NC_Ratio'] >= 0.75][['Metadata_Well', 'ObjectNumber', 'Cells_NC_Ratio']]
outlier_objects.to_csv('outlier_images.csv',index=False)


# In[56]:


# Drop NC Ratio outliers
full_data.drop(full_data[full_data.Cells_NC_Ratio >= 0.75].index, inplace=True)
print("Full data (row,col) after filtering based on NC Ratio: ", full_data.shape)

#Drop Nucleus Solidity outliers
full_data.drop(full_data[full_data.Nuclei_AreaShape_Solidity > 1].index, inplace=True)
print("Full data (row,col) after filtering based on Nucleus Solidity: ", full_data.shape)

# Drop Cell Solidity outliers
full_data.drop(full_data[full_data.Cells_AreaShape_Solidity > 1].index, inplace=True)
print("Full data (row,col) after filtering based on Cell Solidity: ", full_data.shape)


# ## Step 1 - Annotate wells using the platemap file
# 
# In this step, **level 2 profiles** (well-level profiles) are annotated with platemap metadata.

# In[62]:


full_data['Plate_Well'] = full_data['Metadata_Plate'].astype(str) + full_data['Metadata_Well']
print("After =", full_data.shape)
#print(full_data['Plate_Well'])


# In[63]:


full_data_annotated = pd.merge(full_data,platemap_df,left_on='Plate_Well',right_on='Metadata_Plate_Well')
print("After merge =", full_data_annotated.shape)


# In[12]:


# Annotate medians dataframe with platemap/conditions file
full_data['Plate_Well'] = full_data['Metadata_Plate'].astype(str) + full_data['Metadata_Well']
print("After =", full_data.shape)
print(full_data['Plate_Well'])
full_data_annotated = pd.merge(full_data,platemap_df,left_on='Plate_Well',right_on='Metadata_Plate_Well')
print("After merge =", full_data_annotated.shape)

# Drop duplicate columns that are made when appending platemap/conditions dataframe
# Rename 'Metadata_Plate_x' & 'Metadata_Well_x' to 'Metadata_Plate' & 'Metadata_Well', respectively
columns_to_drop = ['Metadata_Plate_y','Metadata_Well_y']
full_data_annotated = full_data_annotated.drop(columns_to_drop,axis=1,inplace=False)
full_data_annotated = full_data_annotated.rename(columns = {'Metadata_Plate_x':'Metadata_Plate','Metadata_Well_x':'Metadata_Well'}, inplace=False)
full_data_annotated.head(5)
print(full_data_annotated.shape)
full_data_annotated.to_csv("full_data_annotated.csv", index=False)


# In[18]:


# Delete any columns that have standard deviation = 0 for normalization to work properly
cols = full_data_annotated.select_dtypes([np.number]).columns
std = full_data_annotated[cols].std()
cols_to_drop = std[std==0].index
cols_to_drop = [i for i in cols_to_drop if not i.startswith('Metadata_')]

std_dev_0_cols = full_data_annotated.loc[:, cols_to_drop]
print("Columns to be removed (std.dev = 0): ", std_dev_0_cols.shape)

full_data_annotated = full_data_annotated.drop(cols_to_drop, axis=1)
print("full data shape after removing std.dev=0 cols: ", full_data_annotated.shape)


# In[19]:


std_dev_0_cols.to_csv('Zero_stddev_cols.csv', index=False)


# In[20]:


print(cols_to_drop)


# In[21]:


#full_data_annotated['Plate_Well']


# ## Step 3 - Single cell aggregation
# 
# In this step, **level 2 profiles** (single cells) are processed to **level 3 profiles** (well-level profiles).

# In[38]:


# Perform the aggregation - output well level profiles
medians = aggregate(
    full_data_annotated,
    strata=["Metadata_Plate", "Metadata_Well", "Metadata_Plate_Well", "Metadata_Treatment","Metadata_Stimulation"],
    features="infer",
    operation="median",
    subset_data_df="none",
)
print("Median profile size (row,col):", medians.shape)
medians.head(12)
medians = pd.concat([N,medians],axis=1,sort=False)
medians.to_csv("cells_medians.csv", index=False)