#rm(list = ls())
library(ggplotify)
library(colorspace)
library(scales)
library(dendextend)
library(ggrepel)
library(ggplot2)
library(gsl)
library(dplyr)
library(metricsgraphics)
library(randomForest)
library(data.table)
library(factoextra) #visualize clustering
library(fpc)
library(forecast)
library(NbClust)
library(reshape)
library(ggdendro)
library(zoo)
library(mclust)
library(psych)
library(cluster)
library(fastcluster)
library(dbscan)
library(LPCM)
library(rgl)
library(ggsci)
library(dplyr)
library(tidyr)
library(corrplot)
library(clustree)
library(FactoMineR)
library(dlookr)
#parallel computing
library(parallel)
library(doSNOW)
library(dplyr)
library(plotly)
library(parallelDist)

setwd("~/ImageJ Pollini/FlowClustR")
giffoni <- read.csv("giffoni_norm.csv", sep = ",", header = TRUE)
#some explorative analysis
colnames(giffoni)
giffoni %>% 
  gather(Attributes, value, 1:ncol(giffoni)) %>% 
  ggplot(aes(x=value)) +
  geom_histogram(fill = "lightblue2", color = "black") + 
  facet_wrap(~Attributes, scales = "free") +
  labs(x = "Value", y = "Frequency")
#trasform infinte values to NA values
giffoni <- do.call(data.frame,lapply(giffoni, function(x) replace(x, is.infinite(x),NA)))
#check presence of NA values
sapply(giffoni, function(x) sum(is.na(x)))
#remove NA
giffoni<- na.omit(giffoni)
#Scale the data
giffoni.scaled <- as.data.frame(scale(giffoni))
#remove features related to object positions
giffoni.scaled <- giffoni.scaled[,-c(1,8:11,13,14,26,28:30)]
#check column names
colnames(giffoni.scaled)
giffoni.scaled %>% 
  gather(Attributes, value, 1:ncol(giffoni.scaled)) %>% 
  ggplot(aes(x=value)) +
  geom_histogram(fill = "lightblue2", color = "black", binwidth = 0.1) + 
  facet_wrap(~Attributes, scales = "free") +
  labs(x = "Value", y = "Frequency") 
#Rename columns
colnames(giffoni.scaled) <- c("Area","Mean","StdDev","Mode","Min","Max","Perim","Width","Height","Major","Minor","Angle",    
                              "Circ","Feret","IntDen","Median","Skew","Kurt","RawIntDen","FeretAngle","MinFeret","AR","Round","Solidity")
#check normality
normality <- normality(giffoni.scaled, sample = 5000)
#Check the correlation between the different features and group by clusters
graphics.off()
corrplot(cor(giffoni.scaled[,1:ncol(giffoni.scaled)], method =  "spearman"), type = "full", method = "color", tl.cex = 0.8,cl.cex=0.8, tl.col="black", order = "hclust", hclust.method = "ward.D2")
giffoni.corr <- cor(giffoni.scaled[,1:ncol(giffoni.scaled)], method =  "spearman")
#how many clusters of correlation distances do we have? 
Nb.res.corr3 <- NbClust(diss = as.dist(1-giffoni.corr),distance = NULL, min.nc=2, max.nc=10, 
                        method = "ward.D2", index = c("cindex")) 
Nb.res.corr3
#Hierchical clustering of correlation distances
cor.res3 <- fastcluster:::hclust(as.dist(1-giffoni.corr), method = "ward.D2")
plot(cor.res3, hang = -1) 
rect.hclust(cor.res3 , k = Nb.res.corr3$Best.nc[1], border = 2:6)
#draw a corrplot based on hierchical clustering
corrplot(giffoni.corr, type = "full", method = "color", tl.cex = 0.8,cl.cex=0.8, tl.col="black", order = "hclust", hclust.method = "ward.D2")
corrRect.hclust(giffoni.corr, k = Nb.res.corr3$Best.nc[1], method = "ward.D2", lwd = 1, col = "red")
#plot dedrogram with ggdendro
#define colors
show_col(colorspace::darken(pal_d3()(10),amount = 0))
cols <- colorspace::darken(pal_d3()(10),amount = 0)
#Extract grouping factor based on clustering of correlaton distances
grp <- cutree(cor.res3, k=Nb.res.corr3$Best.nc[1]) 
grp <- grp[cor.res3$order]
col_cluster <- cols[grp]
show_col(col_cluster)
dend <- as.dendrogram(cor.res3)
plot(dend)
dend1 <- dend %>%
  color_branches(k = ncol(giffoni.scaled), col = col_cluster) %>%
  color_labels(k =  ncol(giffoni.scaled), col = col_cluster) %>%
  dendextend::set("branches_lwd",0.7) %>%
  dendextend::set("labels_cex", 1.2) %>%
  dendextend::set("leaves_pch", 20) %>% 
  dendextend::set("leaves_cex", 1) %>%
  dendextend::set("leaves_col", col_cluster)
plot(dend1, lwd= 5, ylab = "Correlation Distance")

######
#PCA
set.seed(123)
giffoni.pc1 <- PCA(giffoni.scaled[1:ncol(giffoni.scaled)], scale.unit = F, graph = F, ncp = ncol(giffoni.scaled[1:ncol(giffoni.scaled)]))
print(giffoni.pc1)
dimdesc(giffoni.pc1)
#Visualize eigenvalues/variances
fviz_screeplot(giffoni.pc1, addlabels = TRUE, ylim = c(0, 70))
#Control variable colors using their contributions to the principle axis
fviz_pca_var(giffoni.pc1, col.var="contrib",
             gradient.cols = c("#00AFBB", "#E7B800", "#FC4E07"),
             repel = TRUE, # Avoid text overlapping
             xlim = c(-1, 1),
             ylim = c(-1, 1))
#plot individuals and variables by their contributions to the principal components
fviz_pca_biplot(giffoni.pc1, 
                # Individuals
                geom.ind = "point",
                pointshape = 19, pointsize = 1,
                palette = "jco",
                addEllipses = F,
                # Variables
                col.var = "contrib",
                gradient.cols = c("#00AFBB", "#E7B800", "#FC4E07"),
                legend.title = list(color = "Contrib")
)
#group variables based on hierarchical clustering of correlation distances
#extract pc scores for first two component and add to dat dataframe
dat <- giffoni.scaled[1:ncol(giffoni.scaled)]
dat$pc1 <- giffoni.pc1$ind$coord[, 1] # indexing the first column
dat$pc2 <- giffoni.pc1$ind$coord[, 2]  # indexing the second column
#extract the data for the variable contributions to each of the pc axes.
pca.vars <- giffoni.pc1$var$coord %>% data.frame
pca.vars$vars <- rownames(pca.vars)
pca.vars.m <- melt(pca.vars, id.vars = "vars")
#add clusters from correlation distance 
group_pca <- as.data.frame(grp)
group_pca$var <- names(grp)
rownames(group_pca) <- 1:ncol(giffoni.scaled)
group_pca <- group_pca[order(group_pca$var),]
pca.vars <- pca.vars[order(pca.vars$vars),]
pca.vars$cluster <- as.factor(group_pca$grp)
pca.vars.m <- melt(pca.vars, id.vars = c("vars","cluster"))
#make circle in the variable contrib plot
circleFun <- function(center = c(0,0),diameter = 1, npoints = 100){
  r = diameter / 2
  tt <- seq(0,2*pi,length.out = npoints)
  xx <- center[1] + r * cos(tt)
  yy <- center[2] + r * sin(tt)
  return(data.frame(x = xx, y = yy))
}
circ <- circleFun(c(0,0),2,npoints = 500)


pca_var <- ggplot() +
  geom_path(data = circ,aes(x,y), lty = 1, color = "grey", alpha = 0.7) +
  geom_hline(yintercept = 0, lty = 2, color = "grey", alpha = 0.9) +
  geom_vline(xintercept = 0, lty = 2, color = "grey", alpha = 0.9) +
  #geom_segment(data = pca.vars, aes(x = 0, xend = Dim.1, y = 0, yend = Dim.2),
  #arrow = arrow(length = unit(0.025, "npc"), type = "open"),lwd = 1) + 
  geom_text_repel(data = pca.vars,aes(x = Dim.1, y =  Dim.2,
                                label = vars, color = cluster), size = 3, box.padding = 0.25) +
  geom_point(data = pca.vars,aes(x = Dim.1, y =  Dim.2,
                                 color = cluster), size = 1.5)+
  scale_color_manual(values = cols)+
  xlab("PC1 (59.1%)") + 
  ylab("PC2 (14.5%)") +
  #labs(title = "Correlation between variables and PC")+
  coord_equal() +
  theme_classic() +
  theme(legend.position = "none")

##################
#run random forest in unsupervised mode:
#remember that computing time is ntree???mtry???(# objects)log(# objects) 
#Keep rownumbers below 10000 or you will run out of memory (in a system with 16 Gb RAM)
giffoni.scaled.frac <- sample_frac(giffoni.scaled[1:ncol(giffoni.scaled)], 0.8)
set.seed(123)
NumberOfCluster<- detectCores()
cl <- makeCluster(NumberOfCluster) #change the number of clusters you want to use in parallel
registerDoSNOW(cl)

giffoni.rf.fit <- foreach(ntree = rep(400,8), .combine = randomForest::combine,
                  .multicombine=TRUE, .packages = c("dplyr", "randomForest")) %dopar% {
                    randomForest(x = giffoni.scaled.frac, y = NULL, ntree = ntree, proximity = TRUE, oob.prox = TRUE, keep.forest = FALSE)
                  }
stopCluster(cl)
#alternative non parallelized#
#rf.fit <- randomForest(x = sample_frac(giffoni.scaled[1:ncol(giffoni.scaled)], 0.8), y = NULL, ntree = 2000, proximity = TRUE, oob.prox = TRUE, keep.forest = FALSE, do.trace = T)
###extract per-feature importance
giffoni.rf.importance <- setDT(data.frame(giffoni.rf.fit$importance), keep.rownames = TRUE)[]
#check per feature importance
giffoni.rf.importance$rn <- factor(giffoni.rf.importance$rn, levels = giffoni.rf.importance$rn[order(giffoni.rf.importance$MeanDecreaseGini)])
#order grouping variable based on Mean decreasae gini
group_rf <- as.data.frame(grp)
group_rf$var <- names(grp)
rownames(group_rf) <- 1:ncol(giffoni.scaled)
group_rf <- group_rf[order(group_rf$var),]
giffoni.rf.importance$rn <- as.character(giffoni.rf.importance$rn)
giffoni.rf.importance<-giffoni.rf.importance[order(giffoni.rf.importance$rn),]
giffoni.rf.importance$cluster <- as.factor(group_rf$grp)


####Plot of feature importance based on mean decrease gini keeping track of clusters
#identified coorrealtion distances (colors in plot)
giffoni.rf.importance <- giffoni.rf.importance[order(giffoni.rf.importance$MeanDecreaseGini),]
giffoni.rf.importance <- transform(giffoni.rf.importance, rn=reorder(rn, MeanDecreaseGini) ) 

imp_rf <- ggplot(data = giffoni.rf.importance, 
       aes(y = reorder(rn, MeanDecreaseGini) , x = MeanDecreaseGini,
           fill = factor(cluster))) + 
  geom_point(shape =21, size = (giffoni.rf.importance$MeanDecreaseGini/100)+seq(2,0.5,length.out = ncol(giffoni.scaled))) +
  xlab("Mean Decrease in Gini (RF)")+
  scale_x_continuous(breaks = c(0,100,200,300,400,500,600))+
  scale_fill_manual(breaks = c(1:10), labels = paste0(c(1:10)), name = "Correlation clusters", values = cols)+
  guides(fill = guide_legend(override.aes = list(size=2), ncol=2))+
  theme_classic()+theme(legend.position = c(0.15,0.75), legend.background = element_blank(), 
                        axis.title.y = element_blank(), legend.title = element_text(face = "bold"),
                        panel.grid.major.y = element_line(color = "gray80", linetype = "dashed"))

#for the visualization use the most important variable of the first three clusters found
#in correlation distances
plot3d(giffoni.scaled$IntDen, giffoni.scaled$Area, giffoni.scaled$Skew)
##############methods to find the best number of clusters#############
#keep only important features 
giffoni.retain <- giffoni.scaled.frac[,c("IntDen", "Area","Skew")]
######find the best number of clusters using NbClust 
#run the algorithm in parallel
NumberOfCluster<- detectCores()
cl <- makeCluster(NumberOfCluster)
registerDoSNOW(cl)

nb.giffoni <- foreach(i = 1:NumberOfCluster, .combine = "rbind", .packages = c("dplyr", "NbClust")) %dopar% {
  set.seed(i)
  giffoni.retain.sampled <- sample_frac(giffoni.retain, 1)
  giffoni.retain.res<-NbClust(giffoni.retain.sampled, distance = "manhattan", min.nc=2, max.nc=10, 
                      method = "kmeans", index = "all") 
}
stopCluster(cl)
check.res <- data.frame(nb.giffoni[,4])
summary(check.res)
#NbClust non parallelized
giffoni.sample.xNbC <- sample_frac(giffoni.retain, 0.5)
Nb.giffoni <- NbClust(giffoni.sample.xNbC, distance = "manhattan", min.nc=2, max.nc=10, 
                         method = "kmeans", index = c("all"))
giffoni.sample.xNbC$NbClusters <- Nb.giffoni$Best.partition
#plot example results
bestnum <- melt(Nb.giffoni$Best.nc[1,])
bestnum$index <- rownames(bestnum)
count <- bestnum %>%
  group_by(value) %>%
  summarise(counts = n())

count <- count %>%
  arrange(desc(value)) %>%
  mutate(prop = round(counts*100/sum(counts), 1),
         lab.ypos = cumsum(prop) - 0.5*prop)

#make a donut plot
count$ymax = cumsum(count$prop)
count$ymin = c(0, head(count$ymax, n=-1))
count$labelPosition <- (count$ymax + count$ymin) / 2
count$label <- paste0("NC ",count$value," - " , count$counts)

donut <- ggplot(count, aes(ymax=ymax, ymin=ymin, xmax=4, xmin=3, fill=factor(value))) +
  geom_rect(color = "gray20") +
  coord_polar(theta="y") + 
  xlim(c(0, 5))+
  scale_fill_brewer(palette="Paired") +
  scale_color_brewer(palette="Paired") +
  geom_label_repel(x = 4, aes(y = labelPosition, label = label, fill = factor(value)), size = 5, box.padding = 0.25, nudge_x = 1.5,)+
  geom_text(x = 0, y = 0, label = "Majority Rule", size = 6)+
  theme_void()+
  theme(legend.position = "none")
#verify clusters found by nbclust method
plot3d(giffoni.sample.xNbC$IntDen, giffoni.sample.xNbC$Area, giffoni.sample.xNbC$Skew, col =giffoni.sample.xNbC$NbClusters)
################clustering################
##############try clustering using other distance measures###############
graphics.off()
colnames(giffoni.scaled)
giffoni.scaled.dist <- dist(giffoni.scaled[,1:23], "manhattan")
#hierarchical clustering
giffoni.hclust <- fastcluster:::hclust(giffoni.scaled.dist, method = "ward.D2")
plot(giffoni.hclust, hang = -1) 
rect.hclust(giffoni.hclust , k = 3, border = 2:6)
giffoni.cluster.hc <-  cutree(giffoni.hclust, k=3) 
giffoni.scaled$cluster.hc <- as.factor(giffoni.cluster.hc)
#plot
ggplotly(ggplot(giffoni.scaled, aes(x = IntDen, y = Area, colour = cluster.hc)) + 
           geom_point(alpha = 1/4, pch = 1, size = 1)+ 
           theme_classic())

#####filter oultiers#######optional
#library(rrcovHD)
#giffoni.out <- OutlierMahdist(cluster.hc~.,data=giffoni.scaled[,c("IntDen","Area","Skew","cluster.hc")], control = "auto", trace = T)
#giffoni.out.flag <- getFlag(giffoni.out) 
#giffoni.out.flag[giffoni.out.flag == 0] <- NA
#X <- giffoni.out.flag
#X[!is.na(giffoni.out.flag)] <- giffoni.scaled$cluster.hc[!is.na(giffoni.out.flag)]
#X[is.na(X)] <- 0
#giffoni.scaled$cluster.hc.filter <- as.factor(X)
#plot
#ggplotly(ggplot(giffoni.scaled, aes(x = IntDen, y = Area, colour = cluster.hc.filter)) + 
#           geom_point(alpha = 1/4, pch = 1, size = 1)+ 
#           theme_classic())
#plot3d(giffoni.scaled$IntDen, giffoni.scaled$Area, giffoni.scaled$Skew, col =as.numeric(giffoni.scaled$cluster.hc.filter))
#giffoni.scaled <- within(giffoni.scaled, cluster.hc.filter.name <- factor(cluster.hc.filter, labels = c("outliers","dead","sterile","alive")))
plot3d(giffoni.scaled$IntDen, giffoni.scaled$Area, giffoni.scaled$Skew, col =giffoni.scaled$cluster.hc)
giffoni.scaled <- within(giffoni.scaled, cluster.hc.name <- factor(cluster.hc, labels = c("dead","sterile","alive")))