dir.create("data_output")
dir.create("fig_output")

library("BiocGenerics")
library("dada2")
library("phyloseq")
library("Biostrings")
library("ggplot2")
library("dplyr")
library("tidyr")
library("tibble")
library("readxl")
library("readr")
library("stringr")
library("kableExtra")
library("metacoder")
library("phangorn")
library("phyloseqGraphTest")
library("igraph")
library("ggnetwork")
library("vegan")
library("gridExtra")
library("metacoder")
library("microbiome")
library("DECIPHER")
library("DESeq2")
library("Rbowtie2")
library("Rsamtools")
library("ShortRead")
library("AnnotationDbi")
library("GenomicFeatures")
library("biomartr")
library("rtracklayer")
library("DESeq2")
library("TxDb.Hsapiens.UCSC.hg38.knownGene")
library("pvclust")
library("PoiClaClu")
library("ggplot2")
library("pheatmap")
library("BiocGenerics")
library("dada2")
library("phyloseq")
library("Biostrings")
library("ggplot2")
library("dplyr")
library("tidyr")
library("tibble")
library("readxl")
library("readr")
library("stringr")
library("kableExtra")
library("metacoder")
library("phangorn")
library("phyloseqGraphTest")
library("igraph")
library("ggnetwork")
library("vegan")
library("gridExtra")
library("metacoder")
library("microbiome")
library("DECIPHER")
library("DESeq2")
library("ShortRead")
library(MicrobiotaProcess)
library(ggtree)
library(patchwork)
library(decontam)
library(RColorBrewer)

################################################# LOAD DATA FROM EXPERIMENT #################################################
fastq_dir_before_decontam <- "fastq" 
list.files(fastq_dir_before_decontam)
fns_before_decontam <- sort(list.files(fastq_dir_before_decontam, full.names = TRUE))
fns_before_decontam <- fns_before_decontam[str_detect(basename(fns_before_decontam), ".fastq")]
fnFs_before_decontam <- fns_before_decontam[str_detect(basename(fns_before_decontam), "R1")] 
fnRs_before_decontam <- fns_before_decontam[str_detect(basename(fns_before_decontam), "R2")] 
sample_names <- str_split(basename(fnFs_before_decontam), pattern = "_", simplify = TRUE)[, 1]

################################################ Compute number of forward and reverse reads ###############################################
df1 <- data.frame()
df2 <- data.frame()
df<- data.frame()
for (i in 1:length(fnFs_before_decontam)) {
  geom <- fastq.geometry(fnFs_before_decontam[i])
  df_one_row <- data.frame(n_seq = geom[1], file_name = basename(fnFs_before_decontam[i]))
  df1 <- bind_rows(df1, df_one_row)
}
df1
rownames(df1) <- sample_names
colnames(df1) <- c("n_seq1","file_name1")
df1
dfout <- df1
dfout$file_name1 <- NULL
colnames(dfout) <- c("input_host_contaminated")
for (i in 1:length(fnRs_before_decontam)) {
  geom <- fastq.geometry(fnRs_before_decontam[i])
  df_second_row <- data.frame(n_seq = geom[1], file_name = basename(fnRs_before_decontam[i]))
  df2 <- bind_rows(df2, df_second_row)
}
df2
rownames(df2) <- sample_names
colnames(df2) <- c("n_seq2","file_name2")
df2
df <- merge(df1,df2,by="row.names")
df
write.table(df, file ="data_output/n_seq_before_decontam.txt", sep='\t', row.names = FALSE, na='',quote=FALSE)

################################################ Host decontamination ###############################################
for (i in 1:length(fnFs_before_decontam)) {
  read1 <- paste(fnFs_before_decontam[i])
  read2 <- paste(fnRs_before_decontam[i])
  file.exists(read1)
  if(file.exists(file.path("refs/hg38/hg38.1.bt2"))){
    (cmdout<-bowtie2(bt2Index = file.path("refs/hg38/hg38"),
                     samOutput = file.path(paste(sample_names[i],".sam",sep="")),
                     seq1=read1, seq2=read2,overwrite=TRUE,"--threads 16"))
    head(readLines(file.path(paste(sample_names[i],".sam",sep=""))))
  } 
  write.table(cmdout,file=paste(sample_names[i],".txt",sep=""))
}

#samtools
for A in ` ls | grep ".sam" `
do
name=`echo $A | sed 's/_.*//g'`
echo $name
samtools view -bS ${name} > ${name}.bam
samtools view -b -f 12 -F 256 ${name}.bam > ${name}_Unmapped.bam
samtools sort -n -m 5G -@ 2 ${name}_Unmapped.bam -o ${name}_Unmapped_sorted.bam
bedtools bamtofastq -i ${name}_Unmapped_sorted.bam -fq ${name}.R1.fq -fq2 ${name}.R2.fq
done

################################################# LOAD DATA FROM EXPERIMENT AFTER DECONTAMINATION #################################################
fastq_dir <- "fastq_decontam" 
list.files(fastq_dir)
fns <- sort(list.files(fastq_dir, full.names = TRUE))
fns <- fns[str_detect(basename(fns), ".fq")]
fnFs <- fns[str_detect(basename(fns), "R1")] 
fnRs <- fns[str_detect(basename(fns), "R2")] 
sample_names <- str_split(basename(fnFs), pattern = ".sam", simplify = TRUE)[, 1]

################################################ Compute number of forward and reverse reads again ###############################################
df1 <- data.frame()
df2 <- data.frame()
df <- data.frame()
for (i in 1:length(fnFs)) {
  geom <- fastq.geometry(fnFs[i])
  df_one_row <- data.frame(n_seq = geom[1], file_name = basename(fnFs[i]))
  df1 <- bind_rows(df1, df_one_row)
}
df1
rownames(df1) <- sample_names
colnames(df1) <- c("n_seq1","file_name1")
df1
for (i in 1:length(fnRs)) {
  geom <- fastq.geometry(fnRs[i])
  df_second_row <- data.frame(n_seq = geom[1], file_name = basename(fnRs[i]))
  df2 <- bind_rows(df2, df_second_row)
}
df2
rownames(df2) <- sample_names
colnames(df2) <- c("n_seq2","file_name2")
df2
df <- merge(df1,df2,by="row.names")
df
write.table(df, file ="data_output/n_seq_decontam.txt", sep='\t', row.names = FALSE, na='',quote=FALSE)
ggplot(df, aes(x = n_seq1)) + geom_histogram(alpha = 0.5, position = "identity", binwidth = 3000) # + xlim(5000, 20000)
ggsave("fig_output/distrib_decontam.png")

############################################## Inspect read quality profiles ###############################################
plotQualityProfile(fnFs[1:i])
ggsave("fig_output/Forward_quality.png")
plotQualityProfile(fnRs[1:i])
ggsave("fig_output/Reverse_quality.png")

################################################ FILTER and Trim the reads #################################################
filt_path <- "data_output/filtered/"
if(!file_test("-d", filt_path)) dir.create(filt_path)
filtFs <- file.path(filt_path, paste0(sample_names, "_F_filt.fastq.gz"))
filtRs <- file.path(filt_path, paste0(sample_names, "_R_filt.fastq.gz"))
out <- filterAndTrim(fnFs, filtFs, fnRs, filtRs,
                     truncLen=c(150,149),                  
                     trimLeft=c(0,0),                      
                     maxN=0,                               
                     maxEE=c(2,2),                         
                     truncQ=10,                            
                     rm.phix=TRUE,                         
                     compress=TRUE,                        
                     multithread=FALSE,                    
                     verbose=TRUE)                         
head(out)
rownames(out) <- sample_names
out2 <- merge(dfout,out,by="row.names")
head(out2)
row.names(out2) <- out2$Row.names
out2$Row.names <- NULL
write.table(out, file = "data_output/n_filter.txt", sep = "\t", row.names = TRUE, col.names = NA)
out %>% 
  data.frame() %>% 
  mutate(Samples = rownames(.),
         percent_kept = 100*(reads.out/reads.in)) %>%
  select(Samples, everything()) %>%
  summarise(min_remaining = paste0(round(min(percent_kept), 2), "%"), 
            median_remaining = paste0(round(median(percent_kept), 2), "%"),
            mean_remaining = paste0(round(mean(percent_kept), 2), "%"), 
            max_remaining = paste0(round(max(percent_kept), 2), "%"))

################################################ Learn the ERROR RATES ######################################################
loessErrfun_mod <- function(trans) {
  qq <- as.numeric(colnames(trans))
  est <- matrix(0, nrow=0, ncol=length(qq))
  for(nti in c("A","C","G","T")) {
    for(ntj in c("A","C","G","T")) {
      if(nti != ntj) {
        errs <- trans[paste0(nti,"2",ntj),]
        tot <- colSums(trans[paste0(nti,"2",c("A","C","G","T")),])
        rlogp <- log10((errs+1)/tot)  # 1 psuedocount for each err, but if tot=0 will give NA
        rlogp[is.infinite(rlogp)] <- NA
        df <- data.frame(q=qq, errs=errs, tot=tot, rlogp=rlogp)
        mod.lo <- loess(rlogp ~ q, df, weights = log10(tot),span = 2)
        
        pred <- predict(mod.lo, qq)
        maxrli <- max(which(!is.na(pred)))
        minrli <- min(which(!is.na(pred)))
        pred[seq_along(pred)>maxrli] <- pred[[maxrli]]
        pred[seq_along(pred)<minrli] <- pred[[minrli]]
        est <- rbind(est, 10^pred)
      } 
  MAX_ERROR_RATE <- 0.25
  MIN_ERROR_RATE <- 1e-7
  est[est>MAX_ERROR_RATE] <- MAX_ERROR_RATE
  est[est<MIN_ERROR_RATE] <- MIN_ERROR_RATE
  estorig <- est
  est <- est %>%
    data.frame() %>%
    mutate_all(funs(case_when(. < X40 ~ X40,
                              . >= X40 ~ .))) %>% as.matrix()
  rownames(est) <- rownames(estorig)
  colnames(est) <- colnames(estorig)
  err <- rbind(1-colSums(est[1:3,]), est[1:3,],
               est[4,], 1-colSums(est[4:6,]), est[5:6,],
               est[7:8,], 1-colSums(est[7:9,]), est[9,],
               est[10:12,], 1-colSums(est[10:12,]))
  rownames(err) <- paste0(rep(c("A","C","G","T"), each=4), "2", c("A","C","G","T"))
  colnames(err) <- colnames(trans)
  return(err)
}
errF <- learnErrors(
  filtFs,
  multithread = TRUE,
  errorEstimationFunction = loessErrfun_mod,
  verbose = TRUE
)
plotErrors(errF, nominalQ=TRUE)
ggsave(plot = last_plot(), filename = "fig_output/forward_error_rates.pdf",
       device = "pdf", width = 15, height = 15, scale = 1, units = "cm")
errR <- learnErrors(
  filtRs,
  multithread = TRUE,
  errorEstimationFunction = loessErrfun_mod,
  verbose = TRUE
)
plotErrors(errR, nominalQ=TRUE)
ggsave(plot = last_plot(), filename = "fig_output/reverse_error_rates.pdf",
       device = "pdf", width = 15, height = 15, scale = 1, units = "cm")

####################################### Sample INFERENCE and MERGER of paired-end reads #######################################
derep_F <- derepFastq(filtFs, verbose = TRUE)
derep_R <- derepFastq(filtRs, verbose = TRUE)
names(derep_F) <- sample_names
names(derep_R) <- sample_names
dada_F <- dada(derep_F, err = errF, multithread = FALSE, pool = FALSE)
dada_R <- dada(derep_R, err = errR, multithread = FALSE, pool = FALSE)
merger_beta <- mergePairs(dada_F, derep_F, dada_R, derep_R, verbose = TRUE, returnRejects=TRUE)
concat <- mergePairs(dada_F, derep_F, dada_R, derep_R, verbose = TRUE, justConcatenate=TRUE)
mergers <- merger_beta
for(i in seq_along(mergers)) {
  mergers[[i]][!mergers[[i]]$accept,] <- concat[[i]][!mergers[[i]]$accept,]
}
seqtab <- makeSequenceTable(mergers)
dim(seqtab)
saveRDS(seqtab, "data_output/seqtab.rds")
t_seqtab <- t(seqtab)
saveRDS(t_seqtab, "data_output/t_seqtab.rds")
len <- table(nchar(getSequences(seqtab)))
len
write.table(len, file = "data_output/length_distribution.txt", sep = "\t", row.names = TRUE, col.names = NA)

########################################### Remove CHIMERAS and assign TAXNOMOY ###########################################
seqtab.nochim <- removeBimeraDenovo(seqtab, method="consensus", multithread=TRUE, verbose=TRUE)
saveRDS(seqtab.nochim, "data_output/seqtab_nochim.rds")
paste0("% of non chimeras : ", sum(seqtab.nochim)/sum(seqtab) * 100)
paste0("total number of sequences : ", sum(seqtab.nochim))
getN <- function(x) sum(getUniques(x))
sapply()
track <- cbind(out2, sapply(dada_F, getN), sapply(dada_R, getN), sapply(mergers, getN), rowSums(seqtab.nochim))
colnames(track) <- c("all","input", "filtered", "denoisedF", "denoisedR", "merged", "nonchim")
rownames(track) <- sample_names
head(track)
write.table(track, file = "data_output/reads.txt", sep = "\t", row.names = TRUE, col.names = NA)
saveRDS(seqtab.nochim, "data_output/seqtab_final.rds")

#################################################### Give sequences a name ##############################################
seqtab.nochim_trans <- as.data.frame(t(seqtab.nochim)) %>% rownames_to_column(var = "sequence") %>% 
  rowid_to_column(var = "ASVNumber") %>% mutate(ASVNumber = sprintf("asv%04d", 
  ASVNumber)) %>% mutate(sequence = str_replace_all(sequence, "(-|\\.)", ""))
df <- seqtab.nochim_trans
seq_out <- Biostrings::DNAStringSet(df$sequence)
names(seq_out) <- df$ASVNumber
df
write.table(df, file = "data_output/seqtab_nochim.txt", sep = "\t", row.names = TRUE, col.names = NA)

##################################### Complete tracking of reads through the pipeline ##################################### 
track <- cbind(out2, sapply(dada_F, getN), sapply(dada_R, getN), sapply(mergers, getN), rowSums(seqtab.nochim))
colnames(track) <- c("all","input", "filtered", "denoisedF", "denoisedR", "merged", "nonchim")
rownames(track) <- sample_names
head(track)
track_pct <- track %>% 
  data.frame() %>%
  mutate(Sample = rownames(.),
         decontaminated_pct=ifelse(input == 0, 0, 100 * (input/all)),
         filtered_pct = ifelse(filtered == 0, 0, 100 * (filtered/all)),
         denoisedF_pct = ifelse(denoisedF == 0, 0, 100 * (denoisedF/all)),
         denoisedR_pct = ifelse(denoisedR == 0, 0, 100 * (denoisedR/all)),
         merged_pct = ifelse(merged == 0, 0, 100 * (merged/all)),
         nonchim_pct = ifelse(nonchim == 0, 0, 100 * (nonchim/all)),
         total_pct = ifelse(nonchim == 0, 0, 100 * nonchim/all)) %>%
  select(Sample, ends_with("_pct"))
head(track_pct)
track_pct_avg <- track_pct %>% summarize_at(vars(ends_with("_pct")), 
                                            list(avg = mean))
track_pct_med <- track_pct %>% summarize_at(vars(ends_with("_pct")), 
                                            list(avg = stats::median))
head(track_pct_avg)
head(track_pct_med)
track_plot <- track %>% 
  data.frame() %>%
  mutate(Sample = rownames(.)) %>%
  gather(key = "Step", value = "Reads", -Sample) %>%
  mutate(Step = factor(Step, 
                       levels = c("all","input", "filtered", "denoisedF", "denoisedR", "merged", "nonchim"))) %>%
  ggplot(aes(x = Step, y = Reads)) +
  geom_line(aes(group = Sample), alpha = 0.2) +
  geom_point(alpha = 0.5, position = position_jitter(width = 0)) + 
  stat_summary(fun.y = median, geom = "line", group = 1, color = "steelblue", size = 1, alpha = 0.5) +
  stat_summary(fun.y = median, geom = "point", group = 1, color = "steelblue", size = 2, alpha = 0.5) +
  stat_summary(fun.data = median_hilow, fun.args = list(conf.int = 0.5), 
               geom = "ribbon", group = 1, fill = "steelblue", alpha = 0.2) +
  geom_label(data = t(track_pct_avg[1:6]) %>% data.frame() %>% 
               rename(Percent = 1) %>%
               mutate(Step = c("input","filtered", "denoisedF", "denoisedR", "merged", "nonchim"),
                      Percent = paste(round(Percent, 2), "%")),
             aes(label = Percent), y = 1.1 * max(track[,2])) +
  geom_label(data = track_pct_avg[7] %>% data.frame() %>%
               rename(total = 1),
             aes(label = paste("Total\nRemaining:\n", round(track_pct_avg[1,7], 2), "%")), 
             y = mean(track[,7]), x = 8) +
  expand_limits(y = 1.1 * max(track[,2]), x = 9) +
  theme_classic()
track_plot
ggsave("fig_output/track_plot.png")

##################################### Assign taxonomy ##################################### 
tax <- assignTaxonomy(seqtab.nochim, "silva_nr99_v138_wSpecies_train_set.fa.gz", multithread=TRUE, verbose=TRUE)
saveRDS(tax, "data_output/tax_final.rds")
t_tax <- t(tax)
write.table(t_tax, file = "data_output/t_tax.txt", sep = "\t", row.names = TRUE, col.names = NA)
write.table(tax, file = "data_output/tax.txt", sep = "\t", row.names = TRUE, col.names = NA)
tax2 <- as.data.frame(tax)
tax2$sequence <- rownames(tax2)
rownames(tax2) <- NULL
total <- merge(tax2,df,by="sequence")
colnames(total)
total <- total %>% select(ASVNumber, sequence, Kingdom, Phylum,Class,Order,Family,Genus,Species, everything())
total <-total[order(total$ASVNumber),]
write.csv(total,file="data_output/final_table.csv")

############################################## Construct a phylogenetic tree ############################################
seqs <- getSequences(seqtab.nochim)
names(seqs) <- seqs 
alignment <- AlignSeqs(DNAStringSet(seqs), anchor=NA)
phang.align <- phyDat(as(alignment, "matrix"), type="DNA")
dm <- dist.ml(phang.align)
treeNJ <- NJ(dm) 
fit = pml(treeNJ, data=phang.align)
fitGTR <- update(fit, k=4, inv=0.2)
saveRDS(fitGTR, "data_output/tree.rds")

######################################### COMBINE DATA INTO PHYLOSEQ OBJECT ##############################################
samps <- read.csv(file = "samples.csv",
                  header = TRUE,
                  sep = ",",
                  row.names = 1
)
samps
samps$SampleName <- samps$Abb
phy <- phyloseq(tax_table(tax),
                sample_data(samps),
                otu_table(seqtab.nochim, taxa_are_rows = FALSE),
                phy_tree(fitGTR$tree))
saveRDS(phy, "data_output/phy.rds")

######################################### DECONTAM ##############################################
for(i in 1:nrow(samps))
{
if(samps$Group[i]=="voda") {samps$Sample_or_Control[i]="Control"}
else {samps$Sample_or_Control[i]="Sample"}
}

ps <- phyloseq(tax_table(tax),
               sample_data(samps),
               otu_table(seqtab.nochim, taxa_are_rows = FALSE),
               phy_tree(fitGTR$tree))


df <- as.data.frame(sample_data(ps)) 
df$LibrarySize <- sample_sums(ps)
df <- df[order(df$LibrarySize),]
df$Index <- seq(nrow(df))
ggplot(data=df, aes(x=Index, y=LibrarySize, color=Sample_or_Control)) + geom_point()
sample_data(ps)$is.neg <- sample_data(ps)$Sample_or_Control == "Control"
contamdf.prev05 <- isContaminant(ps, method="prevalence", neg="is.neg", threshold=0.5)
table(contamdf.prev05$contaminant)
row_indices <- which(contamdf.prev05$contaminant)
contaminants<- prune_taxa(contamdf.prev05$contaminant, ps)
contaminants
tax.contaminants <- tax_table(contaminants)
write.table(tax.contaminants, file = "data_output/contaminants_tax05.txt", sep = "\t", row.names = TRUE, col.names = NA)
final_biom <- prune_taxa(!contamdf.prev01$contaminant, ps)
final_biom
decontam.seqtab <- otu_table(final_biom)
decontam.seqtab_trans <- as.data.frame(t(decontam.seqtab)) %>% rownames_to_column(var = "sequence") %>% 
  rowid_to_column(var = "ASVNumber") %>% mutate(ASVNumber = sprintf("asv%04d", 
                                                                    ASVNumber)) %>% mutate(sequence = str_replace_all(sequence, "(-|\\.)", ""))
names(seq_out) <- df$ASVNumber
write.table(decontam.seqtab_trans, file = "data_output/decontam_seqtab.txt", sep = "\t", row.names = TRUE, col.names = NA)
tax.decontam <- tax_table(final_biom)
write.table(tax.decontam, file = "data_output/deconatm_tax.txt", sep = "\t", row.names = TRUE, col.names = NA)

tax.decontam2 <- as.data.frame(tax.decontam)
tax.decontam2$sequence <- rownames(tax.decontam2)
rownames(tax.decontam2) <- NULL
total_decontam <- merge(tax.decontam2,decontam.seqtab_trans,by="sequence")
colnames(total_decontam)
total_decontam$sequence <- NULL
total_decontam <- total_decontam %>% select(ASVNumber, Kingdom, Phylum,Class,Order,Family,Genus,Species, everything())
total_decontam <-total_decontam[order(total_decontam$ASVNumber),]
head(total_decontam)
write.csv(total_decontam,file="data_output/decontam_final_table.csv")

################################################### Start with PHYLOSEQ ###################################################
theme_set(theme_bw())
dna <- Biostrings::DNAStringSet(taxa_names(phy))
names(dna) <- taxa_names(phy)
phy <- merge_phyloseq(phy, dna)
taxa_names(phy) <- paste0("ASV", seq(ntaxa(phy)))
phy
phy <- subset_taxa(phy, !is.na(Phylum) & !Phylum %in% c("", "uncharacterized"))
phy <- subset_taxa(phy, !is.na(Kingdom) & !Kingdom%in% c("Archaea"))
plot_bar(phy, x = "SampleName", fill = "Kingdom") 
phy <- subset_samples(phy, Type != "voda")
sample_names(phy)


################################################## RAREFICATION ##################################################
otu_tab <- t(abundances(phy))
vegan::rarecurve(otu_tab, step = 50, label = FALSE, sample = min(rowSums(otu_tab), col = "blue", cex = 0.6), at = seq(1000, 50000, by = 1000))
phy.rare<- rarefy_even_depth(phy, rngseed = 1337, sample.size = 9000, replace = FALSE)
rareres <- get_rarecurve(obj=phy.rare, chunks=1000)
prare1 <- ggrarecurve(obj=rareres, factorNames="Run",
                      indexNames=c("Observe")) +
  scale_fill_manual(values=c("#00AED7", "#FD9347","#ACDF87"),breaks=c("V3","Old","New"), labels=c("V4","V1-V2","V1-V2M" ), name = "Primers")+
  scale_color_manual(values=c("#00AED7", "#FD9347","#ACDF87"),breaks=c("V3","Old","New"), labels=c("V4","V1-V2","V1-V2M") , name = "Primers")+
  ylab("Observe") +
  theme_bw()+
  theme(axis.text=element_text(size=20), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=20)) 
prare1
saveRDS(phy.rare,"data_output/phy.rare.rds")
phy.rare<- readRDS("data_output/phy.rare.rds")
rare.seqtab <- otu_table(phy.rare)
rare.seqtab_trans <- as.data.frame(t(rare.seqtab)) %>% rownames_to_column(var = "sequence") %>% 
  rowid_to_column(var = "ASVNumber") %>% mutate(ASVNumber = sprintf("asv%04d", 
                                                                    ASVNumber)) %>% mutate(sequence = str_replace_all(sequence, "(-|\\.)", ""))

names(dna) <- df$ASVNumber
write.table(rare.seqtab_trans, file = "data_output/rarefied_seqtab.txt", sep = "\t", row.names = TRUE, col.names = NA)
tax.rare <- tax_table(phy.rare)
write.table(tax.rare, file = "data_output/rarefied_tax.txt", sep = "\t", row.names = TRUE, col.names = NA)

tax.rare2 <- as.data.frame(tax.rare)
tax.rare2$sequenceid <- rownames(tax.rare2)
ttrseq <- as.data.frame(phy.rare@refseq)
ttrseq$sequenceid <- rownames(ttrseq)
rownames(tax.rare2) <- NULL
rownames(ttrseq) <- NULL
total_rare_seq <- merge(tax.rare2,ttrseq,by="sequenceid")
total_rare_seq$sequence <- total_rare_seq$sequenceid
total_rare_seq$sequenceid <- NULL
total_rare <- merge(total_rare_seq,rare.seqtab_trans,by="sequence")
colnames(total_rare)
head(total_rare)
total_rare$sequenceid <- NULL
total_rare$sequence <- total_rare$x
total_rare$x <- NULL
total_rare <- total_rare %>% select(ASVNumber, sequence, Kingdom, Phylum,Class,Order,Family,Genus,Species, everything())
total_rare <-total_rare[order(total_rare$ASVNumber),]
rownames(total_rare) <- NULL
head(total_rare)
write_csv(total_rare,file="data_output/final_table_rare_sequences.csv")

################################################## COMPARISON ##################################################
phy.rare_J <- subset_samples(phy.rare, Group2=="J")
Primers_order <- c("V4","V1-V2","V1-V2M")
Alpha_order <-c("Observe","Chao1","Shannon") 
Genus_order <- c("Actinobacillus","Campylobacter","Fusobacterium","Gemella","Haemophilus","Helicobacter","Neisseria","Porphyromonas","Prevotella","Rothia","Streptococcus","Veillonella","Other")
Phylum_order <- c('Actinobacteriota', 'Bacteroidota', 'Campilobacterota',"Cyanobacteria","Firmicutes","Fusobacteriota","Patescibacteria","Proteobacteria","Spirochaetota","Other")

alphaobj_J <- get_alphaindex(phy.rare_J)
alphaobj_J@alpha[["J"]] <- NULL
alphaobj_J@alpha[["ACE"]] <- NULL
alphaobj_J@alpha[["Simpson"]] <- NULL
jicen_alpha <- ggbox(alphaobj_J, geom="violin", factorNames="Primers", signifmap=TRUE, p_textsize=7) +
  theme_bw()+
  scale_fill_brewer(palette="Paired")+
  theme(axis.text=element_text(size=20), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_text(face="bold"),
        text=element_text(size=20),
        legend.position="none",
        axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))
jicen_alpha$data$feature <- factor(jicen_alpha$data$feature, levels = Alpha_order)
jicen_alpha$data$Primers <- factor(jicen_alpha$data$Primers, levels = Primers_order)

phy.rare_J_Run <- merge_samples(phy.rare_J, "Run")
phy.rare_J_Run@sam_data[["Run"]] <- c("New","Old","V3")
phy.rare_J_Run@sam_data[["Primers"]] <- c("V1-V2M","V1-V2","V4")
phy.prop_J_Run <- transform_sample_counts(phy.rare_J_Run, function(otu) otu/sum(otu))
phylumGlommed_J_Run <-  tax_glom(phy.prop_J_Run, "Phylum", NArm = FALSE)
phylumGlommed_J_Run_tmp <- get_top_taxa(phylumGlommed_J_Run, n = 9, relative = TRUE, discard_other = FALSE, other_label = "Other")
jicen_bar <- plot_bar(phylumGlommed_J_Run_tmp, x = "Primers", fill = "Phylum") +
  theme_bw()+ 
  facet_wrap(~Patient, nrow=1) +
  scale_fill_manual(values=phylum_colors)+ 
  labs(x = NULL) +
  theme(axis.text=element_text(size=20), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=20),
        axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))
jicen_bar$data$Primers <- factor(jicen_bar$data$Primers, levels = Primers_order)
jicen_bar$data$Phylum<- factor(jicen_bar$data$Phylum, levels = Phylum_order)

phy.rare_V4_V1V2M <- subset_samples(phy.rare, Run!="Old")
phy.rare_V4_V1V2M_J <- subset_samples(phy.rare_V4_V1V2M, Group2=="J")
phy.rare_V4_V1V2M_G <- subset_samples(phy.rare_V4_V1V2M, Group2=="G")
phy.rare_V4_V1V2M_D <- subset_samples(phy.rare_V4_V1V2M, Group2=="D")

my_paired_palette_custom <- c(brewer.pal(n=12,"Paired"),"#808080")
genus_colors  <- 
  tibble(genus = c("Actinobacillus","Campylobacter","Fusobacterium","Gemella","Haemophilus","Helicobacter","Neisseria","Porphyromonas","Prevotella","Rothia","Streptococcus","Veillonella","Other")) %>%
  mutate(color = my_paired_palette_custom)
genus_colors <-as.data.frame(genus_colors) 
genus_colors <- setNames(genus_colors$color, as.character(genus_colors$genus))

my_paired_palette_custom_phylum <- c(brewer.pal(n=9,"Paired"),"#808080")
phylum_colors  <- 
  tibble(genus = c('Actinobacteriota', 'Bacteroidota', 'Campilobacterota',"Cyanobacteria","Firmicutes","Fusobacteriota","Patescibacteria","Proteobacteria","Spirochaetota","Other")) %>%
  mutate(color = my_paired_palette_custom_phylum)
phylum_colors <-as.data.frame(phylum_colors) 
phylum_colors <- setNames(phylum_colors$color, as.character(phylum_colors$genus))

alphaobj_J_sum <- get_alphaindex(phy.rare_V4_V1V2M_J)
alphaobj_J_sum@alpha[["J"]] <- NULL
alphaobj_J_sum@alpha[["ACE"]] <- NULL
alphaobj_J_sum@alpha[["Simpson"]] <- NULL
jicen_alpha_sum <- ggbox(alphaobj_J_sum, geom="violin", factorNames="Primers", signifmap=TRUE, p_textsize=4) +
  theme_bw()+
  ggtitle("Esophagus")+
  scale_fill_manual(values=c("#A6CEE3","#1F78B4"))+
  theme(axis.text=element_text(size=7), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_text(size=7 ,face="bold"),
        text=element_text(size=7),
        legend.position="none",
        axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1),
        plot.title = element_text(size=12, face="bold"))
jicen_alpha_sum$data$feature <- factor(jicen_alpha_sum$data$feature, levels = Alpha_order)
jicen_alpha_sum$data$Primers <- factor(jicen_alpha_sum$data$Primers, levels = Primers_order)

phy.rare_J_sum_Run <- merge_samples(phy.rare_V4_V1V2M_J, "Run")
sample_data(phy.rare_J_sum_Run )
phy.rare_J_sum_Run@sam_data[["Run"]] <- c("New","V3")
phy.rare_J_sum_Run@sam_data[["Primers"]] <- c("V1-V2M","V4")
phy.prop_J_sum_Run <- transform_sample_counts(phy.rare_J_sum_Run, function(otu) otu/sum(otu))
phylumGlommed_J_sum_Run <-  tax_glom(phy.prop_J_sum_Run, "Phylum", NArm = FALSE)
phylumGlommed_J_sum_Run_tmp <- get_top_taxa(phylumGlommed_J_sum_Run, n = 9, relative = TRUE, discard_other = FALSE, other_label = "Other")
tax_table(phylumGlommed_J_sum_Run_tmp )
jicen_bar_sum <- plot_bar(phylumGlommed_J_sum_Run_tmp, x = "Primers", fill = "Phylum") +
  theme_bw()+ 
  facet_wrap(~Patient, nrow=1) +
  scale_fill_manual(values=phylum_colors)+ 
  labs(x = NULL) +
  guides(fill=guide_legend(ncol=2)) +
  theme(axis.text=element_text(size=10), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=10))
jicen_bar_sum$data$Primers <- factor(jicen_bar_sum$data$Primers, levels = Primers_order)
jicen_bar_sum$data$Phylum<- factor(jicen_bar_sum$data$Phylum, levels = Phylum_order)

genusGlommed_J_sum_Run <-  tax_glom(phy.prop_J_sum_Run, "Genus", NArm = FALSE)
genusGlommed_J_sum_Run_tmp <- get_top_taxa(genusGlommed_J_sum_Run, n = 9, relative = TRUE, discard_other = FALSE, other_label = "Other")
tax_table(genusGlommed_J_sum_Run_tmp )
jicen_bar_sum_genus <- plot_bar(genusGlommed_J_sum_Run_tmp, x = "Primers", fill ="Genus") +
  theme_bw()+ 
  facet_wrap(~Patient, nrow=1) +
  scale_fill_manual(values = genus_colors)+
  labs(x = NULL)+
  guides(fill=guide_legend(ncol=2)) +
  theme(axis.text=element_text(size=10), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=10))
jicen_bar_sum_genus$data$Primers <- factor(jicen_bar_sum_genus$data$Primers, levels = Primers_order)
jicen_bar_sum_genus$data$Genus<- factor(jicen_bar_sum_genus$data$Genus, levels = Genus_order)

phy.ord_J <- ordinate(phy.rare_V4_V1V2M_J, "PCoA", "jaccard")
jicen_beta <- plot_ordination(phy.rare_V4_V1V2M_J, phy.ord_J, color="Primers")+
  theme_bw()+
  scale_fill_manual(values=c("#A6CEE3","#1F78B4"))+
  scale_color_manual(values=c("#A6CEE3","#1F78B4"))+
  geom_point(size = 3)+
  stat_ellipse(aes(fill=Primers), alpha=.2,type='norm',size =1, geom="polygon")+
  xlim(-1.5, 1.5)+
  ylim(-1.5, 1.5)+
  theme(legend.position="bottom")+
  theme(legend.title=element_blank())+
  theme(panel.grid.major = element_line(colour = "gray85"), 
        panel.grid.minor = element_blank(),text=element_text(size=10), panel.background = element_blank(), axis.text.y = element_text(colour = 'black'), axis.text.x = element_text(colour = 'black'), axis.line = element_line(colour = 'black'), axis.ticks = element_line(colour = 'black'))
jicen_beta$data$Primers <- factor(jicen_beta$data$Primers, levels = Primers_order)

alphaobj_G <- get_alphaindex(phy.rare_V4_V1V2M_G)
alphaobj_G@alpha[["J"]] <- NULL
alphaobj_G@alpha[["ACE"]] <- NULL
alphaobj_G@alpha[["Simpson"]] <- NULL
gastrum_alpha<- ggbox(alphaobj_G, geom="violin", factorNames="Primers", signifmap=TRUE, p_textsize=2) +
  theme_bw()+
  ggtitle("Stomach")+
  scale_fill_manual(values=c("#B2DF8A","#33A02C"))+
  theme(axis.text=element_text(size=7), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_text(size=7,face="bold"),
        text=element_text(size=7),
        legend.position="none",
        axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1),
        plot.title = element_text(size=12, face="bold"))
gastrum_alpha$data$feature <- factor(gastrum_alpha$data$feature, levels = Alpha_order)
gastrum_alpha$data$Primers <- factor(gastrum_alpha$data$Primers, levels = Primers_order)

phy.rare_G_Run <- merge_samples(phy.rare_V4_V1V2M_G, "Run")
sample_data(phy.rare_G_Run )
phy.rare_G_Run@sam_data[["Run"]] <- c("New","V3")
phy.rare_G_Run@sam_data[["Primers"]] <- c("V1-V2M","V4")
phy.prop_G_Run <- transform_sample_counts(phy.rare_G_Run, function(otu) otu/sum(otu))
phylumGlommed_G_Run <-  tax_glom(phy.prop_G_Run, "Phylum", NArm = FALSE)
phylumGlommed_G_Run_tmp <- get_top_taxa(phylumGlommed_G_Run, n = 9, relative = TRUE, discard_other = FALSE, other_label = "Other")
tax_table(phylumGlommed_G_Run_tmp )
gastrum_bar <- plot_bar(phylumGlommed_G_Run_tmp, x = "Primers", fill = "Phylum") +
  theme_bw()+ 
  facet_wrap(~Patient, nrow=1) +
  scale_fill_manual(values=phylum_colors)+ 
  labs(x = NULL)+
  guides(fill=guide_legend(ncol=2)) +
  theme(axis.text=element_text(size=10), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=10))
gastrum_bar$data$Primers <- factor(gastrum_bar$data$Primers, levels = Primers_order)
gastrum_bar$data$Phylum<- factor(gastrum_bar$data$Phylum, levels = Phylum_order)

genusGlommed_G_Run <-  tax_glom(phy.prop_G_Run, "Genus", NArm = FALSE)
genusGlommed_G_Run_tmp <- get_top_taxa(genusGlommed_G_Run, n = 9, relative = TRUE, discard_other = FALSE, other_label = "Other")
tax_table(genusGlommed_G_Run_tmp )
gastrum_bar_genus <- plot_bar(genusGlommed_G_Run_tmp, x = "Primers", fill = "Genus") +
  theme_bw()+ 
  facet_wrap(~Patient, nrow=1) +
  scale_fill_manual(values = genus_colors)+
  labs(x = NULL)+
  guides(fill=guide_legend(ncol=2)) +
  theme(axis.text=element_text(size=10), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=10))
gastrum_bar_genus$data$Primers <- factor(gastrum_bar_genus$data$Primers, levels = Primers_order)
gastrum_bar_genus$data$Genus<- factor(gastrum_bar_genus$data$Genus, levels = Genus_order)

phy.ord_G <- ordinate(phy.rare_V4_V1V2M_G, "PCoA", "jaccard")
gastrum_beta <- plot_ordination(phy.rare_V4_V1V2M_G, phy.ord_G, color="Primers")+
  theme_bw()+
  scale_fill_manual(values=c("#B2DF8A","#33A02C"))+
  scale_color_manual(values=c("#B2DF8A","#33A02C"))+
  geom_point(size = 3)+
  stat_ellipse(aes(fill=Primers), alpha=.2,type='norm',size =1, geom="polygon")+
  theme(legend.position="bottom")+
  theme(legend.title=element_blank())+
  xlim(-1.5, 1.5)+
  ylim(-1.5, 1.5)+
  theme(panel.grid.major = element_line(colour = "gray85"), 
        panel.grid.minor = element_blank(),text=element_text(size=10), panel.background = element_blank(), axis.text.y = element_text(colour = 'black'), axis.text.x = element_text(colour = 'black'), axis.line = element_line(colour = 'black'), axis.ticks = element_line(colour = 'black'))
gastrum_beta$data$Primers <- factor(gastrum_beta$data$Primers, levels = Primers_order)

alphaobj_D <- get_alphaindex(phy.rare_V4_V1V2M_D)
alphaobj_D@alpha[["J"]] <- NULL
alphaobj_D@alpha[["ACE"]] <- NULL
alphaobj_D@alpha[["Simpson"]] <- NULL

duodenum_alpha<- ggbox(alphaobj_D, geom="violin", factorNames="Primers", signifmap=TRUE, p_textsize=4) +
  theme_bw()+
  scale_fill_manual(values=c("#FB9A99","#E31A1C"))+
  ggtitle("Duodenum")+
  theme(axis.text=element_text(size=7), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_text(size=7,face="bold"),
        text=element_text(size=7),
        legend.position="none",
        axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1),
        plot.title = element_text(size=12, face="bold")) 
duodenum_alpha$data$feature <- factor(duodenum_alpha$data$feature, levels = Alpha_order)
duodenum_alpha$data$Primers <- factor(duodenum_alpha$data$Primers, levels = Primers_order)

phy.rare_D_Run <- merge_samples(phy.rare_V4_V1V2M_D, "Run")
sample_data(phy.rare_D_Run )
phy.rare_D_Run@sam_data[["Run"]] <- c("New","V3")
phy.rare_D_Run@sam_data[["Primers"]] <- c("V1-V2M","V4")
phy.prop_D_Run <- transform_sample_counts(phy.rare_D_Run, function(otu) otu/sum(otu))
phylumGlommed_D_Run <-  tax_glom(phy.prop_D_Run, "Phylum", NArm = FALSE)
phylumGlommed_D_Run_tmp <- get_top_taxa(phylumGlommed_D_Run, n = 9, relative = TRUE, discard_other = FALSE, other_label = "Other")
tax_table(phylumGlommed_D_Run_tmp )
duodenum_bar <- plot_bar(phylumGlommed_D_Run_tmp, x = "Primers", fill = "Phylum") +
  theme_bw()+ 
  facet_wrap(~Patient, nrow=1) +
  scale_fill_manual(values=phylum_colors)+ 
  labs(x = NULL) +
  guides(fill=guide_legend(ncol=2)) +
  theme(axis.text=element_text(size=10), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=10))
duodenum_bar$data$Primers <- factor(duodenum_bar$data$Primers, levels = Primers_order)
duodenum_bar$data$Phylum<- factor(duodenum_bar$data$Phylum, levels = Phylum_order)


genusGlommed_D_Run <-  tax_glom(phy.prop_D_Run, "Genus", NArm = FALSE)
genusGlommed_D_Run_tmp <- get_top_taxa(genusGlommed_D_Run, n = 9, relative = TRUE, discard_other = FALSE, other_label = "Other")
tax_table(genusGlommed_D_Run_tmp )
duodenum_bar_genus <- plot_bar(genusGlommed_D_Run_tmp, x = "Primers", fill = "Genus") +
  theme_bw()+ 
  facet_wrap(~Patient, nrow=1) +
  scale_fill_manual(values = genus_colors)+
  labs(x = NULL) +
  guides(fill=guide_legend(ncol=2)) +
  theme(axis.text=element_text(size=10), 
        panel.grid=element_blank(),
        strip.background = element_blank(),
        strip.text.x = element_blank(),
        text=element_text(size=10))
duodenum_bar_genus$data$Primers <- factor(duodenum_bar_genus$data$Primers, levels = Primers_order)
duodenum_bar_genus$data$Genus<- factor(duodenum_bar_genus$data$Genus, levels = Genus_order)

phy.ord_D <- ordinate(phy.rare_V4_V1V2M_D, "PCoA", "jaccard")
duodenum_beta <- plot_ordination(phy.rare_V4_V1V2M_D, phy.ord_D, color="Primers")+
  theme_bw()+
  scale_fill_manual(values=c("#FB9A99","#E31A1C"))+
  scale_color_manual(values=c("#FB9A99","#E31A1C"))+
  geom_point(size = 3)+
  stat_ellipse(aes(fill=Primers), alpha=.2,type='norm',size =1, geom="polygon")+
  theme(legend.position="bottom")+
  theme(legend.title=element_blank())+
  xlim(-1.5, 1.5)+
  ylim(-1.5, 1.5)+
  theme(panel.grid.major = element_line(colour = "gray85"), 
        panel.grid.minor = element_blank(),text=element_text(size=10), panel.background = element_blank(), axis.text.y = element_text(colour = 'black'), axis.text.x = element_text(colour = 'black'), axis.line = element_line(colour = 'black'), axis.ticks = element_line(colour = 'black'))
duodenum_beta$data$Primers <- factor(duodenum_beta$data$Primers, levels = Primers_order)


(jicen_alpha_sum + gastrum_alpha + duodenum_alpha) /
  (jicen_bar_sum + gastrum_bar + duodenum_bar + plot_layout(guides = "collect"))/
  (jicen_bar_sum_genus + gastrum_bar_genus + duodenum_bar_genus + plot_layout(guides = "collect"))/
  (jicen_beta + gastrum_beta + duodenum_beta) + plot_layout(nrow = 4,heights = c(4,3, 3, 2))


################################################## STATISTICAL ANALYSIS ##################################################
metadata <- data.frame(sample_data(phy.rare_V4_V1V2M_J))
permanova_pairwise_J <- adonis2(phyloseq::distance(phy.rare_V4_V1V2M_J, method="jaccard", binary = TRUE) ~ Primers, data = metadata)
write.csv(permanova_pairwise_J ,"data_output/permanova_J.csv")

metadata <- data.frame(sample_data(phy.rare_V4_V1V2M_D))
permanova_pairwise_D <- adonis2(phyloseq::distance(phy.rare_V4_V1V2M_D, method="jaccard", binary = TRUE) ~ Primers, data = metadata)
write.csv(permanova_pairwise_D ,"data_output/permanova_D.csv")

metadata <- data.frame(sample_data(phy.rare_V4_V1V2M_G))
permanova_pairwise_G <- adonis2(phyloseq::distance(phy.rare_V4_V1V2M_G, method="jaccard", binary = TRUE) ~ Primers, data = metadata)
write.csv(permanova_pairwise_G ,"data_output/permanova_G.csv")


sample_data(phy.rare_V4_V1V2M)$Primers <- as.factor(sample_data(phy.rare_V4_V1V2M)$Primers)
ps.taxa <- tax_glom(phy.rare_V4_V1V2M, taxrank = "genus", NArm = TRUE)
ps.taxa.pse <- ps.taxa
otu_table(ps.taxa.pse) <- otu_table(ps.taxa)+1
ds <- phyloseq_to_deseq2(ps.taxa.pse, ~ Primers)
ds$Primers <- relevel(ds$Primers, ref = "V4")
ds <- DESeq(ds, test="Wald",fitType = "parametric")
res = results(ds, cooksCutoff = FALSE)
head(res)
write.csv(res, file="data_output/V1-V2M_V4_genus.csv")
alpha = 0.05
sigtab = res[which(res$padj < alpha), ]
sigtab = cbind(as(sigtab, "data.frame"), as(tax_table(ps.taxa.pse)[rownames(sigtab), ], "matrix"))
head(sigtab)
dim(sigtab)
write.csv(sigtab, file="data_output/sig_V1-V2M_V4_genus.csv")