library("metabaR")
library("ggplot2")
library("tidyr")
library("reshape2")
library("kableExtra")
library("ggpubr")


## 1.0 Data import -----

## Note: the following code was adapted from a metabaR tutorial on github: 
## https://github.com/metabaRfactory/metabaR/blob/master/vignettes/metabaRF-vignette.Rmd

inverts <- tabfiles_to_metabarlist(file_reads = "Appendix3A_ReadTable.csv",
                                   file_motus = "Appendix3B_OTU_taxonomy.csv",
                                   file_pcrs = "Appendix3C_PCRs.csv",
                                   file_samples = "Appendix3D_SampleMetadata.csv",
                                   files_sep = ",")

inverts_samples <- subset_metabarlist(inverts,
                                      table = "pcrs",
                                      indices =  is.na(inverts$pcrs$control_type))

inverts_neg <- subset_metabarlist(inverts,
                                      table = "pcrs",
                                      indices =  !is.na(inverts$pcrs$control_type))


summary_metabarlist(inverts)
summary_metabarlist(inverts_samples) ## summarizing reads and OTUs in samples
summary_metabarlist(inverts_neg) ## summarizing reads and OTUs in neg controls


### 2.0 Dataset stats & visualization ---- 

## number of sequence reads per pcr
inverts$pcrs$nb_reads <- rowSums(inverts$reads)

##number of motus (richness) per pcr
inverts$pcrs$nb_motus <- rowSums(inverts$reads>0)

tibble(inverts$pcrs) ## data check

## 2.1 Checking reads/OTUs in negative controls ----

check1 <- melt(inverts$pcrs[,c("control_type", "nb_reads", "nb_motus")])
check1$Type <- check1$control_type %>% replace_na("samples")

check1$Type = factor(check1$Type, levels = c("extraction", "pcr", "sequencing", "samples"))
labels <- c(nb_reads = "Sequence Reads", nb_motus = "OTUs")

ggplot(data <- check1, aes(x=Type, y=value, color=Type)) + 
  geom_boxplot() + theme_classic(14) + 
  geom_jitter(alpha=0.2) + 
  scale_color_manual(values = c("steelblue", "orange", "darkred", "black")) +
  facet_wrap(~variable, labeller = labeller(variable = labels), scales = "free_y") + 
  theme(axis.text.x = element_text(angle=45, h=1)) +
  ylab("Total reads/OTUs") 

## the negative controls have few sequences & OTUs compared to samples (good! low contamination!)

##ggsave("Appendix_plot1_controls.jpg", plot = last_plot())

## 2.2 Assessing sequencing depth ----

## number of reads vs. number of OTUs for controls and samples, lack of correlation 
## between reads and OTUs in samples suggests sequencing depth is sufficient for coverage

inverts$pcrs$Type <- inverts$pcrs$control_type
inverts$pcrs$Type <- inverts$pcrs$Type %>% replace_na("samples")
inverts$pcrs$Type <- factor(inverts$pcrs$Type, levels = c("extraction", "pcr", "sequencing", "samples"))

ggplot(inverts$pcrs, aes(x=nb_reads, y=nb_motus, color = Type)) + 
  geom_point(alpha = 0.5) + theme_classic(14) + 
  scale_y_log10() + scale_x_log10() + 
  scale_color_manual(values = c("steelblue", "orange", "darkred", "black")) +
  xlab("Sequence reads") +
  ylab("OTUs")

##ggsave("Appendix_plot2_controls.jpg", plot = last_plot())

## 2.3 Visualization of reads in samples and controls in PCR plates ----

inverts$pcrs$plate_no <- factor(inverts$pcrs$plate_no, levels = c("M2019", "J2019", "S2019", "redo"))
labels = c(M2019 = "Plate 1 - May", J2019 = "Plate 2 - July", S2019 = "Plate 3 - Sept", redo = "Plate 4 - Redo")

ggpcrplate(inverts, FUN = function(m){rowSums(m$reads)}, legend_title = "# of reads per PCR") +
  theme_bw(14) +
  scale_fill_manual(values = c("steelblue", "orange", "darkred"), na.translate = F) +
  facet_wrap(~plate_no, labeller = labeller(plate_no = labels), scales = "free_y") +
  guides(fill = guide_legend(title="Control Type"))
## can see low reads in most controls (good!) 

##ggsave("Appendix_plot3_controls.jpg", plot = last_plot())

## 2.4 Rarefaction curves for sequencing depth ----

inverts.raref = hill_rarefaction(inverts, nboot = 20, nsteps = 10)
head(inverts.raref$hill_table)

labels = c(D0 = "Species richness", D1 = "Shannon index", D2 = "Inverted Simpson's index", coverage = "Good's coverage index")

gghill_rarefaction(inverts.raref) +
  geom_ribbon(aes(ymin = .data$value - .data$value.sd,
                  ymax = .data$value + .data$value.sd), alpha = 0.1) +
  xlab("Sequence reads") + 
  ylab("Diversity/Coverage index estimate") +
  facet_wrap(~.data$variable, labeller = labeller(variable = labels), ncol = 4, scales = "free_y")

## everything flattens out indicating sequencing depth was sufficient

##ggsave("Appendix_plot4_sequencingdepth_raref.jpg", plot = last_plot())


## 3.0 FLAGGING SPURIOUS SIGNAL ----

## 3.1 Flagging contaminants ----

## First must subset dataset into plates to evaluate respective neg controls

## 3.1.1 May subset ----

inverts_may <- subset_metabarlist(inverts, 
                                  table = "pcrs",
                                  indices = inverts$pcrs$plate_no == "M2019")
                               

summary_metabarlist(inverts_may) ## data check

inverts_may <- contaslayer(inverts_may,
                       control_types = c("pcr", "sequencing", "extraction"),
                       output_col = "not_a_conta")


##OTU flagged as a contaminant (FALSE in not_a_conta) if relative abundance of OTU across whole dataset is highest in neg controls

table(inverts_may$motus$not_a_conta) ## only 1 OTU flagged as a contaminant!

dt <- inverts_may$motus[!inverts_may$motus$not_a_conta,
                    c("seq_count", "occurrence_count", "Similarity", "Genus_species", "sequence")]

colnames(dt) <- c("total # reads", "total occurrence", "% similarity", "taxonomy", "sequence")
rownames(df) <- NULL

## this should order top ten based on frequency 
## here only one contaminant and is a maple tree (Acer saccharinum)

kable(dt[order(dt[,1], decreasing = TRUE)[1:10],], row.names=F) %>%
  kable_styling(bootstrap_options= c("striped", "hover", "condensed"), 
                font_size = 8, full_width = F)

# Identify the most common contaminant (here is only one, Silver maple!) out of 1301 OTUs
# get contaminant ids
id <- !inverts_may$motus$not_a_conta
max.conta <- rownames(inverts_may$motus[id,])[which.max(inverts_may$motus[id, "seq_count"])]

#and its distribution and relative abundance in each pcr
ggpcrplate(inverts_may, legend_title = "#reads of most \nabundant contaminant",
           FUN = function(m) {m$reads[, max.conta]/rowSums(m$reads)})

# Compute relative abundance of all pcr contaminants together - cannot do if only 1
## a <- data.frame(conta.relab = rowSums(inverts_may$reads[,!inverts_may$motus$not_a_conta]) / 
##                  rowSums(inverts_may$reads))


test <- inverts_may$reads[,!inverts_may$motus$not_a_conta, drop = FALSE] / 
  rowSums(inverts_may$reads)

conta.relab <- rowSums(test)

a <- data.frame(conta.relab)

# flag pcrs with total contaminant relative abundance > 10% of reads)
inverts_may$pcrs$low_contamination_level <- 
  ifelse(a$conta.relab[match(rownames(inverts_may$pcrs), rownames(a))]>1e-1,  F, T)

# Proportion of potentially functional (TRUE) vs. failed (FALSE) pcrs
# (controls included) based on this criterion
table(inverts_may$pcrs$low_contamination_level) / nrow(inverts_may$pcrs)

colnames(inverts_may$pcrs) # data check

## 3.1.2 July subset ----

inverts_july <- subset_metabarlist(inverts, 
                                  table = "pcrs",
                                  indices = inverts$pcrs$plate_no == "J2019")

summary_metabarlist(inverts_july) ## data check

inverts_july <- contaslayer(inverts_july,
                           control_types = c("pcr", "extraction", "sequencing"),
                           output_col = "not_a_conta")

table(inverts_july$motus$not_a_conta) ## NO OTUs flagged out of 1215

dt <- inverts_july$motus[!inverts_july$motus$not_a_conta,
                        c("seq_count", "occurrence_count", "Similarity", "Genus_species", "sequence")]

colnames(dt) <- c("total # reads", "total occurrence", "% similarity", "taxonomy", "sequence")
rownames(df) <- NULL

## this should order top ten based on frequency in a bigger list
## here there are none

kable(dt[order(dt[,1], decreasing = TRUE)[1:10],], row.names=F) %>%
  kable_styling(bootstrap_options= c("striped", "hover", "condensed"), 
                font_size = 8, full_width = F)

# Compute relative abundance of all pcr contaminants together - cannot do if 0
#a <- data.frame(conta.relab = rowSums(inverts_july$reads[,!inverts_july$motus$not_a_conta]) / 
                #  rowSums(inverts_july$reads))

test2 <- inverts_july$reads[,!inverts_july$motus$not_a_conta, drop = FALSE] / 
  rowSums(inverts_july$reads)

conta.relab2 <- rowSums(test2)

b <- data.frame(conta.relab2)

# flag pcrs with total contaminant relative abundance > 10% of reads)
inverts_july$pcrs$low_contamination_level <- 
  ifelse(b$conta.relab2[match(rownames(inverts_july$pcrs), rownames(b))]>1e-1,  F, T)

# Proportion of potentially functional (TRUE) vs. failed (FALSE) pcrs
# (controls included) based on this criterion
table(inverts_july$pcrs$low_contamination_level) / nrow(inverts_july$pcrs)

colnames(inverts_july$pcrs)

## 3.1.3 Sept subset ----

inverts_sept <- subset_metabarlist(inverts, 
                                   table = "pcrs",
                                   indices = inverts$pcrs$plate_no == "S2019")


summary_metabarlist(inverts_sept) # data check

##contaminants in pcr controls

inverts_sept <- contaslayer(inverts_sept,
                            control_types = c("pcr", "extraction"),
                            output_col = "not_a_conta")

table(inverts_sept$motus$not_a_conta) ## 3 out of 1375 OTUs flagged

dt <- inverts_sept$motus[!inverts_sept$motus$not_a_conta,
                        c("seq_count", "occurrence_count", "Similarity", "Phylum", "Genus_species", "sequence")]

colnames(dt) <- c("total # reads", "total occurrence", "% similarity", "Phylum", "taxonomy", "sequence")
rownames(df) <- NULL

## OTUs flagged as contaminants match to human, dog and diatom
kable(dt[order(dt[,1], decreasing = TRUE)[1:10],], row.names=F) %>%
  kable_styling(bootstrap_options= c("striped", "hover", "condensed"), 
                font_size = 8, full_width = F)

# Compute relative abundance of all pcr contaminants together
c <- data.frame(conta.relab3 = rowSums(inverts_sept$reads[,!inverts_sept$motus$not_a_conta]) / rowSums(inverts_sept$reads))
           
# flag pcrs with total contaminant relative abundance > 10% of reads)
inverts_sept$pcrs$low_contamination_level <- 
             ifelse(c$conta.relab3[match(rownames(inverts_sept$pcrs), rownames(c))]>1e-1,  F, T)
           
# Proportion of potentially functional (TRUE) vs. failed (FALSE) pcrs
# (controls included) based on this criterion
table(inverts_sept$pcrs$low_contamination_level) / nrow(inverts_sept$pcrs)
           
## 3.1.4 Redo subset ----

inverts_redo <- subset_metabarlist(inverts, 
                                   table = "pcrs",
                                   indices = inverts$pcrs$plate_no == "redo")

summary_metabarlist(inverts_redo) ##data check

##contaminants in controls

inverts_redo <- contaslayer(inverts_redo,
                            control_types = c("pcr", "extraction", "sequencing"),
                            output_col = "not_a_conta")

table(inverts_redo$motus$not_a_conta) ## 2 out of 734 OTUs flagged

dt <- inverts_redo$motus[!inverts_redo$motus$not_a_conta,
                         c("seq_count", "occurrence_count", "Similarity", "Phylum", "Genus_species", "sequence")]

colnames(dt) <- c("total # reads", "total occurrence", "% similarity", "Phylum", "taxonomy", "sequence")
rownames(df) <- NULL

## this should order top ten based on frequency in a bigger list
## here only 2 contaminants (Ascomycota)

kable(dt[order(dt[,1], decreasing = TRUE)[1:10],], row.names=F) %>%
  kable_styling(bootstrap_options= c("striped", "hover", "condensed"), 
                font_size = 8, full_width = F)

# Compute relative abundance of all pcr contaminants together - cannot do if only 1
d <- data.frame(conta.relab4 = rowSums(inverts_redo$reads[,!inverts_redo$motus$not_a_conta]) / rowSums(inverts_redo$reads))

# flag pcrs with total contaminant relative abundance > 10% of reads)
inverts_redo$pcrs$low_contamination_level <- 
  ifelse(d$conta.relab4[match(rownames(inverts_redo$pcrs), rownames(d))]>1e-1,  F, T)

# Proportion of potentially functional (TRUE) vs. failed (FALSE) pcrs
# (controls included) based on this criterion
table(inverts_redo$pcrs$low_contamination_level) / nrow(inverts_redo$pcrs)

## 3.1.5 Combine back together ----

## no sample pcrs were flagged as contaminanted so can combine this way

inverts$pcrs$low_contamination_level <- NA
inverts$pcrs[rownames(inverts_may$pcrs),"low_contamination_level"] <- inverts_may$pcrs$low_contamination_level

inverts$pcrs$low_contamination_level ## it worked, repeat for july/sept/redo

inverts$pcrs[rownames(inverts_july$pcrs),"low_contamination_level"] <- inverts_july$pcrs$low_contamination_level
inverts$pcrs[rownames(inverts_sept$pcrs),"low_contamination_level"] <- inverts_sept$pcrs$low_contamination_level
inverts$pcrs[rownames(inverts_redo$pcrs),"low_contamination_level"] <- inverts_redo$pcrs$low_contamination_level

## repeat for OTU dataframe

inverts$motus$not_a_conta <- NA
inverts$motus[rownames(inverts_may$motus),"not_a_conta"] <- inverts_may$motus$not_a_conta
inverts$motus[rownames(inverts_july$motus),"not_a_conta"] <- inverts_july$motus$not_a_conta
inverts$motus[rownames(inverts_sept$motus),"not_a_conta"] <- inverts_sept$motus$not_a_conta
inverts$motus[rownames(inverts_redo$motus),"not_a_conta"] <- inverts_redo$motus$not_a_conta

inverts$motus$not_a_conta


## 3.2 Flagging non-target OTUs & low quality matches ----

colnames(inverts$pcrs)

inverts$motus$target_taxon <- grepl("Arthropoda|Mollusca|Annelida", inverts$motus$Phylum)

# Proportion of each of these over total number of MOTUs
table(inverts$motus$target_taxon) / nrow(inverts$motus) ### 87.67 OTUs are benthic inverts

inverts$motus$Similarity <- as.numeric(inverts$motus$Similarity)
inverts$motus$Similarity[is.na(inverts$motus$Similarity)] <- 0 ## replace NAs (OTUs with no match) as 0% similarity

# Plot the unweighted distribution OTU sequence similarity scores 
a <- ggplot(inverts$motus, aes(x=Similarity)) + 
  geom_histogram(color="black", fill="grey", bins=20) + 
  geom_vline(xintercept = 90, col="red", lty=2) + ## using 90% but can change
  theme_bw() + 
  theme(panel.grid = element_blank()) + 
  labs(x="Percent similarity against best match", y="Number of OTUs") +
  scale_y_continuous(expand = c(0,0), limits = c(0, 1250))

# Same for the weighted distribution based on total sequence reads of OTU
b <- ggplot(inverts$motus, 
       aes(x=Similarity, weight = seq_count)) + 
  geom_histogram(color="black", fill="grey", bins=20) + 
  geom_vline(xintercept = 90, col="red", lty=2) + 
  theme_bw() + 
  theme(panel.grid = element_blank()) + 
  labs(x="Percent similarity against best match", y="Number of sequence reads") +
  scale_y_continuous(expand = c(0,0), limits = c(0, 50000000))

ggarrange(a,b)

## ggsave("Appendix_plot5_similarity.jpg", plot = last_plot())

inverts$motus$not_degraded <-
  ifelse(inverts$motus$Similarity < 90, F, T) ### flagging OTUs with under 90% match

# Proportion of each of these over total number of MOTUs
table(inverts$motus$not_degraded) / nrow(inverts$motus) ### 78.7% of OTUs had a match of at least 90

## Intersection with target taxa and sequence matching
table(inverts$motus$target_taxon, 
      inverts$motus$not_degraded)

## 3.3 Flagging pcr outliers based on sequencing depth ----

##flagged based on sequencing depth (includes controls)

ggplot(inverts$pcrs, aes(nb_reads)) +
  geom_histogram(bins=40, color="black", fill="grey") + 
  geom_vline(xintercept = 87344, lty=2, color="red") + ## threshold is average sequencing depth minus 1 SD of first 3 plates
  scale_x_log10() + 
  labs(x="Number of sequence reads", 
       y="Number of samples") +
  theme_bw() + 
  theme(panel.grid = element_blank()) +
  scale_y_continuous(expand = c(0,0), limits = c(0, 120))

## ggsave("Appendix_plot6_SeqDepth.jpg", plot = last_plot())

inverts$pcrs$seqdepth_ok <- ifelse(inverts$pcrs$nb_reads < 87344, F, T) ## using one less than SD, 87344, 87.1% of pcrs pass

# Proportion of each of these over total number of pcrs, control excluded
table(inverts$pcrs$seqdepth_ok[inverts$pcrs$type=="sample"]) /
  nrow(inverts$pcrs[inverts$pcrs$type=="sample",]) 


## 3.4 Assessing reproducibility via technical replicates ----

## first need to remove samples without tech reps, pcrs with no reads and neg controls
##subset data


inverts$pcrs$tech_rep

inverts_sub <- subset_metabarlist(inverts, 
                                  table = "pcrs",
                                  indices = !is.na(inverts$pcrs$tech_rep))

inverts_sub$pcrs$tech_rep

inverts_sub$pcrs$sample_id

inverts_sub$pcrs$nb_motus

pcr_within_between(
  inverts_sub,
  replicates = inverts_sub$pcrs$sample_id,
  FUN = FUN_pcrdist_bray_freq,
  method = "centroid"
)

pcrslayer(
  inverts_sub,
  replicates = NULL,
  method = "centroid",
  FUN = FUN_pcrdist_bray_freq,
  thresh.method = "intersect",
  output_col = "functional_pcr",
  plot = T
)
                                

#visualization

comp1 <- pcr_within_between(inverts_sub) 
## detecting PCR replicate outliers based on PCR similarity/reproducibility
## uses bray distance to compare dissimilarities within sample and between samples

check_pcr_thresh(comp1) +
  xlab("Bray-Curtis dissimilarity") +
  ylab("Density") +
  scale_color_manual(values = c("steelblue", "orange")) +
  geom_line(size = 1) +
  theme_classic(14)

##ggsave("Appendix_plot7_techreps.jpg", plot = last_plot())

### very low dissimilarity/distance within samples indicates that tech reps were very consistent


## flags PCR as failed if PCR replicates are outliers by comparing dissimilarities 
## in taxonomic composition within PCR reps vs between samples
## threshold for elimination having distance within samples greater than threshold of intersection

inverts_sub <- pcrslayer(inverts_sub, output_col = "replicating_pcr", method = "pairwise", plot = F) ## flags PCR as failed 

table(inverts_sub$pcrs$replicating_pcr) /
  nrow(inverts_sub$pcrs)

## report flagging in initial metabarlist 
inverts$pcrs$replicating_pcr <- TRUE
inverts$pcrs[rownames(inverts_sub$pcrs),"replicating_pcr"] <- inverts_sub$pcrs$replicating_pcr ## would replace with false if any

inverts$pcrs$replicating_pcr #check

## 3.4 Lowering tag jumps ----

thresholds <- c(0,1e-4,1e-3, 1e-2, 3e-2, 5e-2) 

tests <- lapply(thresholds, function(x) tagjumpslayer(inverts,x))
names(tests) <- paste("t_", thresholds, sep="")

tmp <- melt(as.matrix(do.call("rbind", lapply(tests, function(x) rowSums(x$reads)))))
colnames(tmp) <- c("threshold", "sample", "Sequences")

tmp$OTUs <-
  melt(as.matrix(do.call("rbind", lapply(tests, function(x) {
    rowSums(x$reads > 0)
  }))))$value

# Add control type information on pcrs and make data curation threshold numeric
tmp$controls <- inverts$pcrs$control_type[match(tmp$sample, rownames(inverts$pcrs))]
tmp$threshold <- as.numeric(gsub("t_", "", tmp$threshold))

tmp2 <- melt(tmp, id.vars=colnames(tmp)[-grep("Sequences|OTUs", colnames(tmp))])

unique(tmp2$variable)

tmp2$controls <- tmp2$controls %>% replace_na("samples")
tmp2$controls <- factor(tmp2$controls, levels = c("extraction", "pcr", "sequencing", "samples"))


ggplot(tmp2, aes(x=as.factor(threshold), y=value)) + 
  geom_boxplot(color="black") + 
  geom_vline(xintercept = which(levels(as.factor(tmp2$threshold)) == "0.001"), col="red", lty=2) + 
  geom_jitter(aes(color=controls), width = 0.2, alpha=0.5) + 
  scale_color_manual(values = c("steelblue", "orange", "darkred", "darkgrey")) +
  facet_wrap(~variable + controls, scale="free", ncol=4) + 
  theme_bw() + 
  labs(x="Filtering threshold", y="Reads or OTUs") + 
  theme(panel.grid = element_blank(), 
        axis.text.x = element_text(angle=40, h=1), 
        legend.position = "none")

##ggsave("Appendix_plot8_tagjumps.jpg", plot = last_plot())


## 4.0 Summarizing noise in dataset ----

colnames(inverts$motus)

motus.qual <- !inverts$motus[,c("not_a_conta", "target_taxon", "not_degraded")]
colnames(motus.qual) <- c("Contaminant", "Untargeted", "LowSimilarity")

prop.table(table(apply(motus.qual, 1, sum) > 0)) ###25% OTUs not high quality and/or target taxa

# Proportion of MOTUs and reads artifcat
apply(motus.qual, 2, sum) / nrow(motus.qual)
apply(motus.qual, 2, function(x) sum(inverts$motus$count[x])/sum(inverts$motus$count))

tmp.motus <- 
  apply(sapply(1:ncol(motus.qual), function(x) {
    ifelse(motus.qual[,x]==T, colnames(motus.qual)[x], NA)}), 1, function(x) {
      paste(sort(unique(x)), collapse = "|")
    })
tmp.motus <- as.data.frame(gsub("^$", "Retained", tmp.motus))
colnames(tmp.motus) <-  "artefact_type"

unique(tmp.motus$artefact_type)
count(tmp.motus$artefact_type$Contaminant&LowSimilarity&Untargeted)

ggplot(tmp.motus, aes(x = 1, fill=artefact_type)) +
  geom_bar() + 
  labs(fill="Artifact type") + 
  scale_fill_manual(values = c("black", "black", "black", "darkred", "orange", "steelblue", "darkgrey")) +
  theme_classic(14) + 
  xlim(0, 2) +
  ylab("OTU count") +
  scale_y_continuous(expand = c(0,0), limits = c(0, 2300)) +
  theme(legend.position = "right") +
  theme(axis.title.x=element_blank(),
       axis.text.x=element_blank(),
       axis.ticks.x=element_blank())

##ggsave("Appendix_plot9_OTUartefacts.jpg", plot = last_plot())

# Create a table of pcrs quality criteria 
# noise is identified as FALSE inverts, the "!" transforms it to TRUE
pcrs.qual <- !inverts$pcrs[,c("low_contamination_level", "seqdepth_ok", "replicating_pcr")]
colnames(pcrs.qual) <- c("high_contamination_level", "low_seqdepth", "techrep_outliers")

colnames(inverts$pcrs)
# Proportion of pcrs potentially artifactual (TRUE) based on the criteria used
# excluding controls
prop.table(table(apply(pcrs.qual[inverts$pcrs$type=="sample",], 1, sum) > 0))

# Proportion of MOTUs and reads potentially artifactual for each criterion
apply(pcrs.qual[inverts$pcrs$type=="sample",], 2, sum) / nrow(pcrs.qual[inverts$pcrs$type=="sample",])

tmp.pcrs <- 
  apply(sapply(1:ncol(pcrs.qual), function(x) {
    ifelse(pcrs.qual[inverts$pcrs$type=="sample",x]==T, 
           colnames(pcrs.qual)[x], NA)}), 1, function(x) {
             paste(sort(unique(x)), collapse = "|")
           })
tmp.pcrs <- as.data.frame(gsub("^$", "Retained", tmp.pcrs))

colnames(tmp.pcrs) <- "artefact_type"

ggplot(tmp.pcrs, aes(x = 1, fill=artefact_type)) +
  geom_bar() + 
  labs(fill="Artifact type") + 
  scale_fill_manual(values = c("darkred", "steelblue")) +
  theme_classic(14) + 
  xlim(0, 2) +
  ylab("PCRs in dataset") +
  scale_y_continuous(expand = c(0,0)) +
  theme(legend.position = "right") +
  theme(axis.title.x=element_blank(),
        axis.text.x=element_blank(),
        axis.ticks.x=element_blank())

##ggsave("Appendix_plot10_PCRartefacts.jpg", plot = last_plot())


## 5.0 Data cleaning and aggregation  ----
## 5.1 Removing spurious signals (flags) ----

##subsetting data to remove negs

inverts_s <- subset_metabarlist(inverts, 
                                  table = "pcrs",
                                  indices =  inverts$pcrs$nb_reads > 0 & (
                                    is.na(inverts$pcrs$control_type)
                                  ))

# Subset on MOTUs: we keep motus that are defined as TRUE following the 
# three criteria below (sum of three TRUE is equal to 3 with the rowSums function)

tmp <- tests[["t_0.001"]] ## tagjump threshold we selected

inverts_s <- subset_metabarlist(tmp, 
                                table = "pcrs",
                                indices =  tmp$pcrs$nb_reads > 0 & (
                                  is.na(tmp$pcrs$control_type)
                                ))

tmp$motus$target_taxon

tmp2 <- subset_metabarlist(inverts_s, "motus", 
                          indices = rowSums(inverts_s$motus[,c("not_a_conta", "target_taxon",
                                                                 "not_degraded")]) == 3)
unique(tmp2$motus$Phylum)

summary_metabarlist(tmp) ## sanity check

inverts_clean <- subset_metabarlist(tmp2, "pcrs", 
                                    indices = rowSums(tmp2$pcrs[,c("low_contamination_level", 
                                                                  "seqdepth_ok", "replicating_pcr")]) == 3)


summary_metabarlist(inverts_clean) ## removed 3 samples due to sequencing depth

unique(inverts_clean$motus$Phylum)

#check is this led to empty pcrs or otus
if(sum(colSums(inverts_clean$reads)==0)>0){print("empty motus present")}
if(sum(colSums(inverts_clean$reads)==0)>0){print("empty pcrs present")}

#update parameters with removed motus or reduced read counts
inverts_clean$pcrs$nb_reads_postmetabaR = rowSums(inverts_clean$reads)
inverts_clean$pcrs$nb_motus_postmetabaR = rowSums(ifelse(inverts_clean$reads>0, T, F))

#compare basic characteristics before and after data curation
check <- melt(inverts_clean$pcrs[,c("nb_reads", "nb_reads_postmetabaR", 
                                    "nb_motus", "nb_motus_postmetabaR")])
check$type <- ifelse(grepl("motus", check$variable), "OTUs", "Sequences")

tibble(check)

ggplot(data = check, aes(x = variable, y = value)) +
  geom_boxplot(colour = "black") +
  geom_jitter(alpha=0.5, aes(colour = variable)) +
  scale_colour_manual(values = c("orange", "steelblue", "orange", "steelblue")) +
  theme_bw(14) +
  ylab("Number of OTUs or Sequences") +
  facet_wrap(~type, scales = "free") +
  theme(axis.text.x = element_text(angle=45, h=1)) +
  theme(legend.position = "none") +
  scale_x_discrete(labels = c("nb_motus" = "Pre", "nb_motus_postmetabaR" = "Post", "nb_reads" = "Pre", "nb_reads_postmetabaR" = "Post")) +
  theme(axis.title.x=element_blank())
## not a big difference here


##ggsave("Appendix_plot11_PrePost2.jpg", plot = last_plot())


## clean sequences per plate to check no of sequences per run

may_clean <- subset_metabarlist(inverts_clean, 
                                table = "pcrs",
                                indices = inverts_clean$pcrs$plate_no == "M2019")

summary_metabarlist(may_clean)

july_clean <- subset_metabarlist(inverts_clean, 
                                table = "pcrs",
                                indices = inverts_clean$pcrs$plate_no == "J2019")

summary_metabarlist(july_clean)

sept_clean <- subset_metabarlist(inverts_clean, 
                                table = "pcrs",
                                indices = inverts_clean$pcrs$plate_no == "S2019")

summary_metabarlist(sept_clean)

redo_clean <- subset_metabarlist(inverts_clean, 
                                table = "pcrs",
                                indices = inverts_clean$pcrs$plate_no == "redo")

summary_metabarlist(redo_clean)

## 5.2 Combining tech reps (sum otus across reps) and save final dataset ----

inverts_clean$pcrs$nb_motus

inverts_all <- aggregate_pcrs(inverts_clean, FUN = FUN_agg_pcrs_mean)

inverts_all$pcrs$nb_motus_postmetabaR

tibble(inverts_all$pcrs)

## number of sequence reads per pcr
inverts_all$pcrs$total_reads <- rowSums(inverts_all$reads)

##number of motus (richness) per pcr
inverts_all$pcrs$total_motus <- rowSums(inverts_all$reads>0)

inverts_all$pcrs$nb_reads_postmetabaR
inverts_all$pcrs$total_reads

inverts_all$pcrs$nb_motus_postmetabaR
inverts_all$pcrs$total_motus

summary_metabarlist(inverts_all) 



saveRDS(inverts_all, file = "Appendix5_Clean_Dataset_June12.rds")