##########################################################################################
#
#
#  Representative commands and scripts used for the ATACseqQC case study
#  All analyses were performed in HPC with a bash shell and a LSF job management system
#  except for taking IGV snapshots, which is done in Mac OSX with GUI
#
#
##########################################################################################



##########################################################################################
######### 1. downloading ATAC-seq data from GEO using ascp

~/.aspera/connect/bin/ascp -T -l 100M -m 50M -k 1  -i ~/.aspera/connect/etc/asperaweb_id_dsa.openssh \
    anonftp@ftp.ncbi.nlm.nih.gov:/sra/sra-instant/reads/ByExp/sra/SRX/SRX293/SRX2937358/SRR5720369/SRR5720369.sra   ./




##########################################################################################
######### 2. converting reads from the sra format to the fastq format using fastq-dump

fastq-dump  --split-files  -B -F  --gzip  SRR5720369.sra




##########################################################################################
######### 3. building index for the human reference genome GRCh38 (hg38) using bwa

fasta=/project/umw_mccb/genome/Homo_sapiens/GRCh38/Homo_sapiens.GRCh38.dna.primary_assembly.fa

bwa index  $fasta




##########################################################################################
######### 4. aligning read to the human reference genome using BWA-mem

fasta=/project/umw_mccb/genome/Homo_sapiens/GRCh38/Homo_sapiens.GRCh38.dna.primary_assembly.fa

bwa mem -t 8 -M  $fasta SRR5720369_R1.fastq.gz  SRR5720369_R2.fastq.gz  >  SRR5720369.sam




##########################################################################################
######## 5. filtering alignments using samtools. 
######## This step was done before bamQC function was fully implemented. 

#!/bin/bash

#BSUB -n 8 # minimal numbers of processors required for a parallel job
#BSUB -R rusage[mem=16000] # ask for memory 16G
#BSUB -W 12:00 #limit the job to be finished in 12 hours
#BSUB -J "bwamem[1-25]"
#BSUB -q long # which queue we want to run in
#BSUB -o logs/out.%J.%I.txt # log
#BSUB -e logs/err.%J.%I.txt # error
#BSUB -R "span[hosts=1]" # All hosts on the same chassis"


module load samtools/1.4.1

i=$(($LSB_JOBINDEX - 1))


## "nuclear.chr.txt" is a file containing a list of all human nuclear chromosomes and unplaced 
## scaffolds (no mitochondrial genome included)
grep '^@SQ' SRR3295022.sorted.sam  |cut -f2 | perl -n -e 's/SN://; print if !/MT/'  > nuclear.chr.txt

chromosomes=(`cat nuclear.chr.txt`)

sams=(`ls *.sam `)
names=(`ls *.sam| perl -p -e 's/.sam//' `)

## converting alignment output from SAM to BAM
samtools view -b -h  -o  ${names[${i}]}.bam  -@ 8  -1  ${sams[${i}]}

## sorting the BAM file
samtools sort -l 9 -m 8G  -o  ${names[${i}]}.sorted.bam  -O BAM -@ 8  ${names[${i}]}.bam
samtools index  -@ 1  ${names[${i}]}.sorted.bam

## obtaining statistics of read alignments
samtools flagstat -@ 8  ${names[${i}]}.sorted.bam  > ${names[${i}]}.prefilter.stat


## extracting mitochondrial reads and summarize mapping statistics

samtools view  -h   ${bam[${i}]}   'MT' > ${names[${i}]}.MT.bam
samtools flagstat -@ 8  ${names[${i}]}.MT.bam > ${names[${i}]}.MT.bam.stat


## filtering BAM files to remove mitochondrial reads and other alignments of no interest

samtools view  -h -O SAM  ${bam[${i}]}   ${chromosomes[@]} | awk  'BEGIN{FS=OFS="\t"} \
    function abs(v) {return v < 0 ? -v : v}; /^@/ || ($7 == "="  \
    && ($2 == 81 || $2 == 161|| $2 == 97 || $2 == 145 || $2 ==99 || \
    $2 == 147 || $2 == 83 || $2 ==163) && abs($9) <= 2000 && abs($9) >= 38 && $5 >=20 ) {print}' | \
    samtools view  -h  -b -o ${names[${i}]}.chr.filtered.bam  - 


## sorting and indexing the BAM file

samtools sort -l 9 -m 8G  -o  ${names[${i}]}.chr.filtered.sorted.bam  -O BAM -@ 8  ${names[${i}]}.chr.filtered.bam
samtools index  -@ 1  ${names[${i}]}.chr.filtered.sorted.bam


## obtaining the summary statistics of the BAM file after filtering

samtools flagstat -@ 8  ${names[${i}]}.chr.filtered.sorted.bam > ${names[${i}]}.chr.filtered.sorted.bam.stat
	
## removing duplicates from the filtered BAM file
samtools rmdup  ${names[${i}]}.chr.filtered.sorted.bam  ${names[${i}]}.chr.filtered.sorted.rmdup.bam

## indexing the BAM file
samtools index  -@ 1 ${names[${i}]}.chr.filtered.sorted.rmdup.bam





##########################################################################################
########### 6. performing in silico QC using the ATACseqQC package

## Getting chromosome length information
samtools view -H SRR5720369..sorted.bam | grep -P 'chr\d+|chrX' | cut -f2,3  | perl -n -e 's/[SL]N://g' | \
        awk 'BEGIN{FS=OFS="\t"} {print $1, 1, $2, "hg38"}' > auto.x.chrom.human.txt


##########################################################################################
#################### 6.1 Creating ATACseqQC R scripts: ATACseqQC.R 


## This R script accept a single command line argument - the name of a BAM file for QC analysis. 
## An index file for the BAM file must be included in the same directory.

## Loading all required packages

library(motifStack)
library(ATACseqQC)
library(TxDb.Hsapiens.UCSC.hg38.knownGene)
library(BSgenome.Hsapiens.UCSC.hg38)
library(MotifDb)
library(ChIPpeakAnno)


## Getting the BAM file name and sample ID
args<- commandArgs(trailingOnly=TRUE)
bamfile <- args[1]
bamfile.sample.ID <- gsub(".bam", "", basename(bamfile))

## Plotting size distribution of fragments (Figure 1G)
pdf(paste0(bamfile.sample.ID, ".fragment.size.distribution.pdf"), width =10, height=8) 
fragSize <- fragSizeDist(bamFiles=bamfile, bamFiles.labels=bamfile.sample.ID)
dev.off()


## BAM file tags to be included when read in by the readBamFile
tags <- c("AS", "NM", "MD")

## Create a output directory where all subsequent BAM files will be output
outPath <- paste0(bamfile.sample.ID, ".splited.bam")
if (!dir.exists(outPath)){
   dir.create(outPath)
}

## Build GRanges for the human genome hg38 excluding unplaced scaffoldsand chrY
human.genome <- read.delim("auto.x.chrom.human.txt", header=F)

seqlev <- as.character(human.genome[,1])
gr <- GRanges(as.character(human.genome[,1]), IRanges(human.genome[,2], human.genome[,3]))

## For QC, use read alignments from chromosomes 1 and 2 as representatives
which <- gr[seqnames(gr) %in% c("chr1", "chr2")]
  
## Reading in paired end read alignment
gal <- readBamFile(bamfile, tag=tags, which=which, asMates=TRUE)

## Shifting the coordinates of 5' ends of the aligned reads in the bam file, +4 for reads mapping
## to the positive strand, -5 for reads mapping to the negative strand
gal1 <- shiftGAlignmentsList(gal)
shiftedBamfile <- file.path(outPath, paste0(bamfile,".shifted.bam"))

## Outputting the shifted BAM file
export(gal1, shiftedBamfile)

## Getting information of known transcripts from UCSC genome browser database
txs <- transcripts(TxDb.Hsapiens.UCSC.hg38.knownGene)


## Classifying reads into nucleosome-free, mono-, di- and tri-nucleosome bins based on their fragment sizes.
genome <- Hsapiens
objs <- splitGAlignmentsByCut(gal1, txs=txs, genome=genome)


## outputting split BAM files
null <- writeListOfGAlignments(objs, outPath)

## Generating heatmaps and coverage curves for nucleosome-free, mono-, di- and tri-nucleosome occupied regions

bamfiles <- file.path(outPath,
                      c("NucleosomeFree.bam",
                        "mononucleosome.bam",
                        "dinucleosome.bam",
                        "trinucleosome.bam"))

## extracting TSSs coordinates
TSS <- promoters(txs, upstream=0, downstream=1)
TSS <- unique(TSS)

## estimating the library size for normalization
librarySize <- estLibSize(bamfiles)


## calculating the signals around TSSs.
NTILE <- 101
dws <- ups <- 1010
sigs <- enrichedFragments(bamfiles, TSS=TSS,
                          librarySize=librarySize,
                          seqlev=seqlev,
                          TSS.filter=0.5,
                          n.tile = NTILE,
                          upstream = ups,
                          downstream = dws)
                          
                          
## log2-transforming signals
names(sigs) <- gsub(".bam", "", basename(names(sigs)))
sigs.log2 <- lapply(sigs, function(.ele) log2(.ele+1))

## plotting heatmap showing signals  for nucleosome-free and oligonucleosome-bound regions around TSSs. (Figure 1 H and 1I)
pdf(paste0(bamfile.sample.ID, ".heatmap and averaged coverage.pdf"))
featureAlignedHeatmap(sigs.log2, reCenterPeaks(TSS, width=ups+dws),
                      zeroAt=.5, n.tile=NTILE)

out <- featureAlignedDistribution(sigs, 
                                  reCenterPeaks(TSS, width=ups+dws),
                                  zeroAt=.5, n.tile=NTILE, type="l", 
                                  yalb="Averaged coverage", 
                                  ylab="Averaged coverage")
dev.off()


## Plotting CTCF footprints. (Figure 2C)
CTCF <- query(MotifDb, c("CTCF"))
CTCF <- as.list(CTCF)
print(CTCF[[1]], digits=2)
seqlev <- c("chr1", "chr2")

nucleosome_free_bamfile <- file.path(outPath, "NucleosomeFree.bam")

pdf(paste0(bamfile.sample.ID, ".CTCF.footprint.pdf"))
factorFootprints(nucleosome_free_bamfile, pfm=CTCF[[1]], 
                 genome=genome,
                 min.score="95%", seqlev=seqlev,
                 upstream=100, downstream=100)
dev.off()




##########################################################################################
################ 6.2 running ATACseqQC.R with a filtered BAM file as the argument


#!/bin/bash

#BSUB -n 1 # minimal numbers of processors required for a parallel job
#BSUB -R rusage[mem=32000] # ask for memory 32G
#BSUB -W 72:00 #limit the job to be finished in 72 hours
#BSUB -J "ATACseqQC[1-4]"
#BSUB -q long # which queue we want to run in
#BSUB -o logs/out.%J.%I.txt # log
#BSUB -e logs/err.%J.%I.txt # error
#BSUB -R "span[hosts=1]" # All hosts on the same chassis"

mkdir -p logs
module load R/3.4.0

bams=(`ls *.chr.filtered.sorted.rmdup.bam`)

i=$(($LSB_JOBINDEX -1))

Rscript ./ATACseqQC.R  ${bams[${i}]}






##########################################################################################
###### 7. downsampling for sequencing depth analysis and for testing the effect of sequencing depth 
######    on some diagnostic plots


#########################################################################################
################ 7.1. creating shell scripts for downsampling BAM files: downsampling.sh


#!/bin/sh
#########################################################################################
# Author: Haibo Liu              
# script_name: downsampling.sh
# Usage: sh downsampling.sh    /path/to/BAM_file  suffix  a_sequence_of_fractions_between_0_and_1 
# Example:  sh downsampling.sh    SRR891270.chr.filtered.sorted.rmdup.bam  .chr.filtered.sorted.rmdup.bam  0.1 0.2 0.3 0.4 0.5  0.6  0.7  0.8  0.9
#                                 
#########################################################################################

USAGE="Usage: sh $0 /path/to/BAM_file a_sequence_of_fractions_between_0_and_1"


## checking number of command-line arguments
if [ $# -lt 3 ]; then
    echo "At least three arguments are needed!"
	echo "$USAGE"
	exit 1
fi


## getting the basename of a bam file
base=`basename $1 $2`


## all arguments from $3 to ${N}, holding subsampling percentage
percentage=("${@:3}")

## randomizing alignments by sorting by read names
samtools sort -l 9 -m 8G  -o  ${base}.name.sorted.sam  -O SAM  -n -@ 8  $1

## getting SAM header
grep  -P '^@(HD|SQ|PG)'  ${base}.name.sorted.sam > ${base}.name.sorted.sam.header


## converting paired read alignment to a single line representing a fragment alignment
grep -v -P '^@(HD|SQ|PG)'  ${base}.name.sorted.sam | awk 'BEGIN{FS=OFS="\t"; ORS="\t"; i =1} \
        {if (i==1) {print $0; a=$1; i=i+1} else if ($1==a) {print $0; i= i+1} else if ($1 !=a) \
        {print "\n"; print $0; i= i+1; a= $1}}'| perl -p -e 's/^\t//; s/\t$//' >  ${base}.name.sorted.oneliner.sam


## subsampling by using Unix built-in shuf

total=`wc -l ${base}.name.sorted.oneliner.sam | awk  '{print $1}' `

while (($#)); do
   subsample_size+=( `echo "${total} * ${1} " |bc | perl -p -e 's/\.\d+//'` )
   shift
done

echo "${subsample_size[@]}"


## number of subsamples
num_subsamples=${#subsample_size[@]}

for (( i=0; i<${num_subsamples}; i++ ));

do
	
	file=${base}.name.sorted.${percentage[${i}]}.sam 
	cp ${base}.name.sorted.sam.header   $file
	shuf -n  ${subsample_size[$i]} ${base}.name.sorted.oneliner.sam  >>  $file
	
	base1=`basename $file .sam`
    perl -p -e 's/(HWI.+?)\s+(HWI.+)/$1\n$2/' $file   | samtools view -b -h  -@ 8 - | \
    samtools sort  -l 9 -m 8G  -o  ${base1}.coord.sorted.bam  -O BAM -@ 8 -
    samtools index  -@ 1  ${base1}.coord.sorted.bam

done



#########################################################################################
############# 7.2 downsampling from SRR891270

#!/bin/bash

#BSUB -n 8 # minimal numbers of processors required for a parallel job
#BSUB -R rusage[mem=8000] # ask for memory 8G
#BSUB -W 72:00 #limit the job to be finished in 12 hours
#BSUB -J "subsample"
#BSUB -q long # which queue we want to run in
#BSUB -o logs/out.%J.%I.txt # log
#BSUB -e logs/err.%J.%I.txt # error
#BSUB -R "span[hosts=1]" # All hosts on the same chassis"

mkdir -p logs

module load samtools/1.4.1

sh downsampling.sh  SRR891270.chr.filtered.sorted.rmdup.bam  .chr.filtered.sorted.rmdup.bam \
                    0.1 0.2 0.25 0.3 0.4 0.5  0.6  0.7 0.75 0.8 0.9





#########################################################################################
#############  7.3 calling peaks based on the subsamples (10%, 20%, ..., 90% of aligned fragments)
#############      in SRR891270.chr.filtered.sorted.rmdup.bam)

#!/bin/bash

#BSUB -n 1 # minimal numbers of processors required for a parallel job
#BSUB -R rusage[mem=16000] # ask for memory 32G
#BSUB -W 72:00 #limit the job to be finished in 72 hours
#BSUB -J "MACS2[1-9]"
#BSUB -q long # which queue we want to run in
#BSUB -o logs/out.%J.%I.txt # log
#BSUB -e logs/err.%J.%I.txt # error
#BSUB -R "span[hosts=1]" # All hosts on the same chassis"

mkdir -p logs

module load python/2.7.9
module load python/2.7.9_packages/numpy/1.9.2
module load MACS/2.1.0

i=$(($LSB_JOBINDEX - 1))

bam=(`ls *0.[1-9].coord.sorted.bam`)

peak_dir=MACS2.subsampling.peaks

if [ ! -e  "${peak_dir}"  ]; then
	mkdir ${peak_dir}
fi

macs2 callpeak  --treatment  ${bam[${i}]}  -f BAMPE  -g hs  --keep-dup 1  --outdir  ${peak_dir} \
   --name  ${bam[${i}]}  --bdg  --trackline \
   --SPMR  --shift  0  --tempdir ./  --broad



#########################################################################################
############# 7.4 plotting saturation curves  (Figure 4A and 4B)

library(ggplot2)
peakFiles <- dir("./", "broadPeak$")
total_frags <- 13396741
subsamplingSizes <- seq(0.1, 1, by=0.1)*total_frags
names(subsamplingSizes) <- peakFiles

peakStat <- saturationPlot(subsamplingPeakFiles=peakFiles, subsamplingSizes=subsamplingSizes, sep="\t", 
                           header= FALSE, fdr=0.05, fdrCol=9, startCol=2, 
                           endCol=3, skipLines=1, peakCaller="MACS2", outPrefix="SRR891270")

pdf("Saturation curve for SRR891270.pdf", width=8, height=8)

## total peak number-based curve
p <-ggplot(data= peakStat, aes(x= subsamplingSizes/10^6, y = numPeaks/10^3)) + geom_point(shape=16, size=3) + geom_smooth()
p + expand_limits(y=0) + theme(text=element_text(size=10)) + xlab(expression(Effective~fragments~x~10^6)) + ylab(expression(Peaks~x~10^3)) 

## total peak width-based 
p <-ggplot(data= peakStat, aes(x= subsamplingSizes/10^6, y = breadth/10^6)) + geom_point(shape=16, size=3) + geom_smooth()
p + expand_limits(y=0) + theme(text=element_text(size=10)) + xlab(expression(Effective~fragments~x~10^6)) + ylab(expression(Total~peaks~width~(Mb))) 

dev.off()







#########################################################################################
#############  7.5 library complexity analysis using BAM files with mitochondrial reads removed:
#############   MT-free.SRR891269.bam, MT-free.SRR891270.bam, MT-free.SRR891271.bam, 
#############   MT-free.SRR891272.bam and MT-free.SRR891274.bam

#########################################################################################
#####################  7.5.1 creating R scripts for library complexity analysis: libComplexity.R 


args<- commandArgs(trailingOnly=TRUE)
bamFile <- args[1]

histFile <- readsDupFreq(bamFile=bamFile, index =bamFile)

pdf(paste0(bamFile,".library.complexity.pdf"), width=8, height=5)
complexity <- estimateLibComplexity(histFile=histFile, times=100, 
                                  interpolate.sample.sizes=seq(0.1, 1, by=0.1),
                                  extrapolate.sample.sizes=seq(5, 20, by=5))
dev.off() 
write.table(complexity, paste0(bamFile, ".complexity.txt"), sep ="\t", quote=F, row.names=FALSE)


#########################################################################################
#####################  7.5.2 library complexity analysis

#!/bin/bash

#BSUB -n 1 # minimal numbers of processors required for a parallel job
#BSUB -R rusage[mem=32000] # ask for memory 5G
#BSUB -W 72:00 #limit the job to be finished in 12 hours
#BSUB -J "libCompMTfree[1-5]"
#BSUB -q long # which queue we want to run in
#BSUB -o logs/out.%J.%I.txt # log
#BSUB -e logs/err.%J.%I.txt # error
#BSUB -R "span[hosts=1]" # All hosts on the same chassis"

mkdir -p logs

i=$(($LSB_JOBINDEX - 1))


module load samtools/1.4.1
module load R/3.4.0

bams=(SRR891269.sorted.bam SRR891270.sorted.bam SRR891271.sorted.bam SRR891272.sorted.bam SRR891274.sorted.bam)
chromosomes=(`cat nuclear.chr.txt`)

## Removing MT alignments
samtools view  -h -b  ${bams[${i}]}   ${chromosomes[@]} > MT-free.${bams[${i}]}
samtools index MT-free.${bams[${i}]}

Rscript libComplexity.R  MT-free.${bams[${i}]} 



#########################################################################################
#####################  7.5.2 visualizing library complexity analysis results using R (Figure 4C)

files <- dir("./", "MT-free+.txt$")

distinct <- lapply(files, read.delim, header=TRUE)

names(distinct) <- gsub("MT-free.|.sorted.bam.complexity.txt", "", files)
names(distinct) <- paste(names(distinct), c(rep("50K", 3), rep("500",2)), sep="-")

for (i in seq_along(distinct))
{
    distinct[[i]]$Sample <- names(distinct)[i]
}

distinct.reads <- do.call(rbind, distinct )
str(distinct.reads)
distinct.reads$Sample <- as.factor(distinct.reads$Sample)
distinct.reads$relative.size <- distinct.reads$relative.size/10^6
distinct.reads$values <- distinct.reads$values/10^6

library(ggplot2)

pdf("Greenleaf's replicates Library complexity estimated on the basis of MT-free reads.pdf", width=8, height=4)

p <-ggplot(data= distinct.reads, aes(x= relative.size, y = values, group = Sample, col=Sample)) + geom_point(shape=16, size=3) + geom_line(size=1)
p + expand_limits(y=0) + theme(text=element_text(size=10)) +xlab("Putative sequenced fragments (Million)") + ylab("Distinct fragments (Million)") 

dev.off()





#########################################################################################
############ 8. taking IGV snapshots for genomic regions containing selected housekeeping genes (Figure 2)

IGVSnapshot (maxMem='mm', genomeBuild="hg38", bamFileFullPathOrURLs = c("SRR891270.chr.filtered.sorted.rmdup.bam", 
                        "SRR3295017.chr.filtered.sorted.rmdup.bam", "SRR5920369.chr.filtered.sorted.rmdup.bam",
                         "SRR580802.chr.filtered.sorted.rmdup.bam"), 
                        geneNames=c("GAPDH", "ACTB", "C1orf43", "CHMP2A", "EMC7", "GPI", "PSMB2", "PSMB4", "RAB7A", "REEP5", 
                                    "VCP", "VPS29"), sessionFile="ATAC-seq.session", outDir=getwd())