##################################################################
#
#   Command line script for the data analysis as presented in
#   Hartman et al. (2016) in JOURNAL
# 
###
#   for more information:
#   Kyle  Hartman, kyle.hartman@agroscope.admin.ch
#   Klaus Schlaeppi, klaus.schlaeppi@agroscope.admin.ch 
#
#   06-09-16
#
##################################################################### 

## All necessary input files are located in the "Input" directory in Additional file 3 except for the GOLD v.5 database and the Silva database v119,
## which are used for chimera filtering and taxonomy assignments,respectively.  
## They are available for download from https://gold.jgi.doe.gov and https://www.arb-silva.de/no_cache/download/archive/release_119/

##################################
## Trifolium growth experiments ##
##################################

cd ~/YOURWORKINGDIRECTORY
mkdir a_data
mkdir a_data/gz

## Note: Place Undetermined_S0_L001_R1_001.fastq.gz & Undetermined_S0_L001_R2_001.fastq.gz in the directory a_data/gz to begin analysis

## =======================================================================================
## A | Initial quality control and gzip to FASTQ
## =======================================================================================

## ---------------------------------------------------------------------------------------
## A1 | Quality Control - FastQC v0.11.2
## ---------------------------------------------------------------------------------------
mkdir a_data/qc

fastqc -t 20 -k 8 -q a_data/gz/Undetermined_S0_L001_R1_001.fastq.gz a_data/gz/Undetermined_S0_L001_R2_001.fastq.gz -o a_data/qc

# Remove the zipped fasta file - these files are not needed
rm a_data/qc/*.zip

## ---------------------------------------------------------------------------------------
## A2 | Gzip to FASTQ File Format
## ---------------------------------------------------------------------------------------

mkdir a_data/fq

# Unzip the files but keep a copy
gunzip -c a_data/gz/Undetermined_S0_L001_R1_001.fastq.gz > a_data/fq/Undetermined_S0_L001_R1_001.fastq
gunzip -c a_data/gz/Undetermined_S0_L001_R2_001.fastq.gz > a_data/fq/Undetermined_S0_L001_R2_001.fastq

## =======================================================================================
## B | Trim low quality ends
## =======================================================================================

mkdir b_trim

prinseq-lite.pl -verbose --out_format 3 -ns_max_n 1 --min_len 100 -trim_to_len 280 -fastq a_data/fq/Undetermined_S0_L001_R1_001.fastq -fastq2 a_data/fq/Undetermined_S0_L001_R2_001.fastq -out_good b_trim/CC16S_trim -out_bad b_trim/CC16S_fail -log b_trim/CC16S.log &

## =======================================================================================
## C | Merge overlapping reads - FLASH v1.2.9
## =======================================================================================

mkdir c_merge

flash -t 10 -m 15 -M 250 -x 0.25 -d c_merge -o CC16S b_trim/CC16S_trim_1.fastq b_trim/CC16S_trim_2.fastq

## =======================================================================================
## D | Primer Site
## =======================================================================================

mkdir d_primer

fastx_reverse_complement -i c_merge/CC16S.extendedFrags.fastq -o c_merge/CC16S.extendedFrags_reverse.fastq -Q33 &
cat c_merge/CC16S.extendedFrags.fastq c_merge/CC16S.extendedFrags_reverse.fastq > d_primer/CC16S.fastq &

## De-multiplex samples using bc and primer sequence
mkdir x_exe

## Note: upload demultiplex_CC16S.sh into the newly created directory x_exe

mkdir y_help

## Note: upload CC16S_Sample_BCPrimer1.txt CC16S_Sample_BCPrimer2.txt CC16S_Sample_BCPrimer3.txt and CC16S_Sample_BCPrimer4.txt into newly created directory y_help
## These commands can be run in parallel in separate screens

while read SMPL BCPF BCPR
do
rm y_help/${SMPL}.log
touch y_help/${SMPL}.log
./x_exe/demultiplex_CC16S.sh $SMPL $BCPF $BCPR >> y_help/${SMPL}.log
done < y_help/CC16S_Sample_BCPrimer1.txt &

while read SMPL BCPF BCPR
do
rm y_help/${SMPL}.log
touch y_help/${SMPL}.log
./x_exe/demultiplex_CC16S.sh $SMPL $BCPF $BCPR >> y_help/${SMPL}.log
done < y_help/CC16S_Sample_BCPrimer2.txt &

while read SMPL BCPF BCPR
do
rm y_help/${SMPL}.log
touch y_help/${SMPL}.log
./x_exe/demultiplex_CC16S.sh $SMPL $BCPF $BCPR >> y_help/${SMPL}.log
done < y_help/CC16S_Sample_BCPrimer3.txt &

while read SMPL BCPF BCPR
do
rm y_help/${SMPL}.log
touch y_help/${SMPL}.log
./x_exe/demultiplex_CC16S.sh $SMPL $BCPF $BCPR >> y_help/${SMPL}.log
done < y_help/CC16S_Sample_BCPrimer4.txt &

## =======================================================================================
## E | Quality Filtering - PRINSEQ-lite 0.20.4
## =======================================================================================

mkdir e_qf

## Note: upload CC16S_Samples.txt and prinseq.par into y_help

for SMPL in `cat y_help/CC16S_Samples.txt`
do
   prinseq-lite.pl -verbose -params y_help/prinseq.par -fastq d_primer/CC16S_${SMPL}_trimR_trimF.fastq -out_good e_qf/CC16S_${SMPL}_trimR_trimF_good -out_bad e_qf/CC16S_${SMPL}_trimR_trimF_bad -log e_qf/CC16S_${SMPL}_trimR_trimF_qc.log
done &

cat e_qf/CC16S_*_trimR_trimF_good.fastq > e_qf/CC16s_all.fastq &

## Add barcode label to reads

for SMPL in `cat y_help/CC16S_Samples.txt`
do
   awk -v SMPL=${SMPL} '{if($1~">") print $1";barcodelabel="SMPL";";else print $1}' e_qf/CC16S_${SMPL}_trimR_trimF_good.fasta > e_qf/CC16S_${SMPL}_barcode.fasta
done &

mkdir f_otu

cat e_qf/CC16S_*_barcode.fasta > f_otu/CC16S_barcode.fasta &

#############################
## Magenta Box experiments ##
#############################

## =======================================================================================
## F | Initial quality control and gzip to FASTQ
## =======================================================================================

mkdir m_magenta
mkdir m_magenta/a_data
mkdir m_magenta/a_data/gz

## Place s1-amplicon_S1_L001_R1_001.fastq.gz & s1-amplicon_S1_L001_R2_001.fastq.gz in the directory m_magenta/a_data/gz to begin analysis

## ---------------------------------------------------------------------------------------
## F1 | Quality Control - FastQC v0.11.2
## ---------------------------------------------------------------------------------------
mkdir m_magenta/a_data/qc

fastqc -t 20 -k 8 -q m_magenta/a_data/gz/s1-amplicon_S1_L001_R1_001.fastq.gz m_magenta/a_data/gz/s1-amplicon_S1_L001_R2_001.fastq.gz -o m_magenta/a_data/qc

# Remove the zipped fasta file - these files are not needed
rm m_magenta/a_data/qc/*.zip

## ---------------------------------------------------------------------------------------
## F2 | Gzip to FASTQ File Format
## ---------------------------------------------------------------------------------------

mkdir m_magenta/a_data/fq

# Unzip the files but keep a copy
gunzip -c m_magenta/a_data/gz/s1-amplicon_S1_L001_R1_001.fastq.gz > m_magenta/a_data/fq/s1-amplicon_S1_L001_R1_001.fastq
gunzip -c m_magenta/a_data/gz/s1-amplicon_S1_L001_R2_001.fastq.gz > m_magenta/a_data/fq/s1-amplicon_S1_L001_R2_001.fastq

## =======================================================================================
## G | Trim low quality ends
## =======================================================================================

mkdir m_magenta/b_trim

prinseq-lite.pl -verbose --out_format 3 -ns_max_n 1 --min_len 100 -trim_to_len 280 -fastq m_magenta/a_data/fq/s1-amplicon_S1_L001_R1_001.fastq -fastq2 m_magenta/a_data/fq/s1-amplicon_S1_L001_R2_001.fastq -out_good m_magenta/b_trim/magenta_trim -out_bad m_magenta/b_trim/magenta_fail -log m_magenta/b_trim/magenta.log

## =======================================================================================
## H | Merge overlapping reads - FLASH v1.2.9
## =======================================================================================

mkdir m_magenta/c_merge

flash -t 10 -m 15 -M 250 -x 0.25 -d m_magenta/c_merge -o Magenta  m_magenta/b_trim/magenta_trim_1.fastq m_magenta/b_trim/magenta_trim_2.fastq

mkdir m_magenta/d_primer

fastx_reverse_complement -i m_magenta/c_merge/Magenta.extendedFrags.fastq -o m_magenta/c_merge/Magenta.extendedFrags_reverse.fastq -Q33
cat m_magenta/c_merge/Magenta.extendedFrags.fastq m_magenta/c_merge/Magenta.extendedFrags_reverse.fastq > m_magenta/d_primer/magenta_exps.fastq

## =======================================================================================
## I Magenta experiments sample demultiplex
## =======================================================================================

## De-multiplex samples using bc and primer sequence
mkdir x_exe

## Note: upload demultiplex_magenta.sh into the newly created directory x_exe

mkdir y_help

## Note: upload Magenta1_Sample_BCPrimer.txt Magenta2_Sample_BCPrimer.txt Magenta3_Sample_BCPrimer.txt 
## SoilEx1_Sample_BCPrimer.txt SoilEx2_Sample_BCPrimer.txt and Inoc_samples.txt into newly created directory y_help
## These commands can be run in parallel in separate screens

while read SMPL BCPF BCPR
do
rm m_magenta/y_help/${SMPL}.log # make sure log file does not exist
touch m_magenta/y_help/${SMPL}.log # create log file
./m_magenta/x_exe/demultiplex_magenta.sh $SMPL $BCPF $BCPR >> m_magenta/y_help/${SMPL}.log
done < m_magenta/y_help/Magenta1_Sample_BCPrimer.txt &

while read SMPL BCPF BCPR
do
rm m_magenta/y_help/${SMPL}.log # make sure log file does not exist
touch m_magenta/y_help/${SMPL}.log # create log file
./m_magenta/x_exe/demultiplex_magenta.sh $SMPL $BCPF $BCPR >> m_magenta/y_help/${SMPL}.log
done < m_magenta/y_help/Magenta2_Sample_BCPrimer.txt &

while read SMPL BCPF BCPR
do
rm m_magenta/y_help/${SMPL}.log # make sure log file does not exist
touch m_magenta/y_help/${SMPL}.log # create log file
./m_magenta/x_exe/demultiplex_magenta.sh $SMPL $BCPF $BCPR >> m_magenta/y_help/${SMPL}.log
done < m_magenta/y_help/Magenta3_Sample_BCPrimer.txt &

while read SMPL BCPF BCPR
do
rm m_magenta/y_help/${SMPL}.log # make sure log file does not exist
touch m_magenta/y_help/${SMPL}.log # create log file
./m_magenta/x_exe/demultiplex_magenta.sh $SMPL $BCPF $BCPR >> m_magenta/y_help/${SMPL}.log
done < m_magenta/y_help/SoilEx1_Sample_BCPrimer.txt &

while read SMPL BCPF BCPR
do
rm m_magenta/y_help/${SMPL}.log # make sure log file does not exist
touch m_magenta/y_help/${SMPL}.log # create log file
./m_magenta/x_exe/demultiplex_magenta.sh $SMPL $BCPF $BCPR >> m_magenta/y_help/${SMPL}.log
done < m_magenta/y_help/SoilEx2_Sample_BCPrimer.txt &

while read SMPL BCPF BCPR
do
rm m_magenta/y_help/${SMPL}.log # make sure log file does not exist
touch m_magenta/y_help/${SMPL}.log # create log file
./m_magenta/x_exe/demultiplex_magenta.sh $SMPL $BCPF $BCPR >> m_magenta/y_help/${SMPL}.log
done < m_magenta/y_help/Inoc_samples.txt &

## =======================================================================================
## J | Magenta experiments quality filtering
## =======================================================================================

mkdir m_magenta/e_qf

## Note: upload Magenta_Samples.txt and prinseq_magenta.par into y_help

for SMPL in `cat m_magenta/y_help/Magenta_Samples.txt`
do
   prinseq-lite.pl -verbose -params m_magenta/y_help/prinseq_magenta.par -fastq m_magenta/d_primer/Magenta_${SMPL}_trimR_trimF.fastq -out_good m_magenta/e_qf/Magenta_${SMPL}_trimR_trimF_good -out_bad m_magenta/e_qf/Magenta_${SMPL}_trimR_trimF_bad -log m_magenta/e_qf/Magenta_${SMPL}_trimR_trimF_qc.log
done &

cat m_magenta/e_qf/Magenta_*_trimR_trimF_good.fastq > m_magenta/e_qf/Magenta_all.fastq &

## Add barcode label to reads

for SMPL in `cat m_magenta/y_help/Magenta_Samples.txt`
do
   awk -v SMPL=${SMPL} '{if($1~">") print $1";barcodelabel="SMPL";";else print $1}' m_magenta/e_qf/Magenta_${SMPL}_trimR_trimF_good.fasta > m_magenta/e_qf/Magenta_${SMPL}_barcode.fasta
done &

mkdir m_magenta/f_otu

cat m_magenta/e_qf/Magenta_*_barcode.fasta > m_magenta/f_otu/MagentaExps_barcode.fasta &

## =======================================================================================
## K| OTU clustering of MiSeq and Magenta OTUs 
## =======================================================================================

mkdir r_ref

## Note: GOLD database v.5 (gold.fa) (https://gold.jgi.doe.gov) and Silva v.119 (Silva119_release) (https://www.arb-silva.de) must be downloaded by the user.
## Note: upload gold.fa and Silva119_release to newly created director r_ref

## Set aliases for UPARSE and reference databases
U="/YOUR_PATH_TO/usearch8.1.1812_i86linux64"
REFdb="/YOUR_PATH_TO/r_ref/gold.fa"
TAXdb="/YOUR_PATH_TO/r/ref/Silva119_release/taxonomy/97/taxonomy_97_7_levels.txt"

mkdir g_combined

## Combine Magenta and CC MiSeq fasta files together
cat m_magenta/f_otu/MagentaExps_barcode.fasta f_otu/CC16S_barcode.fasta > g_combined/16S_all_barcode.fasta &

## Combine Magenta and MiSeq fastq files together for read length distribution
cat e_qf/CC16s_all.fastq m_magenta/e_qf/Magenta_all.fastq > g_combined/All_seqlen.fastq &

## Get length report
prinseq-lite.pl -verbose -fastq g_combined/All_seqlen.fastq -graph_data -graph_stats ld &
	
## Trim to fixed length
${U} -fastx_truncate g_combined/16S_all_barcode.fasta -trunclen 360 -label_suffix _360 -fastaout g_combined/16S_all_barcode_360.fasta &

## Sort reads
${U} -sortbylength g_combined/16S_all_barcode_360.fasta -fastaout g_combined/16S_all_barcode_360_sort.fasta &

## De-replicating
${U} -derep_fulllength g_combined/16S_all_barcode_360_sort.fasta -fastaout g_combined/16S_all_barcode_360_sort_unique.fasta -threads 10 -sizeout &

## Abundance sorting
${U} -sortbysize g_combined/16S_all_barcode_360_sort_unique.fasta -fastaout g_combined/16S_all_barcode_360_sort_unique_ab2.fasta -minsize 2 -threads 15 &

## OTU clustering
${U} -cluster_otus g_combined/16S_all_barcode_360_sort_unique_ab2.fasta -otus g_combined/16S_all_barcode_360_sort_unique_ab2_otu.fasta -uparseout g_combined/16S_all_barcode_360_sort_unique_ab2_otu.up -sizein -sizeout &

## Chimera removal
${U} -uchime_ref g_combined/16S_all_barcode_360_sort_unique_ab2_otu.fasta -db ${REFdb} -nonchimeras g_combined/16S_all_barcode_360_sort_unique_ab2_otu_chimerafree.fasta -chimeras g_combined/16S_all_barcode_360_sort_unique_ab2_otu_chimera.fasta -strand plus -threads 10 &

## Label OTU sequences
python ~/4_py/fasta_number.py g_combined/16S_all_barcode_360_sort_unique_ab2_otu_chimerafree.fasta OTU_ > g_combined/16S_all_otu_ab2_id97.fa &

## Map barcoded reads to OTUs
${U} -usearch_global g_combined/16S_all_barcode_360.fasta -db g_combined/16S_all_otu_ab2_id97.fa -strand plus -id 0.97 -uc g_combined/16S_all_360_otu_ab2_id97.uc &

## UC-2-tab

## Note: place uc2otutab.py in y_help

python y_help/uc2otutab.py g_combined/16S_all_360_otu_ab2_id97.uc > g_combined/16S_all_otu_ab2_id97.txt &

## Taxonomy assignment with RDP classifier in QIIME

source /YOUR_PATH_TO/QIIME-1.8.0/activate.sh

TAXdb="/YOUR_PATH_TO/r_ref/Silva119_release/taxonomy/97/taxonomy_97_7_levels.txt"
REFSEQ="/YOUR_PATH_TO/r_ref/Silva119_release/rep_set/97/Silva_119_rep_set97.fna"

assign_taxonomy.py -i g_combined/16S_all_otu_ab2_id97.fa -o g_combined/ -t ${TAXdb} -r ${REFSEQ} -c 0.5 -m rdp --rdp_max_memory 32000 &

## =======================================================================================
## L | Treefile with QIIME 
## =======================================================================================
source /YOUR_PATH_TO/QIIME-1.8.0/activate.sh

mkdir p_pynastalign

## Note: upload otus_subset.txt to p_pynastalign

## Remove OTUs flagged in R from OTU rep set
filter_fasta.py -f g_combined/16S_all_otu_ab2_id97.fa -o p_pynastalign/16S_otu_ab2_id97_filtered.fa -s p_pynastalign/otus_subset.txt &

## OTU sequence alignment with PyNAST
align_seqs.py -m pynast -i p_pynastalign/16S_otu_ab2_id97_filtered.fa -o p_pynastalign -p 0.5 &

## Remove gaps in alignment
filter_alignment.py -v -i  p_pynastalign/16S_otu_ab2_id97_filtered_aligned.fasta --lane_mask_fp /YOUR_PATH_TO/QIIME/lanemask_in_1s_and_0s -o  p_pynastalign &

## Create phylogenetic tree using FastTree
make_phylogeny.py -v -t fasttree --input_fp  p_pynastalign/16S_otu_ab2_id97_filtered_aligned_pfiltered.fasta --tree_method fasttree --log_fp  p_pynastalign/16S_otu_ab2_id97_filtered_aligned_pfiltered_tree.log -o  p_pynastalign/16S_otu_ab2_id97.tre --root_method midpoint &  

##========================================================================================
## M | Alpha diversity curves
##========================================================================================

mkdir q_qiime
mkdir q_qiime/alpha_div
mkdir q_qiime/alpha_div/a_multi_rare
mkdir q_qiime/alpha_div/b_alpha_diversity
mkdir q_qiime/alpha_div/c_alpha_collate

## Note: Upload otu_mat.txt and CC_NS_otu_keep.txt to q_qiime/alpha_div

## Convert tab delimited rarefied OTU table to biom format
biom convert -i q_qiime/alpha_div/otu_mat.txt -o q_qiime/alpha_div/otu_mat.biom --table-type "otu table" &

## Filter tree for CC_NS OTUs only
filter_tree.py -i p_pynastalign/16S_otu_ab2_id97.tre -t q_qiime/alpha_div/CC_NS_otu_keep.txt -o q_qiime/alpha_div/CC_NS_tree.tre

## Multiple rarefactions of rarefied OTU table
multiple_rarefactions.py -i q_qiime/alpha_div/otu_mat.biom -o q_qiime/alpha_div/a_multi_rare -m 2000 -x 100000 -s 2000 -n 100 &

## Alpha diversity of OTU tables (Observed Species, PD whole tree, Shannon Evenness)
alpha_diversity.py -i q_qiime/alpha_div/a_multi_rare -o q_qiime/alpha_div/b_alpha_diversity -m equitability,observed_species,PD_whole_tree -t q_qiime/alpha_div/CC_NS_tree.tre &

## Collate OTU tables together
collate_alpha.py -i q_qiime/alpha_div/b_alpha_diversity -o q_qiime/alpha_div/c_alpha_collate &

##========================================================================================
## N | Filter representative OTUs for "unassigned" root enriched OTUs
##========================================================================================

## Note: upload unassigned_root_enriched.txt to g_combined.  The output file of this step can be used to identify the taxonomy of the OTUs with NCBI BLAST

filter_fasta.py -f g_combined/16S_all_otu_ab2_id97.fa -o g_combined/16S_otu_ab2_id97_unassigned_rootenriched.fa -s g_combined/unassigned_root_enriched.txt &

############################
## Clone library analysis ##
############################


## Note: The raw .AB1 files needed to reproduce this section are available from the authors upon request

source /YOUR_PATH_TO/QIIME-1.8.0/activate.sh

## =======================================================================================
## A | Extract sequenced clone .ab1 files from zipped archive
## =======================================================================================

mkdir h_clone

Note: upload clones_ab1.tar.gz to h_clone.  This file is available in Supplementary data 5.

## Extract files
tar -zxvf h_clone/clones_ab1.tar.gz &

## Remove gz file
rm h_clone/clones_ab1.tar.gz

## =======================================================================================
## B | Convert AB1 files to fastq
## =======================================================================================

## Convert .ab1 to .fastq file
for FILE in h_clone/*.ab1
do                                                                                                                                                                                                                                                             
    seqret -sformat abi -osformat fastq  -auto -stdout -sequence "$FILE" > "$(basename "$FILE" .ab1).fastq"                                                                                                                                                                   
done & 

## =======================================================================================
##  C | Rename FASTQ header with clone name
## =======================================================================================

## Get file names
ls *.fastq > clone_list.txt 

## Note: .fastq file ending was removed in text editor and re-uploaded to h_clone.  This modified file is available in Supplementary Data 2
## and must be uploaded to h_clone for subsequent steps

## Place isolate name in FASTQ file header
for SMPL in `cat h_clone/clone_list.txt`
do
   awk -v SMPL=${SMPL} '{if(NR==1) print "@"SMPL"";else print $1}' h_clone/${SMPL}.fastq > h_clone/${SMPL}_renamed.fastq &
done &

## =======================================================================================
##  D | Split fastq into fasta and qual files
## =======================================================================================

## Split .fastq into .fasta and .qual files
for SMPL in `cat h_clone/clone_list.txt`
do
	convert_fastaqual_fastq.py -f h_clone/${SMPL}_renamed.fastq -c fastq_to_fastaqual -o h_clone
	
done &

cat h_clone/*_renamed.fna > h_clone/clones_all_renamed.fna &

## =======================================================================================
##  E | Trim sequences at 799f primer site
## =======================================================================================

## Note: upload 799_rc.txt to h_clone
 
## Cut 3' at 799_rc primer site
flexbar -r h_clone/clones_all_renamed.fna -b h_clone/799f_rc.txt -u 400 -bt 0 -be RIGHT -bu &

## The resulting file of this step was exported to Text Wrangler, and degenerate nucleotides were set to N manually.
## This file is used for the rev complement

## Note: upload modified flexbar_barcode_799F_rc.fasta to h_clone

## reverse complement with fastx
fastx_reverse_complement -i h_clone/flexbar_barcode_799F_rc.fasta -o h_clone/clone_lib_799F_rc.fasta &

## Make new directory in preparation for clustering clone sequences to climate chamber/natural site reference sequences
mkdir j_cloneotu

## Trim to 360bp for proper OTU clustering
prinseq-lite.pl -verbose --out_format 1 -ns_max_n 1 -trim_to_len 360 -fasta h_clone/clone_lib_799F_rc.fasta -out_good j_cloneotu/clone_lib_360 &

## =======================================================================================
##  D | Cluster against CC reference OTUs
=======================================================================================
## Set alias for UPARSE
U="/YOUR_PATH_TO/usearch8.1.1812_i86linux64"

## Sort reads
${U} -sortbylength j_cloneotu/clone_lib_360.fasta -fastaout j_cloneotu/clone_lib_360_sort.fasta &

## De-replicating
${U} -derep_fulllength j_cloneotu/clone_lib_360_sort.fasta -fastaout j_cloneotu/clone_lib_360_sort_unique.fasta -threads 10 -sizeout &

## Abundance sorting
${U} -sortbysize j_cloneotu/clone_lib_360_sort_unique.fasta -fastaout j_cloneotu/clone_lib_360_sort_unique_ab1.fasta -minsize 1 -threads 15 &

## OTU clustering
${U} -cluster_otus j_cloneotu/clone_lib_360_sort_unique_ab1.fasta -otus j_cloneotu/clone_lib_360_sort_unique_ab1.fasta -uparseout j_cloneotu/clone_lib_360_sort_unique_ab1_otu.up -sizein -sizeout &

## Map barcoded reads to OTUs

## First copy filtered reference sequences from step L to j_cloneotu
cp p_pynastalign/16S_otu_ab2_id97_filtered.fa j_clone_otu

${U} -usearch_global j_cloneotu/clone_lib_360_sort_unique.fasta -db j_cloneotu/16S_otu_ab2_id97_filtered.fa -strand plus -id 0.97 -uc clone_uparse_otus.uc &

######################################
########## Isolate analysis ##########
######################################

source /YOUR_PATH_TO/QIIME-1.8.0/activate.sh

## Note: The raw .AB1 files needed to reproduce this section are available from the authors upon request
## The analysis can be started at section G with the isolate FASTA sequences available in Additional file 5.

## =======================================================================================
## A | Extract isolate .ab1 files from zipped archive
## =======================================================================================

mkdir i_isolate
mkdir i_isolate/a_data
mkdir i_isolate/a_data/1_ab1files

## Note: upload bacteria_isolates_ab1.tar.gz to i_isolate/a_data/1_ab1files.  This file is available from the authors upon request.

## Extract files
tar -zxvf i_isolate/a_data/1_ab1files/bacteria_isolates_ab1.tar.gz

## Remove tar gz file
rm i_isolate/a_data/1_ab1files/bacteria_isolates_ab1.tar.gz

## =======================================================================================
##  B | Make .ab1 files into .fastq
## =======================================================================================
mkdir 2_fastqfiles

## Convert .ab1 to .fastq file
for FILE in i_isolate/a_data/1_ab1files/*.ab1
do                                                                                                                                                                                                                                                             
    seqret -sformat abi -osformat fastq  -auto -stdout -sequence "$FILE" > "$(basename "$FILE" .ab1).fastq"                                                                                                                                                                   
done & 

## Move resulting files to newly created directory
mv *.fastq i_isolate/a_data/2_fastqfiles &

## =======================================================================================
##  C | Rename FASTQ header with isolate name
## =======================================================================================

## Get file names
ls i_isolate/a_data/2_fastqfiles/ > i_isolate/a_data/isolate_list.txt 

## Note: .fastq file ending was removed in text editor manually.  Upload isolate_list.txt to i_isolate/a_data

## Place isolate name in FASTQ file header
for SMPL in `cat i_isolate/a_data/isolate_list.txt`
do
   awk -v SMPL=${SMPL} '{if(NR==1) print "@"SMPL"";else print $1}' i_isolate/a_data/2_fastqfiles/${SMPL}.fastq > a_data/2_fastqfiles/${SMPL}_renamed.fastq &
done &

## =======================================================================================
##  D | Convert ambiguous bases with seqtk
## =======================================================================================

## Because the sequences were generated with 1401R, they are in the 3'-5' direction.  Further analysis
## requires that they be reverse complemented.  However, a few ambiguous nucleotides in the sequences prevent
## FastX from running the RC command.  Therefore, the ambiguous bases have to be randomly assigned based
## upon the IPUAC codes with seqtk.

mkdir i_isolate/a_data/3_fastqualfiles

## Split .fastq into .fasta and .qual files (seqtk drops the qual information with .fastq files)
for SMPL in `cat i_isolate/a_data/isolate_list.txt`
do
	convert_fastaqual_fastq.py -f i_isolate/a_data/2_fastqfiles/${SMPL}_renamed.fastq -c fastq_to_fastaqual -o i_isolate/a_data/3_fastqualfiles
	
done &

## Convert bases with seqtk
for SMPL in `cat i_isolate/a_data/isolate_list.txt`
do
	seqtk randbase i_isolate/a_data/3_fastqualfiles/${SMPL}_renamed.fna > i_isolate/a_data/3_fastqualfiles/${SMPL}_renamed_converted.fna

done &

mkdir i_isolate/a_data/4_fastqconverted

## Reconvert back to .fastq file
for SMPL in `cat i_isolate/a_data/isolate_list.txt`
do
	convert_fastaqual_fastq.py -f i_isolate/a_data/3_fastqualfiles/${SMPL}_renamed_converted.fna -q i_isolate/a_data/3_fastqualfiles/${SMPL}_renamed.qual -c fastaqual_to_fastq -o i_isolate/a_data/4_fastqconverted
	
done &

## Reverse complement 
for SMPL in `cat i_isolate/a_data/isolate_list.txt`
do
	fastx_reverse_complement -i i_isolate/a_data/4_fastqconverted/${SMPL}_renamed_converted.fastq -o i_isolate/a_data/4_fastqconverted/${SMPL}_renamed_converted_rev.fastq -Q33
done &

## Merge all sequences together into one .fastq file
mkdir i_isolate/b_qualfilter

cat i_isolate/a_data/4_fastqconverted/*_renamed_converted_rev.fastq > i_isolate/b_qualfilter/isolate_seqs.fastq &

## =======================================================================================
##  E| Quality Filtering with Prinseq
## =======================================================================================

## Note: upload paramsL.txt and paramsR.txt to i_isolate/b_qualfilter
 
## Trim 5' end
prinseq-lite.pl -verbose -params i_isolate/b_qualfilter/paramsL.txt -fastq i_isolate/b_qualfilter/isolate_seqs.fastq -out_good i_isolate/b_qualfilter/isolate_seqs_good_trimL -out_bad i_isolate/b_qualfilter/isolate_seqs_bad_trimL -log i_isolate/b_qualfilter/isolate_seqs_trimL.log &

## Trim 3' end
prinseq-lite.pl -verbose -params i_isolate/b_qualfilter/paramsR.txt -fastq i_isolate/b_qualfilter/isolate_seqs_good_trimL.fastq -out_good i_isolate/b_qualfilter/isolate_seqs_good_trimL_trimR -out_bad i_isolate/b_qualfilter/isolate_seqs_bad_trimL_trimR -log i_isolate/b_qualfilter/isolate_seqs_trimL_trimR.log &

convert_fastaqual_fastq.py -f i_isolate/b_qualfilter/isolate_seqs_good_trimL_trimR.fastq -c fastq_to_fastaqual -o i_isolate/b_qualfilter &

## Get quality file
prinseq-lite.pl -verbose -fastq i_isolate/b_qualfilter/isolate_seqs_good_trimL_trimR.fastq -graph_data &

## =======================================================================================
##  F| Flexbar
## =======================================================================================

mkdir i_isolate/c_flexbar

## Note: upload 799f.txt and 1193_rc.txt to i_isolate/c_flexbar

## Cut 5' at 799 primer site
flexbar -r i_isolate/b_qualfilter/isolate_seqs_good_trimL_trimR.fastq -b i_isolate/c_flexbar/799f.txt -f sanger -u 13 -bt 2 -be LEFT -bu &

mv flexbar_barcode_799F.fastq flexbar_barcode_unassigned.fastq i_isolate/c_flexbar

## Cut 3' at 1193_rc primer site
flexbar -r i_isolate/c_flexbar/flexbar_barcode_799F.fastq -b i_isolate/c_flexbar/1193_rc.txt -f sanger -u 13 -bt 2 -be RIGHT -bu &

## Move files to c_flexbar
mv *.fastq i_isolate/c_flexbar

## Add discarded read back to fastq file (no Ns, 333bp *high quality*, Unsure why it is discarded during processing)
cat i_isolate/c_flexbar/flexbar_barcode_1193_rc.fastq i_isolate/c_flexbar/flexbar_barcode_unassigned.fastq > i_isolate/c_flexbar/flexbar_barcode_1193_rc_all.fastq &

prinseq-lite.pl -verbose -fastq i_isolate/c_flexbar/flexbar_barcode_1193_rc_all.fastq -graph_data &

convert_fastaqual_fastq.py -f i_isolate/c_flexbar/flexbar_barcode_1193_rc_all.fastq -c fastq_to_fastaqual -o i_isolate/c_flexbar

convert_fastaqual_fastq.py -f i_isolate/c_flexbar/flexbar_barcode_unassigned.fastq -c fastq_to_fastaqual -o i_isolate/c_flexbar

## =======================================================================================
##  G | Cluster Isolated Bacteria Sequences to OTUs
## =======================================================================================
source /YOUR_PATH_TO/QIIME-1.8.0/activate.sh
U="/YOUR_PATH_TO/usearch8.1.1812_i86linux64"

mkdir i_isolate/g_cluster

## Copy reference OTUs and list of CC/NS OTUs to keep to directory g_cluster for filtering of sequences before clustering

cp g_combined/16S_all_otu_ab2_id97.fa i_isolate/g_cluster/
cp q_qiime/alpha_div/CC_NS_otu_keep.txt i_isolate/g_cluster/

filter_fasta.py -f i_isolate/g_cluster/16S_all_otu_ab2_id97.fa -o i_isolate/g_cluster/16S_all_otu_ab2_id97_filtered.fa -s i_isolate/g_cluster/CC_NS_otu_keep.txt &

## Trim to fixed length
${U} -fastx_truncate i_isolate/c_flexbar/flexbar_barcode_1193_rc_all.fna -trunclen 360 -label_suffix _360 -fastaout i_isolate/g_cluster/flexbar_barcode_1193_rc_360.fasta &

## Concatenate removed sequence into fasta file

cat i_isolate/c_flexbar/flexbar_barcode_unassigned.fna i_isolate/g_cluster/flexbar_barcode_1193_rc_360.fasta > i_isolate/g_cluster/flexbar_barcode_1193_rc_360_all.fasta &

mv flexbar_barcode_1193_rc_360_all.fasta g/cluster

## Cluster OTUs with Reference sequences
${U} -usearch_global i_isolate/g_cluster/flexbar_barcode_1193_rc_360_all.fasta -db i_isolate/g_cluster/16S_all_otu_ab2_id97_filtered.fa -strand plus -id 0.97 -uc isolate_uparse_otus_360.uc &

## =======================================================================================
## H | Assign taxonomy with QIIME 
## =======================================================================================
TAXdb="/YOUR_PATH_TO/r_ref/Silva119_release/taxonomy/97/taxonomy_97_7_levels.txt"
REFSEQ="/YOUR_PATH_TO/r_ref/Silva119_release/rep_set/97/Silva_119_rep_set97.fna"

assign_taxonomy.py -i i_isolate/b_qualfilter/isolate_seqs_good_trimL_trimR.fna -o i_isolate/rdp_taxonomy/ -t ${TAXdb} -r ${REFSEQ} -m rdp --rdp_max_memory 32000 &

## =======================================================================================
## I | Treefile with QIIME (v8.0.1623_i86linux32) 
## =======================================================================================

mkdir t_tree

## OTU sequence alignment with PyNAST
align_seqs.py -v -m pynast -e 330 -i i_isolate/b_qualfilter/isolate_seqs_good_trimL_trimR.fna -o i_isolate/t_tree  &

## Remove gaps in alignment
filter_alignment.py -v -i i_isolate/t_tree/isolate_seqs_good_trimL_trimR_aligned.fasta --lane_mask_fp /YOUR_PATH_TO/QIIME/lanemask_in_1s_and_0s -o  i_isolate/t_tree &

## Create phylogenetic tree using FastTree
make_phylogeny.py -v -t fasttree --input_fp i_isolate/t_tree/isolate_seqs_good_trimL_trimR_aligned_pfiltered.fasta --tree_method fasttree --log_fp  i_isolate/t_tree/isolate_seqs_good_trimL_trimR_aligned_pfiltered.log -o  i_isolate/t_tree/isolate_seqs_good_trimL_trimR_aligned_pfiltered.tre --root_method midpoint &

## =======================================================================================
## J | Filter isolate OTUs for "unassigned" isolates 
## =======================================================================================

## Note: upload unassigned_isolates.txt to i_isolate
filter_fasta.py -f i_isolate/b_qualfilter/isolate_seqs_good_trimL_trimR.fna -s i_isolate/unassigned_isolates.txt -o i_isolate/unassigned_isolates.fasta &

The output file of this step is be used to identify the taxonomy of the isolates with NCBI BLAST

##### //END// #####