__author__ = 'Alena Zablotskaya. Laboratory for Cytogenetics and Genome Research (Prof. Dr. Joris R. Vermeesch), KU Leuven, Leuven, Belgium' #This script analyses separate TSSV outputs obtained per locus, makes decisions on the genotypes, analyses partial reads for potential repeat expansions, and combines all the data from all loci into general output files #Usage example: python3.4 /path/to/this/script/TSSVintoGenotype.py -l /path/to/TSSV/library/file/general_TSSV_library.csv -f /path/to/directory/with/TSSV/output/organized/by/locus/perLocus -s /path/to/SAM/file/reads.sam -c=False -d=False -a=0 def count_full_copies(allele_report, unit_length, tag, consensus_mode, alleles_to_align_file): allele_seq = 0 full_allele_seq = 0 new_alleles_dict = {} known_alleles_dict = {} output_string = '' counter = 1 with open(allele_report) as alleles: for line in alleles: if 'allele' not in line: name, forw, rev, reads, allele_seq = line.split("\t") reads = int(reads) allele_seq = allele_seq.strip() if tag == 'new': full_allele_seq = allele_seq allele_length = int(len(allele_seq)) copyNumber = allele_length/unit_length copyNumber = float(Decimal(copyNumber).quantize(Decimal('1e-0'))) if copyNumber in new_alleles_dict: new_alleles_dict[copyNumber] += reads else: new_alleles_dict[copyNumber] = reads elif tag == 'known': pattern, copyNumber = allele_seq[:-1].split("(") full_allele_seq = ''.join([pattern for s in range(int(copyNumber))]) copyNumber = float(copyNumber) allele_length = copyNumber*unit_length known_alleles_dict[copyNumber] = reads output_string += str(copyNumber)+"\t"+str(allele_length)+"\t"+allele_seq+"\t"+str(reads)+"\t" if consensus_mode == "True" and len(allele_seq) > 0: alleles_to_align = open(alleles_to_align_file, 'a') for i in range(reads): alleles_to_align.write('>seq'+str(counter)+"\n"+full_allele_seq+"\n") counter = counter + 1 alleles_to_align.close() if allele_seq == 0: allele_length = 0 copyNumber = 0 reads = 0 output_string += str(Decimal(copyNumber).quantize(Decimal('1e-2')))+"\t"+str(allele_length)+"\t"+str(allele_seq)+"\t"+str(reads)+"\t" output_string += "\n" alleles.close() if tag == 'new': return output_string, new_alleles_dict elif tag == 'known': return output_string, known_alleles_dict def read_fasta_file(input_filename): seq_dict = {} buffer_seq = "" buffer_name = "" not_first_iteration = False with open(input_filename) as fa_input: for line in fa_input: if line.startswith(">"): if not_first_iteration: seq_dict[buffer_name] = buffer_seq buffer_seq = "" buffer_name = line else: not_first_iteration = True buffer_seq += line.strip() seq_dict[buffer_name] = buffer_seq return seq_dict def reverse_complement(seq): alt_map = {'ins':'0'} complement = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'} for k,v in alt_map.items(): seq = seq.replace(k,v) bases = list(seq) bases = reversed([complement.get(base,base) for base in bases]) bases = ''.join(bases) for k,v in alt_map.items(): bases = bases.replace(v,k) return bases def count_partial_copies(seq, flank, number_of_searches, seq_length, adaptor_len, flank_length, unit_length, ref_copies_max, side): read_Extension_buffer = 0 read_NoExtension_buffer = 0 copies_NoExtension = '' copies_Extension = '' counter = 0 while counter < int(number_of_searches): if side == 'left': start_11ntPart = int(flank_length) - 11 - counter flank_11ntPart = flank[start_11ntPart:start_11ntPart+11] flank_11ntPart_location = seq.rfind(flank_11ntPart) elif side == 'right': start_11ntPart = 0 + counter flank_11ntPart = flank[start_11ntPart:start_11ntPart+11] flank_11ntPart_location = seq.find(flank_11ntPart) if flank_11ntPart_location > -1: if side == 'left': partial_locus_length = int(seq_length) - int(flank_11ntPart_location) - int(adaptor_len) + int(start_11ntPart) partial_allele_length = partial_locus_length - int(flank_length) elif side == 'right': partial_allele_length = int(flank_11ntPart_location) - int(adaptor_len) partial_allele_copies = partial_allele_length/int(unit_length) if partial_allele_copies > int(ref_copies_max): read_Extension_buffer = 1 copies_Extension = partial_allele_copies elif partial_allele_copies <= int(ref_copies_max): read_NoExtension_buffer = 1 copies_NoExtension = partial_allele_copies break counter += 1 not_found = '' if flank_11ntPart_location == -1: not_found = 'not_found' return not_found, read_NoExtension_buffer, read_Extension_buffer, copies_NoExtension, copies_Extension def process_partial_alleles(seq_dict, flank, side, adaptor_len, unit_length, ref_copies_max): reads_NoExtension = 0 reads_Extension = 0 copies_Extension_list = [] copies_NoExtension_list = [] flank_length = len(flank) number_of_searches = flank_length - 10 for fasta_entry in seq_dict.keys(): seq = seq_dict[fasta_entry] seq_length = len(seq) result, reads_NoExtension_buffer, reads_Extension_buffer, copies_NoExtension_buffer, copies_Extension_buffer = count_partial_copies(seq, flank, number_of_searches, seq_length, adaptor_len, flank_length, unit_length, ref_copies_max, side) if result == 'not_found': seq_reverse_complement = reverse_complement(seq) result_revcompl, reads_NoExtension_buffer, reads_Extension_buffer, copies_NoExtension_buffer, copies_Extension_buffer = count_partial_copies(seq_reverse_complement, flank, number_of_searches, seq_length, adaptor_len, flank_length, unit_length, ref_copies_max, side) if reads_NoExtension_buffer != 0: reads_NoExtension = reads_NoExtension + int(reads_NoExtension_buffer) copies_NoExtension_list.append(int(copies_NoExtension_buffer)) if reads_Extension_buffer != 0: reads_Extension = reads_Extension + int(reads_Extension_buffer) copies_Extension_list.append(int(copies_Extension_buffer)) return reads_NoExtension, reads_Extension, copies_NoExtension_list, copies_Extension_list def process_input(library_file, tssv_folder, adaptor_len, map_file, consensus_mode, diploidy): #==================DETERMINE_PATHS_TO_PROGRAM_FILES================== EMBOSS_path = '/home/azablo0/EMBOSS-6.6.0/emboss/' #cons #==================================================================== workingDir = os.path.dirname(map_file) sample_name, extension = os.path.basename(map_file).split('.') new_alleles_file = workingDir+'/'+sample_name+'_report_new.txt' if os.path.exists(new_alleles_file): os.remove(new_alleles_file) new_alleles = open(new_alleles_file, 'a') new_alleles.write('''RepeatName Copies(Rounded) AlleleLength(nt) AlleleSequence SuppReads etc'''+"\n") known_alleles_file = workingDir+'/'+sample_name+'_report_known.txt' if os.path.exists(known_alleles_file): os.remove(known_alleles_file) known_alleles = open(known_alleles_file, 'a') known_alleles.write('''RepeatName Copies(Rounded) AlleleLength(nt) AlleleSequence SuppReads etc'''+"\n") comb_alleles_file = workingDir+'/'+sample_name+'_report_comb_CopyNumbers.txt' if os.path.exists(comb_alleles_file): os.remove(comb_alleles_file) comb_alleles = open(comb_alleles_file, 'a') comb_alleles.write('''RepeatName TotalReads Copies SuppReads Copies SuppReads Copies SuppReads etc'''+"\n") comb_consensus_alleles_file = workingDir+'/'+sample_name+'_report_consensus_CopyNumbers.txt' if os.path.exists(comb_consensus_alleles_file): os.remove(comb_consensus_alleles_file) if consensus_mode == "True": consensus_alleles = open(comb_consensus_alleles_file, 'a') consensus_alleles.write('''RepeatName TotalReads Copies'''+"\n") exp_vs_noexp_file = workingDir+'/'+sample_name+'_report_exp_vs_noexp.txt' if os.path.exists(exp_vs_noexp_file): os.remove(exp_vs_noexp_file) exp_vs_noexp = open(exp_vs_noexp_file, 'a') exp_vs_noexp.write('''RepeatName MaxCopies_Ref No_expansion_tag MinCopies_noExp MaxCopies_noExp TotalReads_noExp Expansion_tag MinCopies_Exp MaxCopies_Exp TotalReads_Exp Expansion_Range'''+"\n") unique_tags = [] with open(library_file) as library: for repeat_library in library: name, left_flank, right_flank, variation, unique_tag = repeat_library.split("\t") alleles_to_align_file = workingDir+'/'+name+'_alleles_to_align.fasta' consensus_allele_file = workingDir+'/'+name+'_alleles_aligned.fasta' if consensus_mode == "True": if os.path.exists(alleles_to_align_file): os.remove(alleles_to_align_file) if os.path.exists(consensus_allele_file): os.remove(consensus_allele_file) if unique_tag not in unique_tags: #Regular repeat or first repeat from a cluster -> to process unique_tags.append(unique_tag) unit_seq, ref_copies_min, ref_copies_max = variation.split() ref_copies_max = int(ref_copies_max) unit_length = int(len(unit_seq)) for dirpath, dirnames, files in os.walk(tssv_folder): for dirname in dirnames: if dirname.startswith(name): repeat_dir = os.path.join(dirpath, dirname) break else: repeat_dir = '' continue break #Check if the repeat specific directory was found, and if so - process data in it if repeat_dir != '': #============================================================================================================================ #-----------------------------DERIVING GENOTYPES FROM THE SEQUENCE LENGTH OF DETECTED ALLELES-------------------------------- #============================================================================================================================ #Getting copy numbers for new alleles new_alleles_string, new_alleles_dict = count_full_copies(repeat_dir+'/newalleles.csv', unit_length, 'new', consensus_mode, alleles_to_align_file) new_alleles.write(name+"\t"+new_alleles_string) #print(new_alleles_dict) #Getting copy numbers for known alleles known_alleles_string, known_alleles_dict = count_full_copies(repeat_dir+'/knownalleles.csv', unit_length, 'known', consensus_mode, alleles_to_align_file) known_alleles.write(name+"\t"+known_alleles_string) #print(known_alleles_dict) if consensus_mode == "True" and os.path.exists(alleles_to_align_file): os.chmod(alleles_to_align_file, 0o755) command = EMBOSS_path+"cons -sequence "+alleles_to_align_file+" -outseq "+consensus_allele_file subprocess.call(command, shell=True) os.chmod(consensus_allele_file, 0o755) consensus_allele = open(consensus_allele_file) consensus_length = 0 for line in consensus_allele: if not line.startswith(">"): for i in list(line.strip()): if i != 'n': consensus_length += 1 consensus_allele.close() consensus_copyNumber = consensus_length/unit_length consensus_copyNumber = float(Decimal(consensus_copyNumber).quantize(Decimal('1e-0'))) else: consensus_copyNumber = 0 #Combining all copy numbers in one table comb_alleles_dict = {} for variant in new_alleles_dict.keys(): if variant in known_alleles_dict.keys(): comb_reads = new_alleles_dict[variant] + known_alleles_dict[variant] comb_alleles_dict[variant] = comb_reads else: comb_alleles_dict[variant] = new_alleles_dict[variant] for variant in known_alleles_dict.keys(): if variant not in comb_alleles_dict: comb_alleles_dict[variant] = known_alleles_dict[variant] #Sorting variants by descending number of supporting reads comb_alleles_dict_sorted = {} counter = 0 variants_printed = [] total_reads = 0 while counter < len(comb_alleles_dict): buffer_reads = 0 buffer_variant = '' for variant in comb_alleles_dict.keys(): reads = comb_alleles_dict[variant] if buffer_reads < reads and variant not in variants_printed: buffer_reads = reads buffer_variant = variant comb_alleles_dict_sorted[counter] = {} comb_alleles_dict_sorted[counter][buffer_variant] = buffer_reads variants_printed.append(buffer_variant) total_reads = total_reads + buffer_reads counter = counter + 1 #print(comb_alleles_dict_sorted) #Pairwise comparison of the most represented variants. #If two variants are supported by equal number of reads, # and one of these variants corresponds to the reference sequence, # than such a variant gets displayed with the first priority. counter = 1 priority_variant = {} equiprobable_variants = [] while counter < len(comb_alleles_dict_sorted): reads_for_alleleA = list(comb_alleles_dict_sorted[counter-1].values()) reads_for_alleleA = int(reads_for_alleleA[0]) reads_for_alleleB = list(comb_alleles_dict_sorted[counter].values()) reads_for_alleleB = int(reads_for_alleleB[0]) if reads_for_alleleA == reads_for_alleleB: variantA = list(comb_alleles_dict_sorted[counter-1].keys()) variantA = int(variantA[0]) variantB = list(comb_alleles_dict_sorted[counter].keys()) variantB = int(variantB[0]) if variantA == ref_copies_max: priority_variant = comb_alleles_dict_sorted[counter-1] break elif variantB == ref_copies_max: priority_variant = comb_alleles_dict_sorted[counter] break else: if variantA not in equiprobable_variants: equiprobable_variants.append(variantA) if variantB not in equiprobable_variants: equiprobable_variants.append(variantB) priority_variant_reads = reads_for_alleleA counter = counter + 1 else: break #If reference allele was not found and assigned to the priority variant, # a list of equiprobable variants (=supported by equal read numbers) is analyzed. # A median variant is then chosen as the priority variant for haploid genome. Or the closest longer variant. # Shortest and longest variants are selected for diploid genome. if len(priority_variant) == 0 and len(equiprobable_variants) > 0: equiprobable_variants.sort() if diploidy == "True": shortest_variant = equiprobable_variants[0] longest_variant = equiprobable_variants[-1] else: median_variant = statistics.median(equiprobable_variants) median_variant_rounded = round(median_variant) #Case1: [1, 2, 3, 4], median=2.5, median_rounded=3 if median_variant_rounded in equiprobable_variants: priority_variant[median_variant_rounded] = priority_variant_reads else: #Case2: [1, 2, 5, 6], median=3.5, median_rounded=4 #Case3: [1, 2, 4, 5], median=3, median_rounded=3 for i in equiprobable_variants: if int(i)-median_variant < 0: continue else: priority_variant[i] = priority_variant_reads break #============================================================================================================================ #-----------------------------------ANALYZING ALLELES WHICH WERE NOT SPANNED COMPLETELY-------------------------------------- #============================================================================================================================ left_flank_length = len(left_flank) right_flank_length = len(right_flank) #Searching for a repeat specific directory in the TSSV output directory for dirpath, dirnames, files in os.walk(repeat_dir): for dirname in dirnames: if dirname.startswith(name): repeat_subdir = os.path.join(dirpath, dirname) break else: continue break #Processing nostart and noend files nostart_seq_dict = read_fasta_file(repeat_subdir+'/nostart.fa') reads_NoExtension1, reads_Extension1, copies_NoExtension_list1, copies_Extension_list1 = process_partial_alleles(nostart_seq_dict, right_flank, 'right', adaptor_len, unit_length, ref_copies_max) noend_seq_dict = read_fasta_file(repeat_subdir+'/noend.fa') reads_NoExtension2, reads_Extension2, copies_NoExtension_list2, copies_Extension_list2 = process_partial_alleles(noend_seq_dict, left_flank, 'left', adaptor_len, unit_length, ref_copies_max) #Determine copy number ranges for partially sequenced alleles with no repeat expansion copies_NoExtension_list = copies_NoExtension_list1 + copies_NoExtension_list2 if len(copies_NoExtension_list) > 0: copies_NoExtension_min = min(copies_NoExtension_list) copies_NoExtension_min = Decimal(copies_NoExtension_min).quantize(Decimal('1e-1')) copies_NoExtension_max = max(copies_NoExtension_list) copies_NoExtension_max = Decimal(copies_NoExtension_max).quantize(Decimal('1e-1')) else: copies_NoExtension_min = '' copies_NoExtension_max = '' reads_NoExtension = int(reads_NoExtension1) + int(reads_NoExtension2) #Determine copy number ranges for partially sequenced alleles with a possible repeat expansion copies_Extension_list = copies_Extension_list1 + copies_Extension_list2 if len(copies_Extension_list) > 0: copies_Extension_min = min(copies_Extension_list) copies_Extension_min = Decimal(copies_Extension_min).quantize(Decimal('1e-1')) copies_Extension_max = max(copies_Extension_list) copies_Extension_max = Decimal(copies_Extension_max).quantize(Decimal('1e-1')) else: copies_Extension_min = '' copies_Extension_max = '' reads_Extension = int(reads_Extension1) + int(reads_Extension2) copies_Extension_set = set(copies_Extension_list) #Repeat specific folder does not exist, all values to be printed out are set to 0 or '' else: total_reads = 0 priority_variant = {} equiprobable_variants = {} comb_alleles_dict_sorted = {} copies_NoExtension_min ='' copies_NoExtension_max = '' reads_NoExtension = 0 copies_Extension_min = '' copies_Extension_max = '' reads_Extension = 0 copies_Extension_set = set() #============================================================================================================================ #-----------------------------------------------------PRINTING RESULTS------------------------------------------------------- #============================================================================================================================ #Printing out the priority (=reference) variant to the combined copy number file comb_alleles.write(name+"\t"+str(total_reads)+"\t") if len(priority_variant) > 0: allele = list(priority_variant.keys()) allele = str(allele[0]) reads = list(priority_variant.values()) reads = str(reads[0]) comb_alleles.write(allele+"\t"+reads+"\t") #Printing out the shortest and longest of equiprobable variants to the combined copy number file (in case od diploid genome) elif diploidy == "True" and len(equiprobable_variants) > 0: reads_shortest = str(comb_alleles_dict[shortest_variant]) reads_longest = str(comb_alleles_dict[longest_variant]) comb_alleles.write(str(shortest_variant)+"\t"+reads_shortest+"\t"+str(longest_variant)+"\t"+reads_longest+"\t") #Printing out all the other variants to the combined copy number file counter = 0 counter_0 = '' counter_reported_alleles = 0 while counter < len(comb_alleles_dict_sorted): allele = list(comb_alleles_dict_sorted[counter].keys()) allele = str(allele[0]) #Skip the '0.0' variant as the least probable to be non-artificial if allele == '0.0': counter_0 = counter counter = counter + 1 continue else: reads = list(comb_alleles_dict_sorted[counter].values()) reads = str(reads[0]) if float(allele) not in priority_variant: if diploidy == "True" and len(priority_variant) == 0: #Shortest and longest variants has been already printed if len(equiprobable_variants) > 0: if float(allele) != shortest_variant and float(allele) != longest_variant: comb_alleles.write(allele+"\t"+reads+"\t") #No equiprobable alleles. Stutters need to be distinguished from the real alleles with lower coverage. elif counter_reported_alleles == 0: #Allele with the highest coverage (main allele) first_allele = allele first_allele_reads = reads comb_alleles.write(allele+"\t"+reads+"\t") counter_reported_alleles = counter_reported_alleles+1 elif counter_reported_alleles == 1: #If coverage of the allele with the second highest coverage is less than 50% of the main allele coverage, # it is considered as stutter, and the main allele is printed out again (homozygous genotype). if int(reads)/int(first_allele_reads) < 0.5: comb_alleles.write(first_allele+"\t"+first_allele_reads+"\t") counter_reported_alleles = counter_reported_alleles+1 #Otherwise allele with the second highest coverage is considered as real and printed out. else: comb_alleles.write(allele+"\t"+reads+"\t") counter_reported_alleles = counter_reported_alleles+1 elif counter_reported_alleles > 1: #All the other alleles are not included in the genotype, but printed out as additional variants comb_alleles.write(allele+"\t"+reads+"\t") #Printing out alleles other than priority variant for a haploid genome else: comb_alleles.write(allele+"\t"+reads+"\t") counter = counter + 1 #Printing '0.0' variant in the end as the least probable to be non-artificial if counter_0 != '': allele = list(comb_alleles_dict_sorted[counter_0].keys()) allele = str(allele[0]) reads = list(comb_alleles_dict_sorted[counter_0].values()) reads = str(reads[0]) comb_alleles.write(allele+"\t"+reads+"\t") comb_alleles.write("\n") #Printing out a consensus (self-aligned) copy number to an additional file (if the consensus mode is turned on) if consensus_mode == "True": if not os.path.exists(consensus_allele_file): consensus_copyNumber = 0 consensus_alleles.write(name+"\t"+str(total_reads)+"\t"+str(consensus_copyNumber)+"\n") #Writing to an output Expansion_vs_NoExpansion file exp_vs_noexp.write(name+"\t"+str(ref_copies_max)+"\t"+'''No_repeat_expansion'''+"\t"+str(copies_NoExtension_min)+"\t"+str(copies_NoExtension_max)+"\t"+str(reads_NoExtension)+"\t"+'''Possible_repeat_expansion'''+"\t"+str(copies_Extension_min)+"\t"+str(copies_Extension_max)+"\t"+str(reads_Extension)+"\t") if len(copies_Extension_set) > 0: for copies in copies_Extension_set: exp_vs_noexp.write(str(Decimal(copies).quantize(Decimal('1e-1')))+"\t") else: exp_vs_noexp.write(str(0)+"\t") exp_vs_noexp.write("\n") #Removing unnecessary files if consensus_mode == "True": if os.path.exists(alleles_to_align_file): os.remove(alleles_to_align_file) if os.path.exists(consensus_allele_file): os.remove(consensus_allele_file) elif unique_tag.startswith('cluster'): unique_tag = unique_tag.strip() comb_alleles.write(name+"\t"+unique_tag+"\t\n") exp_vs_noexp.write(name+"\t"+unique_tag+"\t\n") else: continue library.close() new_alleles.close() known_alleles.close() exp_vs_noexp.close() if os.path.exists(alleles_to_align_file): os.remove(consensus_allele_file) import sys import argparse import os from decimal import * import statistics import subprocess def main(): parser = argparse.ArgumentParser() parser.add_argument("-l", "--library", help="Path to input TSSV library file.", default="") parser.add_argument("-f", "--folder", help="Path to report folder.", default="") parser.add_argument("-a", "--adaptor", help="Adaptor length. Set to 0 if not specified otherwise.", default="0") parser.add_argument("-s", "--sam", help="Path to input SAM file.", default="") parser.add_argument("-c", "--consensus", help="Switching on the mode of alternative genotype determination. If =True, variants are aligned to one another, and copy number is determined for the consensus.", default=False) parser.add_argument("-d", "--diploid", help="Ploidy mode. If =True, genotypes are determined for a diploid genome, otherwise - haploid (for example, chrX or chrY in males).", default=False) args = parser.parse_args() print(args) process_input(args.library, args.folder, args.adaptor, args.sam, args.consensus, args.diploid) if __name__ == "__main__": sys.exit(main())