import pickle import seaborn as sns from pandas import DataFrame import os import csv # import matplotlib as plt import matplotlib.pyplot as plt import matplotlib.patches as patches import pysam import numpy as np def find_stats(bamfile, statfile, software): sam_file = pysam.Samfile(bamfile, "rb") seq = {"1": [], "2": []} current_qname = "" uniqmap_uniqmap = 0 uniqmap_multimap = 0 multimap_multimap = 0 uniqmap_unmap = 0 multimap_unmap = 0 unmap_unmap = 0 cnts = 0 for line in sam_file: tags = dict(line.get_tags()) qname = line.qname if current_qname == "": current_qname = qname if qname != current_qname: uniqed_multi_un = {} for fs in ["1", "2"]: if software == 'bowtie2': AS = tags.get("AS") if AS is not None and not any(tag == "XS" for tag in tags): uniqed_multi_un[fs] = 0 elif AS is None: uniqed_multi_un[fs] = 2 else: uniqed_multi_un[fs] = 1 else: NHs = map(lambda x: x[1], seq[fs]) if len(set(NHs)) == 1: NH = NHs[0] if NH == 1: uniqed_multi_un[fs] = 0 elif NH == -1: uniqed_multi_un[fs] = 2 else: uniqed_multi_un[fs] = 1 else: uniqed_multi_un[fs] = 2 # else: # uniqed_multi_un[fs] = 2 if uniqed_multi_un["1"] == 0 and uniqed_multi_un["2"] == 0: uniqmap_uniqmap += 1 elif (uniqed_multi_un["1"] == 0 and uniqed_multi_un["2"] == 1) or ( uniqed_multi_un["1"] == 1 and uniqed_multi_un["2"] == 0): uniqmap_multimap += 1 elif (uniqed_multi_un["1"] == 1 and uniqed_multi_un["2"] == 1): multimap_multimap += 1 elif (uniqed_multi_un["1"] == 0 and uniqed_multi_un["2"] == 2) or ( uniqed_multi_un["1"] == 2 and uniqed_multi_un["2"] == 0): uniqmap_unmap += 1 elif (uniqed_multi_un["1"] == 1 and uniqed_multi_un["2"] == 2) or ( uniqed_multi_un["1"] == 2 and uniqed_multi_un["2"] == 1): multimap_unmap += 1 elif (uniqed_multi_un["1"] == 2 and uniqed_multi_un["2"] == 2): unmap_unmap += 1 else: print ("ERRR3 ", line) # aaaa current_qname = qname seq = {"1": [], "2": []} flag = np.binary_repr(line.flag, 12) tags = dict(line.get_tags()) NH = -1 if "NH" not in tags else tags["NH"] mpd = flag[-3] == "0" pmpd = flag[-4] == "0" first = flag[-7] == "1" second = flag[-8] == "1" if not (first ^ second): print ("ERRR1 ", line) # aaaa if (not mpd) and NH > 0: print ("ERRR1 ", line) # aaaa fs = "1" if first else "2" seq[fs].append([flag, NH, mpd, pmpd]) cnts += 1 with open(statfile, 'wb') as csvfile: spamwriter = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow( ["uniqmap_uniqmap", "uniqmap_multimap", "multimap_multimap", "uniqmap_unmap", "multimap_unmap", "unmap_unmap", "total", "cnts"]) spamwriter.writerow( [uniqmap_uniqmap, uniqmap_multimap, multimap_multimap, uniqmap_unmap, multimap_unmap, unmap_unmap, sum([uniqmap_uniqmap, uniqmap_multimap, multimap_multimap, uniqmap_unmap, multimap_unmap, unmap_unmap]), cnts]) def find_matchstats(bamfile, matchstatfile): sam_file = pysam.Samfile(bamfile, "rb") match_stats = {} for line in sam_file: if line.cigar: codes = {} for k, v in line.cigar: if k not in codes: codes[k] = 0 codes[k] += v for k, v in codes.iteritems(): if k not in match_stats: match_stats[k] = {} if v not in match_stats[k]: match_stats[k][v] = 0 match_stats[k][v] += 1 pickle.dump(match_stats, open(matchstatfile, "w")) def find_NMstats(bamfile, NMstatfile): sam_file = pysam.Samfile(bamfile, "rb") NM_stats = {} for line in sam_file: unmapped = (line.flag / 4) % 2 == 1 if unmapped: continue tags = dict(line.tags) if "NM" in tags: nm = tags["NM"] if nm not in NM_stats: NM_stats[nm] = 0 NM_stats[nm] += 1 elif "nM" in tags: nm = tags["nM"] if nm not in NM_stats: NM_stats[nm] = 0 NM_stats[nm] += 1 else: print (tags) # aaaa pickle.dump(NM_stats, open(NMstatfile, "w")) # # Analysis methods = ["star", "bowtie2", "hisat2"] # id_list=["FZ-1","FZ-2","FZ-3","STJ16-1","STJ16-2","STJ16-3"] # id_list = ["MUC-1","MUC-2","MUC-3","MUT-1","MUT-2","MUT-3"] id_list=["ZT_3d_1","ZT_3d_2","ZT_3d_3","ZT_4d_1","ZT_4d_2","ZT_4d_3","ZT_8d_1","ZT_8d_2","ZT_8d_3","CK_3d_1","CK_3d_2","CK_3d_3","CK_4d_1","CK_4d_2","CK_4d_3","CK_8d_1","CK_8d_2","CK_8d_3","XS11_3day_1","XS11_3day_2","XS11_3day_3","XS11_4day_1","XS11_4day_2","XS11_4day_3","XS11_8day_1","XS11_8day_2","XS11_8day_3" ] for id in id_list: bam_files = {"bowtie2": "/mount_18t/bowtie2/Vdahliae/raw/" + id + ".sort.addRG.bam", "star": "/mount_18t/star/Vdahliae/raw/" + id + ".sortedByCoord.addRG.bam", "hisat2": "/mount_18t/hisat2/Vdahliae/raw/" + id + ".sort.addRG.bam", } statpath = "/mount_18t/stat/Vdahliae/" for method, bamfile in bam_files.iteritems(): statfile = statpath + id + "." + method + ".mystats" if os.path.exists(bamfile): if not os.path.exists(statfile): find_stats(bamfile, statfile, method) for method, bamfile in bam_files.iteritems(): statfile = statpath + id + "." + method + ".mystats_match" if os.path.exists(bamfile): if not os.path.exists(statfile): find_matchstats(bamfile, statfile) for method, bamfile in bam_files.iteritems(): statfile = statpath + id + "." + method + ".mystats_NM" if os.path.exists(bamfile): if not os.path.exists(statfile): print (id, method) find_NMstats(bamfile, statfile) def parse_my_stats(stat_file): mystats = {} with open(stat_file, 'r') as csv_f: spamreader = csv.reader(csv_f, delimiter='\t', quotechar='|') cnt = 0 for row in spamreader: if cnt == 0: keys = row cnt = 1 else: vals = row mystats = {x[0]: int(x[1]) for x in zip(keys, vals)} return mystats return {} alignment_stats = {} for method in methods: alignment_stats[method] = {} for method, bamfile in bam_files.iteritems(): statfile = statpath + id + "." + method + ".mystats" mystats = parse_my_stats(statfile) alignment_stats[method].update(mystats) for method, bamfile in bam_files.iteritems(): statfile = statpath + id + "." + method + ".mystats_match" mystats = pickle.load(open(statfile)) alignment_stats[method].update({"match_stats": mystats}) for method, bamfile in bam_files.iteritems(): statfile = statpath + id + "." + method + ".mystats_NM" mystats = pickle.load(open(statfile)) alignment_stats[method].update({"NM": mystats}) sns.set(style="white", font_scale=1.2) colors = [4] nt = ["A", "C", "G", "T"] etypes = [] for i in nt: for j in nt: if i != j: etypes.append(i + j) print (etypes) bin_labels = ["Both pairs uniquely mapped", "Both pairs multi-mapped", "One pair uniquely, one multi-mapped", "One pair uniquely mapped, one unmapped", "One pair multi-mapped, one unmapped", "Both pairs unmapped"] keys = ['uniqmap_uniqmap', 'multimap_multimap', 'uniqmap_multimap', 'uniqmap_unmap', 'multimap_unmap', 'unmap_unmap'] my_colors = sns.color_palette("Set3", n_colors=10) fig, axes = plt.subplots(1, 3, figsize=(18.5, 2.9)) ax = axes[0] res = [] labels = [] for method in methods: if method not in alignment_stats: continue if "uniqmap_uniqmap" in alignment_stats[method]: myres = [alignment_stats[method][k] / float(alignment_stats[method]["total"]) * 100 for k in keys][::-1] myres = [sum(myres[i:]) for i in range(len(myres))] res.append(myres) label = method labels.append(label) my_data = DataFrame(np.array(res), index=labels, columns=bin_labels) for ii, b in enumerate(bin_labels): cg = sns.barplot(data=my_data, x=b, y=labels, label=b, color=my_colors[ii], ax=ax) ax.set_xlabel("% of " + id + " fragments") ax.set_xlim([0, 100]) sns.despine(left=True) handles, labels = ax.get_legend_handles_labels() # reverse the order ax.legend(handles[::-1], labels, bbox_to_anchor=(-0.35, 1, 1.53, .6), loc="upper left", ncol=2, mode="expand", borderaxespad=0., frameon=False, fontsize=12) plt.tight_layout() plt.subplots_adjust() ax = axes[1] bin_labels = ["1", "2-3", "4-6", "7-10", "11-20", ">20"] bins = [1, 3, 6, 10, 20, 1000] codes = [4] res = [] labels = [] for method in methods: if method not in alignment_stats: continue if "match_stats" not in alignment_stats[method]: continue if set(alignment_stats[method]["match_stats"].keys()) & set(codes): my_res = [] for b in bins[::-1]: my_res.append(sum([v for code in set(alignment_stats[method]["match_stats"].keys()) & set(codes) for k, v in alignment_stats[method]["match_stats"][code].iteritems() if ( k <= b)]) / float(sum(alignment_stats[method]["NM"].values())) * 100) my_res = my_res res.append(my_res) label = method labels.append(label) else: my_res = [] for b in bins: my_res.append(0) my_res = my_res res.append(my_res) label = method labels.append(label) my_data = DataFrame(np.array(res), index=labels, columns=bin_labels) for ii, b in enumerate(bin_labels): cg = sns.barplot(data=my_data, x=b, y=labels, label=b, color=my_colors[ii], ax=ax) ax.set_yticklabels([]) ax.set_xlabel("% of " + id + " mapped fragments") sns.despine(left=True) handles, labels = ax.get_legend_handles_labels() ax.legend(handles[::-1], labels, bbox_to_anchor=(0.2, 1.4, .6, .3), loc=0, ncol=3, mode="expand", borderaxespad=0., frameon=False, fontsize=12, title="Number of soft clipped bases") plt.tight_layout() plt.subplots_adjust() ax = axes[2] bin_labels = ["1", "2", "3-4", "5-6", "7-9", ">9"] bins = [1, 2, 4, 6, 9, 1000] res = [] labels = [] for method in methods: if method not in alignment_stats: continue if "NM" not in alignment_stats[method]: continue my_res = [] for b in bins[::-1]: my_res.append(sum([v / float(sum(alignment_stats[method]["NM"].values())) * 100 for k, v in alignment_stats[method]["NM"].iteritems() if ( 0 < k <= b)])) my_res = my_res res.append(my_res) label = method labels.append(label) my_data = DataFrame(np.array(res), index=labels, columns=bin_labels) for ii, b in enumerate(bin_labels): cg = sns.barplot(data=my_data, x=b, y=labels, label=b, color=my_colors[ii], ax=ax) ax.set_yticklabels([]) ax.set_xlabel("% of " + id + " mapped fragments") sns.despine(left=True) handles, labels = ax.get_legend_handles_labels() # reverse the order ax.legend(handles[::-1], labels, bbox_to_anchor=(0.2, 1, 0.6, .3), loc=0, ncol=3, mode="expand", borderaxespad=0., frameon=False, fontsize=12, title="Number of mismatches") plt.tight_layout() plt.subplots_adjust() plt.show() plt.savefig(statpath + "align." + id + ".raw.pdf", format="pdf")