import copy import re import os from collections import OrderedDict from enum import IntEnum import tempfile import shutil import scipy.io as scio import scipy.stats as scistats import matplotlib.pyplot as plt import seaborn as sns import pydicom as dcm import pandas as pd # to handle relative paths (for calls inside or outside package) from pathlib import Path import SimpleITK as sitk import numpy as np import matplotlib as mpl import datetime from reportlab.lib.pagesizes import A4, A3, landscape from reportlab.lib.units import inch, cm import PIL import io MRBIAS_OUTPUT_FILENAME_MAN_SLICE = "data/MRBIAS_DW_DATA_R8_FT.csv" MRBIAS_OUTPUT_FILENAME = "data/MRBIAS_DW_DATA_R5_H10_FT.csv" # optimal ROI size EMBRACE_OUTPUT_FILENAME = "data/EMBRACE_DW_DATA.csv" EXCLUDE_INST_F = True # due to MR-BIAS not analysing enhanced DICOM data (this had no significant impact on the study reported metrics) EXCLUDE_INST_N_BENCH = True # Outer 50% PVP vial was excluded in original analysis (NaN) values #EXCLUDE_INST_N_INSTITUTE these have been explicitly left out of the datafiles AXES_TITLE_SIZE = 20 AXES_LABEL_SIZE = 17 AXES_TICK_SIZE = 14 DPI = 1200 theme_dict = {**sns.axes_style("white"), "grid.linestyle": ":", #'font.family':['serif','sans-serif'], #'font.serif':'Times New Roman', #'font.sans-serif':'Computer Modern Sans Serif', 'axes.titlesize': AXES_TITLE_SIZE, 'axes.labelsize': AXES_LABEL_SIZE, 'xtick.labelsize': AXES_TICK_SIZE, 'ytick.labelsize': AXES_TICK_SIZE} sns.set_theme(rc=theme_dict) def main(): # parse/load the analysis data df_mrbias_mslc = pd.read_csv(MRBIAS_OUTPUT_FILENAME_MAN_SLICE) df_mrbias_mslc["Field"] = df_mrbias_mslc["Field"].astype(str) df_mrbias = pd.read_csv(MRBIAS_OUTPUT_FILENAME) df_mrbias["Field"] = df_mrbias["Field"].astype(str) df_embrace = pd.read_csv(EMBRACE_OUTPUT_FILENAME) df_embrace["Field"] = df_embrace["Field"].astype(str) for df in [df_mrbias_mslc, df_mrbias, df_embrace]: df = df.sort_values(['Institute', 'ROI', 'SeriesNumber'], ascending=[True, True, True]) for df in [df_mrbias_mslc, df_mrbias, df_embrace]: df.loc[:, "ROILabel"] = "" for roi_num, pos_str in zip(range(13), ['centre', 'in', 'out', 'in', 'out', 'in', 'out', 'in', 'out', 'in', 'out', 'in', 'out']): pvp = df[df.ROI == roi_num].PvpConcentration.unique()[0] df.loc[df.ROI == roi_num, "ROILabel"] = "%s\n%s" % ('{:,.0%}'.format(pvp/100.), pos_str) # seperate benchmark and study/institutional protocol fits # MRBIAS (manual slice) df_mrb_bench_mslc = df_mrbias_mslc[df_mrbias_mslc.Protocol == "benchmark"] # BENCHMARK PROTOCOL df_mrb_bench_mslc = df_mrb_bench_mslc[df_mrb_bench_mslc.FitLabel == "All-bvals"] # ALL B_VALS df_mrb_study_mslc = df_mrbias_mslc[df_mrbias_mslc.Protocol == "study"] # STUDY PROTOCOL df_mrb_study_mslc = df_mrb_study_mslc[df_mrb_study_mslc.FitLabel.isin(["Exclude-b0", "Spec-bvals"])] # [200, 1000] # MRBIAS df_mrb_bench = df_mrbias[df_mrbias.Protocol == "benchmark"] # BENCHMARK PROTOCOL df_mrb_bench = df_mrb_bench[df_mrb_bench.FitLabel == "All-bvals"] # ALL B_VALS df_mrb_study = df_mrbias[df_mrbias.Protocol == "study"] # STUDY PROTOCOL df_mrb_study = df_mrb_study[df_mrb_study.FitLabel.isin(["Exclude-b0", "Spec-bvals"])] # [200, 1000] # EMBRACE df_ece_bench = df_embrace[df_embrace.Protocol == "benchmark"] # BENCHMARK PROTOCOL df_ece_bench = df_ece_bench[df_ece_bench.FitLabel == "All-bvals"] # ALL B_VALS df_ece_study = df_embrace[df_embrace.Protocol == "study"] # STUDY PROTOCOL df_ece_study = df_ece_study[df_ece_study.FitLabel.isin(["Exclude-b0", "Spec-bvals"])] # [200, 1000] # INSTITUTE N (benchmark) # Outer 50% PVP vial was excluded in original analysis (NaN) values # Exclude the same ROI from MRBIAS analysis also if EXCLUDE_INST_N_BENCH: df_ece_bench.loc[(df_ece_bench.Institute=="InstituteN") & (df_ece_bench.ROI==12), "ADC"] = np.nan df_mrb_bench.loc[(df_mrb_bench.Institute=="InstituteN") & (df_mrb_bench.ROI==12), "ADC"] = np.nan df_mrb_bench_mslc.loc[(df_mrb_bench_mslc.Institute=="InstituteN") & (df_mrb_bench_mslc.ROI==12), "ADC"] = np.nan if EXCLUDE_INST_F: # INSTITUTE F (benchmark) # Excluded from MR-BIAS study due to no enhanced DICOM support (looking at difference to embrace metrics if this is excluded) df_ece_bench.loc[df_ece_bench.Institute == "InstituteF", "ADC"] = np.nan df_ece_study.loc[df_ece_study.Institute == "InstituteF", "ADC"] = np.nan f, (ax1, ax2) = plt.subplots(1, 2) # join the benchmark dataframes df_mrb_bench_mslc.loc[:, 'Analysis'] = "MRBIAS_MSL" df_mrb_bench.loc[:, 'Analysis'] = "MRBIAS" df_ece_bench.loc[:, 'Analysis'] = "EMBRACE" df_bench = pd.concat([df_mrb_bench_mslc, df_mrb_bench, df_ece_bench]) compare_analyses(df_bench, "BENCHMARK PROTOCOL", [7.0e-6, 4.0e-6], ax1, "(a)", annotate_axes=True) # join the study dataframes df_mrb_study_mslc.loc[:, 'Analysis'] = "MRBIAS_MSL" df_mrb_study.loc[:, 'Analysis'] = "MRBIAS" df_ece_study.loc[:, 'Analysis'] = "EMBRACE" df_study = pd.concat([df_mrb_study_mslc, df_mrb_study, df_ece_study]) compare_analyses(df_study, "INSTITUTE PROTOCOL", [6.0e-6, 3.0e-6], ax2, "(b)") # f.set_size_inches(14.0, 8.0) f.tight_layout() f.subplots_adjust(wspace=0.175)#, right=0.98) # , hspace=0.15) f.savefig("Figure4.png", dpi=DPI) f.savefig("Figure4.eps") f.savefig("Figure4.pdf") def compare_analyses(df_comp, suptitle, tol_vec, ax1, ax_title, annotate_axes=False, detail_plots=False): df_comp = df_comp.reset_index(drop=True) x_label = r"ADC$_{true}$ (mm$^2$/s x10$^{-3}$)" y_label = r"$\Delta$ADC (mm$^2$/s x10$^{-4}$)" y_label_ADC_um = r"ADC ($\mu$m$^2$/s)" y_label_dADC_um = r"bias ($\mu$m$^2$/s)" #r"$\Delta$ADC$_{\mathrm{meas}-\mathrm{ref}}$ ($\mu$m$^2$/s)" y_label_dADC_um_comp = r"$\Delta$ADC$_{\mathrm{mrbias}-\mathrm{embrace}}$ ($\mu$m$^2$/s)" df_comp.loc[:, x_label] = df_comp['ADC_0deg'] * 1000 df_comp.loc[:, y_label] = (df_comp['ADC'] - df_comp['ADC_0deg']) * 10000 df_comp.loc[:, y_label_dADC_um] = df_comp[y_label] * 100 # for metric calculation for analysis_name in ["MRBIAS", "MRBIAS_MSL", "EMBRACE"]: print(analysis_name, df_comp[df_comp.Analysis == analysis_name].shape) # average the four passes per institute df_subset = df_comp[["Make", "Institute", "ROI", "ROILabel", "Manufacturer", "Field", "Analysis", "ADC_0deg", "ADC"]] df_inst = df_subset.groupby(["Make", "Institute", "ROI", "ROILabel", "Manufacturer", "Field", "Analysis"]) df_comp_mean = df_inst.mean() df_comp_mean = df_comp_mean.droplevel(axis=0, level=0).reset_index() df_comp_mean.loc[:, x_label] = df_comp_mean['ADC_0deg'] * 1000 df_comp_mean.loc[:, y_label] = (df_comp_mean['ADC'] - df_comp_mean['ADC_0deg']) * 10000 df_comp_mean.loc[:, y_label_dADC_um] = df_comp_mean[y_label] * 100 df_comp_mean.loc[:, y_label_ADC_um] = df_comp_mean["ADC"] * 1e6 report_bias(df_comp, y_label_dADC_um, suptitle) report_shortterm_pcntRC(df_comp, suptitle) report_institutue_pcntRDC(df_comp, suptitle) # plot it inst_list = df_comp.Institute.unique() inst_list.sort() hue_order = ["EMBRACE", "MRBIAS_MSL", "MRBIAS"] for df, post_str in zip([df_comp_mean], # df_comp, ["average"]): # "all passes", # mark equivalence/significance on labels tost_analysis(df_comp, y_label, tol_vec, df) for label, ax in zip([y_label_dADC_um], [ax1]): sns.boxplot(df, x="ROILabelEquiv", y=label, hue="Analysis", hue_order=hue_order, fliersize=0, palette={"EMBRACE": 'xkcd:white', "MRBIAS_MSL": 'xkcd:light grey', "MRBIAS": 'xkcd:dusty purple'}, linewidth=1.0, ax=ax) sns.stripplot(df, x="ROILabelEquiv", y=label, hue="Analysis", hue_order=hue_order, palette={"EMBRACE": 'xkcd:black', "MRBIAS_MSL": 'xkcd:black', "MRBIAS": 'xkcd:black'}, ax=ax, dodge=True, alpha=0.5, size=2.5) ax.yaxis.grid(True) # clean up the legend ax1.get_legend().remove() YMIN = -60.0 ax1.set_ylim([YMIN, 75.]) ax1.set_title(ax_title) ax1.set_xlabel("") # add some annotations if annotate_axes: YMIN_AX = -66.0 space = 4.2 ax1.annotate('PVP:', xy=(-2.5, YMIN_AX), annotation_clip=False, fontsize=AXES_TICK_SIZE) ax1.annotate('Location:', xy=(-2.5, YMIN_AX-space), annotation_clip=False, fontsize=AXES_TICK_SIZE) ax1.annotate('Manual:', xy=(-2.5, YMIN_AX-2*space), annotation_clip=False, fontsize=AXES_TICK_SIZE) ax1.annotate('Optimal:', xy=(-2.5, YMIN_AX-3*space), annotation_clip=False, fontsize=AXES_TICK_SIZE) # sns.move_legend(ax2, loc="upper left", bbox_to_anchor=(1, 1), # handles=ax.legend_.legendHandles[:3], # labels=["vanHoudt", "MR-BIAS\n(manual slice)", "MR-BIAS"]) if detail_plots: for inst in inst_list: f, (ax1, ax2) = plt.subplots(1, 2) f.suptitle(inst) df_inst = df_comp[df_comp.Institute.str.match(inst)] sns.stripplot(df_inst, x="ROI", y=y_label, hue="Analysis", hue_order=["EMBRACE", "MRBIAS"], dodge=True, ax=ax1) ax1.set_ylim([-1.5, 1.5]) ax1.grid('on') # add the average into the plot df_inst_av = df_comp_mean[df_comp_mean.Institute.str.match(inst)] #df_plot_av = df_plot_av.droplevel(axis=0, level=0).reset_index() sns.stripplot(df_inst_av, x="ROI", y=y_label, hue="Analysis", hue_order=["EMBRACE", "MRBIAS"], marker="D", dodge=True, ax=ax2) ax2.set_ylim([-1.5, 1.5]) ax2.grid('on') # make a summary figure to highlight the average differences per ROI df_mrb_mean = df_comp_mean[df_comp_mean.Analysis=="MRBIAS"] df_ece_mean = df_comp_mean[df_comp_mean.Analysis=="EMBRACE"] df_comp_av_merge = pd.merge(df_mrb_mean, df_ece_mean, on=["Institute", "ROI", "ROILabel", "Manufacturer", "Field", "ADC_0deg"], suffixes=("_mrb", "_ece")) df_comp_av_merge.loc[:, y_label_dADC_um_comp] = df_comp_av_merge["%s_mrb" % y_label_ADC_um] - df_comp_av_merge["%s_ece" % y_label_ADC_um] f, ax1 = plt.subplots(1, 1) f.suptitle(suptitle) sns.barplot(df_comp_av_merge, x="ROILabel", y=y_label_dADC_um_comp, hue="Institute", hue_order=inst_list, palette=sns.color_palette("hls", 14), ax=ax1) ax1.set_ylim([-0.5*100, 0.5*100]) ax1.yaxis.grid(True) sns.move_legend(ax1, "upper left", bbox_to_anchor=(1, 1)) f.tight_layout() def report_bias(df_comp, y_label_dADC_um, suptitle): # report the bias range (as in van Houdt et al.) # 10.1016/j.radonc.2020.09.013 df_comp = df_comp.dropna(subset=[y_label_dADC_um]) inst_list = df_comp.Institute.unique() inst_list.sort() for df_a, lbl in zip([df_comp], ["ALL"]): mrb_bias_list = [] mrb_msl_bias_list = [] ece_bias_list = [] for inst_name in inst_list: df_inst = df_a[df_a.Institute == inst_name] for bias_list, analysis_name in zip([mrb_bias_list, mrb_msl_bias_list, ece_bias_list], ["MRBIAS", "MRBIAS_MSL", "EMBRACE"]): df_inst_i = df_inst[df_inst.Analysis == analysis_name] bias_list.append(np.nanmean(df_inst_i[y_label_dADC_um])) # print("%s[%s] EMBRACE : bias range [%f, %f]" % (suptitle, lbl, np.min(ece_bias_list), np.max(ece_bias_list))) # ece_bias_list = np.array(ece_bias_list) # ece_bias_list_mtch = ece_bias_list[np.logical_not(np.isnan(mrb_bias_list))] print("%s[%s] EMBRACE :bias range [%d, %d]" % ( suptitle, lbl, np.round(np.nanmin(ece_bias_list)), np.round(np.nanmax(ece_bias_list)))) print("%s[%s] MRBIAS (man slc) : bias range [%d, %d]" % ( suptitle, lbl, np.round(np.nanmin(mrb_msl_bias_list)), np.round(np.nanmax(mrb_msl_bias_list)))) print("%s[%s] MRBIAS : bias range [%d, %d]" % ( suptitle, lbl, np.round(np.nanmin(mrb_bias_list)), np.round(np.nanmax(mrb_bias_list)))) def report_shortterm_pcntRC(df_a, suptitle): # report the short term repeatability (%RC) for each institute (as in van Houdt et al.) # 10.1016/j.radonc.2020.09.013 df_a = df_a.dropna(subset=['ADC']) inst_list = df_a.Institute.unique() inst_list.sort() mrb_RC_list = [] mrb_msl_RC_list = [] ece_RC_list = [] for inst_name in inst_list: df_inst = df_a[df_a.Institute == inst_name] for RC_list, analysis_name in zip([mrb_RC_list, mrb_msl_RC_list, ece_RC_list], ["MRBIAS", "MRBIAS_MSL", "EMBRACE"]): df_inst_i = df_inst[df_inst.Analysis == analysis_name] # calculate the within-subject standard deviation (wCV) # as described in: doi/10.1002/jmri.26518 # =============================================================== # - calculate variance of repeat measurements in each ROI df_i = df_inst_i[ ["Make", "Institute", "ROI", "ROILabel", "Manufacturer", "Field", "Analysis", "ADC_0deg", "ADC"]] df_i = df_i.groupby(["Make", "Institute", "ROI", "ROILabel", "Manufacturer", "Field", "Analysis"]).agg( ADC_var=("ADC", "var"), ADC_std=("ADC", "std"), ADC_mean=("ADC", "mean")) df_i = df_i.droplevel(axis=0, level=0).reset_index() df_i.loc[:, "wCV2"] = df_i.ADC_var / (df_i.ADC_mean * df_i.ADC_mean) # - calculate the mean variance across ROIs wCV2 = df_i.wCV2.mean() wCV = np.sqrt(wCV2) * 100.0 pcntRC = 2.77 * wCV RC_list.append(pcntRC) print("%s EMBRACE :pctRC (median, min-max): %f, %f-%f" % (suptitle, np.nanmedian(ece_RC_list), np.nanmin(ece_RC_list), np.nanmax(ece_RC_list))) print("%s MRB_MAN SLC :pctRC (median, min-max): %f, %f-%f" % (suptitle, np.nanmedian(mrb_msl_RC_list), np.nanmin(mrb_msl_RC_list), np.nanmax(mrb_msl_RC_list))) print("%s MRB_OPTI :pctRC (median, min-max): %f, %f-%f" % (suptitle, np.nanmedian(mrb_RC_list), np.nanmin(mrb_RC_list), np.nanmax(mrb_RC_list))) def report_institutue_pcntRDC(df_a, suptitle): # report the short term repeatability (%RC) for each institute (as in van Houdt et al.) # 10.1016/j.radonc.2020.09.013 df_a = df_a.dropna(subset=['ADC']) inst_list = df_a.Institute.unique() inst_list.sort() # make a dataframe of the first measurements only (using the lowest SeriesNumber per institute) first_scan_data_vector = [] for inst_name in inst_list: df_inst = df_a[df_a.Institute == inst_name] print(inst_name, pd.unique(df_inst.SeriesNumber), ":", pd.unique(df_inst.SeriesNumber)[0]) df_0 = df_inst[df_inst.SeriesNumber==pd.unique(df_inst.SeriesNumber)[0]] assert df_0.shape[0]==39 or \ ((inst_name=="InstituteN") and (df_0.shape[0]==36)) or \ ((inst_name=="InstituteF") and (df_0.shape[0]==13)), "report_institutue_pcntRDC(): %s only has %d" % (inst_name, df_0.shape[0]) first_scan_data_vector.append(df_0) df_first = pd.concat(first_scan_data_vector) # calculate the within-subject standard deviation (wCV) # in this case the wCV is across institutes # wCV as described in: doi/10.1002/jmri.26518 # =============================================================== df_i = df_first[ ["ROILabel", "ROI", "Analysis", "ADC_0deg", "ADC"]] df_i = df_i.dropna(subset=['ADC']) df_i = df_i.groupby(["ROILabel", "ROI", "Analysis", "ADC_0deg"]).agg( ADC_var=("ADC", "var"), ADC_std=("ADC", "std"), ADC_mean=("ADC", "mean")) df_i = df_i.droplevel(axis=0, level=0).reset_index() df_i.loc[:, "wCV2"] = (df_i.ADC_var / (df_i.ADC_mean * df_i.ADC_mean)) for analysis_name in ["EMBRACE", "MRBIAS_MSL", "MRBIAS"]: df_analysis = df_i[df_i.Analysis == analysis_name] wCV2 = df_analysis.wCV2.mean() wCV = np.sqrt(wCV2) * 100.0 pcntRDC = 2.77 * wCV print("%s %s :pctRDC = %f" % (suptitle, analysis_name, pcntRDC)) return None def tost_analysis(df, y_label, tol_vec, df_to_label=None): # make a summary figure to highlight the average differences per ROI df_mrb = df[df.Analysis == "MRBIAS"] df_mrb_msl = df[df.Analysis == "MRBIAS_MSL"] df_ece = df[df.Analysis == "EMBRACE"] how_to_merge = 'inner' # setup comparison of embrace to mrbias df_comp_ece_mrb = pd.merge(df_mrb, df_ece, on=["Institute", "ROI", "ROILabel", "Manufacturer", "Field", "ADC_0deg", "SeriesNumber"], suffixes=("_mrb", "_ece"), how=how_to_merge) df_comp_ece_mrb.loc[:, y_label] = df_comp_ece_mrb["%s_mrb" % y_label] - df_comp_ece_mrb["%s_ece" % y_label] # setup comparison of embrace to mrbias (manual slices) df_comp_ece_mrb_msl = pd.merge(df_mrb_msl, df_ece, on=["Institute", "ROI", "ROILabel", "Manufacturer", "Field", "ADC_0deg", "SeriesNumber"], suffixes=("_mrb", "_ece"), how=how_to_merge) df_comp_ece_mrb_msl.loc[:, y_label] = df_comp_ece_mrb_msl["%s_mrb" % y_label] - df_comp_ece_mrb_msl["%s_ece" % y_label] roi_labels = df.ROILabel.unique() mrb_equiv_dict = OrderedDict() mrb_msl_equiv_dict = OrderedDict() for df_comp, equiv_dict, tol in zip([df_comp_ece_mrb, df_comp_ece_mrb_msl], [mrb_equiv_dict, mrb_msl_equiv_dict], tol_vec): for roi_label in roi_labels: df_roi = df_comp[df_comp.ROILabel == roi_label] # prepare comparison vectors mrbias_adc = df_roi.ADC_mrb embrace_adc = df_roi.ADC_ece # cary out the TOST pval_tost = tost_wilcoxon(embrace_adc, mrbias_adc, tol) sig_str = "" if pval_tost < 0.05: sig_str = "* " if pval_tost < 0.01: sig_str = "** " if pval_tost < 0.005: sig_str = "***" #print("\t\t\t\t\t [%s] pvalue: %0.4f%s" % (roi_label, pval_tost, sig_str)) # add to the label dict equiv_dict[roi_label] = "<%.0f%s" % (tol*1e6, sig_str) # add a modified label to the dataframe if df_to_label is not None: df_to_label.loc[:, "ROILabelEquiv"] = "" for roi_label in roi_labels: equiv_str = "\n%s\n%s" % (mrb_msl_equiv_dict[roi_label], mrb_equiv_dict[roi_label]) df_to_label.loc[df_to_label.ROILabel==roi_label, "ROILabelEquiv"] = df_to_label[df_to_label.ROILabel==roi_label].ROILabel + equiv_str def tost_wilcoxon(a, b, bound): # # TOST # modified from https://rowannicholls.github.io/python/statistics/hypothesis_testing/tost_paired.html # # bound = Magnitude of region of similarity # Paired two-sample wilcoxon signed rank test _, p_greater = scistats.wilcoxon(a + bound, b, alternative='greater') _, p_less = scistats.wilcoxon(a - bound, b, alternative='less') # Choose the maximum p-value pval = max(p_less, p_greater) return pval if __name__ == "__main__": main() print("----------------------------------------- FIN ------------------------------------------") plt.show()