# pcsk9_t2dm_analysis.R
# Comprehensive, publication-grade analysis pipeline for:
# "Serum PCSK9 is not independently associated with dyslipidaemia in type 2 diabetes"
# Inputs: Dat1.csv (lipids by pair: A=Control, B=T2DM), Dat2.csv (PCSK9 for each pair)
# Optional: a metadata CSV with columns pair_id, sex ("M"/"F"), age, BMI, education, etc.
# Outputs: tables (CSV, DOCX) + figures (PNG) in ./outputs

# --------------------------- 0) Setup -----------------------------------------
required_pkgs <- c(
  "tidyverse","janitor","readr","broom","broom.mixed","knitr","kableExtra",
  "lme4","lmerTest","performance","MuMIn","effectsize",
  "survival","survminer","sandwich","lmtest","clubSandwich",
  "epiDisplay","DescTools",
  "pROC","yardstick",
  "ggplot2","patchwork","ggtext","ggrepel",
  "gt","gtsummary","flextable","officer",
  "mice","naniar","VIM","ggpmisc"
)
to_install <- setdiff(required_pkgs, rownames(installed.packages()))
if(length(to_install)) install.packages(to_install, dependencies = TRUE)

suppressPackageStartupMessages({
  library(tidyverse); library(janitor); library(readr); library(broom); library(broom.mixed)
  library(lme4); library(lmerTest); library(performance); library(MuMIn); library(effectsize)
  library(survival); library(sandwich); library(lmtest); library(clubSandwich)
  library(DescTools); library(pROC)
  library(ggplot2); library(patchwork); library(ggrepel); library(ggtext)
  library(gt); library(gtsummary); library(flextable); library(officer)
  library(mice); library(naniar); library(VIM)
})

dir.create("outputs", showWarnings = FALSE)

# --------------------------- 1) User parameters -------------------------------
# Conversion factor for PCSK9 "native assay units" -> ng/mL (set to 1 if already ng/mL)
pcsk9_to_ngml <- 1

# Dyslipidaemia thresholds (primary definition)
ldl_thr  <- 3.37      # mmol/L
hdl_thr  <- 1.03      # mmol/L (sex-agnostic, primary)
tg_thr   <- 1.70      # mmol/L

# Sex-specific sensitivity thresholds (used only if `sex` column exists)
hdl_thr_m <- 1.03
hdl_thr_f <- 1.29

# Optional: path to a metadata file (pair-level) with columns: pair_id, sex ("M"/"F"), age, BMI, education...
# If you have one, set this string to your file name; otherwise leave NULL
meta_path <- NULL   # e.g., "Meta_pairs.csv"

# --------------------------- 2) Import & clean -------------------------------
dat1_raw <- read_csv("Dat1.csv", show_col_types = FALSE)
dat2_raw <- read_csv("Dat2.csv", show_col_types = FALSE)

# Clean names (turn "T.chol (A)" -> "t_chol_a", etc.)
dat1 <- dat1_raw %>%
  clean_names() %>%
  rename(pair_id = s_n) %>%
  mutate(pair_id = as.integer(pair_id))

# Expect columns: t_chol_a, t_chol_b, trig_a, trig_b, hdl_a, hdl_b, ldl_a, ldl_b
stopifnot(all(c("pair_id","t_chol_a","t_chol_b","trig_a","trig_b","hdl_a","hdl_b","ldl_a","ldl_b") %in% names(dat1)))

dat2 <- dat2_raw %>%
  clean_names() %>%
  rename(pcsk9_a = pcsk9_a, pcsk9_b = pcsk9_b) %>%
  mutate(pair_id = row_number())

# Merge
wide <- dat1 %>%
  left_join(dat2, by = "pair_id") %>%
  mutate(pcsk9_a_ng = pcsk9_a * pcsk9_to_ngml,
         pcsk9_b_ng = pcsk9_b * pcsk9_to_ngml)

# Optional metadata
if(!is.null(meta_path) && file.exists(meta_path)){
  meta <- read_csv(meta_path, show_col_types = FALSE) %>% clean_names()
  stopifnot("pair_id" %in% names(meta))
  wide <- wide %>% left_join(meta, by = "pair_id")
}

# --------------------------- 3) Long format -----------------------------------
# Build long person-level dataset: A=Control, B=T2DM
long <- wide %>%
  pivot_longer(cols = -pair_id, names_to = "var", values_to = "value") %>%
  separate(var, into = c("marker","arm"), sep = "_(?=[ab]$)", remove = FALSE) %>%
  mutate(group = ifelse(str_ends(var, "_a"), "Control", ifelse(str_ends(var,"_b"), "T2DM", NA_character_))) %>%
  select(pair_id, marker, arm, group, value) %>%
  pivot_wider(names_from = marker, values_from = value) %>%
  # Rename to tidy columns
  rename(tchol = t, trig = trig, hdl = hdl, ldl = ldl, pcsk9 = pcsk9, pcsk9_ng = pcsk9_ng) %>%
  mutate(group = factor(group, levels = c("Control","T2DM")))

# If sex exists in wide, bring to long
if("sex" %in% names(wide)){
  long <- long %>%
    left_join(wide %>% select(pair_id, sex) %>% distinct(), by="pair_id") %>%
    mutate(sex = factor(sex))
}

# --------------------------- 4) Missingness & analyzable pairs ----------------
miss_tbl <- long %>%
  group_by(group) %>%
  summarize(
    n_rows = n(),
    tchol_nonmiss = sum(!is.na(tchol)),
    ldl_nonmiss = sum(!is.na(ldl)),
    hdl_nonmiss = sum(!is.na(hdl)),
    trig_nonmiss = sum(!is.na(trig)),
    pcsk9_nonmiss = sum(!is.na(pcsk9)),
    .groups="drop"
  )

# Analyzable pairs per variable (both arms present)
count_pairs <- function(a_col, b_col){
  sum(!is.na(a_col) & !is.na(b_col))
}

analyzable <- tibble(
  marker = c("tchol","ldl","hdl","trig","pcsk9"),
  n_pairs = c(
    count_pairs(wide$t_chol_a, wide$t_chol_b),
    count_pairs(wide$ldl_a,    wide$ldl_b),
    count_pairs(wide$hdl_a,    wide$hdl_b),
    count_pairs(wide$trig_a,   wide$trig_b),
    count_pairs(wide$pcsk9_a,  wide$pcsk9_b)
  )
)

write_csv(analyzable, "outputs/analyzable_pairs_by_marker.csv")

# Simple CONSORT-style flow (text table)
flow_df <- tibble(
  Stage = c("Matched pairs recruited","Pairs with lipid data (all 4)",
            "Pairs with PCSK9","Pairs with complete lipid + PCSK9"),
  N = c(nrow(wide),
        sum(complete.cases(wide[,c("t_chol_a","t_chol_b","ldl_a","ldl_b","hdl_a","hdl_b","trig_a","trig_b")])),
        sum(complete.cases(wide[,c("pcsk9_a","pcsk9_b")])),
        sum(complete.cases(wide[,c("t_chol_a","t_chol_b","ldl_a","ldl_b","hdl_a","hdl_b","trig_a","trig_b","pcsk9_a","pcsk9_b")]))
  )
write_csv(flow_df, "outputs/flow_counts.csv")

# --------------------------- 5) Paired differences + tests --------------------
# Helper: paired summary with CI
paired_stats <- function(a, b, name){
  d <- b - a
  d <- d[!is.na(d)]
  n <- length(d)
  md <- mean(d); sd_d <- sd(d)
  sw_p <- if(n >= 3) shapiro.test(d)$p.value else NA_real_
  ttest <- t.test(b, a, paired = TRUE)
  wilc  <- wilcox.test(b, a, paired = TRUE, exact = FALSE)
  tibble(
    outcome = name,
    n_pairs = n,
    mean_diff = md,
    sd_diff = sd_d,
    ci_low = ttest$conf.int[1],
    ci_high = ttest$conf.int[2],
    t_stat = unname(ttest$statistic),
    t_p = ttest$p.value,
    shapiro_p = sw_p,
    wilcox_p = wilc$p.value
  )
}

paired_out <- bind_rows(
  paired_stats(wide$t_chol_a, wide$t_chol_b, "tchol"),
  paired_stats(wide$hdl_a,    wide$hdl_b,    "hdl"),
  paired_stats(wide$ldl_a,    wide$ldl_b,    "ldl"),
  paired_stats(wide$trig_a,   wide$trig_b,   "trig"),
  paired_stats(wide$pcsk9_a,  wide$pcsk9_b,  "pcsk9")
) %>%
  mutate(p_adj_BH = p.adjust(t_p, method = "BH"),
         p_adj_bonf = p.adjust(t_p, method = "bonferroni"))

write_csv(paired_out, "outputs/paired_tests_with_CI.csv")

# --------------------------- 6) Mixed-effects models (continuous) -------------
# Build pair-level long with markers to fit: outcome ~ group + (1 | pair_id)
long_markers <- long %>%
  select(pair_id, group, tchol, hdl, ldl, trig, pcsk9) %>%
  pivot_longer(cols = c(tchol,hdl,ldl,trig,pcsk9), names_to = "marker", values_to = "y") %>%
  drop_na(y)

fit_lmm <- function(df) {
  lmer(y ~ group + (1|pair_id), data = df, REML = FALSE)
}

lmm_by_marker <- long_markers %>%
  group_by(marker) %>%
  group_map(~{
    fit <- fit_lmm(.x)
    out <- broom.mixed::tidy(fit, effects = "fixed", conf.int = TRUE)
    icc <- tryCatch(performance::icc(fit)$ICC_adjusted, error = function(e) NA_real_)
    glance <- broom.mixed::glance(fit)
    tibble(marker = unique(.x$marker)) %>%
      bind_cols(
        out %>% filter(term == "groupT2DM") %>% select(estimate, conf.low, conf.high, p.value)
      ) %>%
      mutate(icc = icc,
             AIC = glance$AIC, BIC = glance$BIC)
  }) %>% bind_rows()

write_csv(lmm_by_marker, "outputs/mixed_effects_group_effect.csv")

# --------------------------- 7) Dyslipidaemia definitions ---------------------
# Primary (sex-agnostic HDL threshold)
indiv <- long %>%
  mutate(
    dyslip = (ldl >= ldl_thr) | (hdl < hdl_thr) | (trig >= tg_thr)
  )

# Sex-specific sensitivity (only if sex exists)
if("sex" %in% names(indiv)){
  indiv <- indiv %>%
    mutate(
      hdl_thr_sex = case_when(
        sex == "F" ~ hdl_thr_f,
        sex == "M" ~ hdl_thr_m,
        TRUE ~ hdl_thr # fallback
      ),
      dyslip_sex = (ldl >= ldl_thr) | (hdl < hdl_thr_sex) | (trig >= tg_thr)
    )
}

# --------------------------- 8) Conditional logistic regression ---------------
# Requires discordant pairs; use primary dyslip first
disc_tbl <- indiv %>%
  select(pair_id, group, dyslip) %>%
  drop_na(dyslip) %>%
  pivot_wider(names_from = group, values_from = dyslip) %>%
  mutate(discordant = Control != T2DM)

n_pairs_all     <- length(unique(indiv$pair_id))
n_pairs_dyslip  <- nrow(disc_tbl)
n_disc          <- sum(disc_tbl$discordant, na.rm = TRUE)

# Prepare data for clogit (each row = person)
clogit_dat <- indiv %>%
  select(pair_id, group, dyslip, pcsk9) %>%
  drop_na(dyslip, pcsk9) %>%
  mutate(group_bin = ifelse(group=="T2DM", 1, 0))

# Conditional logistic regression: dyslip ~ pcsk9 + group | pair_id
if(n_disc >= 5){
  fit_clogit <- clogit(dyslip ~ pcsk9 + group + strata(pair_id), data = clogit_dat, method = "efron")
  clg_tidy <- broom::tidy(fit_clogit, conf.int = TRUE, exponentiate = TRUE) %>%
    select(term, estimate, conf.low, conf.high, p.value) %>%
    rename(OR = estimate, CI_low = conf.low, CI_high = conf.high)
  write_csv(clg_tidy, "outputs/conditional_logistic_primary.csv")
} else {
  clg_tidy <- tibble(
    term = c("pcsk9","groupT2DM"),
    OR = NA_real_, CI_low = NA_real_, CI_high = NA_real_, p.value = NA_real_
  )
  write_csv(clg_tidy, "outputs/conditional_logistic_primary.csv")
}

# AUC (unmatched logistic purely for discrimination reporting; not for causal claim)
glm_unmatched <- glm(dyslip ~ pcsk9 + group, data = clogit_dat, family = binomial)
roc_obj <- pROC::roc(clogit_dat$dyslip, fitted(glm_unmatched))
auc_val <- as.numeric(roc_obj$auc)
write_csv(tibble(model="pcsk9+group", AUC=auc_val), "outputs/auc_unmatched.csv")

# Sensitivity: sex-specific definition (if sex present)
if("sex" %in% names(indiv)){
  clogit_dat_sex <- indiv %>%
    select(pair_id, group, dyslip_sex, pcsk9) %>%
    drop_na(dyslip_sex, pcsk9) %>%
    mutate(group_bin = ifelse(group=="T2DM", 1, 0))
  disc_tbl_sex <- clogit_dat_sex %>%
    select(pair_id, group, dyslip_sex) %>%
    pivot_wider(names_from = group, values_from = dyslip_sex) %>%
    mutate(discordant = Control != T2DM)
  n_disc_sex <- sum(disc_tbl_sex$discordant, na.rm = TRUE)

  if(n_disc_sex >= 5){
    fit_clogit_sex <- clogit(dyslip_sex ~ pcsk9 + group + strata(pair_id),
                             data = clogit_dat_sex, method="efron")
    clg_tidy_sex <- broom::tidy(fit_clogit_sex, conf.int = TRUE, exponentiate = TRUE) %>%
      select(term, estimate, conf.low, conf.high, p.value) %>%
      rename(OR = estimate, CI_low = conf.low, CI_high = conf.high)
    write_csv(clg_tidy_sex, "outputs/conditional_logistic_sex_specific.csv")
  }
}

# --------------------------- 9) Per-lipid linear models -----------------------
# outcome ~ group + pcsk9, separately for TC, HDL, LDL (by reviewer request)
lm_df <- long %>% select(pair_id, group, tchol, hdl, ldl, pcsk9) %>% drop_na(pcsk9)
models <- list(
  tchol = lm(tchol ~ group + pcsk9, data = lm_df),
  hdl   = lm(hdl   ~ group + pcsk9, data = lm_df),
  ldl   = lm(ldl   ~ group + pcsk9, data = lm_df)
)
lm_tidy <- imap(models, ~ broom::tidy(.x, conf.int=TRUE) %>% mutate(outcome=.y))
lm_glance <- imap(models, ~ broom::glance(.x) %>% mutate(outcome=.y))
bind_rows(lm_tidy)   %>% write_csv("outputs/lm_by_lipid_tidy.csv")
bind_rows(lm_glance) %>% write_csv("outputs/lm_by_lipid_glance.csv")

# --------------------------- 10) Plots (publication-grade) --------------------
theme_pub <- theme_minimal(base_size = 12) +
  theme(
    panel.grid.minor = element_blank(),
    plot.title = element_text(face="bold"),
    legend.position = "top"
  )

# A) Spaghetti plots (within-pair)
mk_spaghetti <- function(varname, label){
  dat <- wide %>% select(pair_id, ends_with("_a"), ends_with("_b"))
  gg <- long %>%
    select(pair_id, group, !!sym(varname)) %>%
    drop_na() %>%
    pivot_wider(names_from = group, values_from = !!sym(varname)) %>%
    ggplot(aes(x = Control, y = T2DM)) +
    geom_abline(linetype = 2) +
    geom_point(alpha = .6) +
    geom_segment(aes(xend = Control, yend = T2DM),
                 x = .$Control, y = .$T2DM, alpha = .3) +
    labs(title = paste0("Within-pair changes: ", label),
         x = "Control (A)", y = "T2DM (B)") +
    theme_pub
  ggsave(filename = file.path("outputs", paste0("spaghetti_", varname, ".png")),
         gg, width = 5, height = 4, dpi = 300)
}
walk2(c("tchol","hdl","ldl","trig","pcsk9"), c("Total Cholesterol","HDL-C","LDL-C","Triglycerides","PCSK9"),
      mk_spaghetti)

# B) Box + jitter by group
mk_box <- function(varname, label, ylab){
  gg <- long %>%
    ggplot(aes(x = group, y = !!sym(varname), fill = group)) +
    geom_boxplot(width = .5, alpha = .5, outlier.shape = NA) +
    geom_jitter(width = .15, alpha = .6, size=1) +
    labs(title = paste0(label," by group"),
         x = NULL, y = ylab) +
    scale_fill_manual(values = c("#4C78A8","#F58518")) +
    theme_pub + theme(legend.position = "none")
  ggsave(file.path("outputs", paste0("box_", varname, ".png")),
         gg, width = 5, height = 4, dpi = 300)
}
mk_box("tchol","Total Cholesterol","mmol/L")
mk_box("hdl","HDL-C","mmol/L")
mk_box("ldl","LDL-C","mmol/L")
mk_box("trig","Triglycerides","mmol/L")
mk_box("pcsk9","PCSK9","ng/mL")

# C) Bland–Altman + paired-difference plot for each marker
mk_bland <- function(a, b, name, ylab){
  df <- tibble(A = a, B = b) %>% drop_na()
  df <- df %>%
    mutate(mean_ab = (A+B)/2, diff = B - A)
  md <- mean(df$diff); sdv <- sd(df$diff)
  loa_low <- md - 1.96*sdv; loa_high <- md + 1.96*sdv
  gg <- df %>%
    ggplot(aes(mean_ab, diff)) +
    geom_hline(yintercept = 0, linetype = 2) +
    geom_point(alpha = .7) +
    geom_hline(yintercept = md, color = "black") +
    geom_hline(yintercept = loa_low, linetype=3) +
    geom_hline(yintercept = loa_high, linetype=3) +
    annotate("text", x = Inf, y = md, label = sprintf("Mean: %.2f", md),
             hjust = 1.1, vjust = -0.5, size = 3) +
    annotate("text", x = Inf, y = loa_low, label = sprintf("LoA: %.2f", loa_low),
             hjust = 1.1, vjust = -0.5, size = 3) +
    annotate("text", x = Inf, y = loa_high, label = sprintf("LoA: %.2f", loa_high),
             hjust = 1.1, vjust = -0.5, size = 3) +
    labs(title = paste0("Bland–Altman: ", name),
         x = "Mean of A and B", y = paste0("B − A (", ylab, ")")) +
    theme_pub
  ggsave(file.path("outputs", paste0("bland_altman_", name, ".png")),
         gg, width = 5, height = 4, dpi = 300)

  # Paired-difference with CI
  ttest <- t.test(df$B, df$A, paired = TRUE)
  pd <- tibble(mean_diff = mean(df$diff),
               ci_low = ttest$conf.int[1], ci_high = ttest$conf.int[2])
  gg2 <- ggplot(pd, aes(x = name, y = mean_diff)) +
    geom_point(size = 2) +
    geom_errorbar(aes(ymin = ci_low, ymax = ci_high), width = 0.1) +
    geom_hline(yintercept = 0, linetype = 2) +
    labs(title = paste0("Paired difference (95% CI): ", name),
         x = NULL, y = paste0("B − A (", ylab, ")")) +
    theme_pub
  ggsave(file.path("outputs", paste0("paired_diff_", name, ".png")),
         gg2, width = 4, height = 4, dpi = 300)
}

mk_bland(wide$t_chol_a, wide$t_chol_b, "TC", "mmol/L")
mk_bland(wide$hdl_a,    wide$hdl_b,    "HDL-C", "mmol/L")
mk_bland(wide$ldl_a,    wide$ldl_b,    "LDL-C", "mmol/L")
mk_bland(wide$trig_a,   wide$trig_b,   "Triglycerides", "mmol/L")
mk_bland(wide$pcsk9_a,  wide$pcsk9_b,  "PCSK9", "ng/mL")

# D) Forest of matched ORs (if available)
if(!all(is.na(clg_tidy$OR))){
  forest_df <- clg_tidy %>%
    mutate(term = recode(term, "pcsk9"="PCSK9 (per unit)", "groupT2DM"="T2DM vs Control"))
  gg_forest <- forest_df %>%
    ggplot(aes(y = term, x = OR)) +
    geom_point() +
    geom_errorbarh(aes(xmin = CI_low, xmax = CI_high), height = 0.2) +
    geom_vline(xintercept = 1, linetype = 2) +
    scale_x_log10() +
    labs(title = "Matched odds ratios (conditional logistic regression)",
         x = "Odds ratio (log scale)", y = NULL) +
    theme_pub
  ggsave("outputs/forest_matched_OR.png", gg_forest, width = 6, height = 3.8, dpi = 300)
}

# --------------------------- 11) Balance diagnostics --------------------------
# Standardized mean differences (SMD) for available baseline variables (if any)
baseline_vars <- c("age","bmi","education")  # adjust if present
avail <- intersect(baseline_vars, names(wide))
if(length(avail)){
  base_long <- wide %>%
    select(pair_id, all_of(avail)) %>%
    left_join(long %>% select(pair_id, group) %>% distinct(), by="pair_id")
  smd_tbl <- map_dfr(avail, ~{
    v <- .x
    if(is.numeric(base_long[[v]])){
      smd <- DescTools::StdDiff(base_long[[v]] ~ base_long$group)
    } else {
      # For categorical, approximate by coding as factors, then use StdDiffCat
      smd <- tryCatch(DescTools::StdDiff(as.factor(base_long[[v]]) ~ base_long$group),
                      error=function(e) NA_real_)
    }
    tibble(variable = v, SMD = smd)
  })
  write_csv(smd_tbl, "outputs/standardized_mean_differences.csv")
}

# --------------------------- 12) Tables (DOCX exports) ------------------------
paired_tbl <- paired_out %>%
  mutate(across(c(mean_diff, sd_diff, ci_low, ci_high, t_stat, t_p, shapiro_p, wilcox_p,
                  p_adj_BH, p_adj_bonf), ~round(., 4)))
paired_ft <- flextable(paired_tbl) %>%
  autofit()
doc <- read_docx() %>%
  body_add_par("Paired tests with 95% CI (A=Control, B=T2DM)", style = "heading 1") %>%
  body_add_flextable(paired_ft) %>%
  body_add_par("Analyzable pairs per marker", style = "heading 1") %>%
  body_add_flextable(flextable(analyzable) %>% autofit()) %>%
  body_add_par("Mixed-effects models: group effect (T2DM vs Control)", style = "heading 1") %>%
  body_add_flextable(flextable(lmm_by_marker %>% mutate(across(where(is.numeric), ~round(.,4)))) %>% autofit()) %>%
  body_add_par("Conditional logistic regression (matched ORs)", style = "heading 1") %>%
  body_add_flextable(flextable(clg_tidy %>% mutate(across(where(is.numeric), ~round(.,4)))) %>% autofit()) %>%
  body_add_par(sprintf("AUC (unmatched logistic for discrimination): %.3f", auc_val), style="Normal")
print(doc, target = "outputs/analysis_tables.docx")

# --------------------------- 13) Session info for reproducibility -------------
writeLines(c(capture.output(sessionInfo())), con = "outputs/session_info.txt")

# --------------------------- 14) Console summary ------------------------------
cat("\n==== SUMMARY ====\n")
print(analyzable)
cat(sprintf("\nDiscordant pairs used by conditional logistic (primary): %d\n", n_disc))
cat(sprintf("AUC (unmatched discrimination): %.3f\n", auc_val))
cat("Tables written to outputs/*.csv and outputs/analysis_tables.docx\nFigures written to outputs/*.png\n")