# This scripts processed raw data from 13C label tracing experiments
# recorded by high resolution LC-MS to extract mass distribution vectors
# of label enrichment or isotopologues.


## Adapt these settings according to your local environment

# set path to metabolite list
compoundinfofile <-  '~/Documents/datasets/marialena/AnalyteList_lipid_Exp2.csv'

# set path to mzML files
datapath <- '~/data/metab_results/choline13Clipid_Exp2'

# set maximum number of labeled carbons to be considered
maxlabel <- 3

# maximum deviation from expected retention time (in seconds). 
hardRTcutoff <- 15


## Change these parameters if you need to. 
## The defaults should work well for data recorded on LC-QTOF-MS machines.

minarea <- 10000 # minimum value of peak area to include a peak
masstol <- 3 # mass tolerance for getting EICs in mDa
mingoodfraction <- 0.5 # fraction of samples that must give useful data for a metabolite in order to report the metabolite
add13c <- 1.003355 # mass difference between 12C and 13C in Da
electronmass <- 0.0005486 # mass of electron in Da
maxFCcutoff <- 3 # max % FC of unlabeled control samples
pseudopeakRTsd <- 0.9 # expected standard deviation of retention time. the smaller, the more strict on correct RT when initially finding the correct peak
mzres <- 0.0005 # increment at which profile mz data is created in Da
msres <- 30000 # resolution of MS data
pseudopeaksd <- 2 # the smaller, the more strict on correct mz when initially finding the correct peak
maxpeaks <- 5 # max number of candidate peaks
minhalfpeakrtwidth <- 6 # Minimum width of LC peaks in number of scans. 6 scans is about 3 sec. Peaks must have at least this distance to the boarder of the RT window.
minpeakrtdist <- 12 # Minimum distance between 2 LC peaks in number of scans. 12 scans is about 0.1 min. Peaks that are closer will be merged.



## check if required packages are already installed and if not install them
requiredpackages <- c('mzR', 'openxlsx', 'signal', 'enviPat', 'nnls', 'ggpubr', 'gplots')
new.packages <- requiredpackages[!(requiredpackages %in% installed.packages()[,"Package"])]
if(length(new.packages) > 0) install.packages(new.packages)

library(mzR) # to read mzml files
library(enviPat) # for isopatten for prediction of theoretical MDVs
library(openxlsx) # for write.xlsx
library(signal) # for sgolay filter
library(ggpubr) # for PCA plot
library(nnls) # required in carbon_isotope_correction()


## little helper functions 

# create 3D normal (Gauss) distribution peak peak
normpeak3d <- function(x1, mean1, sd1, x2, mean2, sd2) {
  variance1 <- sd1^2
  variance2 <- sd2^2
  res <- matrix(0, nrow=length(x1), ncol = length(x2))
  for (i1 in 1:length(x1)) {
    for (i2 in 1:length(x2)) {
      res[i1,i2] <- (exp(-(x1[i1]-mean1)^2/(2*variance1)) / sqrt(2*pi*variance1)) * (exp(-(x2[i2]-mean2)^2/(2*variance2)) / sqrt(2*pi*variance2) )
    }
  }
  res
}


# simulate profile spectrum from centroid data
centroid2profile <- function(nowallscans, nowscan, msres, nowmzaxis) {
  nc <- nowallscans[[nowscan]] 
  goodpeaks <- intersect(which(nc[ ,1] >= (min(nowmzaxis) - max(nc[ ,1])/msres ) ), which(nc[ ,1] <= (max(nowmzaxis) + max(nc[ ,1])/msres)) )
  nc <- nc[goodpeaks, ]
  profile <- numeric(length(nowmzaxis))
  if (length(goodpeaks) == 1) {
    var <- nc[1]/(0.25*msres)^2
    profile <- (exp(-(nowmzaxis-nc[1])^2/(2*var)) / sqrt(2*pi*var))
    profile <- nc[2] * profile/max(profile)
  } else if (length(goodpeaks) > 1) {
    for (inc in 1:length(goodpeaks)) {
      var <- nc[inc,1]/(2*msres)^2
      nowdis <- (exp(-(nowmzaxis-nc[inc,1])^2/(2*var)) / sqrt(2*pi*var))
      nowdis <- nc[inc,2] * nowdis/max(nowdis+1E-12) # avoid NaN if nowdis is all 0
      profile <- profile + nowdis
    }             
  }
  profile
}


# smooth matrix with moving average
matrixSmooth <- function(M, npass=1) {
  rownum <- nrow(M)
  colnum <- ncol(M)
  for (i in 1:npass){
    M[2:(rownum-1), ] <- (M[2:(rownum-1), ] + M[1:(rownum-2), ] + M[3:(rownum), ]) / 3
    M[ ,2:(colnum-1)] <- (M[ ,2:(colnum-1)] + M[ ,1:(colnum-2)] + M[ ,3:(colnum)]) / 3
  }
  M
}

# find local maxima
localMaxima <- function(x) {
  # Use -Inf instead if x is numeric (non-integer)
  y <- diff(c(-.Machine$integer.max, x)) > 0L
  rle(y)$lengths
  y <- cumsum(rle(y)$lengths)
  y <- y[seq.int(1L, length(y), 2L)]
  if (x[[1]] == x[[2]]) {
    y <- y[-1]
  }
  y
}

# find maxima in 3D data
localMaxima3D <- function(M,minz=0) {
  M[M<minz] <- 0
  peaks <- matrix(data=0, ncol = ncol(M), nrow = nrow(M))
  for (ic in 1:ncol(M)) {
    nowpeak <- localMaxima(M[ ,ic])
    peaks[nowpeak ,ic] <- 0.5
  }
  for (ir in 1:nrow(M)) {
    nowpeak <- localMaxima(M[ir, ])
    peaks[ir, nowpeak] <- peaks[ir, nowpeak] + 0.5
  }
  peaks[peaks < 1] <- 0
  max <- list()
  max$ind <- which(peaks == 1, arr.ind = TRUE)
  max$mat <- peaks
  max
}

# correct for natural abundance of heavy isotopes.
# modified from accucor package to fit m-0 and pre-compute AtomNumber
# for original code check https://github.com/XiaoyangSu/AccuCor/blob/master/R/accucor_lib.R
carbon_isotope_correction <- function(AtomNumber, datamatrix, label, Resolution,
                                      ResDefAt, purity=0.99, ReportPoolSize=TRUE) {
  
  CarbonNaturalAbundace <- c(0.9893, 0.0107)
  HydrogenNaturalAbundace <- c(0.999885, 0.000115)
  NitrogenNaturalAbundace <- c(0.99636, 0.00364)
  OxygenNaturalAbundace <- c(0.99757, 0.00038, 0.00205)
  SulfurNaturalAbundace <- c(0.9493, 0.00762, 0.0429)
  
  # AtomNumber <- rep(0,6)
  # names(AtomNumber) <- c("C","H","N","O","P","S")
  MassDifference <- abs(c((2.0141-1.00783),(15.00011-14.00307),(16.99913-15.99491),
                          (17.99916-15.99491),(32.97146-31.97207),(33.96787-31.97207))-
                          ((13.00335-12)*c(1,1,1,2,1,2)))
  names(MassDifference) <- c("H2","N15","O17","O18","S33","S34")
  CorrectionLimit <- rep(0,6)
  names(CorrectionLimit) <- c("H2","N15","O17","O18","S33","S34")
  
  MolecularWeight <- sum(AtomNumber*c(12,1,14,16,31,32))
  
  CorrectionLimit <- floor(MolecularWeight^(3/2)*1.66/(Resolution*sqrt(ResDefAt))/MassDifference)
  
  ExpMatrix <- matrix(0, ncol=ncol(datamatrix), nrow=AtomNumber["C"]+1)
  CorrectedMatrix <- matrix(0, ncol=ncol(datamatrix), nrow=AtomNumber["C"]+1)
  for (i in 1:length(label)) {
    ExpMatrix[label[i]+1,] <- datamatrix[i,]
  }
  
  
  PurityMatrix <- diag(AtomNumber["C"]+1)
  CarbonMatrix <- diag(AtomNumber["C"]+1)
  NitrogenMatrix <- diag(AtomNumber["C"]+1)
  HydrogenMatrix <- diag(AtomNumber["C"]+1)
  OxygenMatrix <- matrix(0,ncol=(AtomNumber["C"]+1),nrow=(AtomNumber["C"]+1))
  SulfurMatrix <- matrix(0,ncol=(AtomNumber["C"]+1),nrow=(AtomNumber["C"]+1))
  
  
  for(i in 1:(AtomNumber["C"]+1)){
    PurityMatrix[i,] <- sapply(0:(AtomNumber["C"]), function(x) stats::dbinom(x-i+1, x , (1-purity)))
  }
  
  for(i in 1:(AtomNumber["C"]+1)){
    CarbonMatrix[,i] <- sapply(0:AtomNumber["C"], function(x) stats::dbinom(x-i+1, AtomNumber["C"]-i+1 , CarbonNaturalAbundace[2]))
  }
  
  for(j in 0:min(AtomNumber["N"], CorrectionLimit["N15"], AtomNumber["C"]))
    for(i in 1:(AtomNumber["C"]-j+1)){
      NitrogenMatrix[(i+j),i] <- stats::dbinom(j, AtomNumber["N"], NitrogenNaturalAbundace[2])
    }
  
  for(j in 0:min(AtomNumber["H"], CorrectionLimit["H2"], AtomNumber["C"]))
    for(i in 1:(AtomNumber["C"]-j+1)){
      HydrogenMatrix[(i+j),i] <- stats::dbinom(j, AtomNumber["H"], HydrogenNaturalAbundace[2])
    }
  
  for(i in 0:min(AtomNumber["O"],CorrectionLimit["O17"])) {
    for(j in 0:min(AtomNumber["O"],CorrectionLimit["O18"])){
      k<-(i+j*2)
      if ((i+j)>AtomNumber["O"]|k>AtomNumber["C"]) {
        break
      }
      else {
        for (m in 1:(AtomNumber["C"]-k+1)) {
          OxygenMatrix[(m+k),m] <- OxygenMatrix[(m+k),m] + stats::dmultinom(c((AtomNumber["O"]-i-j),i,j), AtomNumber["O"], OxygenNaturalAbundace)
        }
      }
    }
  }
  
  for(i in 0:min(AtomNumber["S"],CorrectionLimit["S33"])) {
    for(j in 0:min(AtomNumber["S"],CorrectionLimit["S34"])){
      k<-(i+j*2)
      if ((i+j)>AtomNumber["S"]|k>AtomNumber["C"]) {
        break
      }
      else {
        for (m in 1:(AtomNumber["C"]-k+1)) {
          SulfurMatrix[(m+k),m] <- SulfurMatrix[(m+k),m] + stats::dmultinom(c((AtomNumber["S"]-i-j),i,j), AtomNumber["S"], SulfurNaturalAbundace)
        }
      }
    }
  }
  
  
  zeromatrix <- matrix(data=0, ncol(SulfurMatrix)+1, nrow = nrow(SulfurMatrix)+1)
  SulfurMatrix2 <- zeromatrix
  SulfurMatrix2[2:(nrow(SulfurMatrix)+1), 2:(ncol(SulfurMatrix)+1)] <- SulfurMatrix
  OxygenMatrix2 <- zeromatrix
  OxygenMatrix2[2:(nrow(OxygenMatrix)+1), 2:(ncol(OxygenMatrix)+1)] <- OxygenMatrix
  NitrogenMatrix2 <- zeromatrix
  NitrogenMatrix2[2:(nrow(NitrogenMatrix)+1), 2:(ncol(NitrogenMatrix)+1)] <- NitrogenMatrix
  HydrogenMatrix2 <- zeromatrix
  HydrogenMatrix2[2:(nrow(HydrogenMatrix)+1), 2:(ncol(HydrogenMatrix)+1)] <- HydrogenMatrix
  CarbonMatrix2 <- zeromatrix
  CarbonMatrix2[2:(nrow(CarbonMatrix)+1), 2:(ncol(CarbonMatrix)+1)] <- CarbonMatrix
  PurityMatrix2 <- zeromatrix
  PurityMatrix2[2:(nrow(PurityMatrix)+1), 2:(ncol(PurityMatrix)+1)] <- PurityMatrix
  
  # only use part of correction matrices in case not all carbon isotopolouges should be considered.
  SulfurMatrix2 <- SulfurMatrix2[1:length(datamatrix), 1:length(datamatrix)]
  OxygenMatrix2 <- OxygenMatrix2[1:length(datamatrix), 1:length(datamatrix)]
  NitrogenMatrix2 <- NitrogenMatrix2[1:length(datamatrix), 1:length(datamatrix)]
  HydrogenMatrix2 <- HydrogenMatrix2[1:length(datamatrix), 1:length(datamatrix)]
  CarbonMatrix2 <- CarbonMatrix2[1:length(datamatrix), 1:length(datamatrix)]
  PurityMatrix2 <- PurityMatrix2[1:length(datamatrix), 1:length(datamatrix)]
  
  sumresiduals <- numeric()
  for(i in 1:ncol(datamatrix)) {
    res <- nnls::nnls(SulfurMatrix2 %*% OxygenMatrix2 %*% NitrogenMatrix2 %*%
                        HydrogenMatrix2 %*% CarbonMatrix2 %*% PurityMatrix2, datamatrix ) # ExpMatrix[,i]
  }
  return(res) # CorrectedMatrix
  
}



## Start the processing

# set log file
log_con <- file(file.path(datapath ,"R_messages.log"), open="a")

# get files in folder
filelist <- list.files(path = datapath, pattern = '.mzML')
for (i in 1:length(filelist)){
  filelist[i] <- substr(filelist[i],1,nchar(filelist[i])-5)
}
cat('\n Found these files: ', file=log_con)
cat(filelist, file=log_con)

# don't filter for clean control or min good fraction if there is less than 3 samples
if (length(filelist) < 3) {
  maxFCcutoff <- 1000
  mingoodfraction <- 0
}

# get polarity of data set assuming that all measurements have the same polarity
mz <- openMSfile(file.path(datapath, paste0(filelist[1], '.mzML')))
polarity <- c('Negative', 'Positive')[header(mz)$polarity+1][1]

# parse metabolite list and enrich
targetmets <- list()
cmpinfo <- read.csv(compoundinfofile, stringsAsFactors = FALSE)
cmpcols <- colnames(cmpinfo)
cmpcols <- tolower(cmpcols)
colnames(cmpinfo) <- cmpcols
metlist <- list()
data("isotopes")
if ('main.negative.adduct' %in% names(cmpinfo)) {
  cmpinfo$mainnegativeadduct <- cmpinfo$main.negative.adduct
}
if ('main.positive.adduct' %in% names(cmpinfo)) {
  cmpinfo$mainpositiveadduct <- cmpinfo$main.positive.adduct
}

# find useful compounds in list
goodmets <- rep(TRUE, nrow(cmpinfo))
for (i in 1:nrow(cmpinfo)) {
  processmetabolite <- TRUE
  # check if metabolite is a duplicate in AnalyteList
  if (cmpinfo[i,'name'] %in% metlist) {
    processmetabolite <- FALSE
    goodmets[i] <- FALSE
  } 

  # check if an adduct for the current polarity has been defined
  if ( ( (polarity == 'Positive') && (nchar(cmpinfo[i,'mainpositiveadduct']) == 0) ) ||
       ( (polarity == 'Negative') && (nchar(cmpinfo[i,'mainnegativeadduct']) == 0) ) ) {
      processmetabolite <- FALSE
      goodmets[i] <- FALSE
  }
  
  if (processmetabolite) {
    targetmets[[i]] <- as.list(cmpinfo[i, ])
    cat("\r", paste(as.character(targetmets[[i]]$name), '                               ') )
    
    targetmets[[i]]$rt <- as.numeric(targetmets[[i]]$rt)
    targetmets[[i]]$formula <- trimws(targetmets[[i]]$formula)
    
    
    # if there is no explicit number given for the last element append "1" to avoid error in subform/mergeform
    metformula <- as.character(targetmets[[i]]$formula)
    if ( !substr( metformula,nchar(metformula),nchar(metformula)) %in% c('0', '1', '2', '3', '4', '5', '6', '7', '8', '9') ) {
      metformula <- paste0(metformula, '1')
    }
 
    # read main ion and calculate ion formula
    if ( polarity == 'Negative') {
      if (targetmets[[i]]$mainnegativeadduct == '[M-H]-') {
        metformula <- subform(metformula,"H1")
      } else if (targetmets[[i]]$mainnegativeadduct == '[M+Cl]-') {
        metformula <- mergeform(metformula, 'Cl1')
      } else if (targetmets[[i]]$mainnegativeadduct == '[M+CO2H]-') {
        metformula <- mergeform(metformula, 'C1O2H1')
      } else if (targetmets[[i]]$mainnegativeadduct == '[M-H+Si3O]-') {
        metformula <- subform(metformula,"H1")
        metformula <- mergeform(metformula, 'Si3O1')
      }
    } else if ( polarity == 'Positive') {
      if (targetmets[[i]]$mainpositiveadduct == '[M+H]+') {
        metformula <- mergeform(metformula,"H1")
      } else if (targetmets[[i]]$mainpositiveadduct == '[M+NH4]+') {
        metformula <- mergeform(metformula, 'N1H4')
      } else if (targetmets[[i]]$mainpositiveadduct == '[M+Na]+') {
        metformula <- mergeform(metformula, 'Na1')
      }
    } else {
      print(paste('cannot read ion info for', targetmets[[i]]$name))
      goodmets[i] <- FALSE
    }
    targetmets[[i]]$FragmentFormula <- metformula
    if (goodmets[i]) {
      metlist <- c(metlist, targetmets[[i]]$name)
    }
    
    # get maximum number of labels assuming that only 13C labeling was used
    chemform <- check_chemform(isotopes, metformula, get_list=TRUE)
    targetmets[[i]]$AtomNumber <- rep(0,6)
    names(targetmets[[i]]$AtomNumber) <- c("C","H","N","O","P","S")
    for (ic in 1:length(chemform[[1]])) {
      targetmets[[i]]$AtomNumber[[names(chemform[[1]][ic])]] <- chemform[[1]][ic]
    }
    
    targetmets[[i]]$maxlabel <- min(maxlabel, as.numeric(chemform[[metformula]]['C']) )
    
    if ( polarity == 'Negative') {
      targetmets[[i]]$isotopelist <- check_chemform(isotopes, metformula, get_list=FALSE)$monoisotopic_mass - add13c + electronmass
    } else {
      targetmets[[i]]$isotopelist <- check_chemform(isotopes, metformula, get_list=FALSE)$monoisotopic_mass - add13c - electronmass
    }
    for (ic in 0:targetmets[[i]]$maxlabel) {
      targetmets[[i]]$isotopelist[ic+2] <- targetmets[[i]]$isotopelist[1] + (ic+1) * add13c
    }
  } # endif metabolite should be processed
} # endfor go through all metabolites

targetmets <- targetmets[goodmets]


ad <- list()
for (im in 1:length(targetmets)) {
  ad[[im]] <- list()
}
cat(paste(timestamp(), '\n'), file = log_con)
cat('\n Reading raw data ', file=log_con)


plotinfo <- rep( list(  rep(list(list()),length(filelist))) , length(targetmets))
goodfiles = rep(FALSE, length(filelist))

# go through all files and pull out raw data around expected m/z and retention time for all metabolites
for (filenum in 1:length(filelist)){
  fileptm <- proc.time()
  print(filelist[filenum])
  
  filegood <- tryCatch({ 
    mz <- openMSfile(file.path(datapath, paste0( filelist[filenum], '.mzML')))
    cat(paste('Now reading: ', filelist[filenum], '\n'), file=log_con)
    goodfiles[filenum] <- TRUE
    print('good')
    }, error = function(err){
      cat(paste('Failed to read: ', filelist[filenum], '\n'), file=log_con)
      print('bad')
  })
  
  if (filegood == 'good') {
    # get expected retention time
    rt <- header(mz)$retentionTime
  
    # read all scans from mz and save to list. This causes 6x speedup compared to using the peaks function inside centroid2profile
    nowallscans <- list()
    for (ir in 1:length(rt)) {
      nowallscans[[ir]] <- mzR::peaks(mz, ir)
    }
    
    for (im in 1:length(targetmets)) {
      cat("\r", paste(im, '                                        '))
      
      if (!is.na(targetmets[[im]]$rt)) {
        # get start and end scan for stripes
        loscan <- which.min(abs(rt - (targetmets[[im]]$rt*60-hardRTcutoff)))
        hiscan <- which.min(abs(rt - (targetmets[[im]]$rt*60+hardRTcutoff)))
        stripes <- list()
        nowstripmzaxis <- seq(-masstol/1E3 , masstol/1E3, mzres)
        stripmatsum <- matrix(0,ncol = 1+hiscan-loscan, nrow = length(nowstripmzaxis ) )
        
        plotinfo[[im]][[filenum]]$rt <- rt[loscan:hiscan]
        plotinfo[[im]][[filenum]]$plottrace <- list()
        plotinfo[[im]][[filenum]]$intstart <- numeric()
        plotinfo[[im]][[filenum]]$intend <- numeric()
        nowplotstrip <- (-1:1) + ceiling(length(nowstripmzaxis)/2)
        
        for (ist in 1:length(targetmets[[im]]$isotopelist)) {
          stripes[[ist]] <- list()
          stripes[[ist]]$mz <- targetmets[[im]]$isotopelist[ist] + nowstripmzaxis
          stripes[[ist]]$rt <-rt[loscan:hiscan]
          nowmat <- matrix(ncol=length(stripes[[ist]]$rt), nrow = length(stripes[[ist]]$mz))
          
          nowstripmzaxis1 <- stripes[[ist]]$mz 
          
          # get small stripe
          for (ic in 1: (1+hiscan-loscan) ) {
            nowmat[,ic  ] <- centroid2profile(nowallscans, loscan-1+ic, msres, nowstripmzaxis1)
          }
          stripes[[ist]]$mat <- nowmat
          plotinfo[[im]][[filenum]]$plottrace[[ist]] <- as.integer(round(colSums(nowmat[nowplotstrip, ], na.rm = TRUE)))
          # sum stripes and find most probable peak by correct massdefect and absence in M-1 and RT
          stripmatsum <- stripmatsum + stripes[[ist]]$mat
        }
        
        # because we added the M-1 nowmat before, we have to subtract it now twice.
        stripmatsum <- stripmatsum - 2*stripes[[1]]$mat
        # 10x weight for M+0 and M+n because this is usually the most abundant 
        stripmatsum <- stripmatsum + 9*stripes[[2]]$mat
        stripmatsum <- stripmatsum + 9*stripes[[length(stripes)]]$mat
        stripmatsum[stripmatsum < 0] <- 0
        stripes[[1]]$stripmatsum <- stripmatsum
        stripes[[1]]$loscan <- loscan
        stripes[[1]]$hiscan <- hiscan
        
        ad[[im]][[filenum]] <- stripes
  
      } # endif good metabolite
    } # endfor all metabolites
    close(mz)
    fileptime <- proc.time() - fileptm
    print(as.numeric(fileptime[3]/60))
  } # endif mzml file is readable
} # endfor all files

cat('...done reading stripes. ', file=log_con)


# find peaks and build mass distribution vectors 
cat('\n Building MDVs ', file=log_con)
out <- list()
maxmdvlength <- 0
for (im in 1:length(targetmets)){ 
  if (length(ad[[im]]) > 0) {
    if (ad[[im]][[1]][[1]]$hiscan > ad[[im]][[1]][[1]]$loscan) { # if expected RT is within measured range
      # find peak in sum of all strips across all samples
      allstripsum <- 0
      nowstripmz <- 0
      for (filenum in 1:length(filelist)){
        if (goodfiles[filenum]){
          if ( (filenum == 1) || (ncol(allstripsum) == ncol(ad[[im]][[filenum]][[1]]$stripmatsum) )) {
            allstripsum <- allstripsum + ad[[im]][[filenum]][[1]]$stripmatsum
          } else {
            nowlimit <- min(c(ncol(allstripsum), ncol(ad[[im]][[filenum]][[1]]$stripmatsum)))
            allstripsum[ ,1:nowlimit] <- allstripsum[ ,1:nowlimit] + ad[[im]][[filenum]][[1]]$stripmatsum[,1:nowlimit]
          }
          nowstripmz <- nowstripmz + ad[[im]][[filenum]][[2]]$mz
        }
      } 
      nowstripmz <- nowstripmz/filenum
  
      # get start and end scan for stripes assuming that all files have the same RT axis
      loscan <- ad[[im]][[1]][[1]]$loscan
      hiscan <- ad[[im]][[1]][[1]]$hiscan
      
      # idea: find best peak by adding gaussean pseudopeak at expected massdefect and RT to give advantage to peaks closer to expected values and then take the largest peak. This kind of represents the probability that a peak is the correct one.
      pseudopeak <- normpeak3d(1:nrow(allstripsum), ceiling(nrow(allstripsum)/2), pseudopeaksd,  1:ncol(allstripsum), 0.5*(1+hiscan-loscan), pseudopeakRTsd*(1+hiscan-loscan) )
      
      smoothallstripsum <- matrixSmooth(allstripsum, 60)
      smoothpseudostripsum <- matrixSmooth(pseudopeak * allstripsum,60)
      allpeaks <- localMaxima3D(smoothpseudostripsum, 0.1*max(smoothpseudostripsum))
     

      # only take top maxpeaks peaks.
      allpeaks$num <- sum(allpeaks$mat)
      if (allpeaks$num > maxpeaks) {
        peakintens <- numeric(allpeaks$num)
        allpeaks$mat <- 0 * allpeaks$mat # delete matrix representation
        for (ip in 1:allpeaks$num){ # get pseudo intensity values of peaks
          peakintens[ip] <- smoothpseudostripsum[allpeaks$ind[ip,'row'], allpeaks$ind[ip,'col']]
        }
        nowpeakorder <- order(peakintens, decreasing = TRUE)[1:maxpeaks]
        allpeaks$ind <- allpeaks$ind[nowpeakorder, ]
        for (ip in 1:maxpeaks) { # create new matrix representation
          allpeaks$mat[allpeaks$ind[ip,'row'], allpeaks$ind[ip,'col']] <- 1
        }
        allpeaks$num <- sum(allpeaks$mat) # recalculate number of peaks
      }
  
      # peaks need to have a minimum distance of minpeakrtdist. if 2 peaks are closer, take the first one.
      allpeaks$num <- sum(allpeaks$mat)
      if (allpeaks$num > 1) { # if there is more than one candidate peak
        goodallpeaks <- rep(TRUE,allpeaks$num)
        for (iap in 2:allpeaks$num) {
          if (allpeaks$ind[iap,'col'] - allpeaks$ind[iap-1,'col'] < minpeakrtdist) {
            goodallpeaks[iap] <- FALSE
            allpeaks$mat[allpeaks$ind[iap-1,'row'], allpeaks$ind[iap-1,'col'] ] <- 0
          }
        }
        allpeaks$ind <- allpeaks$ind[goodallpeaks, ]
        allpeaks$num <- sum(allpeaks$mat) # recalculate after tidying up
        if (allpeaks$num == 1) {
          allpeaks$ind <- t(as.matrix(allpeaks$ind))
        }
      }
  
      # if there are multiple peaks find valleys between them
      allpeakrtborder <- 1
      if (allpeaks$num > 1) {
        for (ip in 1:(allpeaks$num-1)) {
          # allpeakrtborder[ip+1] <- round((allpeaks$ind[ip,'col'] + allpeaks$ind[ip+1,'col'])/2) # old: use half way between peaks
          nowvalleys <- localMaxima(-smoothallstripsum[ allpeaks$ind[ip,'row']  , ])
          nowborder <- nowvalleys[intersect(which(nowvalleys > allpeaks$ind[ip,'col']), which(nowvalleys < allpeaks$ind[ip+1,'col']))]
          if (length(nowborder) == 1) {
            allpeakrtborder[ip+1] <- nowborder
          } else {
            allpeakrtborder[ip+1] <- round((allpeaks$ind[ip,'col'] + allpeaks$ind[ip+1,'col'])/2) # old: use half way between peaks
          }
        }
      }
      allpeakrtborder <- c(allpeakrtborder, ncol(smoothpseudostripsum))
  
      out[[im]] <- list()
      out[[im]]$peaks <- list()
      out[[im]]$peaklabel <- character()
      
      if (allpeaks$num == 0) {
        ip <- 1
        out[[im]]$peaklabel[ip] <- paste0(targetmets[[im]]$name, ' not found')
        out[[im]]$peaks[[ip]] <- list()
        out[[im]]$peaks[[ip]]$intensity <- numeric(length(filelist))
        out[[im]]$peaks[[ip]]$residual <- numeric(length(filelist))
        out[[im]]$peaks[[ip]]$mz <- numeric(length(filelist))
        out[[im]]$peaks[[ip]]$rt <- numeric(length(filelist))
        out[[im]]$peaks[[ip]]$rtdiff <- numeric(length(filelist))
        out[[im]]$peaks[[ip]]$fc <- numeric(length(filelist))
        out[[im]]$peaks[[ip]]$comment <- character(length(filelist))
        out[[im]]$peaks[[ip]]$mzdiff <- numeric(length(filelist))
        out[[im]]$peaks[[ip]]$mdv <- list(length(filelist))
        out[[im]]$peaks[[ip]]$rawmdv <- list(length(filelist))
        out[[im]]$peaks[[ip]]$m0m1ratio <- numeric(length(filelist))
      } else {
        for (ip in 1:allpeaks$num) {
          out[[im]]$peaklabel[ip] <- paste0(targetmets[[im]]$name, '_', round(ad[[im]][[1]][[1]]$rt[allpeaks$ind[ip,'col']]/60,2) )
          out[[im]]$peaks[[ip]] <- list()
          out[[im]]$peaks[[ip]]$intensity <- numeric(length(filelist))
          out[[im]]$peaks[[ip]]$residual <- numeric(length(filelist))
          out[[im]]$peaks[[ip]]$mz <- numeric(length(filelist))
          out[[im]]$peaks[[ip]]$rt <- numeric(length(filelist))
          out[[im]]$peaks[[ip]]$rtdiff <- numeric(length(filelist))
          out[[im]]$peaks[[ip]]$fc <- numeric(length(filelist))
          out[[im]]$peaks[[ip]]$comment <- character(length(filelist))
          out[[im]]$peaks[[ip]]$mzdiff <- numeric(length(filelist))
          out[[im]]$peaks[[ip]]$mdv <- list(length(filelist))
          out[[im]]$peaks[[ip]]$rawmdv <- list(length(filelist))
          out[[im]]$peaks[[ip]]$m0m1ratio <- numeric(length(filelist))
  
          for (filenum in 1:length(filelist)){
            if (goodfiles[filenum]){
              # reportcount <- reportcount + 1
              cat("\r", paste(im, ip, filenum, '        '))
              
              # now search the peak again in every file
              stripmatsum <- matrixSmooth(ad[[im]][[filenum]][[1]]$stripmatsum,10)
              
              # generate more narrow pseudopeak and add hard cutoffs half way to the next peak on RT
              pseudopeak <- normpeak3d(1:nrow(ad[[im]][[filenum]][[1]]$stripmatsum), allpeaks$ind[ip,'row'], pseudopeaksd/2,  1:ncol(ad[[im]][[filenum]][[1]]$stripmatsum), allpeaks$ind[ip,'col'], pseudopeakRTsd*0.2*(1+hiscan-loscan) )
              # pseudopeak = matrix(data=1, ncol=ncol(stripmatsum), nrow=nrow(stripmatsum))
              pseudopeak[ ,-(allpeakrtborder[ip]:allpeakrtborder[ip+1])] <- 0
              
  
              pseudostripsum <- stripmatsum * pseudopeak
              # find highest peak in pseudostripsum to narrow search in single spectra
              peakpos <- which(pseudostripsum == max(pseudostripsum), arr.ind = TRUE)

              # go back to stripsum , find RT limits of peak in found scan
              sumchromatogram <- sgolayfilt(stripmatsum[peakpos[1,'row'], ], p=3, n=21)
              chromslope <- diff(sumchromatogram)
              peaktop <- as.numeric(peakpos[1,'col'])
              if (peaktop < (1+minhalfpeakrtwidth)) {
                intstart <- 1
                peakstart <- 1
              } else {
                peakstart <- max(c(1, which(chromslope[1:(peaktop-minhalfpeakrtwidth)] < 0)))
                intstart <- max(c(allpeakrtborder[ip], which(sumchromatogram[allpeakrtborder[ip]:(peaktop-minhalfpeakrtwidth)] < (0.2*max(sumchromatogram[allpeakrtborder[ip]:allpeakrtborder[ip+1]],na.rm = TRUE)) ) ) )
                if (is.infinite(peakstart)) {
                  peakstart <- 1
                }
                if (is.infinite(intstart)) {
                  intstart <- 1
                }
                intstart <- max(c(intstart, peakstart))
              }

              if (peaktop > (length(chromslope)-(1+minhalfpeakrtwidth))) {
                intend <- length(chromslope)
                peakend <- length(chromslope)
              } else {
                peakend <- min(c(allpeakrtborder[ip+1], peaktop + minhalfpeakrtwidth + min(which(chromslope[(peaktop+minhalfpeakrtwidth):allpeakrtborder[ip+1]] > 0)) ) )
                intend <- min(c(allpeakrtborder[ip+1], peaktop + minhalfpeakrtwidth + max(which(sumchromatogram[(peaktop+minhalfpeakrtwidth):allpeakrtborder[ip+1]] > (0.2*max(sumchromatogram[allpeakrtborder[ip]:allpeakrtborder[ip+1]],na.rm = TRUE)) )) ) )
                if (is.infinite(peakend)) {
                  peakend <- length(chromslope)
                }
                if (is.infinite(intend)) {
                  intend <- length(chromslope)
                }
                intend <- min(c(intend, peakend))

  
                out[[im]]$peaks[[ip]]$rt[filenum] <- ad[[im]][[filenum]][[1]]$rt[peaktop] 
                out[[im]]$peaks[[ip]]$rtdiff[filenum] <- ad[[im]][[filenum]][[1]]$rt[peaktop] - targetmets[[im]]$rt*60
                out[[im]]$peaks[[ip]]$mz[filenum] <- ad[[im]][[filenum]][[2]]$mz[peakpos[1,'row']]
                out[[im]]$peaks[[ip]]$mzdiff[filenum] <- targetmets[[im]]$isotopelist[2] - ad[[im]][[filenum]][[2]]$mz[peakpos[1,'row']]
                
                # go through all scans and integrate within RT limits to build MDV
                rawmdv <- numeric()
                plotinfo[[im]][[filenum]]$intstart[ip] <- min(c(intstart, ncol(ad[[im]][[filenum]][[2]]$mat))) # [[2]] = M+0 trace
                plotinfo[[im]][[filenum]]$intend[ip] <- min(c(intend, ncol(ad[[im]][[filenum]][[2]]$mat))) # [[2]] = M+0 trace
                
                for (ist in 1:length(targetmets[[im]]$isotopelist)) {
                  nowpeakstart <- min(c(peakstart, ncol(ad[[im]][[filenum]][[ist]]$mat)))
                  nowintstart  <- min(c(intstart, ncol(ad[[im]][[filenum]][[ist]]$mat)))
                  nowpeakend <- min(c(peakend, ncol(ad[[im]][[filenum]][[ist]]$mat)))
                  nowintend  <- min(c(intend, ncol(ad[[im]][[filenum]][[ist]]$mat)))
                  rawmdv[ist] <- mean(ad[[im]][[filenum]][[ist]]$mat[peakpos[1,'row'],nowintstart:nowintend]) - min(ad[[im]][[filenum]][[ist]]$mat[peakpos[1,'row'], nowpeakstart:nowpeakend])  

                }
                if (sum(rawmdv) > 0) {
                  mdv <- rawmdv / sum(rawmdv)
                } else {
                  mdv <- 0*rawmdv
                }
  
                out[[im]]$peaks[[ip]]$intensity[filenum] <- sum(rawmdv[-1])
                out[[im]]$peaks[[ip]]$rawmdv[[filenum]] <- rawmdv[-1]
                out[[im]]$peaks[[ip]]$m0m1ratio[filenum] <- rawmdv[1]/(rawmdv[2]+1) # avoid NaN
                
                # do carbon isotope correction with function from accucor package
                AtomNumber <- targetmets[[im]]$AtomNumber
                datamatrix <- as.matrix(mdv)
                label <- 0:targetmets[[im]]$maxlabel
                Resolution <- 30000
                ResDefAt <- 250
                
                cic <- carbon_isotope_correction(AtomNumber, datamatrix, label, Resolution, ResDefAt, purity=0.99, ReportPoolSize=TRUE)
                fitmdv <- cic$x[2:length(cic$x)]
                if (sum(fitmdv) > 0) {
                  fitmdv <- fitmdv/sum(fitmdv)
                }
                
                
                out[[im]]$peaks[[ip]]$mdv[[filenum]] <- fitmdv
                out[[im]]$peaks[[ip]]$residual[filenum] <- sum(abs(cic$residuals))
                out[[im]]$peaks[[ip]]$fc[filenum] <-  100*sum((1:length(fitmdv)-1) * fitmdv)/(length(fitmdv)-1)
                maxmdvlength <- max(maxmdvlength,length(fitmdv), na.rm = TRUE)
              } # endif peak is not just a shoulder
            } # endif good file
          } # endfor go through all files
        } # endfor go through all alternative peaks
      } # end if there are any peaks 
    } # expeted RT of this target met in measured RT range
  } # data available for this targetmet
} # endfor go through all targetmets

cat('...done. ', file=log_con)
cat(paste(timestamp(), '\n'), file = log_con)



cat('\n Writing output to excel \n', file=log_con)
readmetext <- c('What is in the different sheets?',
                '',
                '"corrected": mass distribution vectors corrected for the natural isotope abundance. This is what you will want to look at.',
                '"raw": intensity values of all relevant isotopologues without any correction or normalization. This is for troubleshooting mostly.',
                '"log": log of the automatic processing',
                '',
                '',
                'What is in the columns?',
                '',
                'Metabolite: Name of the metabolite followed by retention time (RT). For some metabolites there multiple candidate peaks that differ in RT.',
                'File: Filename that contains the raw data. The first 8 characters are the sample ID number.',
                'Type: The type info that you entered at sample submission. Used to identify replicates.',
                'Intensity: The signal intensity (arbitraty unit).',
                'mz: The mass over charge value of the feature. As we only look at singly charged ions, this is the mass of the ion.',
                'Delta.mz: Difference between theoretical and measured mass. The smaller the absolute value the better. Values between -0.001 and 0.001 are acceptable.',
                'RT: retention time of the feature.',
                'Delta.RT: Difference between expected and measured RT. The Delta.RT value should be similar for all correct features.',
                'Residual: Residual of the fitting procedure that corrects for the natural abundance of isotopes. The smaller the better.',
                '%FC: Fractional contribution in %. Basically that is a value for how much of the carbon in the feature is labeled.',
                'The following columns then contain the mass distribution values or the intensity values.',
                '',
                '',
                'How to start?',
                '',
                'You will see that for some metabolites there are multiple entries in the',
                'excel output, each labeled with a different retention time in the name',
                'column. The reason is that the algorithm finds multiple peaks that it ',
                'thinks could be the correct peak. It already filtered out the really bad ',
                'ones. For the ones that are left, it has no way of telling which one is',
                'the most likely one.',
                'Therefore, before you start any real analysis, you should first filter ',
                'the results for crap peaks. Reasons for deleting a whole metabolite_RT set are',
                '* Signal not well above blank',
                '* Labeling not well above unlabeled control sample',
                '* %FC in unlabeled control > 3%',
                '* Retention time shift very different from that of related compounds',
                '',
                'It can well be that you have to delete all candidates of a metabolite.',
                '',
                'If however, after this filtering you are still left with multiple',
                'candidates for a metabolite, you should pick the one that has',
                '1. lower absolute value of Delta.mz',
                '2. lower absolute value of Delta.RT',
                '3. higher intensity',
                '',
                'If you have any questions, please don`t hesitate to contact the metabolomics facilty.')

                
# QC filtering and writing to excel
oddmet <- TRUE
oddtype <- TRUE
nowstyle <- numeric()
labels <- data.frame('Metabolite'=character(),'File'=character(),'Type'=character(),'Intensity'=numeric(),'mz'=numeric(),'Delta mz'=numeric(), 'RT'=numeric(), 'Delta RT'=numeric(), 'Residual'=numeric(),'Comments'=character(),'FC'=numeric(), stringsAsFactors = FALSE )
maxlines <- length(filelist) * length(targetmets) * 5
mdvout <- matrix(data=NA,nrow=maxlines,ncol=maxmdvlength)
rawout <- matrix(data=NA,nrow=maxlines,ncol=maxmdvlength)




rowcount <- 0
for (im in 1:length(targetmets)) {
  for (ip in 1:length(out[[im]]$peaks)) {
    # only report samples if the intensity is at least 10% of minsample and if a possibly present M-1 peak contributes less than 1% to the M+0 peak.
    goodsamples <- intersect(which(out[[im]]$peaks[[ip]]$intensity > (0.1 * minarea)), 
                             which(out[[im]]$peaks[[ip]]$m0m1ratio < (1/targetmets[[im]]$maxlabel) ) ) 
      
    if ( (any(out[[im]]$peaks[[ip]]$fc[goodsamples] < maxFCcutoff ) ) && 
         ( (length(goodsamples) / length(filelist)) >= mingoodfraction ) ) {
      oddmet <- !oddmet

      for (filenum in 1:length(filelist)) { 
        rowcount <- rowcount+1
        
        if (filenum < length(filelist)) { 
          oddtype <- !oddtype
        }
        
        if (oddmet && oddtype) {
          nowstyle[rowcount] <- 3
        } else if (oddmet)  {
          nowstyle[rowcount] <- 2
        } else if (oddtype) {
          nowstyle[rowcount] <- 1
        } else {
          nowstyle[rowcount] <- 0
        }
        
        nowcomment <- ''

        labels[rowcount,1:2] <- c(out[[im]]$peaklabel[ip], filelist[filenum]) 
        labels[rowcount,c('Intensity', 'mz', 'Delta.mz', 'RT', 'Delta.RT', 'Residual')] <- c(out[[im]]$peaks[[ip]]$intensity[filenum], out[[im]]$peaks[[ip]]$mz[filenum], out[[im]]$peaks[[ip]]$mzdiff[filenum], out[[im]]$peaks[[ip]]$rt[filenum]/60, out[[im]]$peaks[[ip]]$rtdiff[filenum]/60, out[[im]]$peaks[[ip]]$residual[filenum])
        
        if (!is.na( sd(out[[im]]$peaks[[ip]]$rtdiff)) ) {
          if (abs(out[[im]]$peaks[[ip]]$rtdiff[filenum] - mean(out[[im]]$peaks[[ip]]$rtdiff)) > 
              (2*sd(out[[im]]$peaks[[ip]]$rtdiff))) {
            nowcomment <- paste(nowcomment, 'WARNING, large RT deviation', sep = ' - ')
          }
        }
        if (out[[im]]$peaks[[ip]]$intensity[filenum] < minarea) {
          nowcomment <- paste(nowcomment, 'WARNING, low signal intensity', sep = ' - ')
        }
        if (out[[im]]$peaks[[ip]]$residual[filenum] > 0.1 ) {
          nowcomment <- paste(nowcomment, 'WARNING, bad fit', sep = ' - ')
        }
        if (abs(out[[im]]$peaks[[ip]]$mzdiff[filenum]) > 0.001 ) {
          nowcomment <- paste(nowcomment, 'WARNING, inaccurate mass', sep = ' - ')
        }
        
        if (!is.na(out[[im]]$peaks[[ip]]$m0m1ratio[filenum])) { # m0m1 ratio can be calculated
          if ( (out[[im]]$peaks[[ip]]$intensity[filenum] > (0.1 * minarea)) && 
               (out[[im]]$peaks[[ip]]$m0m1ratio[filenum] < (1/targetmets[[im]]$maxlabel) ) ) {
            labels[rowcount,'FC'] <- out[[im]]$peaks[[ip]]$fc[filenum]
            mdvout[rowcount,1:(targetmets[[im]]$maxlabel+1)] <- out[[im]]$peaks[[ip]]$mdv[[filenum]]
            rawout[rowcount,1:(targetmets[[im]]$maxlabel+1)] <- out[[im]]$peaks[[ip]]$rawmdv[[filenum]]
          } else {
            nowcomment <- 'Sorry, insufficient data quality'
          }
        } else {
          nowcomment <- 'Sorry, insufficient data quality'
        }
        labels[rowcount,'Comments'] <- nowcomment
      } # go through all files
    } else {
      t <- 0
      
      nowmessage <- paste( out[[im]]$peaklabel[ip] ,'removed because')
      if ((length(goodsamples) / length(filelist)) <= mingoodfraction ) {
        nowmessage <- paste(nowmessage, 'less than', 100*mingoodfraction, '% good samples')
        t <- 1
      }
      if (!any(out[[im]]$peaks[[ip]]$fc < maxFCcutoff ) ) {
        nowmessage <- paste(nowmessage, 'unlabeled control not clean')
        t <- 1
      }
      if (!any(out[[im]]$peaks[[ip]]$m0m1ratio < (1/targetmets[[im]]$maxlabel))) {
        nowmessage <- paste(nowmessage, 'likely contamination from M-1 signal')
        t <- 1
      }
      
      if (t <- 0) {
        stop()
      }
      
      cat(paste0(nowmessage, '\n'), file=log_con)
      # don't plot peak range
      for (id in 1:length(filelist)) {
        plotinfo[[im]][[id]]$intstart[ip] <- NA
        plotinfo[[im]][[id]]$intend[ip] <- NA
      }
    } # endif unlabeled control is ok
  } # endfor all peaks of met
} # endfor all mets
mdvout <- mdvout[1:rowcount, ]
rawout <- rawout[1:rowcount, ]


wb <- createWorkbook()
addWorksheet(wb,'readme')
writeData(wb,'readme', readmetext, startCol = 1, startRow = 1)

addWorksheet(wb,'corrected')
addWorksheet(wb,'raw')
sheets <- c('corrected', 'raw')


for (sheet in sheets) {
  writeData(wb,sheet, labels[ ,c('Metabolite', 'File')],startCol = 1, startRow = 1)
  options("openxlsx.numFmt" = "0")
  writeData(wb,sheet, labels[ ,'Intensity'],startCol = 4, startRow = 2)
  writeData(wb,sheet, 'Intensity',startCol = 4, startRow = 1)
  options("openxlsx.numFmt" = "0.0000")
  writeData(wb,sheet, labels[ ,c('mz', 'Delta.mz', 'RT', 'Delta.RT')],startCol = 5, startRow = 1)
  writeData(wb,sheet, labels[ ,'Comments'],startCol = 10, startRow = 2)
}
writeData(wb,'corrected', labels[ , 'Residual'],startCol = 9, startRow = 2)
writeData(wb,'corrected', 'Residual',startCol = 9, startRow = 1)
writeData(wb,'corrected', labels[ ,'FC'],startCol = 11, startRow = 2)
writeData(wb,'corrected', '%FC', startCol = 11, startRow = 1)
writeData(wb,'corrected', mdvout,startCol = 12, startRow = 1)
writeData(wb,'corrected', t(c('M+0', 'M+1', 'M+2', 'M+3', 'M+4', 'M+5', 'M+6', 'M+7', 'M+8', 'M+9', 'M+10', '...')),startCol = 12, startRow = 1, colNames = FALSE)

writeData(wb,'raw', rawout,startCol = 12, startRow = 1)
writeData(wb,'raw', t(c('M+0', 'M+1', 'M+2', 'M+3', 'M+4', 'M+5', 'M+6', 'M+7', 'M+8', 'M+9', 'M+10', '...')),startCol = 12, startRow = 1, colNames = FALSE)

saveWorkbook(wb, file = file.path(datapath, 'Peakinfo.xlsx'), overwrite = TRUE)

cat('...done ', file=log_con)
cat(paste(timestamp(), '\n'), file = log_con)