#
#  R Code for reproducing results of the article
#
#  "Mixture density networks for the indirect estimation of reference intervals"
#
#  T.Hepp et al.
#
########################################### 

### Main script 

source("data_generating_function.R")
source("utility_functions.R")
source("EM.R")
source("MDN_optim.R")

## Figure 1: Example draw from simulation study

layout(matrix(c(1,1,1,2,3,4),3),widths = c(3,2))

set.seed(12345)

xmpl <- datgenf(seq(0,1,l=1000),FALSE)

par(mar=c(3,3,2,1)+.1)
plot(xmpl$x,xmpl$y,col=c(1,"darkgrey")[xmpl$z],pch=20,las=1,ylab="")
mtext("Example draw",3,.5,font=2,cex=.8)
mtext("x",1,2.5,font=4);mtext("y",2,2.75,font=4,las=1)

for(p in exp(seq(log(.01),log(.5),l=15))){
  polygon(x=c(seq(0,1,l=100),seq(1,0,l=100)),
          y=c(qnorm(p,datgenf(seq(0,1,l=100),smpl=FALSE,param="mu",comp=1),
                    datgenf(seq(0,1,l=100),smpl=FALSE,param="sigma",comp=1)),
              qnorm(1-p,datgenf(seq(1,0,l=100),smpl=FALSE,param="mu",comp=1),
                    datgenf(seq(1,0,l=100),smpl=FALSE,param="sigma",comp=1))),border=NA,col=rgb(0,0,0,.05))
}

par(mar=c(2,.5,2,4.5)+.1)
matplot(seq(0,1,l=100),datgenf(seq(0,1,l=100),FALSE,smpl=FALSE,param="alpha"),col=c(1,"darkgrey"),
        t="l",lty=1,lwd=2,ylim=c(0,1),ylab="",yaxt="n")
matlines(seq(0,1,l=100),datgenf(seq(0,1,l=100),smpl=FALSE,param="alpha"),col=c(1,"darkgrey"),
        t="l",lty=3,lwd=2,ylim=c(0,1),ylab="",yaxt="n")
axis(4,las=1); #mtext("Component weights",3,.5,font=2,cex=.8)
mtext("x",1,2.15,font=4,cex=.8); mtext(expression(alpha[m]),4,2.5,font=4,las=1,cex=.9)

par(mar=c(2.5,.5,1.5,4.5)+.1)
matplot(seq(0,1,l=100),datgenf(seq(0,1,l=100),FALSE,smpl=FALSE,param="mu"),col=c(1,"darkgrey"),
        t="l",lty=1,lwd=2,ylab="",main="",yaxt="n")
axis(4,las=1); #mtext("Location (Mean)",3,.5,font=2,cex=.8)
mtext("x",1,2.15,font=4,cex=.8); mtext(expression(mu[m]),4,2.5,font=4,las=1,cex=.9)

par(mar=c(3,.5,1,4.5)+.1)
matplot(seq(0,1,l=100),datgenf(seq(0,1,l=100),FALSE,smpl=FALSE,param="sigma"),col=c(1,"darkgrey"),
        t="l",lty=1,lwd=2,ylab="",main="",yaxt="n")
axis(4,las=1); #mtext("Scale (Std.Dev.)",3,.5,font=2,cex=.8)
mtext("x",1,2.15,font=4,cex=.8); mtext(expression(sigma[m]),4,2.5,font=4,las=1,cex=.9)


# Load simulation results

# load("Sim_Results.RData")
load("F:/Git-Projects/latent-class-distributional-modeling/Simulation/Sim_Results.RData")

# restructure weights to place component 1 below component 2

ANL10000 <- sortstuff(ANL10000,"mu")
FIX10000 <- sortstuff(FIX10000,"mu")
ANL5000 <- sortstuff(ANL5000,"mu")
FIX5000 <- sortstuff(FIX5000,"mu")

LocX_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    any(diff(sign(apply(predict_and_rescale(r,m,seq(0,1,l=100),"mu"),1,diff)))!=0)
  })
})
LocX_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    any(diff(sign(apply(predict_and_rescale(r,m,seq(0,1,l=100),"mu"),1,diff)))!=0)
  })
})
LocX_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    any(diff(sign(apply(predict_and_rescale(r,m,seq(0,1,l=100),"mu"),1,diff)))!=0)
  })
})
LocX_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    any(diff(sign(apply(predict_and_rescale(r,m,seq(0,1,l=100),"mu"),1,diff)))!=0)
  })
})

colSums(LocX_ANL10000)
colSums(LocX_FIX10000)
colSums(LocX_ANL5000)
colSums(LocX_FIX5000)

## Figure 2: Location/component crossing 

par(oma=c(.5,.5,0,.5))
layout(matrix(c(1,2),1))

par(mar=c(3,3,2,.5)+.1)
for(m in c("BFGS")){
  plot(NA,ylim=c(-15,50),xlim=c(0,1),las=1,ylab="",yaxt="n",cex.axis=.7)
  axis(2,las=1,cex.axis=.7); mtext(m,3,.5,font=2,cex=.9)
  for(i in 1:(length(ANL5000))){
    matlines(seq(0,1,l=100),
             predict_and_rescale(ANL5000[[i]],m,seq(0,1,l=100),"mu"),#-datgenf(seq(0,1,l=100),smpl=FALSE,param="mu",comp=1),
             col=c(rgb(.2,.2,.2,.4),rgb(.6,.6,.6,.4)),lty=1)
    #col=c(rgb(.3,.3,.3,.2),rgb(223/255,83/255,107/255,.2)),lty=1)
  }
  matlines(seq(0,1,l=100),datgenf(seq(0,1,l=100),FALSE,FALSE,"mu"),lwd=2,lty=1,col=c(1,"grey40"))
}
mtext("x",1,2.15,font=4); mtext(expression(mu[m]),2,2.5,font=4,las=1)

par(mar=c(3,.5,2,3)+.1)
for(m in c("ADAM")){
  plot(NA,ylim=c(-15,50),xlim=c(0,1),las=1,ylab="",yaxt="n",cex.axis=.7)
  axis(4,las=1,cex.axis=.7); mtext(m,3,.5,font=2,cex=.9)
  
  for(i in 1:(length(ANL5000))){
    matlines(seq(0,1,l=100),
             predict_and_rescale(ANL5000[[i]],m,seq(0,1,l=100),"mu"),#-datgenf(seq(0,1,l=100),smpl=FALSE,param="mu",comp=1),
             col=c(rgb(.2,.2,.2,.4),rgb(.6,.6,.6,.4)),lty=1)
    #col=c(rgb(.3,.3,.3,.2),rgb(223/255,83/255,107/255,.2)),lty=1)
  }
  
  matlines(seq(0,1,l=100),datgenf(seq(0,1,l=100),FALSE,FALSE,"mu"),lwd=2,lty=1,col=c(1,"grey40"))
}
mtext("x",1,2.15,font=4); mtext(expression(mu[m]),4,2.5,font=4,las=1)


# Failure to detect both components (i.e. "mean(a1) < .05 | mean(a1) > .95")

NOMIX_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  tmp <- colMeans(sapply(ANL10000,function(r){
    predict_and_rescale(r,m,seq(0,1,l=30),"alpha",1)
  }))
  tmp<.05|tmp>.95
})
Excl_ANL10000 <- LocX_ANL10000 | NOMIX_ANL10000
colSums(NOMIX_ANL10000); colSums(!LocX_ANL10000 & NOMIX_ANL10000)
colSums(LocX_ANL10000); colSums(Excl_ANL10000)

NOMIX_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  tmp <- colMeans(sapply(FIX10000,function(r){
    predict_and_rescale(r,m,seq(0,1,l=30),"alpha",1)
  }))
  tmp<.05|tmp>.95
})
Excl_FIX10000 <- LocX_FIX10000 | NOMIX_FIX10000
colSums(NOMIX_FIX10000); colSums(!LocX_FIX10000 & NOMIX_FIX10000)
colSums(LocX_FIX10000); colSums(Excl_FIX10000)

NOMIX_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  tmp <- colMeans(sapply(ANL5000,function(r){
    predict_and_rescale(r,m,seq(0,1,l=30),"alpha",1)
  }))
  tmp<.05|tmp>.95
})
Excl_ANL5000 <- LocX_ANL5000 | NOMIX_ANL5000
colSums(NOMIX_ANL5000); colSums(!LocX_ANL5000 & NOMIX_ANL5000)
colSums(LocX_ANL5000); colSums(Excl_ANL5000)

NOMIX_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  tmp <- colMeans(sapply(FIX5000,function(r){
    predict_and_rescale(r,m,seq(0,1,l=30),"alpha",1)
  }))
  tmp<.05|tmp>.95
})
Excl_FIX5000 <- LocX_FIX5000 | NOMIX_FIX5000
colSums(NOMIX_FIX5000); colSums(!LocX_FIX5000 & NOMIX_FIX5000)
colSums(LocX_FIX5000); colSums(Excl_FIX5000)



## Figure 3: Average prediction error in the non-linear alpha setting (n=10000)

xfill <- seq(0,1,l=50)

BIAS_a1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    (predict_and_rescale(r,m,xfill,"alpha",1)-datgenf(xfill,FALSE,FALSE,"alpha",1))
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})
BIAS_mu1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    (predict_and_rescale(r,m,xfill,"mu",1)-datgenf(xfill,FALSE,FALSE,"mu",1))
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})
BIAS_si1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    (predict_and_rescale(r,m,xfill,"sigma",1)-datgenf(xfill,FALSE,FALSE,"sigma",1))
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})
BIAS_Q1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    (qnorm(.95,predict_and_rescale(r,m,xfill,"mu",1),
           predict_and_rescale(r,m,xfill,"sigma",1))-
       qnorm(.95,datgenf(xfill,FALSE,FALSE,"mu",1),
             datgenf(xfill,FALSE,FALSE,"sigma",1)))
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})

par(oma=c(7,6,3,1.5))

layout(matrix(c(1,3,4,5,2,6,1,7,8,9,2,10),6,2),heights=c(.3,1,1,1,.3,1))

par(mar=c(0,0,0,0))
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Distribution parameters",font=2)
abline(h=1)
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Reference Interval (95%-Quantile)",font=2)
abline(h=1)

mtext("Initialization: ",3,1,adj=-.08,outer=TRUE,cex=.75,font=2)
mtext("random",3,1,adj=.255,outer=TRUE,cex=.75,font=2)
mtext("customized",3,1,adj=.76,outer=TRUE,cex=.75,font=2)

mtext("Average prediction error",2,4,outer=TRUE,font=2,cex=.8)

par(mar=c(1,2.5,0,.5)+.1)
matplot(xfill,BIAS_a1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_a1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(alpha[1]),2,4,font=4,las=1,cex=.9); abline(h=0,lty=3,col="grey40",lwd=2)
matplot(xfill,BIAS_mu1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_mu1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(mu[1]),2,4,font=4,las=1,cex=.9); abline(h=0,lty=3,col="grey40",lwd=2)
matplot(xfill,BIAS_si1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_si1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(sigma[1]),2,4,font=4,las=1,cex=.9); abline(h=0,lty=3,col="grey40",lwd=2)
matplot(xfill,BIAS_Q1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_Q1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(Q[.95*","~1]),2,3,font=4,las=1,cex=.8); abline(h=0,lty=3,col="grey40",lwd=2)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)

par(mar=c(1,.5,0,2.5)+.1)
matplot(xfill,BIAS_a1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_a1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(alpha[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,BIAS_mu1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_mu1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(mu[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,BIAS_si1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_si1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(sigma[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,BIAS_Q1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_Q1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(Q[.95,1]),4,3,font=4,las=1,cex=.8)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)


par(fig = c(0, 1, 0, 1), oma = c(0,6,3,1.5), mar = c(0, 0, 0, 0), new = TRUE)
plot(0, 0, type = "n", bty = "n", xaxt = "n", yaxt = "n")

ltext <- c(expression(bold(EM)), expression(bold(BFGS)[alpha]), expression(bold(BFGS)[alpha(x)]), expression(bold(ADAM)))
legend("bottom", ltext,text.width = sapply(ltext,strwidth)+.1,
       xpd = TRUE, horiz = TRUE, bty = "n",lty=c(2,1,1,1),lwd=2, col = 1:4, cex = 1.25)


## Figure 4: Average prediction error in the setting with constant alphas (n=10000)

xfill <- seq(0,1,l=50)

BIAS_a1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    (predict_and_rescale(r,m,xfill,"alpha",1)-datgenf(xfill,TRUE,FALSE,"alpha",1))
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})
BIAS_mu1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    (predict_and_rescale(r,m,xfill,"mu",1)-datgenf(xfill,TRUE,FALSE,"mu",1))
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})
BIAS_si1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    (predict_and_rescale(r,m,xfill,"sigma",1)-datgenf(xfill,TRUE,FALSE,"sigma",1))
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})
BIAS_Q1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    (qnorm(.95,predict_and_rescale(r,m,xfill,"mu",1),
           predict_and_rescale(r,m,xfill,"sigma",1))-
       qnorm(.95,datgenf(xfill,TRUE,FALSE,"mu",1),
             datgenf(xfill,TRUE,FALSE,"sigma",1)))
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})


par(oma=c(7,6,3,1.5))

layout(matrix(c(1,3,4,5,2,6,1,7,8,9,2,10),6,2),heights=c(.3,1,1,1,.3,1))

par(mar=c(0,0,0,0))
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Distribution parameters",font=2)
abline(h=1)
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Reference Interval (95%-Quantile)",font=2)
abline(h=1)

mtext("Initialization: ",3,1,adj=-.08,outer=TRUE,cex=.75,font=2)
mtext("random",3,1,adj=.255,outer=TRUE,cex=.75,font=2)
mtext("customized",3,1,adj=.76,outer=TRUE,cex=.75,font=2)

mtext("Average prediction error",2,4,outer=TRUE,font=2,cex=.8)

par(mar=c(1,2.5,0,.5)+.1)
matplot(xfill,BIAS_a1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_a1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(alpha[1]),2,4,font=4,las=1,cex=.9); abline(h=0,lty=3,col="grey40",lwd=2)
matplot(xfill,BIAS_mu1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_mu1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(mu[1]),2,4,font=4,las=1,cex=.9); abline(h=0,lty=3,col="grey40",lwd=2)
matplot(xfill,BIAS_si1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_si1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(sigma[1]),2,4,font=4,las=1,cex=.9); abline(h=0,lty=3,col="grey40",lwd=2)
matplot(xfill,BIAS_Q1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_Q1),xaxt="n",yaxt="n")
axis(2,las=1); mtext(expression(Q[.95*","~1]),2,3,font=4,las=1,cex=.8); abline(h=0,lty=3,col="grey40",lwd=2)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)

par(mar=c(1,.5,0,2.5)+.1)
matplot(xfill,BIAS_a1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_a1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(alpha[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,BIAS_mu1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_mu1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(mu[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,BIAS_si1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_si1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(sigma[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,BIAS_Q1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(BIAS_Q1),xaxt="n",yaxt="n")
axis(4,las=1); abline(h=0,lty=3,col="grey40",lwd=2)#; mtext(expression(Q[.95,1]),4,3,font=4,las=1,cex=.8)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)

par(fig = c(0, 1, 0, 1), oma = c(0,6,3,1.5), mar = c(0, 0, 0, 0), new = TRUE)
plot(0, 0, type = "n", bty = "n", xaxt = "n", yaxt = "n")

ltext <- c(expression(bold(EM)), expression(bold(BFGS)[alpha]), expression(bold(BFGS)[alpha(x)]), expression(bold(ADAM)))
legend("bottom", ltext,text.width = sapply(ltext,strwidth)+.1,
       xpd = TRUE, horiz = TRUE, bty = "n",lty=c(2,1,1,1),lwd=2, col = 1:4, cex = 1.25)


## Not included in the article: Average of predicted values (not their errors)

# non-linear alphas

xfill <- seq(0,1,l=50)

FIT_a1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    predict_and_rescale(r,m,xfill,"alpha",1)
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})
FIT_mu1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    predict_and_rescale(r,m,xfill,"mu",1)
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})
FIT_si1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    predict_and_rescale(r,m,xfill,"sigma",1)
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})
FIT_Q1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(ANL10000,function(r){
    qnorm(.95,predict_and_rescale(r,m,xfill,"mu",1),
           predict_and_rescale(r,m,xfill,"sigma",1))
  })
  rowMeans(TMP[,!Excl_ANL10000[,m]])
})

par(oma=c(7,6,3,1.5))

layout(matrix(c(1,3,4,5,2,6,1,7,8,9,2,10),6,2),heights=c(.3,1,1,1,.3,1))

par(mar=c(0,0,0,0))
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Distribution parameters",font=2)
abline(h=1)
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Reference Interval (95%-Quantile)",font=2)
abline(h=1)

mtext("Initialization: ",3,1,adj=-.08,outer=TRUE,cex=.75,font=2)
mtext("random",3,1,adj=.255,outer=TRUE,cex=.75,font=2)
mtext("customized",3,1,adj=.76,outer=TRUE,cex=.75,font=2)

mtext("Average Predictions",2,4,outer=TRUE,font=2,cex=.8)

par(mar=c(1,2.5,0,.5)+.1)
matplot(xfill,FIT_a1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="alpha")),
        xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="alpha"),lty=3)
axis(2,las=1); mtext(expression(alpha[1]),2,4,font=4,las=1,cex=.9)
matplot(xfill,FIT_mu1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_mu1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="mu"),lty=3)
axis(2,las=1); mtext(expression(mu[1]),2,4,font=4,las=1,cex=.9)
matplot(xfill,FIT_si1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_si1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="sigma"),lty=3)
axis(2,las=1); mtext(expression(sigma[1]),2,4,font=4,las=1,cex=.9)
matplot(xfill,FIT_Q1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_Q1),xaxt="n",yaxt="n")
lines(xfill,qnorm(.95,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="mu"),
                  datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="sigma")),lty=3)
axis(2,las=1); mtext(expression(Q[.95*","~1]),2,3,font=4,las=1,cex=.8)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)

par(mar=c(1,.5,0,2.5)+.1)
matplot(xfill,FIT_a1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="alpha")),
        xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="alpha"),lty=3)
axis(4,las=1)#; mtext(expression(alpha[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,FIT_mu1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_mu1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="mu"),lty=3)
axis(4,las=1)#; mtext(expression(mu[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,FIT_si1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_si1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="sigma"),lty=3)
axis(4,las=1)#; mtext(expression(sigma[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,FIT_Q1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_Q1),xaxt="n",yaxt="n")
lines(xfill,qnorm(.95,datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="mu"),
                  datgenf(xfill,afix=FALSE,smpl=FALSE,comp=1,par="sigma")),lty=3)
axis(4,las=1)#; mtext(expression(Q[.95,1]),4,3,font=4,las=1,cex=.8)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)

par(fig = c(0, 1, 0, 1), oma = c(0,6,3,1.5), mar = c(0, 0, 0, 0), new = TRUE)
plot(0, 0, type = "n", bty = "n", xaxt = "n", yaxt = "n")

ltext <- c(expression(bold(EM)), expression(bold(BFGS)[alpha]), expression(bold(BFGS)[alpha(x)]), expression(bold(ADAM)))
legend("bottom", ltext,text.width = sapply(ltext,strwidth)+.1,
       xpd = TRUE, horiz = TRUE, bty = "n",lty=c(2,1,1,1),lwd=2, col = 1:4, cex = 1.25)

# constant alphas

xfill <- seq(0,1,l=50)

FIT_a1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    predict_and_rescale(r,m,xfill,"alpha",1)
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})
FIT_mu1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    predict_and_rescale(r,m,xfill,"mu",1)
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})
FIT_si1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    predict_and_rescale(r,m,xfill,"sigma",1)
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})
FIT_Q1 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  TMP <- sapply(FIX10000,function(r){
    qnorm(.95,predict_and_rescale(r,m,xfill,"mu",1),
          predict_and_rescale(r,m,xfill,"sigma",1))
  })
  rowMeans(TMP[,!Excl_FIX10000[,m]])
})

par(oma=c(7,6,3,1.5))

layout(matrix(c(1,3,4,5,2,6,1,7,8,9,2,10),6,2),heights=c(.3,1,1,1,.3,1))

par(mar=c(0,0,0,0))
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Distribution parameters",font=2)
abline(h=1)
plot(0,0,"n",frame.plot=FALSE,axes=FALSE)
text(0,0,"Reference Interval (95%-Quantile)",font=2)
abline(h=1)

mtext("Initialization: ",3,1,adj=-.08,outer=TRUE,cex=.75,font=2)
mtext("random",3,1,adj=.255,outer=TRUE,cex=.75,font=2)
mtext("customized",3,1,adj=.76,outer=TRUE,cex=.75,font=2)

mtext("Average Predictions",2,4,outer=TRUE,font=2,cex=.8)

par(mar=c(1,2.5,0,.5)+.1)
matplot(xfill,FIT_a1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_a1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,smpl=FALSE,comp=1,par="alpha"),lty=3)
axis(2,las=1); mtext(expression(alpha[1]),2,4,font=4,las=1,cex=.9)
matplot(xfill,FIT_mu1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_mu1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,smpl=FALSE,comp=1,par="mu"),lty=3)
axis(2,las=1); mtext(expression(mu[1]),2,4,font=4,las=1,cex=.9)
matplot(xfill,FIT_si1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_si1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,smpl=FALSE,comp=1,par="sigma"),lty=3)
axis(2,las=1); mtext(expression(sigma[1]),2,4,font=4,las=1,cex=.9)
matplot(xfill,FIT_Q1[,c("EM","BFGS_acons","BFGS","ADAM")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_Q1),xaxt="n",yaxt="n")
lines(xfill,qnorm(.95,datgenf(xfill,smpl=FALSE,comp=1,par="mu"),
                  datgenf(xfill,smpl=FALSE,comp=1,par="sigma")),lty=3)
axis(2,las=1); mtext(expression(Q[.95*","~1]),2,3,font=4,las=1,cex=.8)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)

par(mar=c(1,.5,0,2.5)+.1)
matplot(xfill,FIT_a1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_a1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,smpl=FALSE,comp=1,par="alpha"),lty=3)
axis(4,las=1)#; mtext(expression(alpha[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,FIT_mu1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_mu1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,smpl=FALSE,comp=1,par="mu"),lty=3)
axis(4,las=1)#; mtext(expression(mu[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,FIT_si1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_si1),xaxt="n",yaxt="n")
lines(xfill,datgenf(xfill,smpl=FALSE,comp=1,par="sigma"),lty=3)
axis(4,las=1)#; mtext(expression(sigma[1]),4,3,font=4,las=1,cex=.9)
matplot(xfill,FIT_Q1[,c("EM","BFGS_cinit_acons","BFGS_custom","ADAM_custom")],
        t="l",lty=c(2,1,1,1),lwd=2,col=1:4,ylim=range(FIT_Q1),xaxt="n",yaxt="n")
lines(xfill,qnorm(.95,datgenf(xfill,smpl=FALSE,comp=1,par="mu"),
                  datgenf(xfill,smpl=FALSE,comp=1,par="sigma")),lty=3)
axis(4,las=1)#; mtext(expression(Q[.95,1]),4,3,font=4,las=1,cex=.8)
axis(1,line = 1.5)
mtext("x",1,4,cex=.8,font=4)

par(fig = c(0, 1, 0, 1), oma = c(0,6,3,1.5), mar = c(0, 0, 0, 0), new = TRUE)
plot(0, 0, type = "n", bty = "n", xaxt = "n", yaxt = "n")

ltext <- c(expression(bold(EM)), expression(bold(BFGS)[alpha]), expression(bold(BFGS)[alpha(x)]), expression(bold(ADAM)))
legend("bottom", ltext,text.width = sapply(ltext,strwidth)+.1,
       xpd = TRUE, horiz = TRUE, bty = "n",lty=c(2,1,1,1),lwd=2, col = 1:4, cex = 1.25)


# Calculate the integrated squared erros (ISE)

# "alpha"

ISE_alpha_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="alpha",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_alpha_ANL10000[Excl_ANL10000] <- NA

ISE_alpha_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="alpha",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_alpha_ANL5000[Excl_ANL5000] <- NA

ISE_alpha_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="alpha",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_alpha_FIX10000[Excl_FIX10000] <- NA

ISE_alpha_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="alpha",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_alpha_FIX5000[Excl_FIX5000] <- NA

# "mu"

ISE_mu_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="mu",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_mu_ANL10000[Excl_ANL10000] <- NA

ISE_mu_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="mu",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_mu_ANL5000[Excl_ANL5000] <- NA

ISE_mu_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="mu",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_mu_FIX10000[Excl_FIX10000] <- NA

ISE_mu_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="mu",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_mu_FIX5000[Excl_FIX5000] <- NA


# "sigma"

ISE_sigma_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="sigma",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_sigma_ANL10000[Excl_ANL10000] <- NA

ISE_sigma_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="sigma",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_sigma_ANL5000[Excl_ANL5000] <- NA

ISE_sigma_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="sigma",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_sigma_FIX10000[Excl_FIX10000] <- NA

ISE_sigma_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="sigma",comp=1))^2
    },lower=0,upper=1)$value
  })
})
ISE_sigma_FIX5000[Excl_FIX5000] <- NA



# "q95"

ISE_q95_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=FALSE,FALSE,"mu",1),
               datgenf(x,afix=FALSE,FALSE,"sigma",1)))^2
    },lower=0,upper=1)$value
  })
})
ISE_q95_ANL10000[Excl_ANL10000] <- NA

ISE_q95_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=FALSE,FALSE,"mu",1),
               datgenf(x,afix=FALSE,FALSE,"sigma",1)))^2
    },lower=0,upper=1)$value
  })
})
ISE_q95_ANL5000[Excl_ANL5000] <- NA

ISE_q95_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=TRUE,FALSE,"mu",1),
               datgenf(x,afix=TRUE,FALSE,"sigma",1)))^2
    },lower=0,upper=1)$value
  })
})
ISE_q95_FIX10000[Excl_FIX10000] <- NA

ISE_q95_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=TRUE,FALSE,"mu",1),
               datgenf(x,afix=TRUE,FALSE,"sigma",1)))^2
    },lower=0,upper=1)$value
  })
})
ISE_q95_FIX5000[Excl_FIX5000] <- NA



ISE <- as.data.frame(rbind(ISE_alpha_ANL10000,ISE_alpha_ANL5000,ISE_alpha_FIX10000,ISE_alpha_FIX5000,
                           ISE_mu_ANL10000,ISE_mu_ANL5000,ISE_mu_FIX10000,ISE_mu_FIX5000,
                           ISE_sigma_ANL10000,ISE_sigma_ANL5000,ISE_sigma_FIX10000,ISE_sigma_FIX5000,
                           ISE_q95_ANL10000,ISE_q95_ANL5000,ISE_q95_FIX10000,ISE_q95_FIX5000))


## Table 2: Average integrated squared error (standard deviation not shown in the article)

MISE <- aggregate(ISE,list(set=factor(rep(rep(c("anl10","anl5","fix10","fix5"),each=100),4),levels=c("anl10","anl5","fix10","fix5")),
                           par=factor(rep(c("a","m","s","q"),each=400),levels=c("a","m","s","q"))),function(x) round(mean(x,na.rm=TRUE),4))
SDISE <- aggregate(ISE,list(set=factor(rep(rep(c("anl10","anl5","fix10","fix5"),each=100),4),levels=c("anl10","anl5","fix10","fix5")),
                            par=factor(rep(c("a","m","s","q"),each=400),levels=c("a","m","s","q"))),function(x) round(sd(x,na.rm=TRUE),4))


# Not in the article: Integrated error (not squared)


IE_alpha_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="alpha",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_alpha_ANL10000[Excl_ANL10000] <- NA

IE_alpha_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="alpha",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_alpha_ANL5000[Excl_ANL5000] <- NA

IE_alpha_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="alpha",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_alpha_FIX10000[Excl_FIX10000] <- NA

IE_alpha_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"alpha",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="alpha",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_alpha_FIX5000[Excl_FIX5000] <- NA

# "mu"

IE_mu_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="mu",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_mu_ANL10000[Excl_ANL10000] <- NA

IE_mu_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="mu",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_mu_ANL5000[Excl_ANL5000] <- NA

IE_mu_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="mu",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_mu_FIX10000[Excl_FIX10000] <- NA

IE_mu_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"mu",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="mu",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_mu_FIX5000[Excl_FIX5000] <- NA


# "sigma"

IE_sigma_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="sigma",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_sigma_ANL10000[Excl_ANL10000] <- NA

IE_sigma_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=FALSE,smpl=FALSE,param="sigma",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_sigma_ANL5000[Excl_ANL5000] <- NA

IE_sigma_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="sigma",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_sigma_FIX10000[Excl_FIX10000] <- NA

IE_sigma_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (predict_and_rescale(r,m,x,"sigma",1)-datgenf(x,afix=TRUE,smpl=FALSE,param="sigma",comp=1))
    },lower=0,upper=1)$value
  })
})
IE_sigma_FIX5000[Excl_FIX5000] <- NA



# "q95"

IE_q95_ANL10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL10000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=FALSE,FALSE,"mu",1),
               datgenf(x,afix=FALSE,FALSE,"sigma",1)))
    },lower=0,upper=1)$value
  })
})
IE_q95_ANL10000[Excl_ANL10000] <- NA

IE_q95_ANL5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(ANL5000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=FALSE,FALSE,"mu",1),
               datgenf(x,afix=FALSE,FALSE,"sigma",1)))
    },lower=0,upper=1)$value
  })
})
IE_q95_ANL5000[Excl_ANL5000] <- NA

IE_q95_FIX10000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX10000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=TRUE,FALSE,"mu",1),
               datgenf(x,afix=TRUE,FALSE,"sigma",1)))
    },lower=0,upper=1)$value
  })
})
IE_q95_FIX10000[Excl_FIX10000] <- NA

IE_q95_FIX5000 <- sapply(c("EM","BFGS_acons","BFGS_cinit_acons","BFGS","BFGS_custom","ADAM","ADAM_custom"),function(m){
  sapply(FIX5000,function(r){
    integrate(function(x){
      (qnorm(.95,predict_and_rescale(r,m,x,"mu",1),
             predict_and_rescale(r,m,x,"sigma",1))-
         qnorm(.95,datgenf(x,afix=TRUE,FALSE,"mu",1),
               datgenf(x,afix=TRUE,FALSE,"sigma",1)))
    },lower=0,upper=1)$value
  })
})
IE_q95_FIX5000[Excl_FIX5000] <- NA



IE <- as.data.frame(rbind(IE_alpha_ANL10000,IE_alpha_ANL5000,IE_alpha_FIX10000,IE_alpha_FIX5000,
                           IE_mu_ANL10000,IE_mu_ANL5000,IE_mu_FIX10000,IE_mu_FIX5000,
                           IE_sigma_ANL10000,IE_sigma_ANL5000,IE_sigma_FIX10000,IE_sigma_FIX5000,
                           IE_q95_ANL10000,IE_q95_ANL5000,IE_q95_FIX10000,IE_q95_FIX5000))


MIE <- aggregate(IE,list(set=factor(rep(rep(c("anl10","anl5","fix10","fix5"),each=100),4),levels=c("anl10","anl5","fix10","fix5")),
                           par=factor(rep(c("a","m","s","q"),each=400),levels=c("a","m","s","q"))),function(x) round(mean(x,na.rm=TRUE),4))
SDIE <- aggregate(IE,list(set=factor(rep(rep(c("anl10","anl5","fix10","fix5"),each=100),4),levels=c("anl10","anl5","fix10","fix5")),
                            par=factor(rep(c("a","m","s","q"),each=400),levels=c("a","m","s","q"))),function(x) round(sd(x,na.rm=TRUE),4))


## See "Availability of data and materials" statement regarding the results for the hemoglobin data