library(xlsx)
library(pROC)
library(caret)
library(AppliedPredictiveModeling)
library(dplyr)
library(gbm)
library(h2o)
h2o.init()
library(glmnet)
library(randomForest)
library(e1071)
library(boot)
library(neuralnet)
library(rpart)
library(ggplot2)
library(rsample)
library(car)
library(RColorBrewer)
library(car)
library(mlr)
train_data<-read.xlsx('aidata.xlsx',sheetIndex = 1,header = T)  
test_data1<-read.xlsx('aidata.xlsx',sheetIndex = 2,header = T)
test_data2<-read.xlsx('aidata.xlsx',sheetIndex = 3,header = T)
test_data3<-read.xlsx('aidata.xlsx',sheetIndex = 4,header = T)
test_data4<-read.xlsx('aidata.xlsx',sheetIndex = 5,header = T)
fitControl <- trainControl(method = "repeatedcv",
                           number = 10,
                           repeats = 10,
                           ## Estimate class probabilities
                           classProbs = TRUE,
                           ## Evaluate performance using 
                           ## the following function
                           summaryFunction = twoClassSummary)
plsRglm <- train(group ~ ALDOA+ENO1+p53+NY.ESO.1, data = train_data, 
                 method = "plsRglm", #the name of the training model
                 trControl = fitControl,
                 metric = "ROC")

saveRDS(predicted_probabilities,file = "./predicted_probabilities.rds")
plsRglm <- readRDS("./models/plsRglm.rds")

models_folder <- "E:/submitting/xu/newAI/models"

model_files <- list.files(models_folder, pattern = "\\.rds$", full.names = TRUE)

loaded_models <- lapply(model_files, readRDS)

auc_values <- vector("numeric", length(loaded_models))

for (i in 1:length(loaded_models)) {
  model <- loaded_models[[i]]
  predicted_probs <- predict(model, newdata = train_data, type = "prob")
  roc_data <- roc(train_data$group, predicted_probs$ESCC)
  auc_values[i] <- auc(roc_data)
}
predicted_probabilities <- list()

for (model in loaded_models) {
  predicted_prob <- predict(model, train_data, type = "prob")
  predicted_probabilities <- c(predicted_probabilities, list(predicted_prob))
}
View(predicted_probabilities[[1]]$ESCC)

combined_data <- data.frame(group = train_data$group)

for (i in 1:length(predicted_probabilities)) {
 col_name <- paste(loaded_models[[i]][["method"]])
 combined_data[[col_name]] <- predicted_probabilities[[i]]$ESCC
}

print(combined_data_train)

#DCA curves
library(dcurves)
training_set<-read.xlsx('final_model.xlsx',sheetIndex = 1,header = T)#the data of the final model have been consolidated
test_set1<-read.xlsx('final_model.xlsx',sheetIndex = 2,header = T)
test_set2<-read.xlsx('final_model.xlsx',sheetIndex = 3,header = T)
test_set3<-read.xlsx('final_model.xlsx',sheetIndex = 4,header = T)
dca1 <- dca(group~plsRglm, data = training_set)
dca2 <- dca(group~plsRglm, data = test_set1)
dca3 <- dca(group~plsRglm, data = test_set2)
dca4 <- dca(group~plsRglm, data = test_set3)
dca_list <- list(dca1, dca2, dca3, dca4)
dca_data <- do.call(rbind, lapply(dca_list, function(x) x$net.benefit))
dca_data$dataset <- factor(rep(c("training_set", "test_set1", "test_set2", "test_set3"), each = nrow(dca_data) / 4))
ggplot(dca_data)

#calibration curves
remotes::install_github('ML4LHS/runway')
library(runway)
cal_plot(training_set,
         outcome = 'outcomes',
         prediction = 'predictions',
         positive = '1',
         n_bins = 0,
         show_loess = TRUE)