The permutation was calculated with cyCONDOR v0.1.5. Nevertheless the code was updated to work with cyCONDOR v0.2.0 and the results of the first few permutation was evaluated for consistency with the old version. The metrics show here are explain in details in this nice wiki page https://en.wikipedia.org/wiki/Precision_and_recall.
# # Now running the loops with a seed going from 1 to 100
# for (i in seq(1,100)) {
#
# print(i)
#
# # Set the seed
# set.seed(i)
#
# # Select the sample for the train dataset (80%)
# train_sample <- sample(unique(FCII_meta$Individual),
# size = round(length(unique(FCII_meta$Individual))*0.8),
# replace = FALSE)
# # Select sample for the test dataset (remaining 20%)
# test_sample <- unique(FCII_meta$Individual)[!unique(FCII_meta$Individual) %in% train_sample]
#
# # Exctract sample name and save the annotation file
# FCII_list <- list()
#
# for (pannel in unique(FCII_meta$Tube.number)) {
#
# df <- FCII_meta[FCII_meta$Tube.number == pannel,]
#
# df_train <- df[df$Individual %in% train_sample,]
#
# # Check number of aml in train data
# n_aml <- sum(df_train$Condition == "aml")
# n_normal <- sum(df_train$Condition == "normal")
#
# # Balance the train dataset
# diff <- n_normal - n_aml
#
# df_train <- df_train[!df_train$FCS.file %in% sample(df_train[df_train$Condition == "normal",]$FCS.file, size = diff, replace = FALSE), ]
#
# df_test <- df[df$Individual %in% test_sample,]
#
# FCII_list[[paste("pannel_", pannel, sep = "")]][["train"]] <- df_train
#
# FCII_list[[paste("pannel_", pannel, sep = "")]][["test"]] <- df_test
#
# }
#
# for (pannel in names(FCII_list)) {
#
# tmp <- FCII_list[[pannel]]
#
# for (df in names(tmp)) {
#
# write.csv(file = paste("./data_and_envs/FlowCapII/", pannel, "_", df, ".csv", sep = ""), x = tmp[[df]], row.names = FALSE)
#
# }
#
# }
#
# # Prepare CONDOR object on the train dataset
# condor <- prep_fcd(data_path = "./data_and_envs/Pannel_4/",
# max_cell = 3000,
# useCSV = FALSE,
# transformation = "auto_logi",
# remove_param = c("Time", "InFile"),
# anno_table = "./data_and_envs/FlowCapII/pannel_4_train.csv",
# filename_col = "FCS.file",
# seed = 91)
#
# # Re order the variable to have the prediction for AML
# condor$anno$cell_anno$Condition <- factor(condor$anno$cell_anno$Condition,
# levels = c("normal", "aml"), labels = c("1_normal", "2_aml"))
#
#
# # Train classifier on the training data
# condor <- train_classifier_model(fcd = condor,
# input_type = "expr",
# data_slot = "orig",
# sample_names = "expfcs_filename",
# classification_variable = condor$anno$cell_anno$Condition,
# family = "binomial",
# type1 = "response",
# parallelCore = 1,
# reg = FALSE,
# seed = 91)
#
# # Load test data in a condor object
# condor_test <- prep_fcd(data_path = "./data_and_envs/Pannel_4/",
# max_cell = 3000,
# useCSV = FALSE,
# transformation = "auto_logi",
# remove_param = c("Time", "InFile"),
# anno_table = "./data_and_envs/FlowCapII/pannel_4_test.csv",
# filename_col = "FCS.file",
# seed = 91)
#
# condor_test$anno$cell_anno$Condition <- factor(condor_test$anno$cell_anno$Condition,
# levels = c("normal", "aml"), labels = c("1_normal", "2_aml"))
#
# # Predict the label of the test dataset
# condor_test <- predict_classifier(fcd = condor_test,
# input_type = "expr",
# data_slot = "orig",
# sample_names = "expfcs_filename",
# model_object = condor$extras$classifier_model,
# seed = 91)
#
# # Merge result with original annotation
# anno <- read.csv("./data_and_envs/FlowCapII/pannel_4_test.csv")
#
# tmp <- merge(x = condor_test$extras$classifier_prediction$xNew.Pred.sample, y = anno, by.x = "sample", by.y = "FCS.file")
#
# # Include a column with the predicted labels (according to a probability of 50%)
# tmp$Pred <- ifelse(tmp$y.Pred.s0 >= 0.5, "pred_aml", "pred_normal")
#
# tmp <- tmp[, c("sample", "Condition", "Pred", "y.Pred.s0")]
#
# # Store the results in a list
# collector[[paste0("Pred_seed_", i)]] <- tmp
#
# table <- table(tmp$Condition, tmp$Pred)
#
# # Calculate performance parameters
# P <- sum(table[1,1] + table[1,2]) # Total positive samples
# TP <- table[1,1] # True positive prediction
# FP <- table[2,1] # False positive prediction
#
# N <- sum(table[2,1] + table[2,2]) # Total negative samples
# TN <- table[2,2] # True negative prediction
# FN <- table[1,2] # False negative prediction
#
# SENS <- TP/P # Sensitivity
# SPEC <- ifelse(N > 0, TN/N, 0) # Specificity
# ACC <- (TP+TN)/(TP+FP+FN+TN) # Accuracy
#
# # Store the perfromacne data in a table
# table_collector <- rbind(table_collector, c(P, TP, FP, N, TN, FN, SENS, SPEC, ACC))
#
# # Clean
# rm(anno, condor, condor_test, df_test, df_train, FCII_list, tmp, ACC, df, FN, FP, i, N, P, pannel,
# SENS, SPEC, table, test_sample, TN, TP, train_sample)
#
# # Do a garbage collection to avoid overload of systems memory
# gc()
#
# }## No id variables; using all as measure variables
library(ggplot2)
library(ggsci)
ggplot(df, aes(x = variable, y = value, fill = variable)) +
geom_boxplot(outlier.shape = NA) + scale_y_continuous(limits = c(0.5, 1.01)) +
geom_jitter(width = 0.2, alpha = 0.3, size = 2) +
scale_fill_aaas() +
theme_bw() +
theme(aspect.ratio = 2) +
ggtitle("Figure 7e - FlowCapII performance")## R version 4.3.1 (2023-06-16)
## Platform: x86_64-pc-linux-gnu (64-bit)
## Running under: Ubuntu 22.04.3 LTS
##
## Matrix products: default
## BLAS: /usr/lib/x86_64-linux-gnu/openblas-pthread/libblas.so.3
## LAPACK: /usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.20.so; LAPACK version 3.10.0
##
## locale:
## [1] LC_CTYPE=en_US.UTF-8 LC_NUMERIC=C
## [3] LC_TIME=en_US.UTF-8 LC_COLLATE=en_US.UTF-8
## [5] LC_MONETARY=en_US.UTF-8 LC_MESSAGES=en_US.UTF-8
## [7] LC_PAPER=en_US.UTF-8 LC_NAME=C
## [9] LC_ADDRESS=C LC_TELEPHONE=C
## [11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C
##
## time zone: Etc/UTC
## tzcode source: system (glibc)
##
## attached base packages:
## [1] stats graphics grDevices utils datasets methods base
##
## other attached packages:
## [1] ggsci_3.0.0 ggplot2_3.4.4 reshape2_1.4.4 cyCONDOR_0.2.0
##
## loaded via a namespace (and not attached):
## [1] IRanges_2.34.1 Rmisc_1.5.1
## [3] urlchecker_1.0.1 nnet_7.3-19
## [5] CytoNorm_2.0.1 TH.data_1.1-2
## [7] vctrs_0.6.4 digest_0.6.33
## [9] png_0.1-8 shape_1.4.6
## [11] proxy_0.4-27 slingshot_2.8.0
## [13] ggrepel_0.9.4 parallelly_1.36.0
## [15] MASS_7.3-60 httpuv_1.6.12
## [17] foreach_1.5.2 BiocGenerics_0.46.0
## [19] withr_2.5.1 ggrastr_1.0.2
## [21] xfun_0.40 ggpubr_0.6.0
## [23] ellipsis_0.3.2 survival_3.5-7
## [25] memoise_2.0.1 hexbin_1.28.3
## [27] ggbeeswarm_0.7.2 RProtoBufLib_2.12.1
## [29] princurve_2.1.6 profvis_0.3.8
## [31] zoo_1.8-12 GlobalOptions_0.1.2
## [33] DEoptimR_1.1-3 Formula_1.2-5
## [35] prettyunits_1.2.0 promises_1.2.1
## [37] scatterplot3d_0.3-44 rstatix_0.7.2
## [39] globals_0.16.2 ps_1.7.5
## [41] rstudioapi_0.15.0 miniUI_0.1.1.1
## [43] generics_0.1.3 ggcyto_1.28.1
## [45] base64enc_0.1-3 processx_3.8.2
## [47] curl_5.1.0 S4Vectors_0.38.2
## [49] zlibbioc_1.46.0 flowWorkspace_4.12.2
## [51] polyclip_1.10-6 randomForest_4.7-1.1
## [53] GenomeInfoDbData_1.2.10 RBGL_1.76.0
## [55] ncdfFlow_2.46.0 RcppEigen_0.3.3.9.4
## [57] xtable_1.8-4 stringr_1.5.0
## [59] doParallel_1.0.17 evaluate_0.22
## [61] S4Arrays_1.0.6 hms_1.1.3
## [63] glmnet_4.1-8 GenomicRanges_1.52.1
## [65] irlba_2.3.5.1 colorspace_2.1-0
## [67] harmony_1.1.0 reticulate_1.34.0
## [69] readxl_1.4.3 magrittr_2.0.3
## [71] lmtest_0.9-40 readr_2.1.4
## [73] Rgraphviz_2.44.0 later_1.3.1
## [75] lattice_0.22-5 future.apply_1.11.0
## [77] robustbase_0.99-0 XML_3.99-0.15
## [79] cowplot_1.1.1 matrixStats_1.1.0
## [81] xts_0.13.1 class_7.3-22
## [83] Hmisc_5.1-1 pillar_1.9.0
## [85] nlme_3.1-163 iterators_1.0.14
## [87] compiler_4.3.1 RSpectra_0.16-1
## [89] stringi_1.7.12 gower_1.0.1
## [91] minqa_1.2.6 SummarizedExperiment_1.30.2
## [93] lubridate_1.9.3 devtools_2.4.5
## [95] CytoML_2.12.0 plyr_1.8.9
## [97] crayon_1.5.2 abind_1.4-5
## [99] locfit_1.5-9.8 sp_2.1-1
## [101] sandwich_3.0-2 pcaMethods_1.92.0
## [103] dplyr_1.1.3 codetools_0.2-19
## [105] multcomp_1.4-25 recipes_1.0.8
## [107] openssl_2.1.1 Rphenograph_0.99.1
## [109] TTR_0.24.3 bslib_0.5.1
## [111] e1071_1.7-13 destiny_3.14.0
## [113] GetoptLong_1.0.5 ggplot.multistats_1.0.0
## [115] mime_0.12 splines_4.3.1
## [117] circlize_0.4.15 Rcpp_1.0.11
## [119] sparseMatrixStats_1.12.2 cellranger_1.1.0
## [121] knitr_1.44 utf8_1.2.4
## [123] clue_0.3-65 lme4_1.1-35.1
## [125] fs_1.6.3 listenv_0.9.0
## [127] checkmate_2.3.0 DelayedMatrixStats_1.22.6
## [129] pkgbuild_1.4.2 ggsignif_0.6.4
## [131] tibble_3.2.1 Matrix_1.6-1.1
## [133] rpart.plot_3.1.1 callr_3.7.3
## [135] tzdb_0.4.0 tweenr_2.0.2
## [137] pkgconfig_2.0.3 pheatmap_1.0.12
## [139] tools_4.3.1 cachem_1.0.8
## [141] smoother_1.1 fastmap_1.1.1
## [143] rmarkdown_2.25 scales_1.2.1
## [145] grid_4.3.1 usethis_2.2.2
## [147] broom_1.0.5 sass_0.4.7
## [149] graph_1.78.0 carData_3.0-5
## [151] RANN_2.6.1 rpart_4.1.21
## [153] farver_2.1.1 yaml_2.3.7
## [155] MatrixGenerics_1.12.3 foreign_0.8-85
## [157] ggthemes_4.2.4 cli_3.6.1
## [159] purrr_1.0.2 stats4_4.3.1
## [161] lifecycle_1.0.3 uwot_0.1.16
## [163] askpass_1.2.0 caret_6.0-94
## [165] Biobase_2.60.0 mvtnorm_1.2-3
## [167] lava_1.7.3 sessioninfo_1.2.2
## [169] backports_1.4.1 cytolib_2.12.1
## [171] timechange_0.2.0 gtable_0.3.4
## [173] rjson_0.2.21 umap_0.2.10.0
## [175] ggridges_0.5.4 parallel_4.3.1
## [177] pROC_1.18.5 limma_3.56.2
## [179] jsonlite_1.8.7 edgeR_3.42.4
## [181] RcppHNSW_0.5.0 bitops_1.0-7
## [183] Rtsne_0.16 FlowSOM_2.8.0
## [185] ranger_0.16.0 flowCore_2.12.2
## [187] jquerylib_0.1.4 timeDate_4022.108
## [189] shiny_1.7.5.1 ConsensusClusterPlus_1.64.0
## [191] htmltools_0.5.6.1 diffcyt_1.20.0
## [193] glue_1.6.2 XVector_0.40.0
## [195] VIM_6.2.2 RCurl_1.98-1.13
## [197] gridExtra_2.3 boot_1.3-28.1
## [199] igraph_1.5.1 TrajectoryUtils_1.8.0
## [201] R6_2.5.1 tidyr_1.3.0
## [203] SingleCellExperiment_1.22.0 labeling_0.4.3
## [205] vcd_1.4-11 cluster_2.1.4
## [207] pkgload_1.3.3 GenomeInfoDb_1.36.4
## [209] ipred_0.9-14 nloptr_2.0.3
## [211] DelayedArray_0.26.7 tidyselect_1.2.0
## [213] vipor_0.4.5 htmlTable_2.4.2
## [215] ggforce_0.4.1 CytoDx_1.20.0
## [217] car_3.1-2 future_1.33.0
## [219] ModelMetrics_1.2.2.2 munsell_0.5.0
## [221] laeken_0.5.2 data.table_1.14.8
## [223] htmlwidgets_1.6.2 ComplexHeatmap_2.16.0
## [225] RColorBrewer_1.1-3 rlang_1.1.1
## [227] remotes_2.4.2.1 colorRamps_2.3.1
## [229] ggnewscale_0.4.9 fansi_1.0.5
## [231] hardhat_1.3.0 beeswarm_0.4.0
## [233] prodlim_2023.08.28