########################################################################################################
# Script for article "Internal conceptual replications do not increase independent replication success"#
# Submitted to Psychonomic Bulletin & Review                                                           #
# Responsible for this file: R. Kunert (rikunert@gmail.com)                                            # 
########################################################################################################   

#setwd(choose.dir())

# source functions

if(!require(devtools)){install.packages('devtools')} #RPP functions
library(devtools)
source_url('https://raw.githubusercontent.com/FredHasselman/toolboxR/master/C-3PR.R')
in.IT(c('ggplot2','RColorBrewer','lattice','gridExtra','plyr','dplyr','httr','extrafont'))

if(!require(BayesFactor)){install.packages('BayesFactor')} #Bayesian analysis
library(BayesFactor)

if(!require(BEST)){install.packages('BEST')} #Bayesian estimation of posterior mean and 95% Credible Interval for t-tests
library(BEST)

##########################################################################################################################################################################################
#-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#Figures 1: visualising distributions

#get raw data from OSF website
info <- GET('https://osf.io/fgjvw/?action=download', write_disk('rpp_data.csv', overwrite = TRUE)) #downloads data file from the OSF
RPPdata <- read.csv("rpp_data.csv")[1:167, ]
colnames(RPPdata)[1] <- "ID" # Change first column name to ID to be able to load

#prepare IDs for internally replicated effects and non-internally replicated effects

#for studies with r reported
idIntRepl_r <- RPPdata$ID[!is.na(RPPdata$T_r..O.) & !is.na(RPPdata$T_r..R.) &
                            RPPdata$Successful.conceptual.replications..O. > 0]
idNotIntRepl_r <- RPPdata$ID[!is.na(RPPdata$T_r..O.) & !is.na(RPPdata$T_r..R.) &
                               RPPdata$Successful.conceptual.replications..O. == 0]

#for studies with significant p reported in original publication
idIntRepl_sigp <- RPPdata$ID[!is.na(RPPdata$T_pval_USE..O.) & !is.na(RPPdata$T_pval_USE..R.) &
                               RPPdata$T_sign_O == 1 &
                               RPPdata$Successful.conceptual.replications..O. > 0]
idNotIntRepl_sigp <- RPPdata$ID[!is.na(RPPdata$T_pval_USE..O.) & !is.na(RPPdata$T_pval_USE..R.) &
                                  RPPdata$T_sign_O == 1 &
                                  RPPdata$Successful.conceptual.replications..O. == 0]

# Get ggplot2 themes predefined in C-3PR
mytheme <- gg.theme("clean")

# VQP PANEL A: p-value -------------------------------------------------

dat_vis <- data.frame(ReplicationPValue = as.numeric(c(RPPdata$T_pval_USE..R.[idIntRepl_sigp],
                                                       RPPdata$T_pval_USE..R.[idNotIntRepl_sigp])),
                      grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_sigp)),
                                   rep("Internally Unreplicated Effects",times=length(idNotIntRepl_sigp)))))

# Create some variables for plotting
dat_vis$grp <- as.numeric(dat_vis$grp)
probs   <- seq(0,1,.25)

# Get p-value quantiles and frequencies from data
qtiles <- ldply(unique(dat_vis$grp),
                function(gr) quantile(round(dat_vis$ReplicationPValue[dat_vis$grp==gr],digits=4),probs,na.rm=T,type=3))
freqs  <- ldply(unique(dat_vis$grp),
                function(gr) table(cut(dat_vis$ReplicationPValue[dat_vis$grp==gr],breaks=qtiles[gr,],na.rm=T,include.lowest=T,right=T)))
labels <- sapply(unique(dat_vis$grp),
                 function(gr)levels(cut(round(dat_vis$ReplicationPValue[dat_vis$grp==gr],digits=4), breaks = qtiles[gr,],na.rm=T,include.lowest=T,right=T)))

# Get regular violinplot using package ggplot2
A <- ggplot(dat_vis,aes(x=grp,y=ReplicationPValue)) + 
  geom_violin(aes(group=grp),scale="width",color="grey30",fill="grey30",trim=T,adjust=.7)

# Cut at quantiles using vioQtile() in C-3PR
A0 <- vioQtile(A,qtiles,probs)

# Garnish (I still don't know what this word means!)
A <- A0 + geom_hline(aes(yintercept=.05),linetype=2) +
  xlab("") + ylab("Independent Replication p-value") + 
  xlim("Internal\nReplication  \nPresent", "Internal\n  Replication\nAbsent") +
  mytheme + theme(axis.text.x = element_text(size=12))

# VQP PANEL B: reduction in effect size -------------------------------------------------

#restructure data in data frame
dat_vis <- data.frame(EffectSizeDifference = as.numeric(c(c(RPPdata$T_r..R.[idIntRepl_r]) - c(RPPdata$T_r..O.[idIntRepl_r]),
                                                          c(RPPdata$T_r..R.[idNotIntRepl_r]) - c(RPPdata$T_r..O.[idNotIntRepl_r]))),
                      grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_r)),
                                   rep("Internally Unreplicated Effects",times=length(idNotIntRepl_r)))))

# Create some variables for plotting
dat_vis$grp <- as.numeric(dat_vis$grp)
probs   <- seq(0,1,.25)

# Get effect size difference quantiles and frequencies from data
qtiles <- ldply(unique(dat_vis$grp),
                function(gr) quantile(round(dat_vis$EffectSizeDifference[dat_vis$grp==gr],digits=4),probs,na.rm=T,type=3))
freqs  <- ldply(unique(dat_vis$grp),
                function(gr) table(cut(dat_vis$EffectSizeDifference[dat_vis$grp==gr],breaks=qtiles[gr,],na.rm=T,include.lowest=T,right=T)))
labels <- sapply(unique(dat_vis$grp),
                 function(gr)levels(cut(round(dat_vis$EffectSizeDifference[dat$grp==gr],digits=4), breaks = qtiles[gr,],na.rm=T,include.lowest=T,right=T)))

# Get regular violinplot using package ggplot2
B <- ggplot(dat_vis,aes(x=grp,y=EffectSizeDifference)) + geom_violin(aes(group=grp),scale="width",color="grey30",fill="grey30",trim=T,adjust=.7)

# Cut at quantiles using vioQtile() in C-3PR
B0 <- vioQtile(B,qtiles,probs)
B <- B0 +
  xlab("") + ylab("Replicated - Original Effect Size r") + 
  xlim("Internal\nReplication  \nPresent", "Internal\n  Replication\nAbsent") +
  mytheme + theme(axis.text.x = element_text(size=12))

#put two plots together
multi.PLOT(A,B,cols=2)

#I used RStudio's export function with aspect ratio 800 x 400 to save the figure as eps file, then used Adobe Illustrator for saving in different formats

##########################################################################################################################################################################################
#-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#Table 1: comparing distributions

#How many draws are to be taken from posterior distribution for BF and Credible Interval calculations? The more samples the more precise the estimate and the slower the calculation.
draws = 10000 * 10#BayesFactor package standard = 10000

#custom functions for data transformations
noninversion <- function(d1, d2) {return(list(d1 + abs(min(c(d1, d2))) + 1,
                                              d2 + abs(min(c(d1, d2))) + 1))}
inversion <- function(d1, d2) {return(list(-d1 + abs(min(c(-d1, -d2))) + 1,
                                           -d2 + abs(min(c(-d1, -d2))) + 1))}

#How many studies are considered in this analysis in total?-------------------------------------------------
sprintf('In the reproducibility project s dataset, %d original effects were internally, conceptually  replicated, %d once, %d twice, %d three times, and %d more than three times',
        sum(RPPdata$Successful.conceptual.replications..O.[RPPdata$Completion..R. == 1] > 0),
        sum(RPPdata$Successful.conceptual.replications..O.[RPPdata$Completion..R. == 1] == 1),
        sum(RPPdata$Successful.conceptual.replications..O.[RPPdata$Completion..R. == 1] == 2),
        sum(RPPdata$Successful.conceptual.replications..O.[RPPdata$Completion..R. == 1] == 3),
        sum(RPPdata$Successful.conceptual.replications..O.[RPPdata$Completion..R. == 1] > 3))

#Statistically Significant Replications-------------------------------------------------

#calculate counts
sum(RPPdata$T_pval_USE..R.[idIntRepl_sigp] <= .05)#How many internally replicated effects 'worked' upon replication?
sum(RPPdata$T_pval_USE..R.[idIntRepl_sigp] > .05)#How many internally replicated effects 'did not work' upon replication?
sum(RPPdata$T_pval_USE..R.[idNotIntRepl_sigp] <= .05)#How many internally non-replicated effects 'worked' upon replication?
sum(RPPdata$T_pval_USE..R.[idNotIntRepl_sigp] > .05)#How many internally non-replicated effects 'did not work' upon replication?

#prepare BayesFactor analysis
data_contingency = matrix(c(sum(RPPdata$T_pval_USE..R.[idIntRepl_sigp] <= .05),#row 1, col 1
                            sum(RPPdata$T_pval_USE..R.[idIntRepl_sigp] > .05),#row 2, col 1
                            sum(RPPdata$T_pval_USE..R.[idNotIntRepl_sigp] <= .05),#row 1, col 2
                            sum(RPPdata$T_pval_USE..R.[idNotIntRepl_sigp] > .05)),#row 2, col 2
                          nrow = 2, ncol = 2, byrow = F)#prepare BayesFactor analysis
bf = contingencyTableBF(data_contingency, sampleType = "indepMulti", fixedMargin = "cols")#run BayesFactor comparison
#sprintf('BF01 = %1.2f', 1/exp(bf@bayesFactor$bf))#alternative hypothesis = independent replication success DIFFERENT (two-sided prediction) depending on presence of internal replication, exponentiate BF10 because stored as natural log, turn into BF01

#calculate one-sided BF
chains = posterior(bf, iterations = draws)#draw samples from the posterior
consistent = chains[,"pi[1,1]"] / chains[,"pi[*,1]"] > chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#greater proportion of independent replication when internally replicated compared to not internally replicated
posteriorProb_consistent = sum(consistent)/draws
bf1 = posteriorProb_consistent / (1/2) * exp(bf@bayesFactor$bf)#one-sided posterior / prior odds for x>y * BF two-sided against null
sprintf('BF01 = %1.2f', 1/(bf1))#alternative hypothesis = independent replication success greater in case of presence of internal replication
#result has been verified with JASPv.0.7.5.Beta2, see file contingencyTable_internalReplication.jasp

#Parameter estimation
odds_ratio = (chains[,"omega[1,1]"] * chains[,"omega[2,2]"]) / (chains[,"omega[2,1]"] * chains[,"omega[1,2]"])
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(log(odds_ratio)),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(log(odds_ratio), 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(log(odds_ratio), 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
#plot(mcmc(log(odds_ratio)), main = "Log Odds Ratio")

#Effect Size Reduction (simple subtraction)-------------------------------------------------

#calculate means and standard deviations
mean(c(RPPdata$T_r..R.[idIntRepl_r]) - c(RPPdata$T_r..O.[idIntRepl_r]))#mean ES reduction of internally replicated effects
sd(c(RPPdata$T_r..R.[idIntRepl_r]) - c(RPPdata$T_r..O.[idIntRepl_r]))#Standard Deviation ES reduction of internally replicated effects
mean(c(RPPdata$T_r..R.[idNotIntRepl_r]) - c(RPPdata$T_r..O.[idNotIntRepl_r]))#mean ES reduction of NOT internally replicated effects
sd(c(RPPdata$T_r..R.[idNotIntRepl_r]) - c(RPPdata$T_r..O.[idNotIntRepl_r]))#Standard Deviation ES reduction of NOT internally replicated effects

#perform BayesFactor analysis (one-sided BF)
#problem with normal distribution identified, no transformation performs well, see KunertCodeNormality.R
int_repl_r_diff = c(RPPdata$T_r..R.[idIntRepl_r]) - c(RPPdata$T_r..O.[idIntRepl_r])#untransformed
not_int_repl_r_diff = c(RPPdata$T_r..R.[idNotIntRepl_r]) - c(RPPdata$T_r..O.[idNotIntRepl_r])#untransformed
dat_bf <- data.frame(EffectSizeDifference = as.numeric(c(int_repl_r_diff,
                                                         not_int_repl_r_diff)),
                      grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_r)),
                                   rep("Internally Unreplicated Effects",times=length(idNotIntRepl_r)))))#prepare BayesFactor analysis
bf = ttestBF(formula = EffectSizeDifference ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
#null Interval is positive because effect size reduction is expressed negatively, H1 predicts less reduction in case of internally replicated effects
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF10 because stored as natural log, turn into BF01

##Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(c(RPPdata$T_r..R.[idIntRepl_r]) - c(RPPdata$T_r..O.[idIntRepl_r]),
                   c(RPPdata$T_r..R.[idNotIntRepl_r]) - c(RPPdata$T_r..O.[idNotIntRepl_r]),
                   priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)

#Effect Size Reduction (Cohen's q)-------------------------------------------------

#prepare function to calculate Cohen's q
Cohenq <- function(r1, r2) { 
  fis_r1 = 0.5 * (log((1+r1)/(1-r1)))
  fis_r2 = 0.5 * (log((1+r2)/(1-r2)))
  fis_r1 - fis_r2
  }

#calculate means and standard deviations
mean(Cohenq(c(RPPdata$T_r..R.[idIntRepl_r]), c(RPPdata$T_r..O.[idIntRepl_r])))#mean ES reduction of internally replicated effects
sd(Cohenq(c(RPPdata$T_r..R.[idIntRepl_r]), c(RPPdata$T_r..O.[idIntRepl_r])))#Standard Deviation ES reduction of internally replicated effects
mean(Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_r]), c(RPPdata$T_r..O.[idNotIntRepl_r])))#mean ES reduction of NOT internally replicated effects
sd(Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_r]), c(RPPdata$T_r..O.[idNotIntRepl_r])))#Standard Deviation ES reduction of NOT internally replicated effects

#perform BayesFactor analysis (one-sided BF)
#problem with normal distribution identified, no transformation performs well, see KunertCodeNormality.R
dat_bf <- data.frame(EffectSizeDifference = as.numeric(c(Cohenq(c(RPPdata$T_r..R.[idIntRepl_r]), c(RPPdata$T_r..O.[idIntRepl_r])),
                                                         Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_r]), c(RPPdata$T_r..O.[idNotIntRepl_r])))),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_r)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_r)))))#prepare BayesFactor analysis
bf = ttestBF(formula = EffectSizeDifference ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
#null Interval is positive because effect size reduction is expressed negatively, H1 predicts less reduction in case of internally replicated effects
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF10 because stored as natural log, turn into BF01

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(Cohenq(c(RPPdata$T_r..R.[idIntRepl_r]), c(RPPdata$T_r..O.[idIntRepl_r])),
                   Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_r]), c(RPPdata$T_r..O.[idNotIntRepl_r])),
                   priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)

#Field of study-------------------------------------------------

#calculate counts
sum(RPPdata$Discipline..O.[idIntRepl_sigp] == 'Cognitive')#How many internally replicated effects were in field of Cognitive Psychology?
sum(RPPdata$Discipline..O.[idIntRepl_sigp] == 'Social')#How many internally replicated effects were in field of Social Psychology?
sum(RPPdata$Discipline..O.[idNotIntRepl_sigp] == 'Cognitive')#How many Not internally replicated effects were in field of Cognitive Psychology?
sum(RPPdata$Discipline..O.[idNotIntRepl_sigp] == 'Social')#How many Not internally replicated effects were in field of Social Psychology?

#prepare BayesFactor analysis
data_contingency = matrix(c(sum(RPPdata$Discipline..O.[idIntRepl_sigp] == 'Cognitive'),#row 1, col 1
                            sum(RPPdata$Discipline..O.[idIntRepl_sigp] == 'Social'),#row 2, col 1
                            sum(RPPdata$Discipline..O.[idNotIntRepl_sigp] == 'Cognitive'),#row 1, col 2
                            sum(RPPdata$Discipline..O.[idNotIntRepl_sigp] == 'Social')),#row 2, col 2
                          nrow = 2, ncol = 2, byrow = F)#prepare BayesFactor analysis
bf = contingencyTableBF(data_contingency, sampleType = "indepMulti", fixedMargin = "cols")#run BayesFactor comparison
#sprintf('BF10 = %1.2f', exp(bf@bayesFactor$bf))#two-sided BF, exponentiate BF10 because stored as natural log

#calculate one-sided BF
chains = posterior(bf, iterations = draws)#draw samples from the posterior
consistent = chains[,"pi[1,1]"] / chains[,"pi[*,1]"] < chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#smaller proportion of cognitive classification when internally replicated compared to not internally replicated
posteriorProb_consistent = sum(consistent)/draws
bf1 = posteriorProb_consistent / (1/2) * exp(bf@bayesFactor$bf)#one-sided posterior / prior odds for x<y * BF two-sided against null
sprintf('BF1+ = %1.2f', bf1)#alternative hypothesis = independent replication success greater in case of presence of internal replication

#Parameter estimation
CognitiveGivenIntRepl = chains[,"pi[1,1]"] / chains[,"pi[*,1]"]#cognitive classification proportion in case of internal replication
CognitiveGivenNotIntRepl = chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#cognitive classification proportion in case of absent internal replication
sprintf('M = %1.2f [%1.2f; %1.2f]',
        median(CognitiveGivenNotIntRepl - CognitiveGivenIntRepl),#Median for increase in cognitive classification due to internal replication\n(internally replicated versus not internally replicated)
        quantile(CognitiveGivenNotIntRepl - CognitiveGivenIntRepl, 0.025),#Lower edge of 95% Credible Interval for increase in cognitive classification due to internal replication\n(internally replicated versus not internally replicated)
        quantile(CognitiveGivenNotIntRepl - CognitiveGivenIntRepl, 0.975))#Higher edge of 95% Credible Interval for increase in cognitive classification due to internal replication\n(internally replicated versus not internally replicated)
#plot(mcmc(CognitiveGivenNotIntRepl - CognitiveGivenIntRepl), main = "Increase in cognitive classification \n(internally replicated versus not internally replicated)")

#Effect Type-------------------------------------------------

#calculate counts
sum(RPPdata$Type.of.effect..O.[idIntRepl_sigp] == 'main effect')#How many internally replicated effects were main effects?
sum(RPPdata$Type.of.effect..O.[idIntRepl_sigp] == 'interaction')#How many internally replicated effects were interactions?
sum(RPPdata$Type.of.effect..O.[idNotIntRepl_sigp] == 'main effect')#How many Not internally replicated effects were main effects?
sum(RPPdata$Type.of.effect..O.[idNotIntRepl_sigp] == 'interaction')#How many Not internally replicated effects were interactions?

#prepare BayesFactor analysis
data_contingency = matrix(c(sum(RPPdata$Type.of.effect..O.[idIntRepl_sigp] == 'main effect'),#row 1, col 1
                            sum(RPPdata$Type.of.effect..O.[idIntRepl_sigp] == 'interaction'),#row 2, col 1
                            sum(RPPdata$Type.of.effect..O.[idNotIntRepl_sigp] == 'main effect'),#row 1, col 2
                            sum(RPPdata$Type.of.effect..O.[idNotIntRepl_sigp] == 'interaction')),#row 2, col 2
                          nrow = 2, ncol = 2, byrow = F)#prepare BayesFactor analysis
bf = contingencyTableBF(data_contingency, sampleType = "indepMulti", fixedMargin = "cols")#run BayesFactor comparison
#sprintf('BF01 = %1.2f', 1/exp(bf@bayesFactor$bf))#two-sided bf, exponentiate BF10 because stored as natural log, turn into BF01

#calculate one-sided BF
chains = posterior(bf, iterations = draws)#draw samples from the posterior
consistent = chains[,"pi[1,1]"] / chains[,"pi[*,1]"] < chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#smaller proportion of main effect classification when internally replicated compared to not internally replicated
posteriorProb_consistent = sum(consistent)/draws
bf1 = posteriorProb_consistent / (1/2) * exp(bf@bayesFactor$bf)#one-sided posterior / prior odds for x<y * BF two-sided against null
sprintf('BF0+ = %1.2f', 1/bf1)#alternative hypothesis = independent replication success greater in case of presence of internal replication

#Parameter estimation
MainGivenIntRepl = chains[,"pi[1,1]"] / chains[,"pi[*,1]"]#main effect classification proportion in case of internal replication
MainGivenNotIntRepl = chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#main effect classification proportion in case of absent internal replication
sprintf('M = %1.2f [%1.2f; %1.2f]',
        median(MainGivenNotIntRepl - MainGivenIntRepl),#Median for increase in cognitive classification due to internal replication\n(internally replicated versus not internally replicated)
        quantile(MainGivenNotIntRepl - MainGivenIntRepl, 0.025),#Lower edge of 95% Credible Interval for increase in cognitive classification due to internal replication\n(internally replicated versus not internally replicated)
        quantile(MainGivenNotIntRepl - MainGivenIntRepl, 0.975))#Higher edge of 95% Credible Interval for increase in cognitive classification due to internal replication\n(internally replicated versus not internally replicated)
#plot(mcmc(MainGivenNotIntRepl - MainGivenIntRepl), main = "Increase in main effect classification \n(internally replicated versus not internally replicated)")

#Original study p-value-------------------------------------------------

#calculate means and standard deviations
mean(RPPdata$T_pval_USE..O.[idIntRepl_sigp])#mean original p-values of original studies with were internally replicated
sd(RPPdata$T_pval_USE..O.[idIntRepl_sigp])#Standard Deviation of original p-values of original studies with were internally replicated
mean(RPPdata$T_pval_USE..O.[idNotIntRepl_sigp])#mean original p-value of original studies with were Not internally replicated
sd(RPPdata$T_pval_USE..O.[idNotIntRepl_sigp])#Standard Deviation of original p-values of original studies with were Not internally replicated

#calculate one-sided BF
#problem with normal distribution identified, no transformation performs well, see KunertCodeNormality.R
p_orig_int_repl = RPPdata$T_pval_USE..O.[idIntRepl_sigp]
p_orig_not_int_repl = RPPdata$T_pval_USE..O.[idNotIntRepl_sigp]
dat_bf <- data.frame(P_O = c(p_orig_int_repl,
                             p_orig_not_int_repl),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_sigp)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_sigp)))))#prepare BayesFactor analysis
bf = ttestBF(formula = P_O ~ grp, data = dat_bf, nullInterval=c(0,Inf))#Bayesian t-test to test the difference/similarity between the previous two
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF10 because stored as natural log, turn into BF01

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(p_orig_int_repl, p_orig_not_int_repl, priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in original p-value due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in original p-value success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in original p-value success due to internal replication\n(internally replicated versus not internally replicated)

#Original study effect size-------------------------------------------------

#calculate means and standard deviations
mean(RPPdata$T_r..O.[idIntRepl_r])#mean original r-values of original studies with were internally replicated
sd(RPPdata$T_r..O.[idIntRepl_r])#Standard Deviation of original r-values of original studies with were internally replicated
mean(RPPdata$T_r..O.[idNotIntRepl_r])#mean original r-value of original studies with were Not internally replicated
sd(RPPdata$T_r..O.[idNotIntRepl_r])#Standard Deviation of original r-values of original studies with were Not internally replicated

#calculate one-sided BF
#no severe problem with normal distribution identified, see KunertCodeNormality.R
dat_bf <- data.frame(P_O = c(RPPdata$T_r..O.[idIntRepl_r],
                             RPPdata$T_r..O.[idNotIntRepl_r]),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_r)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_r)))))#prepare BayesFactor analysis
bf = ttestBF(formula = P_O ~ grp, data = dat_bf, nullInterval=c(-Inf,0))#Bayesian t-test to test the difference/similarity between the previous two
sprintf('BF+0 = %1.2f', exp(bf@bayesFactor$bf[1]))#exponentiate BF+0 because stored as natural log, turn into BF01

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(RPPdata$T_r..O.[idIntRepl_r], RPPdata$T_r..O.[idNotIntRepl_r], priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu2 - BESTout$mu1),#Median for increase in original r-value due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu2 - BESTout$mu1, 0.025),#Lower edge of 95% Credible Interval for increase in original r-value success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu2 - BESTout$mu1, 0.975))#Higher edge of 95% Credible Interval for increase in original r-value success due to internal replication\n(internally replicated versus not internally replicated)

#Statistical Power of independent replication-------------------------------------------------

#calculate means and standard deviations
mean(as.numeric(as.character(RPPdata$Power..R.[idIntRepl_sigp])))#average power of internally replicated studies
sd(as.numeric(as.character(RPPdata$Power..R.[idIntRepl_sigp])))#Standard Deviation of internally replicated studies
x = as.numeric(as.character(RPPdata$Power..R.[idNotIntRepl_sigp])); mean(x[!is.na(x)])#average power of NOT internally replicated studies
sd(x[!is.na(x)])#Standard Deviation of power of NOT internally replicated studies

#calculate one-sided BF
#problem with normal distribution identified, no transformation performs well, see KunertCodeNormality.R
dat_bf <- data.frame(Power = c(as.numeric(as.character(RPPdata$Power..R.[idIntRepl_sigp])),
                               x[!is.na(x)]),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_sigp)),
                                  rep("Internally Unreplicated Effects",times=length(x[!is.na(x)])))))#prepare BayesFactor analysis
bf = ttestBF(formula = Power ~ grp, data = dat_bf, nullInterval=c(-Inf,0))#Bayesian t-test to test the difference/similarity between the previous two
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF+0 because stored as natural log, turn into BF0+

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(as.numeric(as.character(RPPdata$Power..R.[idIntRepl_sigp])), x[!is.na(x)], priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu2 - BESTout$mu1),#Median for increase in replication power due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu2 - BESTout$mu1, 0.025),#Lower edge of 95% Credible Interval for increase in replicatioin power due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu2 - BESTout$mu1, 0.975))#Higher edge of 95% Credible Interval for increase in replication power due to internal replication\n(internally replicated versus not internally replicated)

#Surprisingness of original effect-------------------------------------------------

#calculate means and standard deviations
x = as.numeric(as.character(RPPdata$Surprising.result..O.[idIntRepl_sigp])); mean(x[!is.na(x)])#mean surprisingness of effect with internal replication
sd(x[!is.na(x)])#Standard Deviation of surprisingness of effect with internal replication
y = as.numeric(as.character(RPPdata$Surprising.result..O.[idNotIntRepl_sigp])); mean(y[!is.na(y)])#mean surprisingness of effect without internal replication
sd(y[!is.na(y)])#Standard Deviation of surprisingness of effect without internal replication

#calculate one-sided BF
#no severe problem with normal distribution identified, see KunertCodeNormality.R
dat_bf <- data.frame(P_O = c(x[!is.na(x)],
                             y[!is.na(y)]),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(x[!is.na(x)])),
                                  rep("Internally Unreplicated Effects",times=length(y[!is.na(y)])))))#prepare BayesFactor analysis
bf = ttestBF(formula = P_O ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF+0 because stored as natural log, turn into BF0+

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(x[!is.na(x)], y[!is.na(y)], priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in Surprisingness  due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in Surprisingness  due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in Surprisingness  due to internal replication\n(internally replicated versus not internally replicated)

#Challenge of conducting replication-------------------------------------------------

#This is actually a combination of three variables, see master script at https://osf.io/vdnrb/
stand <- function(x) {res <- (x-mean(x, na.rm = TRUE))/sd(x, na.rm = TRUE); return(res)}# Function for standardizing variables
fac.expe <- factor(RPPdata$Methodology.expertise.required..O., levels = c("No expertise required", "Slight expertise required",
                                                                         "Moderate expertise required", "Strong expertise required", 
                                                                         "Extreme expertise required"))# Create the right order of the variable
st.expe <- stand(as.numeric(fac.expe))#standardize variable

fac.oppo.expe <- factor(RPPdata$Opportunity.for.expectancy.bias..O., levels = c("No opportunity for researcher expectations to influence results",
                                                                               "Slight opportunity for researcher expectations to influence results",
                                                                               "Moderate opportunity for researcher expectations to influence results",
                                                                               "Strong opportunity for researcher expectations to influence results",
                                                                               "Extreme opportunity for researcher expectations to influence results"))
st.oppo.expe <- stand(as.numeric(fac.oppo.expe))

fac.oppo.dili <- factor(RPPdata$Opportunity.for.lack.of.diligence..O., levels = c("No opportunity for lack of diligence to affect the results",
                                                                                 "Slight opportunity for lack of diligence to affect the results",
                                                                                 "Moderate opportunity for lack of diligence to affect the results",
                                                                                 "Strong opportunity for lack of diligence to affect the results",
                                                                                 "Extreme opportunity for lack of diligence to affect the results"))
st.oppo.dili <- stand(as.numeric(fac.oppo.dili))

sc.chal <- (st.expe + st.oppo.expe + st.oppo.dili)/3# Create scale

#calculate means and standard deviations
mean(sc.chal[idIntRepl_sigp])#mean number of research sites of publication with internal replication
sd(sc.chal[idIntRepl_sigp])#Standard Deviation of number of research sites of publication with internal replication
mean(sc.chal[idNotIntRepl_sigp])#mean number of research sites of publication without internal replication
sd(sc.chal[idNotIntRepl_sigp])#Standard Deviation of number of research sites of publication without internal replication

#calculate one-sided BF
#problem with normal distribution identified, no transformation performs well, see KunertCodeNormality.R
dat_bf <- data.frame(chal = c(sc.chal[idIntRepl_sigp],
                             sc.chal[idNotIntRepl_sigp]),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_sigp)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_sigp)))))#prepare BayesFactor analysis
bf = ttestBF(formula = chal ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF+0 because stored as natural log, turn into BF0+

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(sc.chal[idIntRepl_sigp], sc.chal[idNotIntRepl_sigp], priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in Challenge  due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in Challenge  due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in Challenge  due to internal replication\n(internally replicated versus not internally replicated)

#Formal Power Analysis in original publication-------------------------------------------------

#not assessed by Open Science Collaboration, so I went into every publication myself, 
#searching for the word 'power' and the justification of the sample size of Exp. 1. 
#If neither turned up a formal power analysis, the publication was considered not to include one.

#calculate counts
x = rep(0, times=length(idIntRepl_sigp)); sum(x == 0)#none of the internally replicated effects were published in articles including a formal power analysis
y = c(rep(0, times = length(idNotIntRepl_sigp) - 2), 1, 1)#Not internally replicated effects. 
#The only publications with a formal power analysis were 15 (Schmidt & Besner, 2008) and 
#37 (Makovski et al., 2008). Note that neither used the power analysis to justify the sample size. 
#Nonetheless, power appears to have played a role in these publications.
sum(y == 0)#Not internally replicated effects: without power analysis
sum(y == 1)#Not internally replicated effects: with power analysis

#prepare BayesFactor analysis
data_contingency = matrix(c(sum(x == 0),#row 1, col 1
                            sum(x == 1),#row 2, col 1
                            sum(y == 0),#row 1, col 2
                            sum(y == 1)),#row 2, col 2
                          nrow = 2, ncol = 2, byrow = F)#prepare BayesFactor analysis
bf = contingencyTableBF(data_contingency, sampleType = "indepMulti", fixedMargin = "cols")#run BayesFactor comparison
#sprintf('BF01 = %1.2f', 1/exp(bf@bayesFactor$bf))#exponentiate BF10 because stored as natural log, turn into BF01

#calculate one-sided BF
chains = posterior(bf, iterations = draws)#draw samples from the posterior
consistent = chains[,"pi[1,1]"] / chains[,"pi[*,1]"] < chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#smaller proportion of power analysis when internally replicated compared to not internally replicated
posteriorProb_consistent = sum(consistent)/draws
bf1 = posteriorProb_consistent / (1/2) * exp(bf@bayesFactor$bf)#one-sided posterior / prior odds for x<y * BF two-sided against null
sprintf('BF0+ = %1.2f', 1/bf1)#alternative hypothesis = independent replication success greater in case of presence of internal replication

#Parameter estimation
PAGivenIntRepl = chains[,"pi[1,1]"] / chains[,"pi[*,1]"]#power analysis proportion in case of internal replication
PAGivenNotIntRepl = chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#power analysis proportion in case of absent internal replication
sprintf('M = %1.2f [%1.2f; %1.2f]',
        median(PAGivenNotIntRepl - PAGivenIntRepl),#Median for increase in PA due to internal replication\n(internally replicated versus not internally replicated)
        quantile(PAGivenNotIntRepl - PAGivenIntRepl, 0.025),#Lower edge of 95% Credible Interval for increase in PA due to internal replication\n(internally replicated versus not internally replicated)
        quantile(PAGivenNotIntRepl - PAGivenIntRepl, 0.975))#Higher edge of 95% Credible Interval for increase in PA due to internal replication\n(internally replicated versus not internally replicated)
#plot(mcmc(PAGivenNotIntRepl - PAGivenIntRepl), main = "Increase in Power Analysis \n(internally replicated versus not internally replicated)")

#Sample Size-------------------------------------------------

#calculate means and standard deviations
mean(RPPdata$T_N_O_for_tables[idIntRepl_r])#mean sample size of original studies with were internally replicated
sd(RPPdata$T_N_O_for_tables[idIntRepl_r])#Standard Deviation sample size of original studies with were internally replicated
x = RPPdata$T_N_O_for_tables[idNotIntRepl_r]; mean(x[x < 100000])#Mean of sample size of original studies with were NOT internally replicated
sd(x[x < 100000])#Standard Deviation of sample size of original studies with were NOT internally replicated

#calculate one-sided BF
#problem with normal distribution identified, natural logarithm transformation performs well, see KunertCodeNormality.R

#BF with raw data
dat_bf <- data.frame(N = c(RPPdata$T_N_O_for_tables[idIntRepl_r],
                           x[x < 100000]),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_r)),
                                  rep("Internally Unreplicated Effects",times=length(x[x < 100000])))))#prepare BayesFactor analysis
bf = ttestBF(formula = N ~ grp, data = dat_bf, nullInterval = c(-Inf, 0))#Bayesian t-test to test the difference/similarity between the previous two
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF+0 because stored as natural log, turn into BF0+

#Bf with logged data
dat_bf <- data.frame(N = c(log(noninversion(RPPdata$T_N_O_for_tables[idIntRepl_r], x[x < 100000])[[1]]),
                           log(noninversion(RPPdata$T_N_O_for_tables[idIntRepl_r], x[x < 100000])[[2]])),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_r)),
                                  rep("Internally Unreplicated Effects",times=length(x[x < 100000])))))#prepare BayesFactor analysis
bf = ttestBF(formula = N ~ grp, data = dat_bf, nullInterval = c(-Inf, 0))#Bayesian t-test to test the difference/similarity between the previous two
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF+0 because stored as natural log, turn into BF0+

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(RPPdata$T_N_O_for_tables[idIntRepl_r], x[x < 100000], priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu2 - BESTout$mu1),#Median for increase in orignal N due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu2 - BESTout$mu1, 0.025),#Lower edge of 95% Credible Interval for increase in original N due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu2 - BESTout$mu1, 0.975))#Higher edge of 95% Credible Interval for increase in original N  due to internal replication\n(internally replicated versus not internally replicated)

##########################################################################################################################################################################################
#-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#Table 2: independent replication success for different fields of study

#Social Psychology-----------------------------------------------------------------------------------------------------------------------------------------
#for studies with r reported
idIntRepl_soc_r <- RPPdata$ID[!is.na(RPPdata$T_r..O.) & !is.na(RPPdata$T_r..R.) &
                            RPPdata$Successful.conceptual.replications..O. > 0 &
                            RPPdata$Discipline..O. == 'Social']
idNotIntRepl_soc_r <- RPPdata$ID[!is.na(RPPdata$T_r..O.) & !is.na(RPPdata$T_r..R.) &
                               RPPdata$Successful.conceptual.replications..O. == 0 &
                               RPPdata$Discipline..O. == 'Social']

#for studies with significant p reported in original publication
idIntRepl_soc_sigp <- RPPdata$ID[!is.na(RPPdata$T_pval_USE..O.) & !is.na(RPPdata$T_pval_USE..R.) &
                               RPPdata$T_sign_O == 1 &
                               RPPdata$Successful.conceptual.replications..O. > 0 &
                               RPPdata$Discipline..O. == 'Social']
idNotIntRepl_soc_sigp <- RPPdata$ID[!is.na(RPPdata$T_pval_USE..O.) & !is.na(RPPdata$T_pval_USE..R.) &
                                  RPPdata$T_sign_O == 1 &
                                  RPPdata$Successful.conceptual.replications..O. == 0 &
                                  RPPdata$Discipline..O. == 'Social']

#Statistically Significant Replications-------------------------------------------------

#calculate counts
sum(RPPdata$T_pval_USE..R.[idIntRepl_soc_sigp] <= .05)#How many internally replicated effects 'worked' upon replication?
sum(RPPdata$T_pval_USE..R.[idIntRepl_soc_sigp] > .05)#How many internally replicated effects 'did not work' upon replication?
sum(RPPdata$T_pval_USE..R.[idNotIntRepl_soc_sigp] <= .05)#How many internally non-replicated effects 'worked' upon replication?
sum(RPPdata$T_pval_USE..R.[idNotIntRepl_soc_sigp] > .05)#How many internally non-replicated effects 'did not work' upon replication?

#prepare BayesFactor analysis
data_contingency = matrix(c(sum(RPPdata$T_pval_USE..R.[idIntRepl_soc_sigp] <= .05),#row 1, col 1
                            sum(RPPdata$T_pval_USE..R.[idIntRepl_soc_sigp] > .05),#row 2, col 1
                            sum(RPPdata$T_pval_USE..R.[idNotIntRepl_soc_sigp] <= .05),#row 1, col 2
                            sum(RPPdata$T_pval_USE..R.[idNotIntRepl_soc_sigp] > .05)),#row 2, col 2
                          nrow = 2, ncol = 2, byrow = F)#prepare BayesFactor analysis
bf = contingencyTableBF(data_contingency, sampleType = "indepMulti", fixedMargin = "cols")#run BayesFactor comparison
#sprintf('BF01 = %1.2f', 1/exp(bf@bayesFactor$bf))#exponentiate BF10 because stored as natural log, turn into BF01

#calculate one-sided BF
chains = posterior(bf, iterations = draws)#draw samples from the posterior
consistent = chains[,"pi[1,1]"] / chains[,"pi[*,1]"] > chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#greater proportion of independent replication success when internally replicated compared to not internally replicated
posteriorProb_consistent = sum(consistent)/draws
bf1 = posteriorProb_consistent / (1/2) * exp(bf@bayesFactor$bf)#one-sided posterior / prior odds for x<y * BF two-sided against null
sprintf('BF0+ = %1.2f', 1/bf1)#alternative hypothesis = independent replication success greater in case of presence of internal replication

#Parameter estimation
odds_ratio = (chains[,"omega[1,1]"] * chains[,"omega[2,2]"]) / (chains[,"omega[2,1]"] * chains[,"omega[1,2]"])
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(log(odds_ratio)),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(log(odds_ratio), 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(log(odds_ratio), 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
#plot(mcmc(log(odds_ratio)), main = "Log Odds Ratio")

#Effect Size Reduction (simple subtraction)-------------------------------------------------

#calculate means and standard deviations
mean(c(RPPdata$T_r..R.[idIntRepl_soc_r]) - c(RPPdata$T_r..O.[idIntRepl_soc_r]))#mean ES reduction of internally replicated effects
sd(c(RPPdata$T_r..R.[idIntRepl_soc_r]) - c(RPPdata$T_r..O.[idIntRepl_soc_r]))#Standard Deviation ES reduction of internally replicated effects
mean(c(RPPdata$T_r..R.[idNotIntRepl_soc_r]) - c(RPPdata$T_r..O.[idNotIntRepl_soc_r]))#mean ES reduction of NOT internally replicated effects
sd(c(RPPdata$T_r..R.[idNotIntRepl_soc_r]) - c(RPPdata$T_r..O.[idNotIntRepl_soc_r]))#Standard Deviation ES reduction of NOT internally replicated effects

# Bayes factor analysis
int_repl_r_diff = c(RPPdata$T_r..R.[idIntRepl_soc_r]) - c(RPPdata$T_r..O.[idIntRepl_soc_r])#untransformed
not_int_repl_r_diff = c(RPPdata$T_r..R.[idNotIntRepl_soc_r]) - c(RPPdata$T_r..O.[idNotIntRepl_soc_r])#untransformed
dat_bf <- data.frame(EffectSizeDifference = as.numeric(c(int_repl_r_diff,
                                                         not_int_repl_r_diff)),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_soc_r)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_soc_r)))))#prepare BayesFactor analysis
bf = ttestBF(formula = EffectSizeDifference ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
#null Interval is positive because effect size reduction is expressed negatively, H1 predicts less reduction in case of internally replicated effects
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF10 because stored as natural log, turn into BF01

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(c(RPPdata$T_r..R.[idIntRepl_soc_r]) - c(RPPdata$T_r..O.[idIntRepl_soc_r]),
                   c(RPPdata$T_r..R.[idNotIntRepl_soc_r]) - c(RPPdata$T_r..O.[idNotIntRepl_soc_r]),
                   priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)

#Effect Size Reduction (Cohen's q)-------------------------------------------------

#calculate means and standard deviations
mean(Cohenq(c(RPPdata$T_r..R.[idIntRepl_soc_r]), c(RPPdata$T_r..O.[idIntRepl_soc_r])))#mean ES reduction of internally replicated effects
sd(Cohenq(c(RPPdata$T_r..R.[idIntRepl_soc_r]), c(RPPdata$T_r..O.[idIntRepl_soc_r])))#Standard Deviation ES reduction of internally replicated effects
mean(Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_soc_r]), c(RPPdata$T_r..O.[idNotIntRepl_soc_r])))#mean ES reduction of NOT internally replicated effects
sd(Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_soc_r]), c(RPPdata$T_r..O.[idNotIntRepl_soc_r])))#Standard Deviation ES reduction of NOT internally replicated effects

#perform BayesFactor analysis (one-sided BF)
dat_bf <- data.frame(EffectSizeDifference = as.numeric(c(Cohenq(c(RPPdata$T_r..R.[idIntRepl_soc_r]), c(RPPdata$T_r..O.[idIntRepl_soc_r])),
                                                         Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_soc_r]), c(RPPdata$T_r..O.[idNotIntRepl_soc_r])))),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_soc_r)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_soc_r)))))#prepare BayesFactor analysis
bf = ttestBF(formula = EffectSizeDifference ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
#null Interval is positive because effect size reduction is expressed negatively, H1 predicts less reduction in case of internally replicated effects
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF10 because stored as natural log, turn into BF01

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(Cohenq(c(RPPdata$T_r..R.[idIntRepl_soc_r]), c(RPPdata$T_r..O.[idIntRepl_soc_r])),
                   Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_soc_r]), c(RPPdata$T_r..O.[idNotIntRepl_soc_r])),
                   priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)

#Cognitive Psychology-----------------------------------------------------------------------------------------------------------------------------------------

#for studies with r reported
idIntRepl_cog_r <- RPPdata$ID[!is.na(RPPdata$T_r..O.) & !is.na(RPPdata$T_r..R.) &
                                RPPdata$Successful.conceptual.replications..O. > 0 &
                                RPPdata$Discipline..O. == 'Cognitive']
idNotIntRepl_cog_r <- RPPdata$ID[!is.na(RPPdata$T_r..O.) & !is.na(RPPdata$T_r..R.) &
                                   RPPdata$Successful.conceptual.replications..O. == 0 &
                                   RPPdata$Discipline..O. == 'Cognitive']

#for studies with significant p reported in original publication
idIntRepl_cog_sigp <- RPPdata$ID[!is.na(RPPdata$T_pval_USE..O.) & !is.na(RPPdata$T_pval_USE..R.) &
                                   RPPdata$T_sign_O == 1 &
                                   RPPdata$Successful.conceptual.replications..O. > 0 &
                                   RPPdata$Discipline..O. == 'Cognitive']
idNotIntRepl_cog_sigp <- RPPdata$ID[!is.na(RPPdata$T_pval_USE..O.) & !is.na(RPPdata$T_pval_USE..R.) &
                                      RPPdata$T_sign_O == 1 &
                                      RPPdata$Successful.conceptual.replications..O. == 0 &
                                      RPPdata$Discipline..O. == 'Cognitive']

#Statistically Significant Replications-------------------------------------------------

#calculate counts
sum(RPPdata$T_pval_USE..R.[idIntRepl_cog_sigp] <= .05)#How many internally replicated effects 'worked' upon replication?
sum(RPPdata$T_pval_USE..R.[idIntRepl_cog_sigp] > .05)#How many internally replicated effects 'did not work' upon replication?
sum(RPPdata$T_pval_USE..R.[idNotIntRepl_cog_sigp] <= .05)#How many internally non-replicated effects 'worked' upon replication?
sum(RPPdata$T_pval_USE..R.[idNotIntRepl_cog_sigp] > .05)#How many internally non-replicated effects 'did not work' upon replication?

#prepare BayesFactor analysis
data_contingency = matrix(c(sum(RPPdata$T_pval_USE..R.[idIntRepl_cog_sigp] <= .05),#row 1, col 1
                            sum(RPPdata$T_pval_USE..R.[idIntRepl_cog_sigp] > .05),#row 2, col 1
                            sum(RPPdata$T_pval_USE..R.[idNotIntRepl_cog_sigp] <= .05),#row 1, col 2
                            sum(RPPdata$T_pval_USE..R.[idNotIntRepl_cog_sigp] > .05)),#row 2, col 2
                          nrow = 2, ncol = 2, byrow = F)#prepare BayesFactor analysis
bf = contingencyTableBF(data_contingency, sampleType = "indepMulti", fixedMargin = "cols")#run BayesFactor comparison
#sprintf('BF01 = %1.2f', 1/exp(bf@bayesFactor$bf))#exponentiate BF10 because stored as natural log, turn into BF01

#calculate one-sided BF
chains = posterior(bf, iterations = draws)#draw samples from the posterior
consistent = chains[,"pi[1,1]"] / chains[,"pi[*,1]"] > chains[,"pi[1,2]"] / chains[,"pi[*,2]"]#greater proportion of independent replication success when internally replicated compared to not internally replicated
posteriorProb_consistent = sum(consistent)/draws
bf1 = posteriorProb_consistent / (1/2) * exp(bf@bayesFactor$bf)#one-sided posterior / prior odds for x<y * BF two-sided against null
sprintf('BF0+ = %1.2f', 1/bf1)#alternative hypothesis = independent replication success greater in case of presence of internal replication

#Parameter estimation
odds_ratio = (chains[,"omega[1,1]"] * chains[,"omega[2,2]"]) / (chains[,"omega[2,1]"] * chains[,"omega[1,2]"])
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(log(odds_ratio)),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(log(odds_ratio), 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(log(odds_ratio), 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
#plot(mcmc(log(odds_ratio)), main = "Log Odds Ratio")

#Effect Size Reduction (simple subtraction)-------------------------------------------------

#calculate means and standard deviations
mean(c(RPPdata$T_r..R.[idIntRepl_cog_r]) - c(RPPdata$T_r..O.[idIntRepl_cog_r]))#mean ES reduction of internally replicated effects
sd(c(RPPdata$T_r..R.[idIntRepl_cog_r]) - c(RPPdata$T_r..O.[idIntRepl_cog_r]))#Standard Deviation ES reduction of internally replicated effects
mean(c(RPPdata$T_r..R.[idNotIntRepl_cog_r]) - c(RPPdata$T_r..O.[idNotIntRepl_cog_r]))#mean ES reduction of NOT internally replicated effects
sd(c(RPPdata$T_r..R.[idNotIntRepl_cog_r]) - c(RPPdata$T_r..O.[idNotIntRepl_cog_r]))#Standard Deviation ES reduction of NOT internally replicated effects

# Bayes factor analysis
int_repl_r_diff = c(RPPdata$T_r..R.[idIntRepl_cog_r]) - c(RPPdata$T_r..O.[idIntRepl_cog_r])#untransformed
not_int_repl_r_diff = c(RPPdata$T_r..R.[idNotIntRepl_cog_r]) - c(RPPdata$T_r..O.[idNotIntRepl_cog_r])#untransformed
dat_bf <- data.frame(EffectSizeDifference = as.numeric(c(int_repl_r_diff,
                                                         not_int_repl_r_diff)),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_cog_r)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_cog_r)))))#prepare BayesFactor analysis
bf = ttestBF(formula = EffectSizeDifference ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
#null Interval is positive because effect size reduction is expressed negatively, H1 predicts less reduction in case of internally replicated effects
sprintf('BF0+ = %1.2f', 1/exp(bf@bayesFactor$bf[1]))#exponentiate BF10 because stored as natural log, turn into BF01

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(c(RPPdata$T_r..R.[idIntRepl_cog_r]) - c(RPPdata$T_r..O.[idIntRepl_cog_r]),
                   c(RPPdata$T_r..R.[idNotIntRepl_cog_r]) - c(RPPdata$T_r..O.[idNotIntRepl_cog_r]),
                   priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)

#Effect Size Reduction (Cohen's q)-------------------------------------------------

#calculate means and standard deviations
mean(Cohenq(c(RPPdata$T_r..R.[idIntRepl_cog_r]), c(RPPdata$T_r..O.[idIntRepl_cog_r])))#mean ES reduction of internally replicated effects
sd(Cohenq(c(RPPdata$T_r..R.[idIntRepl_cog_r]), c(RPPdata$T_r..O.[idIntRepl_cog_r])))#Standard Deviation ES reduction of internally replicated effects
mean(Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_cog_r]), c(RPPdata$T_r..O.[idNotIntRepl_cog_r])))#mean ES reduction of NOT internally replicated effects
sd(Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_cog_r]), c(RPPdata$T_r..O.[idNotIntRepl_cog_r])))#Standard Deviation ES reduction of NOT internally replicated effects

#perform BayesFactor analysis (one-sided BF)
dat_bf <- data.frame(EffectSizeDifference = as.numeric(c(Cohenq(c(RPPdata$T_r..R.[idIntRepl_cog_r]), c(RPPdata$T_r..O.[idIntRepl_cog_r])),
                                                         Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_cog_r]), c(RPPdata$T_r..O.[idNotIntRepl_cog_r])))),
                     grp=factor(c(rep("Internally Replicated Effects",times=length(idIntRepl_cog_r)),
                                  rep("Internally Unreplicated Effects",times=length(idNotIntRepl_cog_r)))))#prepare BayesFactor analysis
bf = ttestBF(formula = EffectSizeDifference ~ grp, data = dat_bf, nullInterval=c(0, Inf))#Bayesian t-test to test the difference/similarity between the previous two
#null Interval is positive because effect size reduction is expressed negatively, H1 predicts less reduction in case of internally replicated effects
sprintf('BF+0 = %1.2f', exp(bf@bayesFactor$bf[1]))#exponentiate BF10 because stored as natural log, turn into BF01

#Parameter estimation: use BEST package to estimate posterior median and 95% Credible Interval
BESTout = BESTmcmc(Cohenq(c(RPPdata$T_r..R.[idIntRepl_cog_r]), c(RPPdata$T_r..O.[idIntRepl_cog_r])),
                   Cohenq(c(RPPdata$T_r..R.[idNotIntRepl_cog_r]), c(RPPdata$T_r..O.[idNotIntRepl_cog_r])),
                   priors=NULL, parallel=FALSE)
#plotAll(BESTout)
sprintf('Median = %1.2f [%1.2f; %1.2f]',
        median(BESTout$mu1 - BESTout$mu2),#Median for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.025),#Lower edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)
        quantile(BESTout$mu1 - BESTout$mu2, 0.975))#Higher edge of 95% Credible Interval for increase in independent replication success due to internal replication\n(internally replicated versus not internally replicated)