\documentclass[man, floatsintext]{apa7}
\usepackage{graphicx} % Required for inserting images
\usepackage{amsmath}
\usepackage[american]{babel}
\usepackage{csquotes}
\usepackage[style=apa,sortcites=true,sorting=nyt,backend=biber]{biblatex}
\usepackage{caption}
\usepackage{xcolor}
\usepackage{subcaption}
\usepackage{longtable}
\usepackage{hyperref}
\usepackage{bm}

\DeclareLanguageMapping{american}{american-apa}
\title{Supplementary Material To Interact or not to Interact: Pros and Cons of Adding Interactions to Linear Regression Models}
\shorttitle{Supplement: Interactions in Linear Regression Models}
\author{A. Rimpler, H.A.L. Kiers, D. van Ravenzwaaij}
\affiliation{Behavioral and Social Sciences, University of Groningen, Groningen, The Netherlands}
\date{April 2024}

\begin{document}
\maketitle
\section{Derivation of Regression Estimates in the Misspecified Model}

Derivation of the population estimates of the regression weights when fitting the main effects only model to  data that incorporate an interaction term in addition to main effects


\underline{Population model}

We assume that in the population we have data for which the following holds:
\begin{equation}
\label{data_pop}
    Y = \beta_0 + \beta_1X+ \beta_2Z+\beta_3XZ+\varepsilon E,
\end{equation}

where
\begin{itemize}
    \item $Y$ is a variable to be predicted
    \item$X$ and $Z$ denote the two predictors of interest
    \item$XZ$ denotes the interaction computed by elementwise multiplication of the scores of $X$ and $Z$
    \item$E$ denotes a noise variable, uncorrelated with $X$ and $Z$
    \item$\beta_0$ denotes the intercept
    \item$\beta_1$, $\beta_2$ and $\beta_3$ denote weights for $X$, $Z$ and $XZ$, respectively
    \item$\varepsilon$ denotes a weight for the noise variable.
\end{itemize}

We further denote 
\begin{itemize}
    \item $m_x$ for the mean of $X$
	\item $m_z$ for the mean of $Z$
	\item $m_y$ for the mean of $Y$
	\item $X_c$ for the centered version of $X$, i.e. $X_c=X-m_x$
	\item $Z_c$ for the centered version of $Z$, i.e. $Z_c=Z-m_z$
	\item $Y_c$ for the centered version of $Y$, i.e. $Y_c=Y-m_y$
\item $X_cZ_c$ for the elementwise multiplication of the scores of $X_c$ and $Z_c$
	\item $r_{xz}$ for the correlation between $X$ and $Z$.
\end{itemize}


Now we assume 
\begin{itemize}
\item $X_c$, $Z_c$ and $E$ have standard normal distributions $\mathcal{N}(0,1)$, hence mean 0
\item $X$  and $Z$ are distributed as $\mathcal{N}(m_x,1)$ and $\mathcal{N}(m_z,)$ respectively
\item$X$ and $Z$ may be correlated, while $E$ is uncorrelated with $X$ and with $Z$.
\end{itemize}
In our derivation below we will use the sum of squares expressions of vectors with $n$ elements and we assume $n\to\infty$, thus representing a population, rather than using stochastic expressions in terms of expectations. We denote these vectors as $\mathbf{y, x, z, xz}$, and $\mathbf{e}$, and we denote their sum of squares by $\|...\|^2$, e.g. $\|\mathbf{x}\|^2$ gives the sums of squares of all elements in the vector $\mathbf{x}$. 

	We wish to derive the expression for the population regression weights when we carry out regression of $Y$ on $X$ and $Z$, by means of the (common) least squares approach. The fitted model is here expressed in vectors as 
	$\mathbf{\hat{y}}=b_0+b_1 \mathbf{x}+b_2\mathbf{z}$,

and expressions for $b_0$, $b_1$ and $b_2$ can be derived from the least squares method that minimizes 
\begin{equation}
\label{least_squares}
    \|\mathbf{y-Xb}\|^2,
\end{equation}
	
where $\mathbf{X}$ is the matrix with columns $\mathbf{1}$ (a vector which has all elements equal to 1), $\mathbf{x}$ and $\mathbf{z}$; $\mathbf{b}$ denotes a vector with the weights $b_0$, $b_1$ and $b_2$ below each other. We use the well known result that the least squares solution for $\mathbf{b}$, that is, the vector $\mathbf{b}$ that minimizes \eqref{least_squares} is given by 
\begin{equation}
\label{min}
    \mathbf{b = (X'X)^{-1}X'y}.
\end{equation}

\underline{Proposition 1}\\
In a regression of $\mathbf{y}$ on $\mathbf{x}$ and $\mathbf{z}$, where $\mathbf{y}$, $\mathbf{x}$ and $\mathbf{z}$, are population data with the properties mentioned above, the estimates for $b_0$, $b_1$ and $b_2$ (as given by $\mathbf{b}$ in \eqref{min}) differ from the population weights $\beta_0$, $\beta_1$, $\beta_2$ by a term related only to $\beta_3$, and specified in \eqref{b_final} below.

\underline{Corollary 1}. In regression solutions for samples, the population based expressions in Proposition 1 give the expected values for the regression weights from the samples.

\underline{Derivation of Proposition 1.}
	The expression for the population data in \eqref{data_pop}, can be written in vectors as 
\begin{equation}
\label{pop_vect}
    \mathbf{y}=\beta_0\mathbf{1}+\beta_1\mathbf{x}+\beta_2\mathbf{z}+ \beta_3\mathbf{xz}+\varepsilon\mathbf{e}
\end{equation}


Writing $\bm{\beta}$ for the vector with weights $\beta_0,\beta_1,\beta_2$, and using that 
\begin{equation}
\label{eq:matrix_X}
    \mathbf{X} = \begin{bmatrix}\mathbf{1}&\mathbf{x}&\mathbf{z}\end{bmatrix},\nonumber
\end{equation}
we can write Equation \eqref{pop_vect} as
\begin{equation}
\label{eq:pop_vect_rwrite}
    \mathbf{y} = \mathbf{X}\bm{\beta} + \beta_3\mathbf{xz} + \varepsilon\mathbf{e}.
\end{equation}

and we can now elaborate the regression estimates  in \eqref{min} as

\begin{align}
\label{min_rewrite}
    \mathbf{b} &= \mathbf{(X'X)}^{-1}\mathbf{X'y} \nonumber \\
    &= \mathbf{(X'X)}^{-1}\mathbf{X'X}\bm{\beta}+ \beta_3\mathbf{(X'X)}^{-1}\mathbf{X'xz} + \varepsilon\mathbf{(X'X)}^{-1}\mathbf{X'e} \nonumber \\
    &= \mathbf{(X'X)}^{-1}\mathbf{X'X}\bm{\beta} + \beta_3\mathbf{(X'X)}^{-1}\mathbf{X'xz},
\end{align}


where we use that $\mathbf{X’e=0}$, because $\mathbf{e}$ is a random variable with mean $0$ and is uncorrelated with $\mathbf{x}$ and $\mathbf{z}$. To see this, we should realize that $\mathbf{X'e}$ has elements $\mathbf{1'e}$, $\mathbf{x'e}$ and $\mathbf{z'e}$. The first term, $\mathbf{1'e}$ equals the sum of all values in $\mathbf{e}$ which is 0 because $\mathbf{e}$ contains infinitely many values from a $\mathcal{N}(0,1$) distribution.  Furthermore, we know that the covariance between $\mathbf{x}$ and $\mathbf{e}$ is 0, so $(\mathbf{x}-m_x\mathbf{1})'\mathbf{e}=0$. It follows that $\mathbf{x'e}-m_x\mathbf{1}'\mathbf{e} =0$, hence $\mathbf{x'e}=m_x\mathbf{1}'\mathbf{e}=0$, where the second equality uses that $\mathbf{1}'\mathbf{e}=0$, as shown just above. Analogously, it can be proven that $\mathbf{z}'\mathbf{e}=0$.
Clearly, the first term in \eqref{min_rewrite} can be simplified, because $\mathbf{(X'X)^{-1}X'X=I}$ , where $\mathbf{I}$ is the identity matrix, and we end up with 
\begin{equation}
\label{b_simplified_M}
    \mathbf{b}  = \bm{\beta} + \beta_3 \mathbf{(X'X)^{-1}X'xz}.
\end{equation}


Noting that $\mathbf{w=(X'X)^{-1}X'xz}$ does not depend on the population weights $\beta_0$, $\beta_1$, and $\beta_2$, we have the simple expression
\begin{equation}
\label{b_simple}
    \mathbf{b}= \bm{\beta}+\beta_3\mathbf{w}
\end{equation}

or elementwise

\begin{subequations}\label{b_simple_elementwise}
\begin{align}
    b_0 &= \beta_0+\beta_3 w_0 \label{eq:b_simple_a}\\
    b_1 &= \beta_1+\beta_3 w_1 \label{eq:b_simple_b}\\
    b_2 &= \beta_2+\beta_3 w_2\label{eq:b_simple_c}
\end{align}
\end{subequations}

We now wish to find explicit expressions for the elements of $w$, without the matrix expression. For this we will use that, because $XZ = (X_c+m_x)(Z_c+m_z) = X_cZ_c + m_xZ_c + m_zX_c + m_xm_z$, hence for its vector version we have

\begin{equation}
\label{xz}
    \mathbf{xz} = \mathbf{x_cz_c}+m_x\mathbf{z_c}+m_z\mathbf{x_c}+m_xm_z\mathbf{1},
\end{equation}
where $\mathbf{x_cz_c}$ denotes the vector version of $X_cZ_c$, and we will use that the vector $\mathbf{w}=\mathbf{(X’X)}^{-1}\mathbf{X’xz}$ actually can be obtained by the regression of $XZ$ on $X$and $Z$, as follows from \eqref{least_squares} and \eqref{min} by simply replacing $\mathbf{y}$ by $\mathbf{xz}$, and $\mathbf{b}$ by $\mathbf{w}$. Thus $\mathbf{w}$ minimizes

\begin{equation}
    \label{w_min}
    f(\mathbf{w})=\|\mathbf{xz-Xw}\|^2=\| \mathbf{x_cz_c}+m_x\mathbf{z_c}+m_z\mathbf{x_c}+m_xm_z\mathbf{1}-w_0\mathbf{1}-w_1\mathbf{x}-w_2\mathbf{z}\|^2
\end{equation}

We now use that the mean of $\mathbf{x_cz_c}$ equals $r_{xz}$, the correlation between $X$ and $Z$. This is because the covariance between $X$ and $Z$ is simply given by the mean of the products of $X_c$ and $Z_c$, and the correlation is given by the covariance divided by the standard deviations of $X$ and $Z$, which are both 1 in this population.  Thus, we can write $\mathbf{x_cz_c} = \mathbf{[x_cz_c]_c} + r_{xz}\mathbf{1}$, where $\mathbf{[x_cz_c]_c}$ denotes the centered version of $\mathbf{x_cz_c}$. Also we use $\mathbf{x} = \mathbf{x_c} + m_x\mathbf{1}$ and $\mathbf{z} = \mathbf{z_c} + m_z\mathbf{1}$. Using these expressions in \eqref{w_min}, we get

\begin{align}
    f(\mathbf{w})&=\|\mathbf{[x_cz_c]}_c+r_{xz}\mathbf{1}+m_x\mathbf{z_c}+m_z\mathbf{x_c}+m_xm_z\mathbf{1}-w_0\mathbf{1}-w_1(\mathbf{x_c}+m_x\mathbf{1})-w_2(\mathbf{z_c}+m_z\mathbf{1})\|^2\nonumber\\
    &=\|r_{xz}\mathbf{1}+m_xm_z\mathbf{1}-w_1m_x\mathbf{1}-w_2m_z\mathbf{1}-w_0\mathbf{1}+m_z\mathbf{x_c}-w_1\mathbf{x_c}+m_x\mathbf{z_c}-w_2\mathbf{z_c}+\mathbf{[x_cz_c]}_c\|^2\nonumber\\
    \label{fw}
    &=\|(r_{xz}+m_xm_z-w_1m_x-w_2m_z-w_0)\mathbf{1}+(m_z-w_1)\mathbf{x_c}+(m_x-w_2)\mathbf{z_c}+\mathbf{[x_cz_c]}_c\|^2.
\end{align}

 Now we use that the mutual inner products between $\mathbf{1}$,  $(m_z-w_1)\mathbf{x_c} + (m_x-w_2)\mathbf{z_c}$ and $\mathbf{[x_cz_c]}_c$ are 0. That is, $\mathbf{1'}[(m_z-w_1)\mathbf{x_c} + (m_x-w_2)\mathbf{z_c}]$ and $\mathbf{1'[x_cz_c]}_c$ are 0 because the means of centered vectors are 0.  The inner product $[(m_z-w_1)\mathbf{x_c} + (m_x-w_2)\mathbf{z_c}]'[\mathbf{x_cz_c}]_c=0$, because $\mathbf{x_c'[x_cz_c]}_c=0$ and $\mathbf{z_c'[x_cz_c]_c}=0$. This is because  $\mathbf{x_c'[x_cz_c]_c}$ is the sum of values $x_i^2z_i$ where $x_i$ and $z_i$ both denote infinitely many values drawn randomly from the standard normal distribution. Since this distribution is symmetric, the sum of values $x_i^2z_i$ with $z_i$ positive will be cancelled by the sum of values $x_i^2z_i$ with $z_i$ negative, and hence the overall sum $\mathbf{x_c'[x_cz_c]}_c=0$; analogously, $\mathbf{z_c'[x_cz_c]}_c=0$  as well.  This implies that we can ‘split the squares’ in \eqref{fw} as follows:
\begin{equation}
    \label{fw_split}
    f(\mathbf{w})=\|(r_{xz}+m_xm_z-w_1m_x-w_2m_z-w_0)\mathbf{1}\|^2+\|(m_z-w_1)\mathbf{x_c}+(m_x-w_2)\mathbf{z_c}\|^2+\|\mathbf{[x_cz_c]}_c\|^2
\end{equation}
The minimum of this function over $\mathbf{w}$ is given by
\begin{subequations}
\begin{align}
    w_0 &= r_{xz}+m_xm_z-w_1m_x-w_2m_z\label{w_0} \\
    w_1 &= m_z \label{w_1}\\
    w_2 &= m_x\label{w_2}
\end{align}
\end{subequations}

Substitutung \eqref{w_1} and \eqref{w_2} in \eqref{w_0} we obtain $w_0=r_{xz}+m_xm_z-m_zm_x-m_xm_z=r_{xz}-m_zm_x$, and using this with \eqref{w_1} and \eqref{w_2} in \eqref{b_simple} we obtain
\begin{subequations}
\label{b_final}
\begin{align}
    b_0 &= \beta_0+\beta_3(r_{xz}-m_zm_x) \\
    b_1 &= \beta_1+\beta_3m_z\\
    b_2 &= \beta_2+\beta_3m_x
\end{align}
\end{subequations}

\underline{Corollary 1.}
Because least squares regression weights are unbiased estimates of the population estimates, the expected values of sample estimates should equal the population estimates. The population estimates for regression of $Y$ on $X$ and $Z$ is given by \eqref{b_final} and should hence also give the expected value for the sample estimates. 



\end{document}