\documentclass[notitlepage]{article} \usepackage{../../Math545} \usepackage{listings} \usepackage{numprint} \usepackage{enumerate} \usepackage{bbm} \usepackage{bm} \usepackage{amsfonts,amsmath} \setlength{\parindent}{0in} \def\Var{\text{Var}} \def\Cov{\text{Cov}} \def\Corr{\text{Corr}} \lstloadlanguages{R} \definecolor{keywordcolor}{rgb}{0,0.6,0.6} \definecolor{delimcolor}{rgb}{0.461,0.039,0.102} \definecolor{Rcommentcolor}{rgb}{0.101,0.043,0.432} \lstdefinestyle{Rsettings}{ basicstyle=\ttfamily, breaklines=true, showstringspaces=false, keywords={if, else, function, theFunction, tmp}, % Write as many keywords otherkeywords={}, commentstyle=\itshape\color{Rcommentcolor}, keywordstyle=\color{keywordcolor}, moredelim=[s][\color{delimcolor}]{"}{"}, } \lstset{basicstyle=\ttfamily, numbers=none, literate={~} {$\sim$}{2}} \def\ctitle{MATH 545: Introduction to Time Series Analysis} \def\ttitle{Winter 2025} \def\stitle{Recurrence Relation Methods for Autoregressive Processes} \begin{document} \maketitle Given an autoregressive process equation which defines the process $\{X_t\}$ implicitly in terms of $\{Z_t\} \sim \WN(0,\sigma_Z^2)$, we typically need to compute an explicit representation. Suppose that \[ X_t = \phi_1 X_{t-1} + \phi_2 X_{t-2} + \cdots + \phi_p X_{t-p} + Z_t \] and we want to obtain $X_t$ explicitly in terms of $\{Z_t\}$, say in the form \[ X_t = \sum_{j=0}^\infty \psi_j Z_{t-j} \] for some constants $\psi_j, j \geq 0$, such that the summation is convergent in mean square (as for the $\AR(1)$ process). \medskip We may use the recursion method to compute such a representation: to begin, we substitute in the equivalent expression for $X_{t-1}$ \begin{align} X_t & = \phi_1 X_{t-1} + \phi_2 X_{t-2} + \cdots + \phi_p X_{t-p} + Z_t\nonumber\\[6pt] & = \phi_1 (\phi_1 X_{t-2} + \phi_2 X_{t-3} + \cdots + \phi_p X_{t-p-1} + Z_{t-1}) + \phi_2 X_{t-2} + \cdots + \phi_p X_{t-p} + Z_t \nonumber \\[6pt] & = \beta_1^{(2)} X_{t-2} + \beta_2^{(2)} X_{t-3} + \cdots + \beta_p^{(2)} X_{t-p-1} + \sum_{j=0}^1 \psi_j Z_{t-j} \label{eq:step1} \end{align} say, where \begin{align*} \beta_1^{(2)} & = \phi_1^2 + \phi_2 \\[6pt] \beta_2^{(2)} & = \phi_1 \phi_2 + \phi_3 \\[6pt] \beta_3^{(2)} & = \phi_1 \phi_3 + \phi_4 \\[6pt] \vdots & \qquad \vdots\\ \beta_{p-1}^{(2)} & = \phi_1 \phi_{p-1} + \phi_p \\[6pt] \beta_p^{(2)} & = \phi_1 \phi_p \end{align*} and $\psi_0 = 1, \psi_1 = \phi_1$. Let $\bmbeta^{(2)} = (\beta_1^{(2)},\beta_2^{(2)},\ldots,\beta_p^{(2)})^\top$ be the vector of coefficients. Notice that, if we define the vector $\bmbeta^{(1)} = (\phi_1,\phi_2,\ldots,\phi_p)^\top$, we can see that in fact \begin{align*} \beta_1^{(2)} & = \phi_1 \beta_1^{(1)} + \beta_2^{(1)} \\[6pt] \beta_2^{(2)} & = \phi_2 \beta_1^{(1)} + \beta_3^{(1)} \\[6pt] \beta_3^{(2)} & = \phi_3 \beta_1^{(1)} + \beta_4^{(1)} \end{align*} and so on, until finally \[ \beta_p^{(2)} = \phi_p \beta_1^{(1)}. \] Therefore we may write \[ \bmbeta^{(2)} = \bA \bmbeta^{(1)} \] for $(p \times p)$ matrix $\bA$, as the relationship between the vectors of coefficients is linear. In fact, we have that \[ \bA = \left[ \begin{array}{cccccc} \phi_1 & 1 & 0 & 0 & \cdots & 0 \\[4pt] \phi_2 & 0 & 1 & 0 & \cdots & 0 \\[4pt] \phi_3 & 0 & 0 & 1 & \cdots &0 \\[4pt] \vdots & \vdots & \vdots & \ddots & \ddots & \vdots \\ \phi_{p-1} & 0 & 0 & 0 & \ddots & 1\\[4pt] \phi_p & 0 & 0 & 0 & \cdots & 0 \end{array} \right] \] We then proceed by applying the same recursive calculation as in \eqref{eq:step1} to obtain \begin{align} X_t & = \beta_1^{(3)} X_{t-3} + \beta_2^{(3)} X_{t-4} + \cdots + \beta_p^{(3)} X_{t-p-2} + \sum_{j=0}^2 \psi_j Z_{t-j} \label{eq:step1} \end{align} where we have an identical recursive relationship \begin{align*} \beta_1^{(3)} & = \phi_1 \beta_1^{(2)} + \beta_2^{(2)} \\[6pt] \beta_2^{(3)} & = \phi_2 \beta_1^{(2)} + \beta_3^{(2)} \\[6pt] \beta_3^{(3)} & = \phi_3 \beta_1^{(2)} + \beta_4^{(2)} \\[6pt] \vdots & \qquad \vdots\\ \beta_{p-1}^{(3)} & = \phi_{p-1} \beta_1^{(2)} + \beta_p^{(2)} \\[6pt] \beta_p^{(3)} & = \phi_{p} \beta_1^{(2)} \end{align*} and with $\psi_2 = \beta_1^{(2)}$. Therefore, we have a recursive relationship \[ \bmbeta^{(3)} =\bA \bmbeta^{(2)} = \bA^2 \bmbeta^{(1)} \] that is given by the linear recurrence relation for $r \geq 2$ \[ \bmbeta^{(r)} = \bA \bmbeta^{(r-1)} \] with $\psi_j = \beta_1^{(j)}$. By repeatedly applying the recursion, we obtain \[ \bmbeta^{(r)} = \bA \bmbeta^{(r-1)} = \bA^2 \bmbeta^{(r-2)} = \cdots = \bA^{r-1} \bmbeta^{(1)}. \] which defines the coefficients in the representation \begin{align} X_t & = \beta_1^{(r)} X_{t-r} + \beta_2^{(r)} X_{t-r-1} + \cdots + \beta_p^{(r)} X_{t-p-r} + \sum_{j=0}^{r-1} \psi_j Z_{t-j} \label{eq:step1} \end{align} Now, as in the $\AR(1)$ case, in order for the representation \[ X_t = \sum_{j=0}^\infty \psi_j Z_{t-j} \] to hold, we need the dependence on the $X_{t-r}, X_{t-r-1},\ldots,X_{t-r-p}$ terms to vanish as $r \longrightarrow \infty$. To establish this, we examine the recursion for the coefficients, and recognize that the components of the vector $\bmbeta^{(r)}$ converge to zero if and only if the matrix $\bA^r$ converges to the zero matrix as $r \longrightarrow \infty$. We can check this by diagonalizing $\bA$, that is, writing \[ \bA = \bV \bD \bV^{-1} \] where $\bD$ a diagonal matrix with diagonal elements equal to the eigenvalues of $\bA$, and $\bV$ is the matrix whose columns are the corresponding eigenvectors. If all the eigenvalues are less than 1 in absolute value, then we have that \[ \bA^r = \bV \bD^r \bV^{-1} \longrightarrow \mathbf{0}_{p,p}. \] Once we have computed \[ \bmbeta^{(r)} = \bA^{r-1} \bmbeta^{(1)} \] for each $r$, we can generate the $\psi_j$ values by recalling that $\psi_j = \beta_1^{(j)}$. \bigskip \textbf{Example:} For $p=3$, consider the process \[ X_t = X_{t-1} - 0.19 X_{t-2} - 0.03 X_{t-3} + Z_t \] so that $(\phi_1,\phi_2,\phi_3) = (1,-0.19,-0.03)$. Here the matrix $\bA$ is \[ \bA = \left[ \begin{array}{rcc} 1.00 & 1 & 0\\[4pt] -0.19 & 0 & 1\\[4pt] -0.03 & 0 & 0\\[4pt] \end{array} \right] \] We can check the eigen decomposition in \texttt{R} <>= A<-matrix(c(1,-0.19,-0.03,1,0,0,0,1,0),3,3) eigen(A) @ indicating that the eigen values are 0.6, 0.5 and -0.1. Thus, $\bA^r$ should converge to the zero matrix as $r \longrightarrow \infty$; we can check that also <>= A.r<-diag(c(1,1,1)) #Identity matrix for(r in 1:100){A.r<-A.r %*% A} #A.r stores the A^r matrix A.r @ This demonstrates that the process can be represented \[ X_t = \sum_{j=0}^\infty \psi_j Z_{t-j} \] and that this process is well-defined (in mean-square terms). To compute the $\psi$ values, we may use the recursive formula. We perform the calculation out to 1000 steps: <>= nlags<-1000; psi.vec<-numeric(nlags) psi0<-1 #psi_0 = 1 psi.vec[1]<-1 #psi_1 = 1 alpha.mat<-matrix(0,ncol=nlags,nrow=3) alpha.mat[,1]<-c(1,-0.19,-0.03) #initialize the recursion for(r in 2:nlags){ alpha.mat[,r]<-A %*% alpha.mat[,r-1] #apply the recursive formula psi.vec[r]<-alpha.mat[1,r] #psi_r, r > 1 } psi.vec[1:12] #The first 12 psi values starting at psi_1 ARMAtoMA(ar = c(1,-0.19,-0.03),lag.max=12) #Check psi.vec[nlags] #psi_{999} value @ Once we have the representation in terms of $\psi$ values, we may compute from first principles the autocovariances. Recall that for the zero mean process, by standard calculations \begin{align*} \Cov[X_t,X_{t+h}] = \E[X_{t} X_{t+h}] = \E \left[\left(\sum_{j=0}^\infty \psi_j Z_{t-j} \right) \left(\sum_{k=0}^\infty \psi_k Z_{t-k} \right) \right] & = \sum_{j=0}^\infty \sum_{k=0}^\infty \psi_j \psi_k \E[Z_{t-j} Z_{t+h-k}]\\[6pt] & = \sigma_Z^2 \sum_{j=0}^\infty \psi_j \psi_{j+h} \end{align*} where, as usual $\E[Z_{t-j} Z_{t+h-k}] = \sigma_Z^2$ if $k=j+h$, and is zero otherwise. Truncating the infinite sum by assuming that $\psi_j = 0$ for $j \geq \Sexpr{nlags}$ yields a computational formula for the covariances. <>= acvs<-numeric(nlags) psi0.vec<-c(psi0,psi.vec) acvs[1]<-sum(psi0.vec^2) #h=0 for(i in 2:nlags){ acvs[i]<-sum(psi0.vec[1:(nlags-i+1)]*psi0.vec[i:nlags]) #h=i } acfs<-acvs/acvs[1] acvs[1:10] #The first 10 acvs values acfs[1:10] #The first 10 acfs values @ \end{document}