\documentclass[11pt]{article}

\usepackage{../Math545}

\usepackage{amsmath,amssymb}

\pagestyle{plain}

\renewcommand{\labelenumi}{\arabic{enumi}.}
\renewcommand{\labelenumii}{\arabic{enumi}.\arabic{enumii}}
\renewcommand{\labelenumiii}{\arabic{enumi}.\arabic{enumii}.\arabic{enumiii}}

\setlength{\topmargin}{-0.75 in} \setlength{\textheight}{9.5 in}

\parindent0in

\newenvironment{packed_enum}{
\begin{enumerate}
  \setlength{\itemsep}{12pt}
  \setlength{\parskip}{0pt}
  \setlength{\parsep}{0pt}
  \setlength{\topsep}{0pt}
}{\end{enumerate}}


\begin{document}

<<setup, cache=FALSE, echo=FALSE>>=
library(knitr)
# global chunk options
opts_chunk$set(cache=TRUE, autodep=TRUE)
options(scipen=999)
options(repos=c(CRAN="https://cloud.r-project.org/"))
@

\begin{center}
\textsclarge{MATH 545: Introduction to Time Series Analysis}

\smallskip
\textsc{The $\AR(2)$ process}
\end{center}

Consider the stationary, causal $\AR(2)$ process defined in terms of white noise process $\{Z_t\}$ via
\[
X_t = \phi_1 X_{t-1} + \phi_2 X_{t-2} + Z_t
\]
or equivalently
\[
(1-\alpha_1 \B) (1-\alpha_2 \B) X_t = (1-(\alpha_1+\alpha_2) \B + \alpha_1 \alpha_2 \B^2) = Z_t
\]
for reciprocal roots $\alpha_1,\alpha_2$, so that
\begin{equation}
\alpha_1+\alpha_2 = \phi_1 \qquad \alpha_1 \alpha_2 = -\phi_2. \label{eq:rroots}
\end{equation}
We have by the usual calculation method (multiplying by $X_{t-h}$, taking expectations) that
\begin{align}
h=0 \qquad & \gamma_X(0) = \phi_1 \gamma_X(1) + \phi_2 \gamma_X(2) + \sigma_Z^2 & \label{eq:h0}\\[6pt]
h=1 \qquad & \gamma_X(1) = \phi_1 \gamma_X(0) + \phi_2 \gamma_X(1) & \label{eq:h1} \\[6pt]
h=2 \qquad & \gamma_X(2) = \phi_1 \gamma_X(1) + \phi_2 \gamma_X(0) & \label{eq:h2} \\[6pt]
h\geq 3 \qquad & \gamma_X(h) =  \phi_1 \gamma_X(h-1) + \phi_2 \gamma_X(h-2) & \label{eq:h3}
\end{align}
We solve this system for $\gamma_X(0),\gamma_X(1),\gamma_X(2)$ by direct algebraic manipulation.  From \eqref{eq:h1} and thus from \eqref{eq:h2} we have
\[
\gamma_X(1) = \frac{\phi_1}{1-\phi_2} \gamma_X(0) \qquad \therefore \qquad \gamma_X(2) = \frac{\phi_1^2 + (1-\phi_2) \phi_2}{1-\phi_2} \gamma_X(0)
\]
so from \eqref{eq:h0}
\[
\gamma_X(0) =  \frac{(1-\phi_2)}{1- \phi_2 - \phi_1^2 - \phi_2(\phi_1^2 + (1-\phi_2) \phi_2)} \sigma_Z^2 .
\]
Subsequently we have for $h \geq 3$ the linear recurrence relation given by \eqref{eq:h3}, that is
\[
\begin{bmatrix}
\gamma_X(h) \\[3pt] \gamma_X(h-1)
\end{bmatrix} =
\begin{bmatrix}
\phi_1 & \phi_2 \\[3pt]
1 & 0
\end{bmatrix} \begin{bmatrix}
\gamma_X(h-1) \\[3pt] \gamma_X(h-2)
\end{bmatrix} = \begin{bmatrix}
\phi_1 & \phi_2 \\[3pt]
1 & 0
\end{bmatrix}^{2} \begin{bmatrix}
\gamma_X(h-2) \\[3pt] \gamma_X(h-3)
\end{bmatrix} = \cdots = \begin{bmatrix}
\phi_1 & \phi_2 \\[3pt]
1 & 0
\end{bmatrix}^{h-2} \begin{bmatrix}
\gamma_X(2) \\[3pt] \gamma_X(1)
\end{bmatrix}
\]

\medskip

To solve the recursion, we diagonalize the matrix using an eigen decomposition, and write
\[
\begin{bmatrix}
\phi_1 & \phi_2 \\[3pt]
1 & 0
\end{bmatrix} =  \begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}  \begin{bmatrix}
\lambda_1 & 0 \\[3pt]
0 & \lambda_2
\end{bmatrix} \begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}^{-1}.
\]
For the eigenvalues, we have the characteristic equation
\[
-\lambda (\phi_1 - \lambda) - \phi_2 = 0 \quad \text{that is} \quad \lambda^2 - \lambda \phi_1 - \phi_2 = 0.
\]
Hence we have
\[
\lambda_1 = \frac{\phi_1 + \sqrt{\phi_1^2 + 4 \phi_2}}{2} \qquad \lambda_2 = \frac{\phi_1 - \sqrt{\phi_1^2 + 4 \phi_2}}{2}
\]
and therefore
\[
\lambda_1 + \lambda_2 = \phi_1  \qquad \lambda_1 \lambda_2 = -\phi_2.
\]
We can therefore conclude by inspecting equation \eqref{eq:rroots} that $(\lambda_1,\lambda_2) \equiv (\alpha_1,\alpha_2)$.

\medskip
From the diagonalization, we have that
\[
\begin{bmatrix}
\phi_1 & \phi_2 \\[3pt]
1 & 0
\end{bmatrix}^{h-2} =  \begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}  \begin{bmatrix}
\lambda_1 & 0 \\[3pt]
0 & \lambda_2
\end{bmatrix}^{h-2} \begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}^{-1}
\]
and hence
\begin{eqnarray*}
\begin{bmatrix}
\gamma_X(h) \\[3pt] \gamma_X(h-1)
\end{bmatrix} & = &
\begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}  \begin{bmatrix}
\lambda_1 & 0 \\[3pt]
0 & \lambda_2
\end{bmatrix}^{h-2} \begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}^{-1}
 \begin{bmatrix}
\gamma_X(2) \\[3pt] \gamma_X(1)
\end{bmatrix} \\[6pt]
& = &
\begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}  \begin{bmatrix}
\lambda_1^{h-2} & 0 \\[3pt]
0 & \lambda_2^{h-2}
\end{bmatrix}
\frac{1}{v_{11} v_{22} - v_{12} v_{21}}
\begin{bmatrix}
v_{22} \gamma_X(2) - v_{12} \gamma_X(1) \\[3pt] v_{11} \gamma_X(1) - v_{21} \gamma_X(2)
\end{bmatrix} \\[6pt]
& = &
\begin{bmatrix}
v_{11} & v_{12} \\[3pt]
v_{21} & v_{22}
\end{bmatrix}
\frac{1}{v_{11} v_{22} - v_{12} v_{21}}
\begin{bmatrix}
(v_{22} \gamma_X(2) - v_{12} \gamma_X(1))\lambda_1^{h-2} \\[3pt] (v_{11} \gamma_X(1) - v_{21} \gamma_X(2))\lambda_2^{h-2}
\end{bmatrix} \\[6pt]
\end{eqnarray*}
so therefore for $h \geq 3$
\[
\gamma_X(h) = \frac{1}{v_{11} v_{22} - v_{12} v_{21}} \left[v_{11} (v_{22} \gamma_X(2) - v_{12} \gamma_X(1))\lambda_1^{h-2} + v_{12} (v_{11} \gamma_X(1) - v_{21} \gamma_X(2)) \lambda_2^{h-2}  \right]
\]
which can be re-written
\[
\gamma_X(h) = c_1 \alpha_1^{h-2} + c_2 \alpha_2^{h-2}
\]
as $\alpha_1 = \lambda_1$ and $\alpha_2 = \lambda_2$, where
\[
c_1 = \frac{v_{11} (v_{22} \gamma_X(2) - v_{12} \gamma_X(1))}{v_{11} v_{22} - v_{12} v_{21}} \qquad c_2 = \frac{v_{12} (v_{11} \gamma_X(1) - v_{21} \gamma_X(2))}{v_{11} v_{22} - v_{12} v_{21}}.
\]
To compute $v_{11},v_{12},v_{21},v_{22}$ we need the eigenvectors; by a standard result
\[
\begin{bmatrix}
  a & b\\[3pt]
  c & d
\end{bmatrix} \qquad \text{has eigenvectors} \qquad \begin{bmatrix}
  b \\[3pt]
  \lambda_1-a
\end{bmatrix} \quad \text{and} \quad \begin{bmatrix}
  \lambda_2 - d \\[3pt]
 c
\end{bmatrix}
\]
so in our case these are proportional to 
\[
\begin{bmatrix}
  \phi_2 \\[3pt]
  \lambda_1 - \phi_1
\end{bmatrix} \qquad \text{and} \qquad \begin{bmatrix}
  \lambda_2 \\[3pt]
  1
\end{bmatrix}.
\]
We then normalize these vectors to give
\[
v_{11} = \frac{\phi_2}{\sqrt{\phi_2^2+(\lambda_1-\phi_1)^2}} \qquad v_{21} = \frac{(\lambda_1-\phi_1)}{\sqrt{\phi_2^2+(\lambda_1-\phi_1)^2}} \qquad v_{12} = \frac{\lambda_2}{\sqrt{\lambda_2^2 + 1}} \qquad v_{22} = \frac{1}{\sqrt{\lambda_2^2 + 1}}
\]
We therefore have an explicit formula for $\gamma_X(h)$, $h=0,1,2,\ldots$.
\bigskip

\textbf{Note:} if $\alpha_1 = \alpha_2 = \alpha$, and we have repeated reciprocal roots, then the eigenvectors are identical.  In this case, the general solution to \eqref{eq:h3} for $h \geq 3$ takes the form
\[
\gamma_X(h) = (c_1 + c_2 (h-2)) \alpha^{h-2}
\]
from which we deduce
\begin{align*}
\gamma_X(3) & = (c_1 + c_2) \alpha \equiv \phi_1 \gamma_X(2) + \phi_2 \gamma_X(1) \\[6pt]
\gamma_X(4) & = (c_1 + 2 c_2) \alpha^2  \equiv \phi_1 \gamma_X(3) + \phi_2 \gamma_X(2) = (\phi_1^2 + \phi_2) \gamma_X(2) + \phi_1 \phi_2 \gamma_X(1)
\end{align*}
which are two equations in the two unknown constants which can be used to identify $c_1$ and $c_2$, that is
\[
\begin{bmatrix} 
\alpha & \alpha \\[3pt]
\alpha^2 & 2 \alpha^2
\end{bmatrix} \begin{bmatrix} c_1 \\[3pt] c_2 \end{bmatrix} = \begin{bmatrix} \phi_1 \gamma_X(2) + \phi_2 \gamma_X(1) \\[3pt] (\phi_1^2 + \phi_2) \gamma_X(2) + \phi_1 \phi_2 \gamma_X(1) \end{bmatrix}.
\]

\bigskip

\textbf{EXAMPLE}: Consider the $\AR(2)$ process
\[
X_t = 1.2 X_{t-1} - 0.35 X_{t-2} + Z_t.
\]
Then
\[
\Phi(z) = (1-1.2z+0.35z^2) = (1-0.7z)(1-0.5z)
\]
so $\alpha_1 = 0.7$, $\alpha_2 = 0.5$.

<<AR2a,comment='+', fig.width=7.25, fig.height=4.1, fig.align='center'>>=
phi1<-1.2;phi2<--0.35
gam0<-(1-phi2)/(1-phi2-phi1^2-phi2*(phi1^2+(1-phi2)*phi2))
gam1<-phi1/(1-phi2)*gam0
gam2<-((phi1^2+(1-phi2)*phi2)/(1-phi2))*gam0

A<-matrix(c(phi1,1,phi2,0),2,2);eigen(A)

al1<-eigen(A)$values[1];al2<-eigen(A)$values[2]
v1<-c(A[1,2],al1-A[1,1]);v1<-v1/sqrt(sum(v1^2))
v2<-c(al2-A[2,2],A[2,1]);v2<-v2/sqrt(sum(v2^2))
cbind(v1,v2)

v11<-v1[1];v21<-v1[2]
v12<-v2[1];v22<-v2[2]

P<-matrix(c(v11,v21,v12,v22),2,2)
Pinv<-solve(P)
P %*% diag(c(al1,al2)) %*% Pinv

c1<-v11*(v22*gam2-v12*gam1)/(v11*v22-v12*v21)
c2<-v12*(v11*gam1-v21*gam2)/(v11*v22-v12*v21)

gammaX<-rep(0,11)
gammaX[(3+1):(10+1)]<-c1*al1^(c(3:10)-2)+c2*al2^(c(3:10)-2)
gammaX[1:3]<-c(gam0,gam1,gam2)

rhoX<-gammaX/gammaX[1];rhoX
true.rhoX<-ARMAacf(ar=c(phi1,phi2),lag.max=10);as.numeric(true.rhoX)


@

\textbf{EXAMPLE}: Consider the $\AR(2)$ process
\[
X_t = X_{t-1} - 0.25 X_{t-2} + Z_t.
\]
Then
\[
\Phi(z) = (1-z+0.25z^2) = (1-0.5z)(1-0.5z)
\]
so $\alpha_1 = \alpha_2 = 0.5$.

<<AR2b,comment='+', fig.width=7.25, fig.height=4.1, fig.align='center'>>=
phi1<-1.0;phi2<--0.25
gam0<-(1-phi2)/(1-phi2-phi1^2-phi2*(phi1^2+(1-phi2)*phi2))
gam1<-phi1/(1-phi2)*gam0
gam2<-((phi1^2+(1-phi2)*phi2)/(1-phi2))*gam0

A<-matrix(c(phi1,1,phi2,0),2,2);eigen(A)

al1<-eigen(A)$values[1];al2<-eigen(A)$values[2]
v1<-c(A[1,2],al1-A[1,1]);v1<-v1/sqrt(sum(v1^2))
v2<-c(al2-A[2,2],A[2,1]);v2<-v2/sqrt(sum(v2^2))
cbind(v1,v2)

A<-matrix(c(al1,al1^2,al1,2*al1^2),2,2)
bvec<-c(phi1*gam2+phi2*gam1,(phi1^2+phi2)*gam2 + phi1*phi2*gam1)
cvec<-solve(A,bvec)
cvec

c1<-cvec[1]
c2<-cvec[2]

gammaX<-rep(0,11)
gammaX[1:3]<-c(gam0,gam1,gam2)
gammaX[(3+1):(10+1)]<-(c1+c2*(c(3:10)-2))*al1^(c(3:10)-2)

rhoX<-gammaX/gammaX[1];rhoX
true.rhoX<-ARMAacf(ar=c(phi1,phi2),lag.max=10);as.numeric(true.rhoX)


@
\end{document}