\documentclass[notitlepage]{article} \usepackage{Math} %\usepackage{../../Math598} \usepackage{listings} \usepackage{numprint} \usepackage{enumerate} \usepackage{bm} \usepackage{bbm} \usepackage{verbatim} \usepackage{mathtools} \usepackage{dsfont} \usepackage{scalerel} \usepackage{amsfonts,amsmath,amssymb} \usepackage{tikz} \usetikzlibrary{shapes,decorations,arrows,calc,arrows.meta,fit,positioning} \usepackage{pgfplots} \pgfplotsset{compat=1.17} \newcommand*\circled[1]{\tikz[baseline=(char.base)]{ \node[shape=circle,draw,inner sep=2pt] (char) {#1};}} \usepackage{xcolor} \newtheorem{alg}{Algorithm} \newtheorem{ex}{Example} \newtheorem{note}{Note} \newenvironment{proof}{\par\noindent{\normalfont{\bf{Proof }}}} {\hfill \small$\blacksquare$\\[2mm]} \setlength{\parindent}{0in} \def\E{\mathbbmss{E}} \def\Var{\text{Var}} \def\Cov{\text{Cov}} \def\Corr{\text{Corr}} \def\bW{\mathbf{W}} \def\bH{\mathbf{H}} \newcommand{\bs}[1]{\bm{#1}} \newcommand{\Rho}{\mathrm{P}} \newcommand{\cif}{\text{ if }} \newcommand{\Ra}{\Longrightarrow} \def\bphi{\boldsymbol \phi} \lstloadlanguages{R} \definecolor{keywordcolor}{rgb}{0,0.6,0.6} \definecolor{delimcolor}{rgb}{0.461,0.039,0.102} \definecolor{Rcommentcolor}{rgb}{0.101,0.043,0.432} \lstdefinestyle{Rsettings}{ basicstyle=\ttfamily, breaklines=true, showstringspaces=false, keywords={if, else, function, theFunction, tmp}, % Write as many keywords otherkeywords={}, commentstyle=\itshape\color{Rcommentcolor}, keywordstyle=\color{keywordcolor}, moredelim=[s][\color{delimcolor}]{"}{"}, } \lstset{basicstyle=\ttfamily, numbers=none, literate={~} {$\sim$}{2}} \def\Xb{\mathbf{X}_{\beta}} \def\Xp{\mathbf{X}_{\psi}} \def\beps{\boldsymbol{\epsilon}} \def\betahat{\widehat \beta} \def\psihat{\widehat \psi} \begin{document} <>= library(knitr) # global chunk options opts_chunk$set(cache=TRUE, autodep=TRUE) options(scipen=6) options(repos=c(CRAN="https://cloud.r-project.org/")) knit_hooks$set(document = function(x) {sub('\\usepackage[]{color}', '\\usepackage{xcolor}', x, fixed = TRUE)}) @ \begin{center} {\textsclarge{The Geometry of Least Squares}} \end{center} \tikzset{ -Latex,auto,node distance =1 cm and 1 cm,semithick, state/.style ={circle, draw, minimum width = 0.7 cm}, box/.style ={rectangle, draw, minimum width = 0.7 cm, fill=lightgray}, point/.style = {circle, draw, inner sep=0.08cm,fill,node contents={}}, bidirected/.style={Latex-Latex,dashed}, el/.style = {inner sep=3pt, align=left, sloped} } Suppose \begin{itemize} \item $\by \in \R^3$ \item $\bx_1, \bx_2 \in \R^2$ linearly independent \item Linear subspace $\calX \subset \R^3$ defined by \[ \calX \equiv \{\bx = \beta_1 \bx_1 + \beta_2 \bx_2, \beta_1,\beta_2 \in \R\}. \] \item $\bX$ a $3 \times 2$ matrix \[ \bX = \begin{bmatrix} x_{11} & x_{21} \\[6pt] x_{12} & x_{22} \\[6pt] 0 & 0 \end{bmatrix} \] \item $\widehat \by \in \calX$ is the projection of $\by$ onto $\calX$, and satisfies \[ \widehat \by = \arg \min_{\by^\ast \in \calX} \| \by - \by^\ast \|^2. \] We have that $\by^\ast = \bX \beta$ for some $\beta$ and hence $\widehat \by = \bX \widehat \beta$ where \[ \bX^\top (\by - \bX \widehat \beta) = \zerovec. \] \item It follows that \[ \widehat \beta = (\bX^\top \bX)^{-1} \bX^\top \by \qquad \qquad \widehat \by = \bX (\bX^\top \bX)^{-1} \bX^\top \by = \bH \by. \] \item Can repeat for linear subspaces $\calX_1, \calX_2 \subset \R^3$ \[ \calX_1 \equiv \{\bx : \bx = \beta_1 \bx_1, \beta_1 \in \R\} \qquad \calX_2 \equiv \{\bx : \bx = \beta_2 \bx_2, \beta_2 \in \R\} \] with matrices \[ \bX_1 = \begin{bmatrix} x_{11} \\[6pt] x_{12} \\[6pt] 0 \end{bmatrix} \qquad \bX_2 = \begin{bmatrix} x_{21} \\[6pt] x_{22} \\[6pt] 0 \end{bmatrix} \] and estimates and fitted values \[ \widehat \beta_1 = (\bX_1^\top \bX_1)^{-1} \bX_1^\top \by \qquad \qquad \widehat \by_1 = \bX_1 (\bX_1^\top \bX_1)^{-1} \bX_1^\top \by = \bH_1 \by. \] and \[ \widehat \beta_2 = (\bX_2^\top \bX_2)^{-1} \bX_2^\top \by \qquad \qquad \widehat \by_2 = \bX_2 (\bX_2^\top \bX_2)^{-1} \bX_2^\top \by = \bH_2 \by. \] respectively. \end{itemize} <>= library(plot3D) th<-30 O<-c(0,0,0) Y<-c(2,2.75,3) X1<-c(0.25,1,0) X2<-c(1,0.3,0) X<-cbind(X1,X2) be<-solve(t(X) %*% X) %*% t(X) %*% Y Yhat<-X %*% be X1<-X[,1] Y1hat<-X1 %*% (solve(t(X1) %*% X1) %*% t(X1) %*% Y) X2<-X[,2] Y2hat<-X2 %*% (solve(t(X2) %*% X2) %*% t(X2) %*% Y) eps<-0.25 p0<-cbind(O,Y,Yhat) par(mar=c(2,2,2,2)) polygon3D(p0[1,],p0[2,],p0[3,], border = "red", lwd = 1, alpha = 0.1, theta=30, bty='b2',main = "",xlim=range(-0.5,6),ylim=range(-.5,6),zlim=range(0,5), xlab="",ylab="",zlab=expression(y),axes=FALSE,plot=FALSE) text3D(O[1],O[2]-0.35,O[3],c(expression(O)),add=T) arrows3D(O[1],O[2],O[3],5*X1[1],5*X1[2],X1[3],lty=2,add=T) text3D(5*X1[1]-eps,5*X1[2],X1[3],c(expression(x[1])),add=T) arrows3D(O[1],O[2],O[3],5*X2[1],5*X2[2],X2[3],lty=2,add=T) text3D(5*X2[1],5*X2[2],X2[3],c(expression(x[2])),add=T) points3D(Y[1],Y[2],Y[3],add=T,pch=19,col='red',cex=0.75) #segments3D(O[1],O[2],O[3],Y[1],Y[2],Y[3],add=T,col='red',lwd=2) text3D(Y[1],Y[2],Y[3]+eps,c(expression(y)),add=T) #points3D(Yhat[1],Yhat[2],Yhat[3],add=T,pch=19,col='red') segments3D(Yhat[1],Yhat[2],Yhat[3],Yhat[1],Yhat[2],Yhat[3]+0.2,add=T,col='black',lwd=1) segments3D(0.95*Yhat[1],0.95*Yhat[2],Yhat[3],0.95*Yhat[1],0.95*Yhat[2],Yhat[3]+0.2,add=T,col='black',lwd=1) segments3D(0.95*Yhat[1],0.95*Yhat[2],Yhat[3]+0.2,Yhat[1],Yhat[2],Yhat[3]+0.2,add=T,col='black',lwd=1) segments3D(0.95*Yhat[1],0.95*Yhat[2],Yhat[3],Yhat[1],Yhat[2],Yhat[3],add=T,col='black',lwd=1) text3D(Yhat[1],Yhat[2]+eps,Yhat[3],c(expression(hat(y))),add=T) @ <>= p1<-cbind(O,Y,Y1hat) par(mar=c(2,2,2,2)) polygon3D(p1[1,],p1[2,],p1[3,], border = "red", lwd = 1, alpha = 0.1, theta=30, bty='b2',main = "",xlim=range(-0.5,6),ylim=range(-.5,6),zlim=range(0,5), xlab="",ylab="",zlab=expression(y),axes=FALSE,plot=FALSE) text3D(O[1],O[2]-0.35,O[3],c(expression(O)),add=T) arrows3D(O[1],O[2],O[3],5*X1[1],5*X1[2],X1[3],lty=2,add=T) text3D(5*X1[1]-eps,5*X1[2],X1[3],c(expression(x[1])),add=T) arrows3D(O[1],O[2],O[3],5*X2[1],5*X2[2],X2[3],lty=2,add=T) text3D(5*X2[1],5*X2[2],X2[3],c(expression(x[2])),add=T) points3D(Y[1],Y[2],Y[3],add=T,pch=19,col='red',cex=0.75) #segments3D(O[1],O[2],O[3],Y[1],Y[2],Y[3],add=T,col='red',lwd=2) text3D(Y[1],Y[2],Y[3]+eps,c(expression(y)),add=T) #points3D(Y1hat[1],Y1hat[2],Y1hat[3],add=T,pch=19,col='red') #segments3D(O[1],O[2],O[3],Y1hat[1],Y1hat[2],Y1hat[3],add=T,col='red',lwd=2) #segments3D(Y[1],Y[2],Y[3],Y1hat[1],Y1hat[2],Y1hat[3],add=T,col='red',lwd=2) points3D(Yhat[1],Yhat[2],Yhat[3],add=T,pch=19,col='red',cex=0.75) text3D(Yhat[1],Yhat[2]+eps,Yhat[3],c(expression(hat(y))),add=T) points3D(Y1hat[1],Y1hat[2],Y1hat[3],add=T,pch=19,col='red',cex=0.75) text3D(Y1hat[1]-0.5,Y1hat[2]+0.5,Y1hat[3],c(expression(hat(y)[1])),add=T) d1<-Y-Y1hat segments3D(Y1hat[1],Y1hat[2],Y1hat[3],Y1hat[1]+0.1*d1[1],Y1hat[2]+0.1*d1[2],Y1hat[3]+0.1*d1[3],add=T,col='black',lwd=1) segments3D(0.95*Y1hat[1],0.95*Y1hat[2],0.95*Y1hat[3],Y1hat[1],Y1hat[2],Y1hat[3],add=T,col='black',lwd=1) segments3D(0.95*Y1hat[1],0.95*Y1hat[2],0.95*Y1hat[3],0.95*Y1hat[1]+0.1*d1[1],0.95*Y1hat[2]+0.1*d1[2],0.95*Y1hat[3]+0.1*d1[3],add=T,col='black',lwd=1) segments3D(0.95*Y1hat[1]+0.1*d1[1],0.95*Y1hat[2]+0.1*d1[2],0.95*Y1hat[3]+0.1*d1[3],Y1hat[1]+0.1*d1[1],Y1hat[2]+0.1*d1[2],Y1hat[3]+0.1*d1[3],add=T,col='black',lwd=1) @ <>= p2<-cbind(O,Y,Y2hat) par(mar=c(2,2,2,2)) polygon3D(p2[1,],p2[2,],p2[3,], border = "red", lwd = 1, alpha = 0.1, theta=30, bty='b2',main = "",xlim=range(-0.5,6),ylim=range(-.5,6),zlim=range(0,5), xlab="",ylab="",zlab=expression(y),axes=FALSE,plot=FALSE) text3D(O[1],O[2]-0.35,O[3],c(expression(O)),add=T) arrows3D(O[1],O[2],O[3],5*X1[1],5*X1[2],X1[3],lty=2,add=T) text3D(5*X1[1]-eps,5*X1[2],X1[3],c(expression(x[1])),add=T) arrows3D(O[1],O[2],O[3],5*X2[1],5*X2[2],X2[3],lty=2,add=T) text3D(5*X2[1],5*X2[2],X2[3],c(expression(x[2])),add=T) points3D(Y[1],Y[2],Y[3],add=T,pch=19,col='red',cex=0.75) #segments3D(O[1],O[2],O[3],Y[1],Y[2],Y[3],add=T,col='red',lwd=2) text3D(Y[1],Y[2],Y[3]+eps,c(expression(y)),add=T) #points3D(Y2hat[1],Y2hat[2],Y2hat[3],add=T,pch=19,col='red') #segments3D(O[1],O[2],O[3],Y2hat[1],Y2hat[2],Y2hat[3],add=T,col='red',lwd=2) #segments3D(Y[1],Y[2],Y[3],Y2hat[1],Y2hat[2],Y2hat[3],add=T,col='red',lwd=2) d2<-Y-Y2hat segments3D(Y2hat[1],Y2hat[2],Y2hat[3],Y2hat[1]+0.075*d2[1],Y2hat[2]+0.075*d2[2],Y2hat[3]+0.075*d2[3],add=T,col='black',lwd=1) segments3D(0.925*Y2hat[1],0.925*Y2hat[2],0.925*Y2hat[3],Y2hat[1],Y2hat[2],Y2hat[3],add=T,col='black',lwd=1) segments3D(0.925*Y2hat[1],0.925*Y2hat[2],0.925*Y2hat[3],0.925*Y2hat[1]+0.075*d2[1],0.925*Y2hat[2]+0.075*d2[2],0.925*Y2hat[3]+0.075*d2[3],add=T,col='black',lwd=1) segments3D(0.925*Y2hat[1]+0.075*d2[1],0.925*Y2hat[2]+0.075*d2[2],0.925*Y2hat[3]+0.075*d2[3],Y2hat[1]+0.075*d2[1],Y2hat[2]+0.075*d2[2],Y2hat[3]+0.075*d2[3],add=T,col='black',lwd=1) points3D(Yhat[1],Yhat[2],Yhat[3],add=T,pch=19,col='red',cex=0.75) text3D(Yhat[1],Yhat[2]+eps,Yhat[3],c(expression(hat(y))),add=T) points3D(Y1hat[1],Y1hat[2],Y1hat[3],add=T,pch=19,col='red',cex=0.75) text3D(Y1hat[1]-0.5,Y1hat[2]+0.5,Y1hat[3],c(expression(hat(y)[1])),add=T) text3D(Y2hat[1],Y2hat[2]+eps,Y2hat[3],c(expression(hat(y)[2])),add=T) @ <>= p3<-cbind(O,Yhat,Y1hat,Y2hat,Y) par(mar=c(2,2,2,2)) scatter3D(p3[1,],p3[2,],p3[3,], theta=30, colkey=FALSE, pch=19, col='red', bty='b2',main = "",xlim=range(-0.5,6),ylim=range(-.5,6),zlim=range(0,5), xlab="",ylab="",zlab=expression(y),axes=FALSE) text3D(O[1],O[2]-0.35,O[3],c(expression(O)),add=T) arrows3D(O[1],O[2],O[3],5*X1[1],5*X1[2],X1[3],lty=2,add=T) text3D(5*X1[1]-eps,5*X1[2],X1[3],c(expression(x[1])),add=T) arrows3D(O[1],O[2],O[3],5*X2[1],5*X2[2],X2[3],lty=2,add=T) text3D(5*X2[1],5*X2[2],X2[3],c(expression(x[2])),add=T) text3D(Yhat[1],Yhat[2]+eps,Yhat[3],c(expression(hat(y))),add=T) text3D(Y1hat[1]-0.5,Y1hat[2]+0.5,Y1hat[3],c(expression(hat(y)[1])),add=T) text3D(Y2hat[1],Y2hat[2]+eps,Y2hat[3],c(expression(hat(y)[2])),add=T) text3D(Y[1],Y[2],Y[3]+eps,c(expression(y)),add=T) polygon3D(p3[1,c(1,2,3)],p3[2,c(1,2,3)],p3[3,c(1,2,3)], col='gray',border = "red", lwd = 1, alpha = 0.25, theta=30, add=TRUE) polygon3D(p3[1,c(1,2,4)],p3[2,c(1,2,4)],p3[3,c(1,2,4)], col='gray',border = "red", lwd = 1, alpha = 0.25, theta=30, add=TRUE) @ <>= t(Yhat) %*% (Y-Yhat) t(Y1hat) %*% (Y-Y1hat) t(Y2hat) %*% (Y-Y2hat) t(Y1hat) %*% (Y-Y1hat) t(Y2hat) %*% (Y-Y2hat) @ \end{document}