veepi.tex

\documentclass[DIV=calc]{scrartcl}
\usepackage{mathpazo}
\begin{document}
\[
V^{\pi}\left(s\right)=R\left(s,\pi(s)\right)+\gamma\sum_{s'}P\left(s'|s,\pi\left(s\right)\right)V^{\pi}\left(s'\right)
\]

We can then form a system of equations:

\begin{eqnarray*}
\left(1-\gamma P\left(s_{1}|s_{1},\pi\left(s_{1}\right)\right)\right)V^{\pi}\left(s_{1}\right)-\gamma P\left(s_{2}|s_{1},\pi\left(s_{1}\right)\right)V^{\pi}\left(s_{2}\right)-\ldots & = & -R\left(s_{1},\pi\left(s_{1}\right)\right)\\
-\gamma P\left(s_{1}|s_{2},\pi\left(s_{2}\right)\right)V^{\pi}\left(s_{1}\right)+\left(1-\gamma P\left(s_{2}|s_{2},\pi\left(s_{2}\right)\right)\right)V^{\pi}\left(s_{2}\right)-\ldots & = & -R\left(s_{2},\pi\left(s_{2}\right)\right)\\
 & \ldots
\end{eqnarray*}
which can be written as 
\[
c_{1,1}V^{\pi}\left(s_{1}\right)+c_{1,2}V^{\pi}(s_{2})+\ldots=r_{1}
\]

Policy iteration requires finding the best policy for a given $V^{\pi}$.
\[
\pi\left(s\right)=\arg\max_{a}\sum_{s'}P\left(s'|s,a\right)V^{\pi}\left(s'\right)
\]
for all $s\in S$.
\end{document}