fix minor bugs

examples/monte.py

@@ -9,9 +9,9 @@ GAMMA = 0.999
 
 def first_visit_mc(policy, env, T):
     V = np.zeros((N_S))
-    Returns = [] * N_S
+    Returns = [[]] * N_S
 
-    for k in range(1000):
+    for k in range(100):
         observation, info = env.reset()
 
         rewards = []
@@ -25,11 +25,18 @@ def first_visit_mc(policy, env, T):
                 break
 
         G = 0
-        for j in range(i):
-            G = GAMMA * G + reward[i]
+        for j in range(i+1):
+            G = GAMMA * G + rewards[i]
 
             s_t = observations[j]
 
             if j == 0 or s_t not in observations[: j - 1]:
                 Returns[s_t].append(G)
                 V[s_t] = np.mean(Returns[s_t])
+    return V
+
+def main():
+    env = gym.make("FrozenLake-v1", is_slippery=False)
+    policy = [2] * 16
+
+    print(first_visit_mc(policy, env, 16))