Chapter 04

Value Iteration

NotebookPython 35 cells

In [3]python · cell 1

python

import numpy as np
import pprint
import sys
if "../" not in sys.path:
  sys.path.append("../") 
from lib.envs.gridworld import GridworldEnv

In [4]python · cell 2

python

pp = pprint.PrettyPrinter(indent=2)
env = GridworldEnv()

In [5]python · cell 3

python

def value_iteration(env, theta=0.0001, discount_factor=1.0):
    """
    Value Iteration Algorithm.
    
    Args:
        env: OpenAI env. env.P represents the transition probabilities of the environment.
            env.P[s][a] is a list of transition tuples (prob, next_state, reward, done).
            env.nS is a number of states in the environment. 
            env.nA is a number of actions in the environment.
        theta: We stop evaluation once our value function change is less than theta for all states.
        discount_factor: Gamma discount factor.
        
    Returns:
        A tuple (policy, V) of the optimal policy and the optimal value function.        
    """
    

    V = np.zeros(env.nS)
    policy = np.zeros([env.nS, env.nA])
    
    # Implement!
    return policy, V

In [6]python · cell 4

python

policy, v = value_iteration(env)

print("Policy Probability Distribution:")
print(policy)
print("")

print("Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):")
print(np.reshape(np.argmax(policy, axis=1), env.shape))
print("")

print("Value Function:")
print(v)
print("")

print("Reshaped Grid Value Function:")
print(v.reshape(env.shape))
print("")

Output

Policy Probability Distribution:
[[ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]]

Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):
[[0 0 0 0]
 [0 0 0 0]
 [0 0 0 0]
 [0 0 0 0]]

Value Function:
[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]

Reshaped Grid Value Function:
[[ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]]

In [7]python · cell 5

python

# Test the value function
expected_v = np.array([ 0, -1, -2, -3, -1, -2, -3, -2, -2, -3, -2, -1, -3, -2, -1,  0])
np.testing.assert_array_almost_equal(v, expected_v, decimal=2)

Output

[0;31m---------------------------------------------------------------------------[0m
[0;31mAssertionError[0m                            Traceback (most recent call last)
[0;32m<ipython-input-7-55581f8eb5c9>[0m in [0;36m<module>[0;34m()[0m
[1;32m      1[0m [0;31m# Test the value function[0m[0;34m[0m[0;34m[0m[0m
[1;32m      2[0m [0mexpected_v[0m [0;34m=[0m [0mnp[0m[0;34m.[0m[0marray[0m[0;34m([0m[0;34m[[0m [0;36m0[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m  [0;36m0[0m[0;34m][0m[0;34m)[0m[0;34m[0m[0m
[0;32m----> 3[0;31m [0mnp[0m[0;34m.[0m[0mtesting[0m[0;34m.[0m[0massert_array_almost_equal[0m[0;34m([0m[0mv[0m[0;34m,[0m [0mexpected_v[0m[0;34m,[0m [0mdecimal[0m[0;34m=[0m[0;36m2[0m[0;34m)[0m[0;34m[0m[0m
[0m
[0;32m/Users/dennybritz/venvs/tf/lib/python3.5/site-packages/numpy/testing/utils.py[0m in [0;36massert_array_almost_equal[0;34m(x, y, decimal, err_msg, verbose)[0m
[1;32m    914[0m     assert_array_compare(compare, x, y, err_msg=err_msg, verbose=verbose,
[1;32m    915[0m              [0mheader[0m[0;34m=[0m[0;34m([0m[0;34m'Arrays are not almost equal to %d decimals'[0m [0;34m%[0m [0mdecimal[0m[0;34m)[0m[0;34m,[0m[0;34m[0m[0m
[0;32m--> 916[0;31m              precision=decimal)
[0m[1;32m    917[0m [0;34m[0m[0m
[1;32m    918[0m [0;34m[0m[0m

[0;32m/Users/dennybritz/venvs/tf/lib/python3.5/site-packages/numpy/testing/utils.py[0m in [0;36massert_array_compare[0;34m(comparison, x, y, err_msg, verbose, header, precision)[0m
[1;32m    735[0m                                 names=('x', 'y'), precision=precision)
[1;32m    736[0m             [0;32mif[0m [0;32mnot[0m [0mcond[0m[0;34m:[0m[0;34m[0m[0m
[0;32m--> 737[0;31m                 [0;32mraise[0m [0mAssertionError[0m[0;34m([0m[0mmsg[0m[0;34m)[0m[0;34m[0m[0m
[0m[1;32m    738[0m     [0;32mexcept[0m [0mValueError[0m[0;34m:[0m[0;34m[0m[0m
[1;32m    739[0m         [0;32mimport[0m [0mtraceback[0m[0;34m[0m[0m

[0;31mAssertionError[0m: 
Arrays are not almost equal to 2 decimals

(mismatch 87.5%)
 x: array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.])
 y: array([ 0, -1, -2, -3, -1, -2, -3, -2, -2, -3, -2, -1, -3, -2, -1,  0])

def value_iteration(env, theta=0.0001, discount_factor=1.0): """ Value Iteration Algorithm. Args: env: OpenAI env. env.P represents the transition probabilities of the environment. env.P[s][a] is a list of transition tuples (prob, next_state, reward, done). env.nS is a number of states in the environment. env.nA is a number of actions in the environment. theta: We stop evaluation once our value function change is less than theta for all states. discount_factor: Gamma discount factor. Returns: A tuple (policy, V) of the optimal policy and the optimal value function. """ V = np.zeros(env.nS) policy = np.zeros([env.nS, env.nA]) # Implement! return policy, V

policy, v = value_iteration(env) print("Policy Probability Distribution:") print(policy) print("") print("Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):") print(np.reshape(np.argmax(policy, axis=1), env.shape)) print("") print("Value Function:") print(v) print("") print("Reshaped Grid Value Function:") print(v.reshape(env.shape)) print("")

Policy Probability Distribution: [[ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.]] Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left): [[0 0 0 0] [0 0 0 0] [0 0 0 0] [0 0 0 0]] Value Function: [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.] Reshaped Grid Value Function: [[ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.] [ 0. 0. 0. 0.]]

[0;31m---------------------------------------------------------------------------[0m [0;31mAssertionError[0m Traceback (most recent call last) [0;32m<ipython-input-7-55581f8eb5c9>[0m in [0;36m<module>[0;34m()[0m [1;32m 1[0m [0;31m# Test the value function[0m[0;34m[0m[0;34m[0m[0m [1;32m 2[0m [0mexpected_v[0m [0;34m=[0m [0mnp[0m[0;34m.[0m[0marray[0m[0;34m([0m[0;34m[[0m [0;36m0[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m [0;34m-[0m[0;36m3[0m[0;34m,[0m [0;34m-[0m[0;36m2[0m[0;34m,[0m [0;34m-[0m[0;36m1[0m[0;34m,[0m [0;36m0[0m[0;34m][0m[0;34m)[0m[0;34m[0m[0m [0;32m----> 3[0;31m [0mnp[0m[0;34m.[0m[0mtesting[0m[0;34m.[0m[0massert_array_almost_equal[0m[0;34m([0m[0mv[0m[0;34m,[0m [0mexpected_v[0m[0;34m,[0m [0mdecimal[0m[0;34m=[0m[0;36m2[0m[0;34m)[0m[0;34m[0m[0m [0m [0;32m/Users/dennybritz/venvs/tf/lib/python3.5/site-packages/numpy/testing/utils.py[0m in [0;36massert_array_almost_equal[0;34m(x, y, decimal, err_msg, verbose)[0m [1;32m 914[0m assert_array_compare(compare, x, y, err_msg=err_msg, verbose=verbose, [1;32m 915[0m [0mheader[0m[0;34m=[0m[0;34m([0m[0;34m'Arrays are not almost equal to %d decimals'[0m [0;34m%[0m [0mdecimal[0m[0;34m)[0m[0;34m,[0m[0;34m[0m[0m [0;32m--> 916[0;31m precision=decimal) [0m[1;32m 917[0m [0;34m[0m[0m [1;32m 918[0m [0;34m[0m[0m [0;32m/Users/dennybritz/venvs/tf/lib/python3.5/site-packages/numpy/testing/utils.py[0m in [0;36massert_array_compare[0;34m(comparison, x, y, err_msg, verbose, header, precision)[0m [1;32m 735[0m names=('x', 'y'), precision=precision) [1;32m 736[0m [0;32mif[0m [0;32mnot[0m [0mcond[0m[0;34m:[0m[0;34m[0m[0m [0;32m--> 737[0;31m [0;32mraise[0m [0mAssertionError[0m[0;34m([0m[0mmsg[0m[0;34m)[0m[0;34m[0m[0m [0m[1;32m 738[0m [0;32mexcept[0m [0mValueError[0m[0;34m:[0m[0;34m[0m[0m [1;32m 739[0m [0;32mimport[0m [0mtraceback[0m[0;34m[0m[0m [0;31mAssertionError[0m: Arrays are not almost equal to 2 decimals (mismatch 87.5%) x: array([ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) y: array([ 0, -1, -2, -3, -1, -2, -3, -2, -2, -3, -2, -1, -3, -2, -1, 0])