强化学习:策略迭代求解FrozenLake-v1滑冰问题代码

import numpy as np
np.random.seed(0)
import gym
def play_policy(env, policy, render=False):
    total_reward = 0.
    observation = env.reset()
    while True:
        if render:
            env.render() # 此行可显示
        action = np.random.choice(env.action_space.n,
                p=policy[observation])
        observation, reward, done, _ = env.step(action)
        total_reward += reward  # 统计回合奖励
        if done: # 游戏结束
            break
    return total_reward

def policy_v2q(env,v,s=None,gamma=1.0):
    if s is not None: # 计算第v[s]
        q = np.zeros(env.action_space.n)
        for action in range(env.action_space.n):
            for p,next_state,reward,done in env.unwrapped.P[s][action]:
                q[action] += p*(reward + gamma*v[next_state]*(1-done))
    else:
        q = np.zeros((env.observation_space.n,env.action_space.n))
        for state in range(env.observation_space.n):
            q[state] = policy_v2q(env,v,state,gamma)
    return q

# 下面是策略评估
def policy_evluate(env,policy,gamma=1.0,tolerant=1e-6):
    v = np.zeros(env.observation_space.n)
    vs = np.zeros(env.observation_space.n)
    q = np.zeros(env.action_space.n)
    while True:
        delta = 0
        for state in range(env.observation_space.n):
            vs[state] = sum(policy_v2q(env,v,state,gamma)*policy[state])
            delta = max(delta,vs[state]-v[state])
            v[state] = vs[state]
        if delta