fixed some bugs

Author: huangshiyu13

examples/selfplay/human_vs_agent.py

@@ -27,7 +27,7 @@ def get_human_env(env_num):
     env = make(
         "tictactoe_v3",
         env_num=env_num,
-        asynchronous=True,
+        asynchronous=False,
         opponent_wrappers=[TictactoeRender, HumanOpponentWrapper],
         env_wrappers=[FlattenObservation],
         auto_reset=False,

examples/selfplay/opponent_templates/tictactoe_opponent/opponent.py

@@ -47,7 +47,7 @@ def process_obs(self, observation, termination, truncation, info):
         return new_obs, termination, truncation, new_info
 
     def process_action(self, action):
-        return action[0][0][0]
+        return action[0][0]
 
 
 class Opponent(NetworkOpponent):

examples/selfplay/tictactoe_utils/tictactoe_render.py

@@ -45,7 +45,7 @@ def reset(self, seed: Optional[int] = None, options: Optional[dict] = None):
 
     def step(self, action: ActionType) -> None:
         result = super().step(action)
-        self.last_action = action
+        self.last_action = action[0]
         return result
 
     def observe(self, agent: str) -> Optional[ObsType]:

examples/snake/jidi_eval.py

@@ -36,7 +36,10 @@ def run_arena(
     )
 
     agent1 = JiDiAgent("./submissions/rule_v1", player_num=player_num)
-    agent2 = JiDiAgent("./submissions/rl", player_num=player_num)
+    if player_num == 3:
+        agent2 = JiDiAgent("./submissions/rl", player_num=player_num)
+    else:
+        agent2 = JiDiAgent("./submissions/rule_v1", player_num=player_num)
 
     arena.reset(
         agents={"agent1": agent1, "agent2": agent2},
@@ -51,4 +54,4 @@ def run_arena(
 
 
 if __name__ == "__main__":
-    run_arena(render=False, parallel=True, seed=0, total_games=100, max_game_onetime=5)
+    run_arena(render=False, parallel=True, seed=0, total_games=10, max_game_onetime=5)

examples/snake/train_selfplay.py

@@ -15,7 +15,10 @@ def train():
 
     # Create environment
     env_num = 10
+
     render_model = None
+
+    # ConvertObs can only be used for snakes_1v1, if you want to train snakes_3v3, you need to write your own wrapper
     env = make(
         "snakes_1v1",
         render_mode=render_model,
@@ -32,6 +35,7 @@ def train():
     agent = Agent(net)
     # Begin training
     agent.train(total_time_steps=100000)
+
     env.close()
     agent.save("./selfplay_agent/")
     return agent
@@ -71,6 +75,7 @@ def evaluation():
         while not np.any(done):
             # predict next action based on the observation
             action, _ = agent.act(obs, info, deterministic=True)
+
             obs, r, done, info = env.step(action)
             step += 1
 

examples/snake/wrappers.py

@@ -23,14 +23,14 @@
 
 
 def raw2vec(raw_obs, n_player=2):
-    control_index = raw_obs["controlled_snake_index"][0]
+    control_index = raw_obs["controlled_snake_index"]
 
-    width = raw_obs["board_width"][0]
-    height = raw_obs["board_height"][0]
-    beans = raw_obs[1][0]
+    width = raw_obs["board_width"]
+    height = raw_obs["board_height"]
+    beans = raw_obs[1]
 
-    ally_pos = raw_obs[control_index][0]
-    enemy_pos = raw_obs[5 - control_index][0]
+    ally_pos = raw_obs[control_index]
+    enemy_pos = raw_obs[5 - control_index]
 
     obs = np.zeros(width * height * n_player, dtype=int)
 
@@ -59,7 +59,7 @@ def raw2vec(raw_obs, n_player=2):
     obs_ = np.array([])
     for i in obs:
         obs_ = np.concatenate([obs_, np.eye(6)[i]])
-    obs_ = obs_.reshape(-1, width * height * n_player * 6)
+    obs_ = obs_.reshape(width * height * n_player * 6)
 
     return obs_
 
@@ -87,4 +87,8 @@ def observation(self, observation):
             The flattened observation
         """
 
-        return raw2vec(observation)
+        new_obs = []
+        for obs in observation:
+            new_obs.append(raw2vec(obs))
+
+        return new_obs

openrl/algorithms/dqn.py

@@ -167,9 +167,7 @@ def prepare_loss(
         )
 
         q_targets = rewards_batch + self.gamma * max_next_q_values * next_masks_batch
-        q_loss = torch.mean(
-            F.mse_loss(q_values, q_targets.detach())
-        )  # 均方误差损失函数
+        q_loss = torch.mean(F.mse_loss(q_values, q_targets.detach()))  # 均方误差损失函数
 
         loss_list.append(q_loss)
 

openrl/algorithms/vdn.py

@@ -211,9 +211,7 @@ def prepare_loss(
         rewards_batch = rewards_batch.reshape(-1, self.n_agent, 1)
         rewards_batch = torch.sum(rewards_batch, dim=1, keepdim=True).view(-1, 1)
         q_targets = rewards_batch + self.gamma * max_next_q_values * next_masks_batch
-        q_loss = torch.mean(
-            F.mse_loss(q_values, q_targets.detach())
-        )  # 均方误差损失函数
+        q_loss = torch.mean(F.mse_loss(q_values, q_targets.detach()))  # 均方误差损失函数
 
         loss_list.append(q_loss)
         return loss_list

openrl/arena/__init__.py

@@ -28,7 +28,6 @@ def make_arena(
     render: Optional[bool] = False,
     **kwargs,
 ):
-    print(openrl.envs.PettingZoo.registration.pettingzoo_env_dict.keys())
     if custom_build_env is None:
         if (
             env_id in pettingzoo_all_envs

openrl/envs/PettingZoo/__init__.py

@@ -20,7 +20,7 @@
 
 from openrl.envs.common import build_envs
 from openrl.envs.PettingZoo.registration import pettingzoo_env_dict, register
-from openrl.envs.wrappers.pettingzoo_wrappers import SeedEnv
+from openrl.envs.wrappers.pettingzoo_wrappers import CheckAgentNumber, SeedEnv
 
 
 def PettingZoo_make(id, render_mode, disable_env_checker, **kwargs):
@@ -46,8 +46,9 @@ def make_PettingZoo_env(
     **kwargs,
 ):
     env_num = 1
-    env_wrappers = [SeedEnv]
+    env_wrappers = [CheckAgentNumber, SeedEnv]
     env_wrappers += copy.copy(kwargs.pop("env_wrappers", []))
+
     env_fns = build_envs(
         make=PettingZoo_make,
         id=id,
@@ -65,16 +66,15 @@ def make_PettingZoo_envs(
     render_mode: Optional[Union[str, List[str]]] = None,
     **kwargs,
 ):
-    from openrl.envs.wrappers import (  # AutoReset,; DictWrapper,
+    from openrl.envs.wrappers import (  # AutoReset,; DictWrapper,; Single2MultiAgentWrapper,
         MoveActionMask2InfoWrapper,
         RemoveTruncated,
-        Single2MultiAgentWrapper,
     )
 
-    env_wrappers = [SeedEnv]
+    env_wrappers = [CheckAgentNumber, SeedEnv]
     env_wrappers += copy.copy(kwargs.pop("opponent_wrappers", []))
     env_wrappers += [
-        Single2MultiAgentWrapper,
+        # Single2MultiAgentWrapper,
         RemoveTruncated,
         MoveActionMask2InfoWrapper,
     ]