i think i did the converstion correct
This commit is contained in:
@@ -221,7 +221,7 @@ class BaseEnvironment(ABC):
|
|||||||
# Set n_agents
|
# Set n_agents
|
||||||
self.agent_composition=agent_composition
|
self.agent_composition=agent_composition
|
||||||
n_agents=0
|
n_agents=0
|
||||||
for k,v in agent_composition:
|
for k,v in agent_composition.items():
|
||||||
n_agents+=v
|
n_agents+=v
|
||||||
|
|
||||||
# Number of agents must be an integer and there must be at least 2 agents
|
# Number of agents must be an integer and there must be at least 2 agents
|
||||||
@@ -323,7 +323,7 @@ class BaseEnvironment(ABC):
|
|||||||
# now that we know all the entities we'll use.
|
# now that we know all the entities we'll use.
|
||||||
self.world = World(
|
self.world = World(
|
||||||
self.world_size,
|
self.world_size,
|
||||||
self.n_agents,
|
self.agent_composition,
|
||||||
self.resources,
|
self.resources,
|
||||||
self.landmarks,
|
self.landmarks,
|
||||||
self.multi_action_mode_agents,
|
self.multi_action_mode_agents,
|
||||||
|
|||||||
@@ -381,12 +381,13 @@ class World:
|
|||||||
self.agent_composition=agent_composition
|
self.agent_composition=agent_composition
|
||||||
self.n_agents=0
|
self.n_agents=0
|
||||||
self._agents = []
|
self._agents = []
|
||||||
for k,v in agent_composition:
|
for k,v in agent_composition.items():
|
||||||
self._agent_class_idx_map[k]=[]
|
self._agent_class_idx_map[k]=[]
|
||||||
for offset in range(v):
|
for offset in range(v):
|
||||||
agent_class=agent_registry.get(k)
|
agent_class=agent_registry.get(k)
|
||||||
self._agents.append(agent_class(self.n_agents,multi_action_mode_agents=self.multi_action_mode_agents))
|
agent=agent_class(self.n_agents,self.multi_action_mode_agents)
|
||||||
self._agent_class_idx_map[k].append(self.n_agents)
|
self._agents.append(agent)
|
||||||
|
self._agent_class_idx_map[k].append(str(self.n_agents))
|
||||||
self.n_agents+=1
|
self.n_agents+=1
|
||||||
self.maps = Maps(world_size, self.n_agents, world_resources, world_landmarks)
|
self.maps = Maps(world_size, self.n_agents, world_resources, world_landmarks)
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -2,4 +2,4 @@
|
|||||||
from . import (
|
from . import (
|
||||||
simple_market,
|
simple_market,
|
||||||
econ_wrapper
|
econ_wrapper
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,6 +3,10 @@ import numpy as np
|
|||||||
from stable_baselines3.common.vec_env import vec_frame_stack
|
from stable_baselines3.common.vec_env import vec_frame_stack
|
||||||
from stable_baselines3.common.evaluation import evaluate_policy
|
from stable_baselines3.common.evaluation import evaluate_policy
|
||||||
import envs
|
import envs
|
||||||
|
import wrapper
|
||||||
|
from wrapper.base_econ_wrapper import BaseEconWrapper
|
||||||
|
from wrapper.reciever_econ_wrapper import RecieverEconWrapper
|
||||||
|
from wrapper.sb3_econ_converter import SB3EconConverter
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
import components
|
import components
|
||||||
from stable_baselines3.common.env_checker import check_env
|
from stable_baselines3.common.env_checker import check_env
|
||||||
@@ -43,7 +47,7 @@ env_config = {
|
|||||||
|
|
||||||
# ===== STANDARD ARGUMENTS ======
|
# ===== STANDARD ARGUMENTS ======
|
||||||
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
||||||
'n_agents': 20, # Number of non-planner agents (must be > 1)
|
'agent_composition': {"BasicMobileAgent": 20}, # Number of non-planner agents (must be > 1)
|
||||||
'world_size': [1, 1], # [Height, Width] of the env world
|
'world_size': [1, 1], # [Height, Width] of the env world
|
||||||
'episode_length': 256, # Number of timesteps per episode
|
'episode_length': 256, # Number of timesteps per episode
|
||||||
'allow_observation_scaling': True,
|
'allow_observation_scaling': True,
|
||||||
@@ -95,7 +99,7 @@ eval_env_config = {
|
|||||||
|
|
||||||
# ===== STANDARD ARGUMENTS ======
|
# ===== STANDARD ARGUMENTS ======
|
||||||
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
||||||
'n_agents': 20, # Number of non-planner agents (must be > 1)
|
'agent_composition': {"BasicMobileAgent": 20}, # Number of non-planner agents (must be > 1)
|
||||||
'world_size': [1, 1], # [Height, Width] of the env world
|
'world_size': [1, 1], # [Height, Width] of the env world
|
||||||
'episode_length': 100, # Number of timesteps per episode
|
'episode_length': 100, # Number of timesteps per episode
|
||||||
'allow_observation_scaling': True,
|
'allow_observation_scaling': True,
|
||||||
@@ -200,9 +204,14 @@ def printReplay(econ,agentid):
|
|||||||
print("Reward: {}".format(reward))
|
print("Reward: {}".format(reward))
|
||||||
|
|
||||||
#Setup Env Objects
|
#Setup Env Objects
|
||||||
|
econ=foundation.make_env_instance(**env_config)
|
||||||
|
baseEconWrapper=BaseEconWrapper(econ)
|
||||||
|
baseEconWrapper.run()
|
||||||
|
mobileRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="BasicMobileAgent")
|
||||||
|
sb3Converter=SB3EconConverter(mobileRecieverEconWrapper,econ,"BasicMobileAgent")
|
||||||
|
obs=sb3Converter.reset()
|
||||||
vecenv=EconVecEnv(env_config=env_config)
|
vecenv=EconVecEnv(env_config=env_config)
|
||||||
econ=vecenv.env
|
|
||||||
monenv=VecMonitor(venv=vecenv,info_keywords=["social/productivity","trend/productivity"])
|
monenv=VecMonitor(venv=vecenv,info_keywords=["social/productivity","trend/productivity"])
|
||||||
normenv=VecNormalize(monenv,norm_reward=False,clip_obs=1)
|
normenv=VecNormalize(monenv,norm_reward=False,clip_obs=1)
|
||||||
stackenv=vec_frame_stack.VecFrameStack(venv=monenv,n_stack=10)
|
stackenv=vec_frame_stack.VecFrameStack(venv=monenv,n_stack=10)
|
||||||
|
|||||||
+3
-3
@@ -1,3 +1,3 @@
|
|||||||
import base_econ_wrapper
|
from . import(
|
||||||
import reciever_econ_wrapper
|
utils
|
||||||
import utils
|
)
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
from ai_economist.foundation.base import base_env
|
from ai_economist.foundation.base import base_env
|
||||||
from threading import Event, Lock, Thread
|
from threading import Event, Lock, Thread
|
||||||
from queue import Queue
|
from queue import Queue
|
||||||
class BaseEconVecEnv():
|
class BaseEconWrapper():
|
||||||
"""Base class for connecting reciever wrapper to a multi threaded econ simulation and training session"""
|
"""Base class for connecting reciever wrapper to a multi threaded econ simulation and training session"""
|
||||||
|
|
||||||
base_notification=Event() #Notification for Base
|
base_notification=Event() #Notification for Base
|
||||||
@@ -33,11 +33,12 @@ class BaseEconVecEnv():
|
|||||||
|
|
||||||
def register_vote(self):
|
def register_vote(self):
|
||||||
"""Register reciever on base. Base now knows"""
|
"""Register reciever on base. Base now knows"""
|
||||||
|
self.n_voters+=1
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""Start the base wrapper"""
|
"""Start the base wrapper"""
|
||||||
thr=Thread(target=self._run,daemon=True)
|
thr=Thread(target=self._run,daemon=True)
|
||||||
thr.run()
|
thr.start()
|
||||||
return thr
|
return thr
|
||||||
|
|
||||||
def _run(self):
|
def _run(self):
|
||||||
@@ -46,16 +47,20 @@ class BaseEconVecEnv():
|
|||||||
self.reset_notification.clear()
|
self.reset_notification.clear()
|
||||||
self.step_notification.clear()
|
self.step_notification.clear()
|
||||||
|
|
||||||
self.stop_edit_lock.release()
|
if self.stop_edit_lock.locked():
|
||||||
|
self.stop_edit_lock.release()
|
||||||
self.stop=False
|
self.stop=False
|
||||||
self.action_edit_lock.release()
|
|
||||||
|
if self.action_edit_lock.locked():
|
||||||
|
self.action_edit_lock.release()
|
||||||
self.actor_actions={}
|
self.actor_actions={}
|
||||||
self.vote_lock.release()
|
if self.vote_lock.locked():
|
||||||
|
self.vote_lock.release()
|
||||||
self.reset_notification.clear()
|
self.reset_notification.clear()
|
||||||
self.n_votes_reset=0
|
self.n_votes_reset=0
|
||||||
self.n_votes_step=0
|
self.n_votes_step=0
|
||||||
|
if self.env_data_lock.locked():
|
||||||
self.env_data_lock.release()
|
self.env_data_lock.release()
|
||||||
self.obs=None
|
self.obs=None
|
||||||
self.rew=None
|
self.rew=None
|
||||||
self.done=None
|
self.done=None
|
||||||
|
|||||||
@@ -6,16 +6,15 @@ from ai_economist.foundation.base import base_env
|
|||||||
import gym
|
import gym
|
||||||
import gym.spaces
|
import gym.spaces
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from base_econ_wrapper import BaseEconVecEnv
|
|
||||||
|
|
||||||
from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices, VecEnvObs, VecEnvStepReturn
|
from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices, VecEnvObs, VecEnvStepReturn
|
||||||
from stable_baselines3.common.vec_env.util import copy_obs_dict, dict_to_obs, obs_space_info
|
from stable_baselines3.common.vec_env.util import copy_obs_dict, dict_to_obs, obs_space_info
|
||||||
|
from wrapper.base_econ_wrapper import BaseEconWrapper
|
||||||
from ai_economist import foundation
|
from ai_economist import foundation
|
||||||
|
|
||||||
class RecieverEconVecEnv(gym.Env):
|
class RecieverEconWrapper(gym.Env):
|
||||||
"""Reciever part of BaseEconVecEnv. Filters by agent class and presents gym api to RL algos. Enables multi threading learning for different agent types."""
|
"""Reciever part of BaseEconVecEnv. Filters by agent class and presents gym api to RL algos. Enables multi threading learning for different agent types."""
|
||||||
def __init__(self, base_econ: BaseEconVecEnv, agent_classname: str):
|
def __init__(self, base_econ: BaseEconWrapper, agent_classname: str):
|
||||||
self.base_econ=base_econ
|
self.base_econ=base_econ
|
||||||
base_econ.register_vote()
|
base_econ.register_vote()
|
||||||
self.econ=base_econ.env
|
self.econ=base_econ.env
|
||||||
@@ -23,7 +22,7 @@ class RecieverEconVecEnv(gym.Env):
|
|||||||
self.agnet_idx=list(self.econ.world._agent_class_idx_map[agent_classname])
|
self.agnet_idx=list(self.econ.world._agent_class_idx_map[agent_classname])
|
||||||
self.idx_to_index={}
|
self.idx_to_index={}
|
||||||
#create idx to index map
|
#create idx to index map
|
||||||
for i in range(len(self.agnet_idx)):
|
for i in range(len(self.agnet_idx)):
|
||||||
self.idx_to_index[self.agnet_idx[i]]=i
|
self.idx_to_index[self.agnet_idx[i]]=i
|
||||||
first_idx=self.agnet_idx[0]
|
first_idx=self.agnet_idx[0]
|
||||||
|
|
||||||
@@ -36,13 +35,16 @@ class RecieverEconVecEnv(gym.Env):
|
|||||||
def _dict_idx_to_index(self, data):
|
def _dict_idx_to_index(self, data):
|
||||||
data_out={}
|
data_out={}
|
||||||
for k,v in data.items():
|
for k,v in data.items():
|
||||||
data_out[self.idx_to_index[k]]=v
|
if k in self.idx_to_index:
|
||||||
|
index=self.idx_to_index[k]
|
||||||
|
data_out[index]=v
|
||||||
return data_out
|
return data_out
|
||||||
|
|
||||||
def _dict_index_to_idx(self, data):
|
def _dict_index_to_idx(self, data):
|
||||||
data_out={}
|
data_out={}
|
||||||
for k,v in data.items():
|
for k,v in data.items():
|
||||||
data_out[self.agnet_idx[k]]=v
|
idx=self.agnet_idx[k]
|
||||||
|
data_out[idx]=v
|
||||||
return data_out
|
return data_out
|
||||||
|
|
||||||
def step_wait(self):
|
def step_wait(self):
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
import gym
|
import gym
|
||||||
import gym.spaces
|
import gym.spaces
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import utils
|
from wrapper import utils
|
||||||
from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices, VecEnvObs, VecEnvStepReturn
|
from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices, VecEnvObs, VecEnvStepReturn
|
||||||
from ai_economist.foundation.base import base_env,base_agent
|
from ai_economist.foundation.base import base_env,base_agent
|
||||||
|
from typing import Any, Callable, List, Optional, Sequence, Type, Union
|
||||||
|
|
||||||
class SB3EconConverter(VecEnv, gym.Env):
|
class SB3EconConverter(VecEnv, gym.Env):
|
||||||
|
|
||||||
@@ -15,7 +16,7 @@ class SB3EconConverter(VecEnv, gym.Env):
|
|||||||
self.num_envs=len(obs.keys())
|
self.num_envs=len(obs.keys())
|
||||||
#get action and obervation space
|
#get action and obervation space
|
||||||
self.action_space=self._get_action_space_by_class(agentclass)
|
self.action_space=self._get_action_space_by_class(agentclass)
|
||||||
self.packager=utils.build_packager(obs[0],put_in_both=["time"])
|
self.packager=utils.build_packager(obs[0])
|
||||||
#flatten obervation of first agent
|
#flatten obervation of first agent
|
||||||
obs0=utils.package(obs[0],*self.packager)
|
obs0=utils.package(obs[0],*self.packager)
|
||||||
self.observation_space=gym.spaces.Box(low=-np.inf,high=np.inf,shape=(len(obs0),1),dtype=np.float32)
|
self.observation_space=gym.spaces.Box(low=-np.inf,high=np.inf,shape=(len(obs0),1),dtype=np.float32)
|
||||||
@@ -23,8 +24,9 @@ class SB3EconConverter(VecEnv, gym.Env):
|
|||||||
|
|
||||||
|
|
||||||
def _get_action_space_by_class(self,agentClass: str):
|
def _get_action_space_by_class(self,agentClass: str):
|
||||||
idx=self.econ.world._agent_class_idx_map[agentClass]
|
idx_list=self.econ.world._agent_class_idx_map[agentClass]
|
||||||
agent=base_agent.BaseAgent(self.econ.world.agents[idx[0]])
|
idx=int(idx_list[0])
|
||||||
|
agent=self.econ.world.agents[idx]
|
||||||
return gym.spaces.Discrete(agent.action_spaces)
|
return gym.spaces.Discrete(agent.action_spaces)
|
||||||
|
|
||||||
def step_async(self, actions: np.ndarray) -> None:
|
def step_async(self, actions: np.ndarray) -> None:
|
||||||
@@ -34,12 +36,12 @@ class SB3EconConverter(VecEnv, gym.Env):
|
|||||||
def step_wait(self) -> VecEnvStepReturn:
|
def step_wait(self) -> VecEnvStepReturn:
|
||||||
obs,rew,done,info=self.env.step_wait()
|
obs,rew,done,info=self.env.step_wait()
|
||||||
#flatten obs
|
#flatten obs
|
||||||
f_obs=utils.package(obs,*self.packager)
|
f_obs={}
|
||||||
#convert to flat
|
for k,v in obs.items():
|
||||||
g_obs={}
|
o=utils.package(v,*self.packager)
|
||||||
for k,v in f_obs.items():
|
f_obs[k]=o["flat"]
|
||||||
g_obs[k]=v["flat"]
|
|
||||||
c_obs=utils.convert_econ_to_gym(g_obs)
|
c_obs=utils.convert_econ_to_gym(f_obs)
|
||||||
c_rew=utils.convert_econ_to_gym(rew)
|
c_rew=utils.convert_econ_to_gym(rew)
|
||||||
c_done=utils.convert_econ_to_gym(done)
|
c_done=utils.convert_econ_to_gym(done)
|
||||||
c_info=utils.convert_econ_to_gym(info)
|
c_info=utils.convert_econ_to_gym(info)
|
||||||
@@ -53,9 +55,59 @@ class SB3EconConverter(VecEnv, gym.Env):
|
|||||||
return c_obs,c_rew,c_done,c_info
|
return c_obs,c_rew,c_done,c_info
|
||||||
def reset(self) -> VecEnvObs:
|
def reset(self) -> VecEnvObs:
|
||||||
obs=self.env.reset()
|
obs=self.env.reset()
|
||||||
f_obs=utils.package(obs,*self.packager)
|
f_obs={}
|
||||||
|
for k,v in obs.items():
|
||||||
|
f_obs[k]=utils.package(v,*self.packager)
|
||||||
g_obs={}
|
g_obs={}
|
||||||
for k,v in f_obs.items():
|
for k,v in f_obs.items():
|
||||||
g_obs[k]=v["flat"]
|
g_obs[k]=v["flat"]
|
||||||
c_obs=utils.convert_econ_to_gym(g_obs)
|
c_obs=utils.convert_econ_to_gym(g_obs)
|
||||||
return c_obs
|
return c_obs
|
||||||
|
|
||||||
|
def seed(self, seed: Optional[int] = None) -> List[Union[None, int]]:
|
||||||
|
if seed is None:
|
||||||
|
seed = np.random.randint(0, 2**32 - 1)
|
||||||
|
seeds = []
|
||||||
|
for idx, env in enumerate(self.envs):
|
||||||
|
seeds.append(env.seed(seed + idx))
|
||||||
|
return seeds
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def get_attr(self, attr_name: str, indices: VecEnvIndices = None) -> List[Any]:
|
||||||
|
"""Return attribute from vectorized environment (see base class)."""
|
||||||
|
target_envs = self._get_target_envs(indices)
|
||||||
|
return [getattr(env_i, attr_name) for env_i in target_envs]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def set_attr(self, attr_name: str, value: Any, indices: VecEnvIndices = None) -> None:
|
||||||
|
"""Set attribute inside vectorized environments (see base class)."""
|
||||||
|
target_envs = self._get_target_envs(indices)
|
||||||
|
for env_i in target_envs:
|
||||||
|
setattr(env_i, attr_name, value)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def env_method(self, method_name: str, *method_args, indices: VecEnvIndices = None, **method_kwargs) -> List[Any]:
|
||||||
|
"""Call instance methods of vectorized environments."""
|
||||||
|
target_envs = self._get_target_envs(indices)
|
||||||
|
return [getattr(env_i, method_name)(*method_args, **method_kwargs) for env_i in target_envs]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def env_is_wrapped(self, wrapper_class: Type[gym.Wrapper], indices: VecEnvIndices = None) -> List[bool]:
|
||||||
|
"""Check if worker environments are wrapped with a given wrapper"""
|
||||||
|
target_envs = self._get_target_envs(indices)
|
||||||
|
# Import here to avoid a circular import
|
||||||
|
from stable_baselines3.common import env_util
|
||||||
|
|
||||||
|
return [env_util.is_wrapped(env_i, wrapper_class) for env_i in target_envs]
|
||||||
|
|
||||||
|
|
||||||
|
def _get_target_envs(self, indices: VecEnvIndices) -> List[gym.Env]:
|
||||||
|
indices = self._get_indices(indices)
|
||||||
|
return [self.envs[i] for i in indices]
|
||||||
+23
-21
@@ -44,26 +44,28 @@ def build_packager(sub_obs, put_in_both=None):
|
|||||||
|
|
||||||
|
|
||||||
def package(obs_dict, keep_as_is, flatten, wrap_as_list):
|
def package(obs_dict, keep_as_is, flatten, wrap_as_list):
|
||||||
"""Flattens observation with packagers."""
|
"""Flattens observation with packagers."""
|
||||||
new_obs = {k: obs_dict[k] for k in keep_as_is}
|
|
||||||
if len(flatten) == 1:
|
new_obs = {k: obs_dict[k] for k in keep_as_is}
|
||||||
k = flatten[0]
|
if len(flatten) == 1:
|
||||||
o = obs_dict[k]
|
k = flatten[0]
|
||||||
if wrap_as_list[k]:
|
o = obs_dict[k]
|
||||||
o = [o]
|
if wrap_as_list[k]:
|
||||||
new_obs["flat"] = np.array(o, dtype=np.float32)
|
o = [o]
|
||||||
else:
|
new_obs["flat"] = np.array(o, dtype=np.float32)
|
||||||
to_flatten = [
|
else:
|
||||||
[obs_dict[k]] if wrap_as_list[k] else obs_dict[k] for k in flatten
|
to_flatten = [
|
||||||
]
|
[obs_dict[k]] if wrap_as_list[k] else obs_dict[k] for k in flatten
|
||||||
try:
|
]
|
||||||
new_obs["flat"] = np.concatenate(to_flatten).astype(np.float32)
|
try:
|
||||||
except ValueError:
|
new_obs["flat"] = np.concatenate(to_flatten).astype(np.float32)
|
||||||
for k, v in zip(flatten, to_flatten):
|
except ValueError:
|
||||||
print(k, np.array(v).shape)
|
for k, v in zip(flatten, to_flatten):
|
||||||
print(v)
|
print(k, np.array(v).shape)
|
||||||
print("")
|
print(v)
|
||||||
raise
|
print("")
|
||||||
return new_obs
|
raise
|
||||||
|
|
||||||
|
return new_obs
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user