Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a137f37603 | ||
|
|
8c36302edc | ||
|
|
f1982d1fe4 | ||
|
|
1ee1c2eb04 | ||
|
|
bfd6d1e37b |
+3
-1
@@ -1,4 +1,6 @@
|
|||||||
logs/*
|
logs/*
|
||||||
_pycache_
|
_pycache_
|
||||||
*.pyc
|
*.pyc
|
||||||
*tfevents*
|
*tfevents*
|
||||||
|
db/econ_data/*
|
||||||
|
db/metabase_data/*
|
||||||
Binary file not shown.
@@ -0,0 +1,42 @@
|
|||||||
|
version: "3.5"
|
||||||
|
|
||||||
|
services:
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
image: postgres:10
|
||||||
|
volumes:
|
||||||
|
- "./metabase_data:/var/lib/postgresql/data"
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: metabase
|
||||||
|
POSTGRES_USER: metabase_usr
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||||
|
|
||||||
|
metabase:
|
||||||
|
depends_on:
|
||||||
|
- postgres
|
||||||
|
image: metabase/metabase
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
MB_ENCRYPTION_SECRET_KEY: ${MB}
|
||||||
|
MB_DB_TYPE: postgres
|
||||||
|
MB_DB_DBNAME: metabase
|
||||||
|
MB_DB_PORT: 5432
|
||||||
|
MB_DB_USER: metabase_usr
|
||||||
|
MB_DB_PASS: ${POSTGRES_PASSWORD}
|
||||||
|
MB_DB_HOST: postgres
|
||||||
|
|
||||||
|
mongo:
|
||||||
|
image: mongo
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
MONGO_INITDB_DATABASE: econ
|
||||||
|
MONGO_INITDB_ROOT_USERNAME: root
|
||||||
|
MONGO_INITDB_ROOT_PASSWORD: econ
|
||||||
|
ports:
|
||||||
|
- '27017:27017'
|
||||||
|
volumes:
|
||||||
|
- ./econ_data:/data/db
|
||||||
|
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
from pymongo import MongoClient
|
||||||
|
import datetime
|
||||||
|
cmt="current_market_trades"
|
||||||
|
cma="current_market_asks"
|
||||||
|
cmb="current_market_bids"
|
||||||
|
|
||||||
|
class MongoExporter():
|
||||||
|
|
||||||
|
def __init__(self,conn_string,db,econ) -> None:
|
||||||
|
|
||||||
|
# Provide the mongodb atlas url to connect python to mongodb using pymongo
|
||||||
|
|
||||||
|
|
||||||
|
# Create a connection using MongoClient. You can import MongoClient or use pymongo.MongoClient
|
||||||
|
client = MongoClient(conn_string)
|
||||||
|
|
||||||
|
# Create the database for our example (we will use the same database throughout the tutorial
|
||||||
|
self.client=client[db]
|
||||||
|
self.econ=econ
|
||||||
|
|
||||||
|
def reset_market(self):
|
||||||
|
"""Resets the current market data in the analytics stack"""
|
||||||
|
c=self.client
|
||||||
|
c[cmt].drop()
|
||||||
|
c[cma].drop()
|
||||||
|
c[cmb].drop()
|
||||||
|
|
||||||
|
def submit_full_market(self):
|
||||||
|
c=self.client
|
||||||
|
market=self.econ.get_component("ContinuousDoubleAuction")
|
||||||
|
self.reset_market()
|
||||||
|
base=datetime.datetime.fromtimestamp(0)
|
||||||
|
delta=datetime.timedelta(days=1)
|
||||||
|
for i in range(len(market.executed_trades)):
|
||||||
|
step=market.executed_trades[i]
|
||||||
|
timestep=base+(i*delta)
|
||||||
|
if len(step)>0:
|
||||||
|
for transaction in step:
|
||||||
|
transaction["step"]=i
|
||||||
|
c[cmt].insert_one(transaction)
|
||||||
|
for com,asks in market.asks.items():
|
||||||
|
for ask in asks:
|
||||||
|
ask["commodity"]=com
|
||||||
|
c[cma].insert_one(ask)
|
||||||
|
for com,bids in market.bids.items():
|
||||||
|
for bid in bids:
|
||||||
|
bid["commodity"]=com
|
||||||
|
c[cmb].insert_one(bid)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# This is added so that many files can reuse the function get_database()
|
||||||
@@ -0,0 +1,362 @@
|
|||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from ai_economist import foundation
|
||||||
|
from stable_baselines3.common.vec_env import vec_frame_stack
|
||||||
|
from stable_baselines3.common.evaluation import evaluate_policy
|
||||||
|
from sb3_contrib.ppo_mask import MaskablePPO
|
||||||
|
import envs
|
||||||
|
import wrapper
|
||||||
|
import resources
|
||||||
|
import pprint
|
||||||
|
from agents import trading_agent
|
||||||
|
from wrapper.base_econ_wrapper import BaseEconWrapper
|
||||||
|
from wrapper.reciever_econ_wrapper import RecieverEconWrapper
|
||||||
|
from wrapper.sb3_econ_converter import SB3EconConverter
|
||||||
|
from tqdm import tqdm
|
||||||
|
import components
|
||||||
|
from stable_baselines3.common.env_checker import check_env
|
||||||
|
from stable_baselines3 import PPO
|
||||||
|
from stable_baselines3.common.vec_env.vec_monitor import VecMonitor
|
||||||
|
from stable_baselines3.common.vec_env.vec_normalize import VecNormalize
|
||||||
|
from sb3_contrib import RecurrentPPO
|
||||||
|
from envs.econ_wrapper import EconVecEnv
|
||||||
|
from stable_baselines3.common.callbacks import BaseCallback
|
||||||
|
import yaml
|
||||||
|
import time
|
||||||
|
from threading import Thread
|
||||||
|
|
||||||
|
env_config = {
|
||||||
|
# ===== SCENARIO CLASS =====
|
||||||
|
# Which Scenario class to use: the class's name in the Scenario Registry (foundation.scenarios).
|
||||||
|
# The environment object will be an instance of the Scenario class.
|
||||||
|
'scenario_name': 'econ',
|
||||||
|
|
||||||
|
# ===== COMPONENTS =====
|
||||||
|
# Which components to use (specified as list of ("component_name", {component_kwargs}) tuples).
|
||||||
|
# "component_name" refers to the Component class's name in the Component Registry (foundation.components)
|
||||||
|
# {component_kwargs} is a dictionary of kwargs passed to the Component class
|
||||||
|
# The order in which components reset, step, and generate obs follows their listed order below.
|
||||||
|
'components': [
|
||||||
|
# (1) Building houses
|
||||||
|
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
||||||
|
# (2) Trading collectible resources
|
||||||
|
('ContinuousDoubleAuction', {'max_num_orders': 10}),
|
||||||
|
# (3) Movement and resource collection
|
||||||
|
('SimpleGather', {}),
|
||||||
|
('ExternalMarket',{'market_demand':{
|
||||||
|
'Gem': 15
|
||||||
|
}}),
|
||||||
|
],
|
||||||
|
|
||||||
|
# ===== SCENARIO CLASS ARGUMENTS =====
|
||||||
|
# (optional) kwargs that are added by the Scenario class (i.e. not defined in BaseEnvironment)
|
||||||
|
|
||||||
|
'starting_agent_coin': 50,
|
||||||
|
'fixed_four_skill_and_loc': True,
|
||||||
|
|
||||||
|
# ===== STANDARD ARGUMENTS ======
|
||||||
|
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
||||||
|
'agent_composition': {"BasicMobileAgent": 20,"TradingAgent":5}, # Number of non-planner agents (must be > 1)
|
||||||
|
'world_size': [5, 5], # [Height, Width] of the env world
|
||||||
|
'episode_length': 256, # Number of timesteps per episode
|
||||||
|
'isoelastic_eta':0.001,
|
||||||
|
'allow_observation_scaling': True,
|
||||||
|
'dense_log_frequency': 100,
|
||||||
|
'world_dense_log_frequency':1,
|
||||||
|
'energy_cost':0,
|
||||||
|
'energy_warmup_method': "auto",
|
||||||
|
'energy_warmup_constant': 4000,
|
||||||
|
|
||||||
|
# In multi-action-mode, the policy selects an action for each action subspace (defined in component code).
|
||||||
|
# Otherwise, the policy selects only 1 action.
|
||||||
|
'multi_action_mode_agents': False,
|
||||||
|
'multi_action_mode_planner': False,
|
||||||
|
|
||||||
|
# When flattening observations, concatenate scalar & vector observations before output.
|
||||||
|
# Otherwise, return observations with minimal processing.
|
||||||
|
'flatten_observations': False,
|
||||||
|
# When Flattening masks, concatenate each action subspace mask into a single array.
|
||||||
|
# Note: flatten_masks = True is required for masking action logits in the code below.
|
||||||
|
'flatten_masks': True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
eval_env_config = {
|
||||||
|
# ===== SCENARIO CLASS =====
|
||||||
|
# Which Scenario class to use: the class's name in the Scenario Registry (foundation.scenarios).
|
||||||
|
# The environment object will be an instance of the Scenario class.
|
||||||
|
'scenario_name': 'econ',
|
||||||
|
|
||||||
|
# ===== COMPONENTS =====
|
||||||
|
# Which components to use (specified as list of ("component_name", {component_kwargs}) tuples).
|
||||||
|
# "component_name" refers to the Component class's name in the Component Registry (foundation.components)
|
||||||
|
# {component_kwargs} is a dictionary of kwargs passed to the Component class
|
||||||
|
# The order in which components reset, step, and generate obs follows their listed order below.
|
||||||
|
'components': [
|
||||||
|
# (1) Building houses
|
||||||
|
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
||||||
|
# (2) Trading collectible resources
|
||||||
|
('ContinuousDoubleAuction', {'max_num_orders': 10}),
|
||||||
|
# (3) Movement and resource collection
|
||||||
|
('SimpleGather', {}),
|
||||||
|
('ExternalMarket',{'market_demand':{
|
||||||
|
'Gem': 15
|
||||||
|
}}),
|
||||||
|
],
|
||||||
|
|
||||||
|
# ===== SCENARIO CLASS ARGUMENTS =====
|
||||||
|
# (optional) kwargs that are added by the Scenario class (i.e. not defined in BaseEnvironment)
|
||||||
|
|
||||||
|
'starting_agent_coin': 50,
|
||||||
|
'fixed_four_skill_and_loc': True,
|
||||||
|
|
||||||
|
# ===== STANDARD ARGUMENTS ======
|
||||||
|
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
||||||
|
'agent_composition': {"BasicMobileAgent": 20,"TradingAgent":5}, # Number of non-planner agents (must be > 1)
|
||||||
|
'world_size': [1, 1], # [Height, Width] of the env world
|
||||||
|
'episode_length': 256, # Number of timesteps per episode
|
||||||
|
'allow_observation_scaling': True,
|
||||||
|
'isoelastic_eta':0.001,
|
||||||
|
'dense_log_frequency': 1,
|
||||||
|
'world_dense_log_frequency':1,
|
||||||
|
'energy_cost':0,
|
||||||
|
'energy_warmup_method': "auto",
|
||||||
|
'energy_warmup_constant': 4000,
|
||||||
|
|
||||||
|
# In multi-action-mode, the policy selects an action for each action subspace (defined in component code).
|
||||||
|
# Otherwise, the policy selects only 1 action.
|
||||||
|
'multi_action_mode_agents': False,
|
||||||
|
'multi_action_mode_planner': False,
|
||||||
|
|
||||||
|
# When flattening observations, concatenate scalar & vector observations before output.
|
||||||
|
# Otherwise, return observations with minimal processing.
|
||||||
|
'flatten_observations': False,
|
||||||
|
# When Flattening masks, concatenate each action subspace mask into a single array.
|
||||||
|
# Note: flatten_masks = True is required for masking action logits in the code below.
|
||||||
|
'flatten_masks': True,
|
||||||
|
}
|
||||||
|
|
||||||
|
num_frames=1
|
||||||
|
|
||||||
|
class TensorboardCallback(BaseCallback):
|
||||||
|
"""
|
||||||
|
Custom callback for plotting additional values in tensorboard.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self,econ, verbose=0):
|
||||||
|
super().__init__(verbose)
|
||||||
|
self.econ=econ
|
||||||
|
self.metrics=econ.scenario_metrics()
|
||||||
|
def _on_step(self) -> bool:
|
||||||
|
# Log scalar value (here a random variable)
|
||||||
|
if econ.world.timestep==0:
|
||||||
|
prev_metrics=self.metrics
|
||||||
|
if self.econ.previous_episode_metrics is None:
|
||||||
|
self.metrics=self.econ.scenario_metrics()
|
||||||
|
else:
|
||||||
|
self.metrics=self.econ.previous_episode_metrics
|
||||||
|
curr_prod=self.metrics["social/productivity"]
|
||||||
|
trend_pord=curr_prod/np.maximum(0.0001,prev_metrics["social/productivity"])
|
||||||
|
self.logger.record("social/total_productivity", curr_prod)
|
||||||
|
self.logger.record("social/delta_productivity", trend_pord)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
min_at_target_basic=0.5
|
||||||
|
min_lr_basic=5e-6
|
||||||
|
start_lr_basic=9e-4
|
||||||
|
|
||||||
|
min_at_target_trade=0.5
|
||||||
|
min_lr_trade=5e-6
|
||||||
|
start_lr_trade=9e-4
|
||||||
|
|
||||||
|
def learning_rate_adj_basic(x) -> float:
|
||||||
|
diff=start_lr_basic-min_lr_basic
|
||||||
|
lr=min_lr_basic+x*diff
|
||||||
|
return lr
|
||||||
|
|
||||||
|
def learning_rate_adj_trade(x) -> float:
|
||||||
|
diff=start_lr_trade-min_lr_trade
|
||||||
|
lr=min_lr_basic+x*diff
|
||||||
|
return lr
|
||||||
|
|
||||||
|
def printMarket(market):
|
||||||
|
for i in range(len(market)):
|
||||||
|
step=market[i]
|
||||||
|
if len(step)>0:
|
||||||
|
print("=== Step {} ===".format(i))
|
||||||
|
for transaction in step:
|
||||||
|
t=transaction
|
||||||
|
transstring = "({}) {} -> {} | [{}/{}] {} Coins\n".format(t["commodity"],t["seller"],t["buyer"],t["ask"],t["bid"],t["price"])
|
||||||
|
print(transstring)
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def printBuilds(builds):
|
||||||
|
for i in range(len(builds)):
|
||||||
|
step=builds[i]
|
||||||
|
if len(step)>0:
|
||||||
|
for build in step:
|
||||||
|
t=build
|
||||||
|
transstring = "({}) Builder: {}, Skill: {}, Income {} ".format(i,t["builder"],t["build_skill"],t["income"])
|
||||||
|
print(transstring)
|
||||||
|
return ""
|
||||||
|
def printReplay(econ,agentid):
|
||||||
|
worldmaps=["Stone","Wood"]
|
||||||
|
|
||||||
|
log=econ.previous_episode_dense_log
|
||||||
|
agent=econ.world.agents[agentid]
|
||||||
|
|
||||||
|
agentid=str(agentid)
|
||||||
|
maxsetp=len(log["states"])-1
|
||||||
|
|
||||||
|
for step in range(maxsetp):
|
||||||
|
print()
|
||||||
|
print("=== Step {} ===".format(step))
|
||||||
|
# state
|
||||||
|
print("--- World ---")
|
||||||
|
world=log['world'][step]
|
||||||
|
for res in worldmaps:
|
||||||
|
print("{}: {}".format(res,world[res][0][0]))
|
||||||
|
print("--- State ---")
|
||||||
|
state=log['states'][step][agentid]
|
||||||
|
|
||||||
|
pprint.pprint(state)
|
||||||
|
print("--- Action ---")
|
||||||
|
action=log["actions"][step][agentid]
|
||||||
|
|
||||||
|
|
||||||
|
if action=={}:
|
||||||
|
print("Action: 0 -> NOOP")
|
||||||
|
else:
|
||||||
|
for k in action:
|
||||||
|
formats="Action: {}({})".format(k,action[k])
|
||||||
|
print(formats)
|
||||||
|
print("--- Reward ---")
|
||||||
|
reward=log["rewards"][step][agentid]
|
||||||
|
print("Reward: {}".format(reward))
|
||||||
|
|
||||||
|
#Setup Env Objects
|
||||||
|
econ=foundation.make_env_instance(**env_config)
|
||||||
|
|
||||||
|
market=econ.get_component("ContinuousDoubleAuction")
|
||||||
|
action=market.get_n_actions("TradingAgent")
|
||||||
|
baseEconWrapper=BaseEconWrapper(econ)
|
||||||
|
baseEconWrapper.run()
|
||||||
|
time.sleep(0.5)
|
||||||
|
mobileRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="BasicMobileAgent")
|
||||||
|
tradeRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="TradingAgent")
|
||||||
|
sb3_traderConverter=SB3EconConverter(tradeRecieverEconWrapper,econ,"TradingAgent",True)
|
||||||
|
sb3Converter=SB3EconConverter(mobileRecieverEconWrapper,econ,"BasicMobileAgent",True)
|
||||||
|
# attach sb3 wrappers
|
||||||
|
|
||||||
|
monenv=VecMonitor(venv=sb3Converter,info_keywords=["social/productivity","trend/productivity"])
|
||||||
|
montraidingenv=VecMonitor(venv=sb3_traderConverter)
|
||||||
|
|
||||||
|
stackenv_basic=vec_frame_stack.VecFrameStack(venv=monenv,n_stack=num_frames)
|
||||||
|
stackenv_traid=vec_frame_stack.VecFrameStack(venv=montraidingenv,n_stack=num_frames)
|
||||||
|
# Model setup complete
|
||||||
|
|
||||||
|
# Setup Eval Env
|
||||||
|
econ_eval=foundation.make_env_instance(**eval_env_config)
|
||||||
|
|
||||||
|
|
||||||
|
baseEconWrapper_eval=BaseEconWrapper(econ_eval)
|
||||||
|
baseEconWrapper_eval.run()
|
||||||
|
time.sleep(0.5)
|
||||||
|
mobileRecieverEconWrapper_eval=RecieverEconWrapper(base_econ=baseEconWrapper_eval,agent_classname="BasicMobileAgent")
|
||||||
|
tradeRecieverEconWrapper_eval=RecieverEconWrapper(base_econ=baseEconWrapper_eval,agent_classname="TradingAgent")
|
||||||
|
sb3_traderConverter_eval=SB3EconConverter(tradeRecieverEconWrapper_eval,econ_eval,"TradingAgent",False)
|
||||||
|
sb3Converter_eval=SB3EconConverter(mobileRecieverEconWrapper_eval,econ_eval,"BasicMobileAgent",False)
|
||||||
|
# attach sb3 wrappers
|
||||||
|
|
||||||
|
monenv_eval=VecMonitor(venv=sb3Converter_eval,info_keywords=["social/productivity","trend/productivity"])
|
||||||
|
montraidingenv_eval=VecMonitor(venv=sb3_traderConverter_eval)
|
||||||
|
|
||||||
|
stackenv_basic_eval=vec_frame_stack.VecFrameStack(venv=monenv_eval,n_stack=num_frames)
|
||||||
|
stackenv_traid_eval=vec_frame_stack.VecFrameStack(venv=montraidingenv_eval,n_stack=num_frames)
|
||||||
|
|
||||||
|
|
||||||
|
obs=monenv.reset()
|
||||||
|
|
||||||
|
|
||||||
|
# define training functions
|
||||||
|
def train(model,timesteps, econ_call,process_bar,name,db,index):
|
||||||
|
db[index]=model.learn(total_timesteps=timesteps,progress_bar=process_bar,reset_num_timesteps=False,tb_log_name=name,callback=TensorboardCallback(econ_call))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# prepare training
|
||||||
|
run_number=int(np.random.rand()*100)
|
||||||
|
runname="run_{}".format(run_number)
|
||||||
|
model_db=[None,None] # object for storing model
|
||||||
|
|
||||||
|
|
||||||
|
model = MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.001, vf_coef=0.5 ,gamma=0.99, learning_rate=learning_rate_adj_basic,env=stackenv_basic, seed=445,verbose=1,device="cuda",tensorboard_log="./log")
|
||||||
|
model_trade=MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.001, vf_coef=0.5 ,gamma=0.99, learning_rate=learning_rate_adj_trade,env=stackenv_traid, seed=445,verbose=1,device="cuda",tensorboard_log="./log")
|
||||||
|
|
||||||
|
n_agents=econ.n_agents
|
||||||
|
|
||||||
|
total_required_for_episode_basic=len(mobileRecieverEconWrapper.agnet_idx)*env_config['episode_length']
|
||||||
|
total_required_for_episode_traid=len(tradeRecieverEconWrapper.agnet_idx)*env_config['episode_length']
|
||||||
|
|
||||||
|
print("this is run {}".format(runname))
|
||||||
|
|
||||||
|
while True:
|
||||||
|
|
||||||
|
|
||||||
|
#Train
|
||||||
|
runname="run_{}_{}".format(run_number,"basic")
|
||||||
|
|
||||||
|
thread_model=Thread(target=train,args=(model,total_required_for_episode_basic*150,econ,True,runname,model_db,0))
|
||||||
|
runname="run_{}_{}".format(run_number,"trader")
|
||||||
|
thread_model_traid=Thread(target=train,args=(model_trade,total_required_for_episode_traid*150,econ,False,runname,model_db,1))
|
||||||
|
|
||||||
|
thread_model.start()
|
||||||
|
thread_model_traid.start()
|
||||||
|
thread_model.join()
|
||||||
|
thread_model_traid.join()
|
||||||
|
#normenv.save("temp-normalizer.ai")
|
||||||
|
model=model_db[0]
|
||||||
|
model_trade=model_db[1]
|
||||||
|
model.save("basic.ai")
|
||||||
|
model_trade.save("trade.ai")
|
||||||
|
|
||||||
|
## Run Eval
|
||||||
|
print("### EVAL ###")
|
||||||
|
obs_basic=stackenv_basic_eval.reset()
|
||||||
|
obs_trade=stackenv_traid_eval.reset()
|
||||||
|
done=False
|
||||||
|
for i in tqdm(range(eval_env_config['episode_length'])):
|
||||||
|
#create masks
|
||||||
|
masks_basic=stackenv_basic_eval.action_masks()
|
||||||
|
masks_trade=stackenv_traid_eval.action_masks()
|
||||||
|
# get actions
|
||||||
|
action_basic=model.predict(obs_basic,action_masks=masks_basic)
|
||||||
|
action_trade=model_trade.predict(obs_trade,action_masks=masks_trade)
|
||||||
|
#submit async directly for non blocking operation
|
||||||
|
sb3Converter_eval.step_async(action_basic[0])
|
||||||
|
sb3_traderConverter_eval.step_async(action_trade[0])
|
||||||
|
# retieve full results
|
||||||
|
obs_basic,rew_basic,done_e,info=stackenv_basic_eval.step(action_basic[0])
|
||||||
|
obs_trade,rew_trade,done_e,info=stackenv_traid_eval.step(action_trade[0])
|
||||||
|
done=done_e[0]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
market=econ_eval.get_component("ContinuousDoubleAuction")
|
||||||
|
craft=econ_eval.get_component("Craft")
|
||||||
|
# trades=market.get_dense_log()
|
||||||
|
build=craft.get_dense_log()
|
||||||
|
met=econ.previous_episode_metrics
|
||||||
|
printReplay(econ_eval,0)
|
||||||
|
# printMarket(trades)
|
||||||
|
# printBuilds(builds=build)
|
||||||
|
print("social/productivity: {}".format(met["social/productivity"]))
|
||||||
|
print("labor/weighted_cost: {}".format(met["labor/weighted_cost"]))
|
||||||
|
print("labor/warmup_integrator: {}".format(met["labor/warmup_integrator"]))
|
||||||
|
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -41,11 +41,11 @@ env_config = {
|
|||||||
# (1) Building houses
|
# (1) Building houses
|
||||||
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
||||||
# (2) Trading collectible resources
|
# (2) Trading collectible resources
|
||||||
('ContinuousDoubleAuction', {'max_num_orders': 10}),
|
('ContinuousDoubleAuction', {'max_num_orders': 10,'max_bid_ask':100}),
|
||||||
# (3) Movement and resource collection
|
# (3) Movement and resource collection
|
||||||
('SimpleGather', {}),
|
('SimpleGather', {}),
|
||||||
('ExternalMarket',{'market_demand':{
|
('ExternalMarket',{'market_demand':{
|
||||||
'Gem': 15
|
'Gem': 65
|
||||||
}}),
|
}}),
|
||||||
],
|
],
|
||||||
|
|
||||||
@@ -57,14 +57,14 @@ env_config = {
|
|||||||
|
|
||||||
# ===== STANDARD ARGUMENTS ======
|
# ===== STANDARD ARGUMENTS ======
|
||||||
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
||||||
'agent_composition': {"BasicMobileAgent": 20,"TradingAgent":5}, # Number of non-planner agents (must be > 1)
|
'agent_composition': {"BasicMobileAgent": 10,"TradingAgent":2}, # Number of non-planner agents (must be > 1)
|
||||||
'world_size': [5, 5], # [Height, Width] of the env world
|
'world_size': [5, 5], # [Height, Width] of the env world
|
||||||
'episode_length': 256, # Number of timesteps per episode
|
'episode_length': 256, # Number of timesteps per episode
|
||||||
'isoelastic_eta':0.001,
|
'isoelastic_eta':0.001,
|
||||||
'allow_observation_scaling': True,
|
'allow_observation_scaling': True,
|
||||||
'dense_log_frequency': 100,
|
'dense_log_frequency': 100,
|
||||||
'world_dense_log_frequency':1,
|
'world_dense_log_frequency':1,
|
||||||
'energy_cost':0,
|
'energy_cost':0.2,
|
||||||
'energy_warmup_method': "auto",
|
'energy_warmup_method': "auto",
|
||||||
'energy_warmup_constant': 4000,
|
'energy_warmup_constant': 4000,
|
||||||
|
|
||||||
@@ -97,7 +97,7 @@ eval_env_config = {
|
|||||||
# (1) Building houses
|
# (1) Building houses
|
||||||
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
||||||
# (2) Trading collectible resources
|
# (2) Trading collectible resources
|
||||||
('ContinuousDoubleAuction', {'max_num_orders': 10}),
|
('ContinuousDoubleAuction', {'max_num_orders': 10,'max_bid_ask':100}),
|
||||||
# (3) Movement and resource collection
|
# (3) Movement and resource collection
|
||||||
('SimpleGather', {}),
|
('SimpleGather', {}),
|
||||||
('ExternalMarket',{'market_demand':{
|
('ExternalMarket',{'market_demand':{
|
||||||
@@ -113,14 +113,14 @@ eval_env_config = {
|
|||||||
|
|
||||||
# ===== STANDARD ARGUMENTS ======
|
# ===== STANDARD ARGUMENTS ======
|
||||||
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
||||||
'agent_composition': {"BasicMobileAgent": 20,"TradingAgent":5}, # Number of non-planner agents (must be > 1)
|
'agent_composition': {"BasicMobileAgent": 10,"TradingAgent":10}, # Number of non-planner agents (must be > 1)
|
||||||
'world_size': [1, 1], # [Height, Width] of the env world
|
'world_size': [1, 1], # [Height, Width] of the env world
|
||||||
'episode_length': 256, # Number of timesteps per episode
|
'episode_length': 256, # Number of timesteps per episode
|
||||||
'allow_observation_scaling': True,
|
'allow_observation_scaling': True,
|
||||||
'isoelastic_eta':0.001,
|
'isoelastic_eta':0.001,
|
||||||
'dense_log_frequency': 1,
|
'dense_log_frequency': 1,
|
||||||
'world_dense_log_frequency':1,
|
'world_dense_log_frequency':1,
|
||||||
'energy_cost':0,
|
'energy_cost':0.2,
|
||||||
'energy_warmup_method': "auto",
|
'energy_warmup_method': "auto",
|
||||||
'energy_warmup_constant': 4000,
|
'energy_warmup_constant': 4000,
|
||||||
|
|
||||||
@@ -138,6 +138,7 @@ eval_env_config = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
num_frames=1
|
num_frames=1
|
||||||
|
CONNECTION_STRING = "mongodb+srv://root:econ@localhost:/econ"
|
||||||
|
|
||||||
class TensorboardCallback(BaseCallback):
|
class TensorboardCallback(BaseCallback):
|
||||||
"""
|
"""
|
||||||
@@ -157,7 +158,7 @@ class TensorboardCallback(BaseCallback):
|
|||||||
else:
|
else:
|
||||||
self.metrics=self.econ.previous_episode_metrics
|
self.metrics=self.econ.previous_episode_metrics
|
||||||
curr_prod=self.metrics["social/productivity"]
|
curr_prod=self.metrics["social/productivity"]
|
||||||
trend_pord=curr_prod-prev_metrics["social/productivity"]
|
trend_pord=curr_prod/np.maximum(0.0001,prev_metrics["social/productivity"])
|
||||||
self.logger.record("social/total_productivity", curr_prod)
|
self.logger.record("social/total_productivity", curr_prod)
|
||||||
self.logger.record("social/delta_productivity", trend_pord)
|
self.logger.record("social/delta_productivity", trend_pord)
|
||||||
|
|
||||||
@@ -241,7 +242,7 @@ econ=foundation.make_env_instance(**env_config)
|
|||||||
|
|
||||||
market=econ.get_component("ContinuousDoubleAuction")
|
market=econ.get_component("ContinuousDoubleAuction")
|
||||||
action=market.get_n_actions("TradingAgent")
|
action=market.get_n_actions("TradingAgent")
|
||||||
baseEconWrapper=BaseEconWrapper(econ)
|
baseEconWrapper=BaseEconWrapper(econ,True)
|
||||||
baseEconWrapper.run()
|
baseEconWrapper.run()
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
mobileRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="BasicMobileAgent")
|
mobileRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="BasicMobileAgent")
|
||||||
@@ -261,7 +262,7 @@ stackenv_traid=vec_frame_stack.VecFrameStack(venv=montraidingenv,n_stack=num_fra
|
|||||||
econ_eval=foundation.make_env_instance(**eval_env_config)
|
econ_eval=foundation.make_env_instance(**eval_env_config)
|
||||||
|
|
||||||
|
|
||||||
baseEconWrapper_eval=BaseEconWrapper(econ_eval)
|
baseEconWrapper_eval=BaseEconWrapper(econ_eval,False)
|
||||||
baseEconWrapper_eval.run()
|
baseEconWrapper_eval.run()
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
mobileRecieverEconWrapper_eval=RecieverEconWrapper(base_econ=baseEconWrapper_eval,agent_classname="BasicMobileAgent")
|
mobileRecieverEconWrapper_eval=RecieverEconWrapper(base_econ=baseEconWrapper_eval,agent_classname="BasicMobileAgent")
|
||||||
@@ -292,8 +293,10 @@ runname="run_{}".format(run_number)
|
|||||||
model_db=[None,None] # object for storing model
|
model_db=[None,None] # object for storing model
|
||||||
|
|
||||||
|
|
||||||
model = MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.1, vf_coef=0.5 ,gamma=0.99, learning_rate=learning_rate_adj_basic,env=stackenv_basic, seed=445,verbose=1,device="cuda",tensorboard_log="./log")
|
model = MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.001, vf_coef=0.5 ,gamma=0.99, learning_rate=4e-4,env=stackenv_basic, seed=500,verbose=1,device="cuda",tensorboard_log="./log")
|
||||||
model_trade=MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.1, vf_coef=0.5 ,gamma=0.99, learning_rate=learning_rate_adj_trade,env=stackenv_traid, seed=445,verbose=1,device="cuda",tensorboard_log="./log")
|
model_trade=MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.001, vf_coef=0.5 ,gamma=0.99, learning_rate=4e-4,env=stackenv_traid, seed=500,verbose=1,device="cuda",tensorboard_log="./log")
|
||||||
|
model=model.load("basic.ai",stackenv_basic)
|
||||||
|
model_trade=model_trade.load("trade.ai",stackenv_traid)
|
||||||
|
|
||||||
n_agents=econ.n_agents
|
n_agents=econ.n_agents
|
||||||
|
|
||||||
@@ -308,9 +311,9 @@ while True:
|
|||||||
#Train
|
#Train
|
||||||
runname="run_{}_{}".format(run_number,"basic")
|
runname="run_{}_{}".format(run_number,"basic")
|
||||||
|
|
||||||
thread_model=Thread(target=train,args=(model,total_required_for_episode_basic*150,econ,True,runname,model_db,0))
|
thread_model=Thread(target=train,args=(model,total_required_for_episode_basic*50,econ,True,runname,model_db,0))
|
||||||
runname="run_{}_{}".format(run_number,"trader")
|
runname="run_{}_{}".format(run_number,"trader")
|
||||||
thread_model_traid=Thread(target=train,args=(model_trade,total_required_for_episode_traid*150,econ,False,runname,model_db,1))
|
thread_model_traid=Thread(target=train,args=(model_trade,total_required_for_episode_traid*50,econ,False,runname,model_db,1))
|
||||||
|
|
||||||
thread_model.start()
|
thread_model.start()
|
||||||
thread_model_traid.start()
|
thread_model_traid.start()
|
||||||
@@ -324,6 +327,7 @@ while True:
|
|||||||
|
|
||||||
## Run Eval
|
## Run Eval
|
||||||
print("### EVAL ###")
|
print("### EVAL ###")
|
||||||
|
baseEconWrapper_eval.force_reset()
|
||||||
obs_basic=stackenv_basic_eval.reset()
|
obs_basic=stackenv_basic_eval.reset()
|
||||||
obs_trade=stackenv_traid_eval.reset()
|
obs_trade=stackenv_traid_eval.reset()
|
||||||
done=False
|
done=False
|
||||||
|
|||||||
+2
-1
@@ -1,3 +1,4 @@
|
|||||||
|
pandas
|
||||||
|
pymongo
|
||||||
gym
|
gym
|
||||||
ray[rllib]
|
ray[rllib]
|
||||||
@@ -25,6 +25,7 @@ from stable_baselines3.common.callbacks import BaseCallback
|
|||||||
import yaml
|
import yaml
|
||||||
import time
|
import time
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
|
from db.export_to_mongo import MongoExporter
|
||||||
|
|
||||||
env_config = {
|
env_config = {
|
||||||
# ===== SCENARIO CLASS =====
|
# ===== SCENARIO CLASS =====
|
||||||
@@ -41,29 +42,30 @@ env_config = {
|
|||||||
# (1) Building houses
|
# (1) Building houses
|
||||||
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
||||||
# (2) Trading collectible resources
|
# (2) Trading collectible resources
|
||||||
('ContinuousDoubleAuction', {'max_num_orders': 10}),
|
('ContinuousDoubleAuction', {'max_num_orders': 10,'max_bid_ask':100}),
|
||||||
# (3) Movement and resource collection
|
# (3) Movement and resource collection
|
||||||
('SimpleGather', {}),
|
('SimpleGather', {}),
|
||||||
('ExternalMarket',{'market_demand':{
|
('ExternalMarket',{'market_demand':{
|
||||||
'Gem': 15
|
'Gem': 90
|
||||||
}}),
|
}}),
|
||||||
],
|
],
|
||||||
|
|
||||||
# ===== SCENARIO CLASS ARGUMENTS =====
|
# ===== SCENARIO CLASS ARGUMENTS =====
|
||||||
# (optional) kwargs that are added by the Scenario class (i.e. not defined in BaseEnvironment)
|
# (optional) kwargs that are added by the Scenario class (i.e. not defined in BaseEnvironment)
|
||||||
|
|
||||||
'starting_agent_coin': 10,
|
'starting_agent_coin': 50,
|
||||||
'fixed_four_skill_and_loc': True,
|
'fixed_four_skill_and_loc': True,
|
||||||
|
|
||||||
# ===== STANDARD ARGUMENTS ======
|
# ===== STANDARD ARGUMENTS ======
|
||||||
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
||||||
'agent_composition': {"BasicMobileAgent": 20,"TradingAgent":5}, # Number of non-planner agents (must be > 1)
|
'agent_composition': {"BasicMobileAgent": 2,"TradingAgent":5}, # Number of non-planner agents (must be > 1)
|
||||||
'world_size': [5, 5], # [Height, Width] of the env world
|
'world_size': [5, 5], # [Height, Width] of the env world
|
||||||
'episode_length': 256, # Number of timesteps per episode
|
'episode_length': 256, # Number of timesteps per episode
|
||||||
|
'isoelastic_eta':0.001,
|
||||||
'allow_observation_scaling': True,
|
'allow_observation_scaling': True,
|
||||||
'dense_log_frequency': 100,
|
'dense_log_frequency': 100,
|
||||||
'world_dense_log_frequency':1,
|
'world_dense_log_frequency':1,
|
||||||
'energy_cost':0,
|
'energy_cost':0.2,
|
||||||
'energy_warmup_method': "auto",
|
'energy_warmup_method': "auto",
|
||||||
'energy_warmup_constant': 4000,
|
'energy_warmup_constant': 4000,
|
||||||
|
|
||||||
@@ -80,63 +82,8 @@ env_config = {
|
|||||||
'flatten_masks': True,
|
'flatten_masks': True,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
num_frames=1
|
||||||
eval_env_config = {
|
CONNECTION_STRING = "mongodb://root:econ@localhost:27017/"
|
||||||
# ===== SCENARIO CLASS =====
|
|
||||||
# Which Scenario class to use: the class's name in the Scenario Registry (foundation.scenarios).
|
|
||||||
# The environment object will be an instance of the Scenario class.
|
|
||||||
'scenario_name': 'econ',
|
|
||||||
|
|
||||||
# ===== COMPONENTS =====
|
|
||||||
# Which components to use (specified as list of ("component_name", {component_kwargs}) tuples).
|
|
||||||
# "component_name" refers to the Component class's name in the Component Registry (foundation.components)
|
|
||||||
# {component_kwargs} is a dictionary of kwargs passed to the Component class
|
|
||||||
# The order in which components reset, step, and generate obs follows their listed order below.
|
|
||||||
'components': [
|
|
||||||
# (1) Building houses
|
|
||||||
('Craft', {'skill_dist': "pareto", 'commodities': ["Gem"],'max_skill_amount_benefit':1.5}),
|
|
||||||
# (2) Trading collectible resources
|
|
||||||
('ContinuousDoubleAuction', {'max_num_orders': 10}),
|
|
||||||
# (3) Movement and resource collection
|
|
||||||
('SimpleGather', {}),
|
|
||||||
('ExternalMarket',{'market_demand':{
|
|
||||||
'Gem': 15
|
|
||||||
}}),
|
|
||||||
],
|
|
||||||
|
|
||||||
# ===== SCENARIO CLASS ARGUMENTS =====
|
|
||||||
# (optional) kwargs that are added by the Scenario class (i.e. not defined in BaseEnvironment)
|
|
||||||
|
|
||||||
'starting_agent_coin': 10,
|
|
||||||
'fixed_four_skill_and_loc': True,
|
|
||||||
|
|
||||||
# ===== STANDARD ARGUMENTS ======
|
|
||||||
# kwargs that are used by every Scenario class (i.e. defined in BaseEnvironment)
|
|
||||||
'agent_composition': {"BasicMobileAgent": 20,"TradingAgent":5}, # Number of non-planner agents (must be > 1)
|
|
||||||
'world_size': [1, 1], # [Height, Width] of the env world
|
|
||||||
'episode_length': 256, # Number of timesteps per episode
|
|
||||||
'allow_observation_scaling': True,
|
|
||||||
'dense_log_frequency': 1,
|
|
||||||
'world_dense_log_frequency':1,
|
|
||||||
'energy_cost':0,
|
|
||||||
'energy_warmup_method': "auto",
|
|
||||||
'energy_warmup_constant': 4000,
|
|
||||||
|
|
||||||
# In multi-action-mode, the policy selects an action for each action subspace (defined in component code).
|
|
||||||
# Otherwise, the policy selects only 1 action.
|
|
||||||
'multi_action_mode_agents': False,
|
|
||||||
'multi_action_mode_planner': False,
|
|
||||||
|
|
||||||
# When flattening observations, concatenate scalar & vector observations before output.
|
|
||||||
# Otherwise, return observations with minimal processing.
|
|
||||||
'flatten_observations': False,
|
|
||||||
# When Flattening masks, concatenate each action subspace mask into a single array.
|
|
||||||
# Note: flatten_masks = True is required for masking action logits in the code below.
|
|
||||||
'flatten_masks': True,
|
|
||||||
}
|
|
||||||
|
|
||||||
num_frames=5
|
|
||||||
|
|
||||||
class TensorboardCallback(BaseCallback):
|
class TensorboardCallback(BaseCallback):
|
||||||
"""
|
"""
|
||||||
Custom callback for plotting additional values in tensorboard.
|
Custom callback for plotting additional values in tensorboard.
|
||||||
@@ -161,6 +108,23 @@ class TensorboardCallback(BaseCallback):
|
|||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
min_at_target_basic=0.5
|
||||||
|
min_lr_basic=5e-6
|
||||||
|
start_lr_basic=9e-4
|
||||||
|
|
||||||
|
min_at_target_trade=0.5
|
||||||
|
min_lr_trade=5e-6
|
||||||
|
start_lr_trade=9e-4
|
||||||
|
|
||||||
|
def learning_rate_adj_basic(x) -> float:
|
||||||
|
diff=start_lr_basic-min_lr_basic
|
||||||
|
lr=min_lr_basic+x*diff
|
||||||
|
return lr
|
||||||
|
|
||||||
|
def learning_rate_adj_trade(x) -> float:
|
||||||
|
diff=start_lr_trade-min_lr_trade
|
||||||
|
lr=min_lr_basic+x*diff
|
||||||
|
return lr
|
||||||
|
|
||||||
def printMarket(market):
|
def printMarket(market):
|
||||||
for i in range(len(market)):
|
for i in range(len(market)):
|
||||||
@@ -219,16 +183,15 @@ def printReplay(econ,agentid):
|
|||||||
|
|
||||||
#Setup Env Objects
|
#Setup Env Objects
|
||||||
econ=foundation.make_env_instance(**env_config)
|
econ=foundation.make_env_instance(**env_config)
|
||||||
|
|
||||||
market=econ.get_component("ContinuousDoubleAuction")
|
market=econ.get_component("ContinuousDoubleAuction")
|
||||||
action=market.get_n_actions("TradingAgent")
|
action=market.get_n_actions("TradingAgent")
|
||||||
baseEconWrapper=BaseEconWrapper(econ)
|
baseEconWrapper=BaseEconWrapper(econ,False)
|
||||||
baseEconWrapper.run()
|
baseEconWrapper.run()
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
mobileRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="BasicMobileAgent")
|
mobileRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="BasicMobileAgent")
|
||||||
tradeRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="TradingAgent")
|
tradeRecieverEconWrapper=RecieverEconWrapper(base_econ=baseEconWrapper,agent_classname="TradingAgent")
|
||||||
sb3_traderConverter=SB3EconConverter(tradeRecieverEconWrapper,econ,"TradingAgent",True)
|
sb3_traderConverter=SB3EconConverter(tradeRecieverEconWrapper,econ,"TradingAgent",False)
|
||||||
sb3Converter=SB3EconConverter(mobileRecieverEconWrapper,econ,"BasicMobileAgent",True)
|
sb3Converter=SB3EconConverter(mobileRecieverEconWrapper,econ,"BasicMobileAgent",False)
|
||||||
# attach sb3 wrappers
|
# attach sb3 wrappers
|
||||||
|
|
||||||
monenv=VecMonitor(venv=sb3Converter,info_keywords=["social/productivity","trend/productivity"])
|
monenv=VecMonitor(venv=sb3Converter,info_keywords=["social/productivity","trend/productivity"])
|
||||||
@@ -238,28 +201,8 @@ stackenv_basic=vec_frame_stack.VecFrameStack(venv=monenv,n_stack=num_frames)
|
|||||||
stackenv_traid=vec_frame_stack.VecFrameStack(venv=montraidingenv,n_stack=num_frames)
|
stackenv_traid=vec_frame_stack.VecFrameStack(venv=montraidingenv,n_stack=num_frames)
|
||||||
# Model setup complete
|
# Model setup complete
|
||||||
|
|
||||||
# Setup Eval Env
|
|
||||||
econ_eval=foundation.make_env_instance(**eval_env_config)
|
|
||||||
|
|
||||||
|
|
||||||
baseEconWrapper_eval=BaseEconWrapper(econ_eval)
|
|
||||||
baseEconWrapper_eval.run()
|
|
||||||
time.sleep(0.5)
|
|
||||||
mobileRecieverEconWrapper_eval=RecieverEconWrapper(base_econ=baseEconWrapper_eval,agent_classname="BasicMobileAgent")
|
|
||||||
tradeRecieverEconWrapper_eval=RecieverEconWrapper(base_econ=baseEconWrapper_eval,agent_classname="TradingAgent")
|
|
||||||
sb3_traderConverter_eval=SB3EconConverter(tradeRecieverEconWrapper_eval,econ_eval,"TradingAgent",False)
|
|
||||||
sb3Converter_eval=SB3EconConverter(mobileRecieverEconWrapper_eval,econ_eval,"BasicMobileAgent",False)
|
|
||||||
# attach sb3 wrappers
|
|
||||||
|
|
||||||
monenv_eval=VecMonitor(venv=sb3Converter_eval,info_keywords=["social/productivity","trend/productivity"])
|
|
||||||
montraidingenv_eval=VecMonitor(venv=sb3_traderConverter_eval)
|
|
||||||
|
|
||||||
stackenv_basic_eval=vec_frame_stack.VecFrameStack(venv=monenv_eval,n_stack=num_frames)
|
|
||||||
stackenv_traid_eval=vec_frame_stack.VecFrameStack(venv=montraidingenv_eval,n_stack=num_frames)
|
|
||||||
|
|
||||||
|
|
||||||
obs=monenv.reset()
|
obs=monenv.reset()
|
||||||
|
obs=montraidingenv.reset()
|
||||||
|
|
||||||
# define training functions
|
# define training functions
|
||||||
def train(model,timesteps, econ_call,process_bar,name,db,index):
|
def train(model,timesteps, econ_call,process_bar,name,db,index):
|
||||||
@@ -273,66 +216,47 @@ runname="run_{}".format(run_number)
|
|||||||
model_db=[None,None] # object for storing model
|
model_db=[None,None] # object for storing model
|
||||||
|
|
||||||
|
|
||||||
model = MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.1, vf_coef=0.5 ,gamma=0.99, learning_rate=1e-5,env=stackenv_basic, seed=300,verbose=1,device="cuda",tensorboard_log="./log")
|
model = MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.1, vf_coef=0.5 ,gamma=0.99, learning_rate=learning_rate_adj_basic,env=stackenv_basic, seed=445,verbose=1,device="cuda",tensorboard_log="./log")
|
||||||
model_trade=MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.1, vf_coef=0.5 ,gamma=0.99, learning_rate=1e-5,env=stackenv_traid, seed=300,verbose=1,device="cuda",tensorboard_log="./log")
|
model_trade=MaskablePPO("MlpPolicy",n_steps=int(env_config['episode_length']*2),ent_coef=0.1, vf_coef=0.5 ,gamma=0.99, learning_rate=learning_rate_adj_trade,env=stackenv_traid, seed=445,verbose=1,device="cuda",tensorboard_log="./log")
|
||||||
|
# Setup complete
|
||||||
|
|
||||||
n_agents=econ.n_agents
|
# Load models
|
||||||
|
model=model.load("basic.ai")
|
||||||
|
model_trade=model_trade.load("trade.ai")
|
||||||
|
|
||||||
total_required_for_episode_basic=len(mobileRecieverEconWrapper.agnet_idx)*env_config['episode_length']
|
exporter=MongoExporter(CONNECTION_STRING,"econ",econ)
|
||||||
total_required_for_episode_traid=len(tradeRecieverEconWrapper.agnet_idx)*env_config['episode_length']
|
|
||||||
|
|
||||||
print("this is run {}".format(runname))
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
|
||||||
|
|
||||||
#Train
|
|
||||||
runname="run_{}_{}".format(run_number,"basic")
|
|
||||||
|
|
||||||
thread_model=Thread(target=train,args=(model,total_required_for_episode_basic*50,econ,True,runname,model_db,0))
|
|
||||||
runname="run_{}_{}".format(run_number,"trader")
|
|
||||||
thread_model_traid=Thread(target=train,args=(model_trade,total_required_for_episode_traid*50,econ,False,runname,model_db,1))
|
|
||||||
|
|
||||||
thread_model.start()
|
|
||||||
thread_model_traid.start()
|
|
||||||
thread_model.join()
|
|
||||||
thread_model_traid.join()
|
|
||||||
#normenv.save("temp-normalizer.ai")
|
|
||||||
model=model_db[0]
|
|
||||||
model_trade=model_db[1]
|
|
||||||
model.save("basic.ai")
|
|
||||||
model_trade.save("trade.ai")
|
|
||||||
|
|
||||||
## Run Eval
|
|
||||||
print("### EVAL ###")
|
print("### EVAL ###")
|
||||||
obs_basic=stackenv_basic_eval.reset()
|
baseEconWrapper.force_reset()
|
||||||
obs_trade=stackenv_traid_eval.reset()
|
obs_basic=stackenv_basic.reset()
|
||||||
|
obs_trade=stackenv_traid.reset()
|
||||||
done=False
|
done=False
|
||||||
for i in tqdm(range(eval_env_config['episode_length'])):
|
for i in tqdm(range(env_config['episode_length'])):
|
||||||
#create masks
|
#create masks
|
||||||
masks_basic=stackenv_basic_eval.action_masks()
|
masks_basic=stackenv_basic.action_masks()
|
||||||
masks_trade=stackenv_traid_eval.action_masks()
|
masks_trade=stackenv_traid.action_masks()
|
||||||
# get actions
|
# get actions
|
||||||
action_basic=model.predict(obs_basic,action_masks=masks_basic)
|
action_basic=model.predict(obs_basic,action_masks=masks_basic)
|
||||||
action_trade=model_trade.predict(obs_trade,action_masks=masks_trade)
|
action_trade=model_trade.predict(obs_trade,action_masks=masks_trade)
|
||||||
#submit async directly for non blocking operation
|
#submit async directly for non blocking operation
|
||||||
sb3Converter_eval.step_async(action_basic[0])
|
sb3Converter.step_async(action_basic[0])
|
||||||
sb3_traderConverter_eval.step_async(action_trade[0])
|
sb3_traderConverter.step_async(action_trade[0])
|
||||||
# retieve full results
|
# retieve full results
|
||||||
obs_basic,rew_basic,done_e,info=stackenv_basic_eval.step(action_basic[0])
|
obs_basic,rew_basic,done_e,info=stackenv_basic.step(action_basic[0])
|
||||||
obs_trade,rew_trade,done_e,info=stackenv_traid_eval.step(action_trade[0])
|
obs_trade,rew_trade,done_e,info=stackenv_traid.step(action_trade[0])
|
||||||
done=done_e[0]
|
done=done_e[0]
|
||||||
|
|
||||||
|
|
||||||
|
exporter.submit_full_market()
|
||||||
market=econ_eval.get_component("ContinuousDoubleAuction")
|
market=econ.get_component("ContinuousDoubleAuction")
|
||||||
craft=econ_eval.get_component("Craft")
|
craft=econ.get_component("Craft")
|
||||||
# trades=market.get_dense_log()
|
# trades=market.get_dense_log()
|
||||||
build=craft.get_dense_log()
|
build=craft.get_dense_log()
|
||||||
met=econ.previous_episode_metrics
|
met=econ.previous_episode_metrics
|
||||||
printReplay(econ_eval,0)
|
#printReplay(econ,5)
|
||||||
# printMarket(trades)
|
#printMarket(trades)
|
||||||
# printBuilds(builds=build)
|
#printBuilds(builds=build)
|
||||||
print("social/productivity: {}".format(met["social/productivity"]))
|
print("social/productivity: {}".format(met["social/productivity"]))
|
||||||
print("labor/weighted_cost: {}".format(met["labor/weighted_cost"]))
|
print("labor/weighted_cost: {}".format(met["labor/weighted_cost"]))
|
||||||
print("labor/warmup_integrator: {}".format(met["labor/warmup_integrator"]))
|
print("labor/warmup_integrator: {}".format(met["labor/warmup_integrator"]))
|
||||||
|
|||||||
Binary file not shown.
@@ -17,7 +17,7 @@ class BaseEconWrapper():
|
|||||||
info=None
|
info=None
|
||||||
n_data_retrieved=0
|
n_data_retrieved=0
|
||||||
|
|
||||||
def __init__(self, econ: base_env.BaseEnvironment):
|
def __init__(self, econ: base_env.BaseEnvironment,auto_reset):
|
||||||
self.env=econ
|
self.env=econ
|
||||||
self.vote_lock=Lock()
|
self.vote_lock=Lock()
|
||||||
|
|
||||||
@@ -26,6 +26,8 @@ class BaseEconWrapper():
|
|||||||
self.action_edit_lock=Lock()
|
self.action_edit_lock=Lock()
|
||||||
self.stop_edit_lock=Lock()
|
self.stop_edit_lock=Lock()
|
||||||
self.env_data_lock=Lock()
|
self.env_data_lock=Lock()
|
||||||
|
self.auto_reset=auto_reset
|
||||||
|
|
||||||
|
|
||||||
def register_vote(self):
|
def register_vote(self):
|
||||||
"""Register reciever on base. Returns ID of Voter to pass on during blocking"""
|
"""Register reciever on base. Returns ID of Voter to pass on during blocking"""
|
||||||
@@ -79,7 +81,7 @@ class BaseEconWrapper():
|
|||||||
|
|
||||||
#check for reset
|
#check for reset
|
||||||
self.vote_lock.acquire() # we might edit votes
|
self.vote_lock.acquire() # we might edit votes
|
||||||
if self.n_voters==self.n_votes_reset:
|
if self.n_voters<=self.n_votes_reset:
|
||||||
## perform reset
|
## perform reset
|
||||||
self.n_votes_reset=0
|
self.n_votes_reset=0
|
||||||
self._reset()
|
self._reset()
|
||||||
@@ -96,6 +98,7 @@ class BaseEconWrapper():
|
|||||||
# release actions
|
# release actions
|
||||||
# we are done
|
# we are done
|
||||||
|
|
||||||
|
|
||||||
def stop_env(self):
|
def stop_env(self):
|
||||||
"""Stops the wrapper"""
|
"""Stops the wrapper"""
|
||||||
self.stop_edit_lock.acquire()
|
self.stop_edit_lock.acquire()
|
||||||
@@ -110,13 +113,17 @@ class BaseEconWrapper():
|
|||||||
self.n_votes_reset=0
|
self.n_votes_reset=0
|
||||||
self.obs=self.env.reset() #Reset env
|
self.obs=self.env.reset() #Reset env
|
||||||
self.rew=None
|
self.rew=None
|
||||||
self.done=None
|
self.done={'__all__': False}
|
||||||
self.info=None
|
self.info=None
|
||||||
self.env_data_lock.release() #Release lock
|
self.env_data_lock.release() #Release lock
|
||||||
# Notify for reset
|
# Notify for reset
|
||||||
self.reset_notification.set()
|
self.reset_notification.set()
|
||||||
for v in self.step_notifications:
|
for v in self.step_notifications:
|
||||||
v.clear() # unlock stepping
|
v.clear() # unlock stepping
|
||||||
|
def force_reset(self):
|
||||||
|
"""Force a reset with no votes"""
|
||||||
|
self._reset()
|
||||||
|
#self.n_votes_reset=-self.n_voters
|
||||||
|
|
||||||
def _step(self):
|
def _step(self):
|
||||||
"""Steping interaly"""
|
"""Steping interaly"""
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ class SB3EconConverter(VecEnv, gym.Env):
|
|||||||
obs0=utils.package(obs[0],*self.packager)
|
obs0=utils.package(obs[0],*self.packager)
|
||||||
obs0["flat"]
|
obs0["flat"]
|
||||||
self.step_request_send=False
|
self.step_request_send=False
|
||||||
|
self.reset_request_send=False
|
||||||
self.auto_reset=auto_reset
|
self.auto_reset=auto_reset
|
||||||
self.observation_space=gym.spaces.Box(low=0,high=10,shape=(len(obs0["flat"]),),dtype=np.float32)
|
self.observation_space=gym.spaces.Box(low=0,high=10,shape=(len(obs0["flat"]),),dtype=np.float32)
|
||||||
super().__init__(self.num_envs, self.observation_space, self.action_space)
|
super().__init__(self.num_envs, self.observation_space, self.action_space)
|
||||||
@@ -73,7 +74,7 @@ class SB3EconConverter(VecEnv, gym.Env):
|
|||||||
|
|
||||||
def reset(self) -> VecEnvObs:
|
def reset(self) -> VecEnvObs:
|
||||||
obs=self.env.reset()
|
obs=self.env.reset()
|
||||||
self.step_request_send=False
|
|
||||||
f_obs={}
|
f_obs={}
|
||||||
self.curr_obs=obs
|
self.curr_obs=obs
|
||||||
for k,v in obs.items():
|
for k,v in obs.items():
|
||||||
|
|||||||
Reference in New Issue
Block a user