adding ai_economist for modding
This commit is contained in:
@@ -0,0 +1,638 @@
|
||||
# Copyright (c) 2021, salesforce.com, inc.
|
||||
# All rights reserved.
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# For full license text, see the LICENSE file in the repo root
|
||||
# or https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
import itertools
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
_NP_DTYPE = np.float32
|
||||
|
||||
|
||||
def all_agents_export_experiment_template(
|
||||
NUMFIRMS, NUMCONSUMERS, NUMGOVERNMENTS, episodes_const=30000
|
||||
):
|
||||
consumption_choices = [
|
||||
np.array([0.0 + 1.0 * c for c in range(11)], dtype=_NP_DTYPE)
|
||||
]
|
||||
work_choices = [
|
||||
np.array([0.0 + 20 * 13 * h for h in range(5)], dtype=_NP_DTYPE)
|
||||
] # specify dtype -- be consistent?
|
||||
|
||||
consumption_choices = np.array(
|
||||
list(itertools.product(*consumption_choices)), dtype=_NP_DTYPE
|
||||
)
|
||||
work_choices = np.array(list(itertools.product(*work_choices)), dtype=_NP_DTYPE)
|
||||
|
||||
price_choices = np.array([0.0 + 500.0 * c for c in range(6)], dtype=_NP_DTYPE)
|
||||
wage_choices = np.array([0.0, 11.0, 22.0, 33.0, 44.0], dtype=_NP_DTYPE)
|
||||
capital_choices = np.array([0.1], dtype=_NP_DTYPE)
|
||||
price_and_wage = np.array(
|
||||
list(itertools.product(price_choices, wage_choices, capital_choices)),
|
||||
dtype=_NP_DTYPE,
|
||||
)
|
||||
|
||||
# government action discretization
|
||||
income_taxation_choices = np.array(
|
||||
[0.0 + 0.2 * c for c in range(6)], dtype=_NP_DTYPE
|
||||
)
|
||||
corporate_taxation_choices = np.array(
|
||||
[0.0 + 0.2 * c for c in range(6)], dtype=_NP_DTYPE
|
||||
)
|
||||
tax_choices = np.array(
|
||||
list(itertools.product(income_taxation_choices, corporate_taxation_choices)),
|
||||
dtype=_NP_DTYPE,
|
||||
)
|
||||
global_state_dim = (
|
||||
NUMFIRMS # prices
|
||||
+ NUMFIRMS # wages
|
||||
+ NUMFIRMS # stocks
|
||||
+ NUMFIRMS # was good overdemanded
|
||||
+ 2 * NUMGOVERNMENTS # tax rates
|
||||
+ 1
|
||||
) # time
|
||||
|
||||
global_state_digit_dims = list(
|
||||
range(2 * NUMFIRMS, 3 * NUMFIRMS)
|
||||
) # stocks are the only global state var that can get huge
|
||||
consumer_state_dim = (
|
||||
global_state_dim + 1 + 1
|
||||
) # budget # theta, the disutility of work
|
||||
|
||||
firm_state_dim = (
|
||||
global_state_dim
|
||||
+ 1 # budget
|
||||
+ 1 # capital
|
||||
+ 1 # production alpha
|
||||
+ NUMFIRMS # onehot specifying which firm
|
||||
)
|
||||
|
||||
episodes_to_anneal_firm = 100000
|
||||
episodes_to_anneal_government = 100000
|
||||
government_phase1_start = 100000
|
||||
government_state_dim = global_state_dim
|
||||
DEFAULT_CFG_DICT = {
|
||||
# actions_array key will be added below
|
||||
"agents": {
|
||||
"num_consumers": NUMCONSUMERS,
|
||||
"num_firms": NUMFIRMS,
|
||||
"num_governments": NUMGOVERNMENTS,
|
||||
"global_state_dim": global_state_dim,
|
||||
"consumer_state_dim": consumer_state_dim,
|
||||
# action vectors are how much consume from each firm,
|
||||
# how much to work, and which firm to choose
|
||||
"consumer_action_dim": NUMFIRMS + 1 + 1,
|
||||
"consumer_num_consume_actions": consumption_choices.shape[0],
|
||||
"consumer_num_work_actions": work_choices.shape[0],
|
||||
"consumer_num_whichfirm_actions": NUMFIRMS,
|
||||
"firm_state_dim": firm_state_dim, # what are observations?
|
||||
# actions are price and wage for own firm, and capital choices
|
||||
"firm_action_dim": 3,
|
||||
"firm_num_actions": price_and_wage.shape[0],
|
||||
"government_state_dim": government_state_dim,
|
||||
"government_action_dim": 2,
|
||||
"government_num_actions": tax_choices.shape[0],
|
||||
"max_possible_consumption": float(consumption_choices.max()),
|
||||
"max_possible_hours_worked": float(work_choices.max()),
|
||||
"max_possible_wage": float(wage_choices.max()),
|
||||
"max_possible_price": float(price_choices.max()),
|
||||
# these are dims which, due to being on a large scale,
|
||||
# have to be expanded to a digit representation
|
||||
"consumer_digit_dims": global_state_digit_dims
|
||||
+ [global_state_dim], # global state + consumer budget
|
||||
# global state + firm budget (do we need capital?)
|
||||
"firm_digit_dims": global_state_digit_dims + [global_state_dim],
|
||||
# govt only has global state
|
||||
"government_digit_dims": global_state_digit_dims,
|
||||
"firm_reward_scale": 10000,
|
||||
"government_reward_scale": 100000,
|
||||
"consumer_reward_scale": 50.0,
|
||||
"firm_anneal_wages": {
|
||||
"anneal_on": True,
|
||||
"start": 22.0,
|
||||
"increase_const": float(wage_choices.max() - 22.0)
|
||||
/ (episodes_to_anneal_firm),
|
||||
"decrease_const": (22.0) / episodes_to_anneal_firm,
|
||||
},
|
||||
"firm_anneal_prices": {
|
||||
"anneal_on": True,
|
||||
"start": 1000.0,
|
||||
"increase_const": float(price_choices.max() - 1000.00)
|
||||
/ episodes_to_anneal_firm,
|
||||
"decrease_const": (1000.0) / episodes_to_anneal_firm,
|
||||
},
|
||||
"government_anneal_taxes": {
|
||||
"anneal_on": True,
|
||||
"start": 0.0,
|
||||
"increase_const": 1.0 / episodes_to_anneal_government,
|
||||
},
|
||||
"firm_begin_anneal_action": 0,
|
||||
"government_begin_anneal_action": government_phase1_start,
|
||||
"consumer_anneal_theta": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
},
|
||||
"consumer_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"firm_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"govt_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"consumer_noponzi_eta": 0.0,
|
||||
"consumer_penalty_scale": 1.0,
|
||||
"firm_noponzi_eta": 0.0,
|
||||
"firm_training_start": episodes_to_anneal_firm,
|
||||
"government_training_start": government_phase1_start
|
||||
+ episodes_to_anneal_government,
|
||||
"consumer_training_start": 0,
|
||||
"government_counts_firm_reward": 0,
|
||||
"should_boost_firm_reward": False,
|
||||
"firm_reward_for_government_factor": 0.0025,
|
||||
},
|
||||
"world": {
|
||||
"maxtime": 10,
|
||||
"initial_firm_endowment": 22.0 * 1000 * NUMCONSUMERS,
|
||||
"initial_consumer_endowment": 2000,
|
||||
"initial_stocks": 0.0,
|
||||
"initial_prices": 1000.0,
|
||||
"initial_wages": 22.0,
|
||||
"interest_rate": 0.1,
|
||||
"consumer_theta": 0.01,
|
||||
"crra_param": 0.1,
|
||||
"production_alpha": "fixed_array", # only works for exactly 10 firms, kluge
|
||||
"initial_capital": "twolevel",
|
||||
"paretoscaletheta": 4.0,
|
||||
"importer_price": 500.0,
|
||||
"importer_quantity": 100.0,
|
||||
"use_importer": 1,
|
||||
},
|
||||
"train": {
|
||||
"batch_size": 8,
|
||||
"base_seed": 1234,
|
||||
"save_dense_every": 2000,
|
||||
"save_model_every": 10000,
|
||||
"num_episodes": 500000,
|
||||
"infinite_episodes": False,
|
||||
"lr": 0.01,
|
||||
"gamma": 0.9999,
|
||||
"entropy": 0.0,
|
||||
"value_loss_weight": 1.0,
|
||||
"digit_representation_size": 10,
|
||||
"lagr_num_steps": 1,
|
||||
"boost_firm_reward_factor": 1.0,
|
||||
},
|
||||
}
|
||||
return (
|
||||
DEFAULT_CFG_DICT,
|
||||
consumption_choices,
|
||||
work_choices,
|
||||
price_and_wage,
|
||||
tax_choices,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def all_agents_short_export_experiment_template(
|
||||
NUMFIRMS, NUMCONSUMERS, NUMGOVERNMENTS, episodes_const=10000
|
||||
):
|
||||
consumption_choices = [
|
||||
np.array([0.0 + 1.0 * c for c in range(11)], dtype=_NP_DTYPE)
|
||||
]
|
||||
work_choices = [
|
||||
np.array([0.0 + 20 * 13 * h for h in range(5)], dtype=_NP_DTYPE)
|
||||
] # specify dtype -- be consistent?
|
||||
|
||||
consumption_choices = np.array(
|
||||
list(itertools.product(*consumption_choices)), dtype=_NP_DTYPE
|
||||
)
|
||||
work_choices = np.array(list(itertools.product(*work_choices)), dtype=_NP_DTYPE)
|
||||
|
||||
price_choices = np.array([0.0 + 500.0 * c for c in range(6)], dtype=_NP_DTYPE)
|
||||
wage_choices = np.array([0.0, 11.0, 22.0, 33.0, 44.0], dtype=_NP_DTYPE)
|
||||
capital_choices = np.array([0.1], dtype=_NP_DTYPE)
|
||||
price_and_wage = np.array(
|
||||
list(itertools.product(price_choices, wage_choices, capital_choices)),
|
||||
dtype=_NP_DTYPE,
|
||||
)
|
||||
|
||||
# government action discretization
|
||||
income_taxation_choices = np.array(
|
||||
[0.0 + 0.2 * c for c in range(6)], dtype=_NP_DTYPE
|
||||
)
|
||||
corporate_taxation_choices = np.array(
|
||||
[0.0 + 0.2 * c for c in range(6)], dtype=_NP_DTYPE
|
||||
)
|
||||
tax_choices = np.array(
|
||||
list(itertools.product(income_taxation_choices, corporate_taxation_choices)),
|
||||
dtype=_NP_DTYPE,
|
||||
)
|
||||
global_state_dim = (
|
||||
NUMFIRMS # prices
|
||||
+ NUMFIRMS # wages
|
||||
+ NUMFIRMS # stocks
|
||||
+ NUMFIRMS # was good overdemanded
|
||||
+ 2 * NUMGOVERNMENTS # tax rates
|
||||
+ 1
|
||||
) # time
|
||||
|
||||
global_state_digit_dims = list(
|
||||
range(2 * NUMFIRMS, 3 * NUMFIRMS)
|
||||
) # stocks are the only global state var that can get huge
|
||||
consumer_state_dim = (
|
||||
global_state_dim + 1 + 1
|
||||
) # budget # theta, the disutility of work
|
||||
|
||||
firm_state_dim = (
|
||||
global_state_dim
|
||||
+ 1 # budget
|
||||
+ 1 # capital
|
||||
+ 1 # production alpha
|
||||
+ NUMFIRMS # onehot specifying which firm
|
||||
)
|
||||
|
||||
episodes_to_anneal_firm = 30000
|
||||
episodes_to_anneal_government = 30000
|
||||
government_phase1_start = 30000
|
||||
government_state_dim = global_state_dim
|
||||
DEFAULT_CFG_DICT = {
|
||||
# actions_array key will be added below
|
||||
"agents": {
|
||||
"num_consumers": NUMCONSUMERS,
|
||||
"num_firms": NUMFIRMS,
|
||||
"num_governments": NUMGOVERNMENTS,
|
||||
"global_state_dim": global_state_dim,
|
||||
"consumer_state_dim": consumer_state_dim,
|
||||
# action vectors are how much consume from each firm,
|
||||
# how much to work, and which firm to choose
|
||||
"consumer_action_dim": NUMFIRMS + 1 + 1,
|
||||
"consumer_num_consume_actions": consumption_choices.shape[0],
|
||||
"consumer_num_work_actions": work_choices.shape[0],
|
||||
"consumer_num_whichfirm_actions": NUMFIRMS,
|
||||
"firm_state_dim": firm_state_dim, # what are observations?
|
||||
# actions are price and wage for own firm, and capital choices
|
||||
"firm_action_dim": 3,
|
||||
"firm_num_actions": price_and_wage.shape[0],
|
||||
"government_state_dim": government_state_dim,
|
||||
"government_action_dim": 2,
|
||||
"government_num_actions": tax_choices.shape[0],
|
||||
"max_possible_consumption": float(consumption_choices.max()),
|
||||
"max_possible_hours_worked": float(work_choices.max()),
|
||||
"max_possible_wage": float(wage_choices.max()),
|
||||
"max_possible_price": float(price_choices.max()),
|
||||
# these are dims which, due to being on a large scale,
|
||||
# have to be expanded to a digit representation
|
||||
"consumer_digit_dims": global_state_digit_dims
|
||||
+ [global_state_dim], # global state + consumer budget
|
||||
"firm_digit_dims": global_state_digit_dims
|
||||
+ [global_state_dim], # global state + firm budget (do we need capital?)
|
||||
# govt only has global state
|
||||
"government_digit_dims": global_state_digit_dims,
|
||||
"firm_reward_scale": 10000,
|
||||
"government_reward_scale": 100000,
|
||||
"consumer_reward_scale": 50.0,
|
||||
"firm_anneal_wages": {
|
||||
"anneal_on": True,
|
||||
"start": 22.0,
|
||||
"increase_const": float(wage_choices.max() - 22.0)
|
||||
/ (episodes_to_anneal_firm),
|
||||
"decrease_const": (22.0) / episodes_to_anneal_firm,
|
||||
},
|
||||
"firm_anneal_prices": {
|
||||
"anneal_on": True,
|
||||
"start": 1000.0,
|
||||
"increase_const": float(price_choices.max() - 1000.00)
|
||||
/ episodes_to_anneal_firm,
|
||||
"decrease_const": (1000.0) / episodes_to_anneal_firm,
|
||||
},
|
||||
"government_anneal_taxes": {
|
||||
"anneal_on": True,
|
||||
"start": 0.0,
|
||||
"increase_const": 1.0 / episodes_to_anneal_government,
|
||||
},
|
||||
"firm_begin_anneal_action": 0,
|
||||
"government_begin_anneal_action": government_phase1_start,
|
||||
"consumer_anneal_theta": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
},
|
||||
"consumer_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"firm_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"govt_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"consumer_noponzi_eta": 0.0,
|
||||
"consumer_penalty_scale": 1.0,
|
||||
"firm_noponzi_eta": 0.0,
|
||||
"firm_training_start": episodes_to_anneal_firm,
|
||||
"government_training_start": government_phase1_start
|
||||
+ episodes_to_anneal_government,
|
||||
"consumer_training_start": 0,
|
||||
"government_counts_firm_reward": 0,
|
||||
"should_boost_firm_reward": False,
|
||||
"firm_reward_for_government_factor": 0.0025,
|
||||
},
|
||||
"world": {
|
||||
"maxtime": 10,
|
||||
"initial_firm_endowment": 22.0 * 1000 * NUMCONSUMERS,
|
||||
"initial_consumer_endowment": 2000,
|
||||
"initial_stocks": 0.0,
|
||||
"initial_prices": 1000.0,
|
||||
"initial_wages": 22.0,
|
||||
"interest_rate": 0.1,
|
||||
"consumer_theta": 0.01,
|
||||
"crra_param": 0.1,
|
||||
"production_alpha": "fixed_array", # only works for exactly 10 firms, kluge
|
||||
"initial_capital": "twolevel",
|
||||
"paretoscaletheta": 4.0,
|
||||
"importer_price": 500.0,
|
||||
"importer_quantity": 100.0,
|
||||
"use_importer": 1,
|
||||
},
|
||||
"train": {
|
||||
"batch_size": 8,
|
||||
"base_seed": 1234,
|
||||
"save_dense_every": 2000,
|
||||
"save_model_every": 10000,
|
||||
"num_episodes": 200000,
|
||||
"infinite_episodes": False,
|
||||
"lr": 0.01,
|
||||
"gamma": 0.9999,
|
||||
"entropy": 0.0,
|
||||
"value_loss_weight": 1.0,
|
||||
"digit_representation_size": 10,
|
||||
"lagr_num_steps": 1,
|
||||
"boost_firm_reward_factor": 1.0,
|
||||
},
|
||||
}
|
||||
return (
|
||||
DEFAULT_CFG_DICT,
|
||||
consumption_choices,
|
||||
work_choices,
|
||||
price_and_wage,
|
||||
tax_choices,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def very_short_test_template(
|
||||
NUMFIRMS, NUMCONSUMERS, NUMGOVERNMENTS, episodes_const=30000
|
||||
):
|
||||
consumption_choices = [
|
||||
np.array([0.0 + 1.0 * c for c in range(11)], dtype=_NP_DTYPE)
|
||||
]
|
||||
work_choices = [
|
||||
np.array([0.0 + 20 * 13 * h for h in range(5)], dtype=_NP_DTYPE)
|
||||
] # specify dtype -- be consistent?
|
||||
|
||||
consumption_choices = np.array(
|
||||
list(itertools.product(*consumption_choices)), dtype=_NP_DTYPE
|
||||
)
|
||||
work_choices = np.array(list(itertools.product(*work_choices)), dtype=_NP_DTYPE)
|
||||
|
||||
price_choices = np.array([0.0 + 500.0 * c for c in range(6)], dtype=_NP_DTYPE)
|
||||
wage_choices = np.array([0.0, 11.0, 22.0, 33.0, 44.0], dtype=_NP_DTYPE)
|
||||
capital_choices = np.array([0.1], dtype=_NP_DTYPE)
|
||||
price_and_wage = np.array(
|
||||
list(itertools.product(price_choices, wage_choices, capital_choices)),
|
||||
dtype=_NP_DTYPE,
|
||||
)
|
||||
|
||||
# government action discretization
|
||||
income_taxation_choices = np.array(
|
||||
[0.0 + 0.2 * c for c in range(6)], dtype=_NP_DTYPE
|
||||
)
|
||||
corporate_taxation_choices = np.array(
|
||||
[0.0 + 0.2 * c for c in range(6)], dtype=_NP_DTYPE
|
||||
)
|
||||
tax_choices = np.array(
|
||||
list(itertools.product(income_taxation_choices, corporate_taxation_choices)),
|
||||
dtype=_NP_DTYPE,
|
||||
)
|
||||
global_state_dim = (
|
||||
NUMFIRMS # prices
|
||||
+ NUMFIRMS # wages
|
||||
+ NUMFIRMS # stocks
|
||||
+ NUMFIRMS # was good overdemanded
|
||||
+ 2 * NUMGOVERNMENTS # tax rates
|
||||
+ 1
|
||||
) # time
|
||||
|
||||
global_state_digit_dims = list(
|
||||
range(2 * NUMFIRMS, 3 * NUMFIRMS)
|
||||
) # stocks are the only global state var that can get huge
|
||||
consumer_state_dim = (
|
||||
global_state_dim + 1 + 1
|
||||
) # budget # theta, the disutility of work
|
||||
|
||||
firm_state_dim = (
|
||||
global_state_dim
|
||||
+ 1 # budget
|
||||
+ 1 # capital
|
||||
+ 1 # production alpha
|
||||
+ NUMFIRMS # onehot specifying which firm
|
||||
)
|
||||
|
||||
episodes_to_anneal_firm = 10
|
||||
episodes_to_anneal_government = 10
|
||||
government_phase1_start = 10
|
||||
government_state_dim = global_state_dim
|
||||
DEFAULT_CFG_DICT = {
|
||||
# actions_array key will be added below
|
||||
"agents": {
|
||||
"num_consumers": NUMCONSUMERS,
|
||||
"num_firms": NUMFIRMS,
|
||||
"num_governments": NUMGOVERNMENTS,
|
||||
"global_state_dim": global_state_dim,
|
||||
"consumer_state_dim": consumer_state_dim,
|
||||
# action vectors are how much consume from each firm,
|
||||
# how much to work, and which firm to choose
|
||||
"consumer_action_dim": NUMFIRMS + 1 + 1,
|
||||
"consumer_num_consume_actions": consumption_choices.shape[0],
|
||||
"consumer_num_work_actions": work_choices.shape[0],
|
||||
"consumer_num_whichfirm_actions": NUMFIRMS,
|
||||
"firm_state_dim": firm_state_dim, # what are observations?
|
||||
# actions are price and wage for own firm, and capital choices
|
||||
"firm_action_dim": 3,
|
||||
"firm_num_actions": price_and_wage.shape[0],
|
||||
"government_state_dim": government_state_dim,
|
||||
"government_action_dim": 2,
|
||||
"government_num_actions": tax_choices.shape[0],
|
||||
"max_possible_consumption": float(consumption_choices.max()),
|
||||
"max_possible_hours_worked": float(work_choices.max()),
|
||||
"max_possible_wage": float(wage_choices.max()),
|
||||
"max_possible_price": float(price_choices.max()),
|
||||
# these are dims which, due to being on a large scale,
|
||||
# have to be expanded to a digit representation
|
||||
"consumer_digit_dims": global_state_digit_dims
|
||||
+ [global_state_dim], # global state + consumer budget
|
||||
"firm_digit_dims": global_state_digit_dims
|
||||
+ [global_state_dim], # global state + firm budget (do we need capital?)
|
||||
# govt only has global state
|
||||
"government_digit_dims": global_state_digit_dims,
|
||||
"firm_reward_scale": 10000,
|
||||
"government_reward_scale": 100000,
|
||||
"consumer_reward_scale": 50.0,
|
||||
"firm_anneal_wages": {
|
||||
"anneal_on": True,
|
||||
"start": 22.0,
|
||||
"increase_const": float(wage_choices.max() - 22.0)
|
||||
/ (episodes_to_anneal_firm),
|
||||
"decrease_const": (22.0) / episodes_to_anneal_firm,
|
||||
},
|
||||
"firm_anneal_prices": {
|
||||
"anneal_on": True,
|
||||
"start": 1000.0,
|
||||
"increase_const": float(price_choices.max() - 1000.00)
|
||||
/ episodes_to_anneal_firm,
|
||||
"decrease_const": (1000.0) / episodes_to_anneal_firm,
|
||||
},
|
||||
"government_anneal_taxes": {
|
||||
"anneal_on": True,
|
||||
"start": 0.0,
|
||||
"increase_const": 1.0 / episodes_to_anneal_government,
|
||||
},
|
||||
"firm_begin_anneal_action": 0,
|
||||
"government_begin_anneal_action": government_phase1_start,
|
||||
"consumer_anneal_theta": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
},
|
||||
"consumer_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"firm_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"govt_anneal_entropy": {
|
||||
"anneal_on": True,
|
||||
"exp_decay_length_in_steps": episodes_const,
|
||||
"coef_floor": 0.1,
|
||||
},
|
||||
"consumer_noponzi_eta": 0.0,
|
||||
"consumer_penalty_scale": 1.0,
|
||||
"firm_noponzi_eta": 0.0,
|
||||
"firm_training_start": episodes_to_anneal_firm,
|
||||
"government_training_start": government_phase1_start
|
||||
+ episodes_to_anneal_government,
|
||||
"consumer_training_start": 0,
|
||||
"government_counts_firm_reward": 0,
|
||||
"should_boost_firm_reward": False,
|
||||
"firm_reward_for_government_factor": 0.0025,
|
||||
"train_firms_every": 2,
|
||||
"train_consumers_every": 1,
|
||||
"train_government_every": 5,
|
||||
},
|
||||
"world": {
|
||||
"maxtime": 10,
|
||||
"initial_firm_endowment": 22.0 * 1000 * NUMCONSUMERS,
|
||||
"initial_consumer_endowment": 2000,
|
||||
"initial_stocks": 0.0,
|
||||
"initial_prices": 1000.0,
|
||||
"initial_wages": 22.0,
|
||||
"interest_rate": 0.1,
|
||||
"consumer_theta": 0.01,
|
||||
"crra_param": 0.1,
|
||||
"production_alpha": "fixed_array", # only works for exactly 10 firms, kluge
|
||||
"initial_capital": "twolevel",
|
||||
"paretoscaletheta": 4.0,
|
||||
"importer_price": 500.0,
|
||||
"importer_quantity": 100.0,
|
||||
"use_importer": 1,
|
||||
},
|
||||
"train": {
|
||||
"batch_size": 8,
|
||||
"base_seed": 1234,
|
||||
"save_dense_every": 2000,
|
||||
"save_model_every": 10000,
|
||||
"num_episodes": 100,
|
||||
"infinite_episodes": False,
|
||||
"lr": 0.01,
|
||||
"gamma": 0.9999,
|
||||
"entropy": 0.0,
|
||||
"value_loss_weight": 1.0,
|
||||
"digit_representation_size": 10,
|
||||
"lagr_num_steps": 1,
|
||||
"boost_firm_reward_factor": 1.0,
|
||||
},
|
||||
}
|
||||
return (
|
||||
DEFAULT_CFG_DICT,
|
||||
consumption_choices,
|
||||
work_choices,
|
||||
price_and_wage,
|
||||
tax_choices,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def global_state_scaling_factors(cfg_dict):
|
||||
max_wage = cfg_dict["agents"]["max_possible_wage"]
|
||||
max_price = cfg_dict["agents"]["max_possible_price"]
|
||||
num_firms = cfg_dict["agents"]["num_firms"]
|
||||
num_governments = cfg_dict["agents"]["num_governments"]
|
||||
maxtime = cfg_dict["world"]["maxtime"]
|
||||
|
||||
digit_size = cfg_dict["train"]["digit_representation_size"]
|
||||
|
||||
return torch.tensor(
|
||||
# prices, wages, stocks, overdemanded
|
||||
([max_price] * num_firms)
|
||||
+ ([max_wage] * num_firms)
|
||||
+ ([1.0] * num_firms * digit_size) # stocks are expanded to digit form
|
||||
+ ([1.0] * num_firms)
|
||||
+ ([1.0] * (2 * num_governments))
|
||||
+ [maxtime]
|
||||
)
|
||||
|
||||
|
||||
def consumer_state_scaling_factors(cfg_dict):
|
||||
global_state_scales = global_state_scaling_factors(cfg_dict)
|
||||
digit_size = cfg_dict["train"]["digit_representation_size"]
|
||||
consumer_scales = torch.tensor(
|
||||
([1.0] * digit_size) + [cfg_dict["world"]["consumer_theta"]]
|
||||
)
|
||||
return torch.cat((global_state_scales, consumer_scales)).cuda()
|
||||
|
||||
|
||||
def firm_state_scaling_factors(cfg_dict):
|
||||
num_firms = cfg_dict["agents"]["num_firms"]
|
||||
global_state_scales = global_state_scaling_factors(cfg_dict)
|
||||
digit_size = cfg_dict["train"]["digit_representation_size"]
|
||||
# budget, capital, alpha, one-hot
|
||||
firm_scales = torch.tensor(
|
||||
([1.0] * digit_size) + [10000.0, 1.0] + ([1.0] * num_firms)
|
||||
)
|
||||
return torch.cat((global_state_scales, firm_scales)).cuda()
|
||||
|
||||
|
||||
def govt_state_scaling_factors(cfg_dict):
|
||||
return global_state_scaling_factors(cfg_dict).cuda()
|
||||
Reference in New Issue
Block a user