manuel/ai-econ
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adding ai_economist for modding

Browse Source
This commit is contained in:
manuel
2023-01-12 16:41:38 +01:00
parent 0479a4f6a4
commit f177f8f0ba
85 changed files with 19373 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_economist/datasets/covid19_datasets/README.md
+27
View File
@@ -0,0 +1,27 @@
## List of COVID-19 datasources used
1. **US state government policies** (Oxford Covid-19 Government Response Tracker (OxCGRT))
https://github.com/OxCGRT/USA-covid-policy
2. **US federal government direct payments** (Committee for a Responsible Federal Budget)
https://www.covidmoneytracker.org/
https://docs.google.com/spreadsheets/d/1Nr_J5wLfUT4IzqSXkYbdOXrRgEkBxhX0/edit#gid=682404301
3. **US deaths data** (COVID-19 Data Repository by the Center for Systems Science and Engineering (CSSE) at Johns Hopkins University)
https://github.com/CSSEGISandData/COVID-19
4. **US unemployment** (Bureau of Labor and Statistics)
https://www.bls.gov/lau/
5. **US vaccinations** (Our World in Data)
https://ourworldindata.org/covid-vaccinations
ai_economist/datasets/covid19_datasets/__init__.py
+5
View File
@@ -0,0 +1,5 @@
# Copyright (c) 2021, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root
# or https://opensource.org/licenses/BSD-3-Clause
ai_economist/datasets/covid19_datasets/data_and_fitted_params/fitted_params.json
+1
View File
File diff suppressed because one or more lines are too long
ai_economist/datasets/covid19_datasets/data_and_fitted_params/model_constants.json
+1
View File
@@ -0,0 +1 @@
{"DATE_FORMAT": "%Y-%m-%d", "STRINGENCY_POLICY_KEY": "StringencyIndex", "NUM_STRINGENCY_LEVELS": 10, "SIR_SMOOTHING_STD": 10, "SIR_MORTALITY": 0.02, "SIR_GAMMA": 0.07142857142857142, "US_STATE_IDX_TO_STATE_NAME": {"0": "Alabama", "1": "Alaska", "2": "Arizona", "3": "Arkansas", "4": "California", "5": "Colorado", "6": "Connecticut", "7": "Delaware", "8": "District of Columbia", "9": "Florida", "10": "Georgia", "11": "Hawaii", "12": "Idaho", "13": "Illinois", "14": "Indiana", "15": "Iowa", "16": "Kansas", "17": "Kentucky", "18": "Louisiana", "19": "Maine", "20": "Maryland", "21": "Massachusetts", "22": "Michigan", "23": "Minnesota", "24": "Mississippi", "25": "Missouri", "26": "Montana", "27": "Nebraska", "28": "Nevada", "29": "New Hampshire", "30": "New Jersey", "31": "New Mexico", "32": "New York", "33": "North Carolina", "34": "North Dakota", "35": "Ohio", "36": "Oklahoma", "37": "Oregon", "38": "Pennsylvania", "39": "Rhode Island", "40": "South Carolina", "41": "South Dakota", "42": "Tennessee", "43": "Texas", "44": "Utah", "45": "Vermont", "46": "Virginia", "47": "Washington", "48": "West Virginia", "49": "Wisconsin", "50": "Wyoming"}, "US_STATE_POPULATION": [4903185, 740995, 7278717, 3017804, 39512223, 5758736, 3565287, 973764, 705749, 21477737, 10617423, 1415872, 1787065, 12671821, 6732219, 3155070, 2913314, 4467673, 4648794, 1344212, 6045680, 6892503, 9986857, 5639632, 2976149, 6626371, 1068778, 1934408, 3080156, 1359711, 8882190, 2096829, 19453561, 10488084, 762062, 11689100, 3956971, 4217737, 12801989, 1059361, 5148714, 884659, 6829174, 28995881, 3205958, 623989, 8535519, 7614893, 1792147, 5822434, 578759], "US_POPULATION": 328737916, "GDP_PER_CAPITA": 65300}
ai_economist/datasets/covid19_datasets/data_and_fitted_params/real_world_data.npz
BIN
View File
Binary file not shown.
ai_economist/datasets/covid19_datasets/fit_model_parameters.ipynb
+1450
View File
File diff suppressed because it is too large Load Diff
ai_economist/datasets/covid19_datasets/gather_real_world_data.ipynb
+846
View File
@@ -0,0 +1,846 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Copyright (c) 2021, salesforce.com, inc. \n",
"All rights reserved. \n",
"SPDX-License-Identifier: BSD-3-Clause \n",
"For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# This notebook will be used to gather real-world data and perform data processing in order to use it in the covid-19 simulation.\n",
"\n",
"### All the downloaded data will be formatted into pandas dataframes."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Below is the list of COVID-19 data sources used in this notebook\n",
"\n",
"1. **US state government policies** (Oxford Covid-19 Government Response Tracker (OxCGRT))\n",
"\n",
" https://github.com/OxCGRT/USA-covid-policy\n",
"\n",
"\n",
"2. **US federal government direct payments** (Committee for a Responsible Federal Budget)\n",
"\n",
" https://www.covidmoneytracker.org/\n",
" \n",
" https://docs.google.com/spreadsheets/d/1Nr_J5wLfUT4IzqSXkYbdOXrRgEkBxhX0/edit#gid=682404301\n",
" \n",
"\n",
"3. **US deaths data** (COVID-19 Data Repository by the Center for Systems Science and Engineering (CSSE) at Johns Hopkins University)\n",
"\n",
" https://github.com/CSSEGISandData/COVID-19\n",
"\n",
"\n",
"4. **US vaccinations** (Our World in Data)\n",
" \n",
" https://ourworldindata.org/covid-vaccinations\n",
" \n",
" \n",
"5. **US unemployment** (Bureau of Labor and Statistics)\n",
"\n",
" https://www.bls.gov/lau/"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Dependencies"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime, timedelta\n",
"import json\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import os\n",
"import pandas as pd\n",
"import pickle\n",
"import scipy\n",
"from scipy.signal import convolve"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Classes to fetch the real-world data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from ai_economist.datasets.covid19_datasets.us_policies import DatasetCovidPoliciesUS\n",
"from ai_economist.datasets.covid19_datasets.us_deaths import DatasetCovidDeathsUS\n",
"from ai_economist.datasets.covid19_datasets.us_vaccinations import DatasetCovidVaccinationsUS\n",
"from ai_economist.datasets.covid19_datasets.us_unemployment import DatasetCovidUnemploymentUS"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Set a base directory where you would like to download real world data. The latest data will be downloaded into a folder within the base directory, named using the current date"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"BASE_DATA_DIR_PATH = \"/tmp/covid19_data\" # SPECIFY A BASE DIRECTORY TO STORE ALL THE DOWNLOADED DATA"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"DOWNLOAD_LATEST_DATA = True # Download the latest data or use whatever is saved earlier \n",
"CURRENT_DATE = datetime.now()\n",
"DATE_FORMAT = \"%Y-%m-%d\"\n",
"date_string = CURRENT_DATE.strftime(DATE_FORMAT).replace('/','-')\n",
"data_dir = os.path.join(BASE_DATA_DIR_PATH, date_string)\n",
"\n",
"print(\"All the data will be downloaded to the directory: '{}'.\".format(data_dir))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Set up dictionary to write model constants\n",
"model_constants = {}\n",
"model_constants_filename = \"model_constants.json\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Gather real-world data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 1. COVID-19 US State Government Policies\n",
"### Source: Oxford Covid-19 Government Response Tracker (OxCGRT) \n",
"(https://github.com/OxCGRT/USA-covid-policy)\n",
"\n",
"**NOTE:** All data will use the same format as **policy_df** (below) and use the same date index"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"covid_policies_us = DatasetCovidPoliciesUS(\n",
" data_dir=data_dir,\n",
" download_latest_data=DOWNLOAD_LATEST_DATA\n",
")\n",
"\n",
"# Which of the policy indicators to treat as the open/close level\n",
"STRINGENCY_POLICY_KEY = 'StringencyIndex'\n",
"# Number of levels to discretize the stringency policy into. \n",
"# In the context of reinforcement learning, this also determines the action space of the agents.\n",
"NUM_STRINGENCY_LEVELS = 10\n",
"\n",
"policies_us_df = covid_policies_us.process_policy_data(\n",
" stringency_policy_key=STRINGENCY_POLICY_KEY,\n",
" num_stringency_levels=NUM_STRINGENCY_LEVELS\n",
")\n",
"\n",
"print(\"Policy data are available between {} and {}\".format(policies_us_df[\"Date\"].min(), \n",
" policies_us_df[\"Date\"].max()))\n",
"\n",
"policy_df = policies_us_df.pivot(\n",
" index=\"Date\", columns=\"RegionName\", values=STRINGENCY_POLICY_KEY\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# This is the common date index that all the dataframes will use\n",
"COMMON_DATE_INDEX = policy_df.index"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# This is the list of states (in order) all the dataframes will use\n",
"US_STATE_ORDER = policy_df.columns.values"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Visualize the stringency level for a specified US state\n",
"state = \"California\"\n",
"policy_df[state].plot(figsize=(15,5), x='Date', title=\"Stringency Level for {}\".format(state), grid=True);"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 2. COVID-19 Federal government subsidies (direct payments) to the states\n",
"### Source: Committee For A Responsible Federal Budget\n",
"https://www.covidmoneytracker.org/\n",
"\n",
"### Direct payments provided by the Federal Government so far are recorded in this google spreadsheet\n",
"https://docs.google.com/spreadsheets/d/1Nr_J5wLfUT4IzqSXkYbdOXrRgEkBxhX0/edit#gid=682404301\n",
"### Read as (date: direct payment amount)\n",
"2020-04-15: 274B\n",
"\n",
"2020-12-27: 142B\n",
"\n",
"2021-03-11: 386B"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"subsidy_df = pd.DataFrame(policy_df.index).set_index(\"Date\")\n",
"subsidy_df[\"USA\"] = 0.0\n",
"\n",
"subsidy_df.loc[\"2020-04-15\", \"USA\"] = 274e9\n",
"subsidy_df.loc[\"2020-12-27\", \"USA\"] = 142e9\n",
"subsidy_df.loc[\"2021-03-11\", \"USA\"] = 386e9"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 3. COVID-19 Deaths data\n",
"### Source: COVID-19 Data Repository by the Center for Systems Science and Engineering (CSSE) at Johns Hopkins University \n",
"(https://github.com/CSSEGISandData/COVID-19)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"deaths_us_df = DatasetCovidDeathsUS(\n",
" data_dir=data_dir,\n",
" download_latest_data=DOWNLOAD_LATEST_DATA\n",
").df\n",
"\n",
"print(\"COVID-19 death data for the US is available between {} and {}\".format(\n",
" deaths_us_df.columns[12], deaths_us_df.columns[-1]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Retain just the states in US_STATE_ORDER\n",
"deaths_us_df = deaths_us_df[deaths_us_df.Province_State.isin(US_STATE_ORDER)]\n",
"\n",
"# We will visualize this later in the notebook"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 4. COVID-19 Vaccination Data\n",
"### Source: Our World in Data\n",
"(https://ourworldindata.org/covid-vaccinations)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vaccinations_us_df = DatasetCovidVaccinationsUS(\n",
" data_dir=data_dir,\n",
" download_latest_data=DOWNLOAD_LATEST_DATA\n",
").df\n",
"\n",
"vaccination_dates = sorted(vaccinations_us_df.date.unique())\n",
"print(\"Vaccination data is available between {} and {}\".format(min(vaccination_dates), max(vaccination_dates)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vaccinated_df = vaccinations_us_df.pivot(\n",
" index=\"date\", columns=\"location\", values=\"people_fully_vaccinated\"\n",
")[US_STATE_ORDER]\n",
"\n",
"vaccinated_df.index = pd.to_datetime(vaccinated_df.index)\n",
"vaccinated_df = vaccinated_df.reindex(COMMON_DATE_INDEX).fillna(0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Visualize the vaccinations for a specified US state\n",
"# Warning: the last value may not be updated (may show it to be 0)\n",
"\n",
"state = \"California\"\n",
"vaccinated_df[state].plot(figsize=(15,5), x='Date', title=\"Vaccinations for {}\".format(state), grid=True);"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Using deaths and vaccinations to compute the susceptible-infected-recovered (SIR) numbers\n",
"\n",
"Our SIR data will only treat **deaths** as ground-truth.\n",
"\n",
"Given death data and some assumed constants about the _death rate_ and _recovery rate_ , we can apply some \"SIR algebra\" (i.e. solve for unknowns using the SIR equations) to _infer_ quantities like total \"recovered\", number of infected people, and ultimately **Beta**, which is the rate of transmission times the number of people an infected person comes into contact with."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# For data representation, we will want to build a dataframe for...\n",
"# ... deaths...\n",
"deaths_df = pd.DataFrame(COMMON_DATE_INDEX, columns=['Date']).set_index('Date')\n",
"smoothed_deaths_df = pd.DataFrame(COMMON_DATE_INDEX, columns=['Date']).set_index('Date')\n",
"# ... (inferred) SIR states...\n",
"susceptible_df = pd.DataFrame(COMMON_DATE_INDEX, columns=['Date']).set_index('Date')\n",
"infected_df = pd.DataFrame(COMMON_DATE_INDEX, columns=['Date']).set_index('Date')\n",
"recovered_df = pd.DataFrame(COMMON_DATE_INDEX, columns=['Date']).set_index('Date')\n",
"# ... and (inferred) Beta.\n",
"beta_df = pd.DataFrame(COMMON_DATE_INDEX, columns=['Date']).set_index('Date')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# STD of the Gaussian smoothing window applied to the death data.\n",
"SIR_SMOOTHING_STD = 10"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Fill the death dataframe from (smoothed) raw data\n",
"\n",
"def smooth(x, gauss_std=10):\n",
" \"\"\"\n",
" gauss_std: standard deviation of the Gaussian smoothing window applied to the death data.\n",
" \"\"\"\n",
" if gauss_std <= 0:\n",
" return x\n",
" # To invalidate the near-edge results, bookend the input x with nans\n",
" x = np.concatenate([[np.nan], np.array(x), [np.nan]])\n",
" \n",
" kernel = scipy.stats.norm.pdf(\n",
" np.linspace(-3*gauss_std, 3*gauss_std, 1+6*gauss_std),\n",
" scale=gauss_std\n",
" )\n",
" normer = np.ones_like(x)\n",
" smoothed_x = convolve(x, kernel, mode='same') / convolve(normer, kernel, mode='same')\n",
" \n",
" # Remove the indices added by the nan padding\n",
" return smoothed_x[1:-1]\n",
"\n",
"for us_state_name in US_STATE_ORDER:\n",
" state_deaths = deaths_us_df[deaths_us_df['Province_State']==us_state_name]\n",
" cumulative_state_deaths = []\n",
" for d in COMMON_DATE_INDEX:\n",
" date_string = '{d.month}/{d.day}/{y}'.format(d=d, y=d.year % 2000)\n",
" if date_string in state_deaths:\n",
" cumulative_state_deaths.append(\n",
" state_deaths[date_string].sum()\n",
" )\n",
" else:\n",
" cumulative_state_deaths.append(\n",
" np.nan\n",
" )\n",
" \n",
" # Store raw numbers (for direct comparison)\n",
" deaths_df[us_state_name] = cumulative_state_deaths\n",
" \n",
" # Store smoothed numbers (for beta analysis)\n",
" smoothed_cumulative_state_deaths = smooth(cumulative_state_deaths, gauss_std=SIR_SMOOTHING_STD)\n",
" smoothed_deaths_df[us_state_name] = smoothed_cumulative_state_deaths"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"state_deaths = deaths_us_df[deaths_us_df['Province_State']==\"California\"]\n",
"cumulative_state_deaths = []\n",
"for d in COMMON_DATE_INDEX:\n",
" date_string = '{d.month}/{d.day}/{y}'.format(d=d, y=d.year % 2000)\n",
" if date_string in state_deaths:\n",
" cumulative_state_deaths.append(\n",
" state_deaths[date_string].sum()\n",
" )\n",
" else:\n",
" cumulative_state_deaths.append(\n",
" np.nan\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Visualize the deaths for a specified US state\n",
"state = \"California\"\n",
"\n",
"# Some values near the ends may be \"missing\" because of smoothing\n",
"deaths_df[state].plot(figsize=(15,5), x='Date', ylim=[0, 65000]);\n",
"smoothed_deaths_df[state].plot(figsize=(15,5), x='Date', title=\"COVID deaths in {}\".format(state), ylim=[0, 65000], grid=True);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Death rate: fraction of infected persons who die\n",
"SIR_MORTALITY = 0.02\n",
"\n",
"# Recovery rate: the inverse of expected time someone remains infected\n",
"SIR_GAMMA = 1 / 14"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# This is the core \"SIR algebra\" used to infer S, I, R, and Beta at each date.\n",
"\n",
"def infer_sir_and_beta(us_state_name):\n",
" state_population = deaths_us_df[deaths_us_df['Province_State']==us_state_name]['Population'].sum()\n",
" \n",
" # Helpful to do this math in normalized numbers\n",
" dead = np.array(smoothed_deaths_df[us_state_name]) / state_population\n",
" vaccinated = np.array(vaccinated_df[us_state_name]) / state_population\n",
" \n",
" # Dead is the fraction of \"recovered\" that did not survive\n",
" # Also, the vaccinated lot is part of the recovered\n",
" recovered = dead / SIR_MORTALITY + vaccinated\n",
" \n",
" # The daily change in recovered (ignoring the vaccinated) is a fraction of the infected population on the previous day\n",
" infected = np.nan * np.zeros_like(dead)\n",
" infected[:-1] = (recovered[1:] - recovered[:-1] - (vaccinated[1:] - vaccinated[:-1])) / SIR_GAMMA\n",
" \n",
" # S+I+R must always = 1\n",
" susceptible = 1 - infected - recovered\n",
" \n",
" # Here's where things get interesting. The change in infected is due to...\n",
" change_in_i = infected[1:] - infected[:-1]\n",
" # ... infected people that transition to the recovered state (decreases I)...\n",
" expected_change_from_recovery = -infected[:-1] * SIR_GAMMA\n",
" # ... and susceptible people that transition to the infected state (increases I).\n",
" new_infections = change_in_i - expected_change_from_recovery\n",
" \n",
" # With these pieces, we can solve for Beta.\n",
" beta_ = new_infections / (infected[:-1] * susceptible[:-1] + 1e-6)\n",
" beta_ = np.clip(beta_, 0, 1)\n",
" # Apply a threshold in terms of normalized daily deaths (if too low, beta estimates are bad)\n",
" normalized_daily_deaths = dead[1:]-dead[:-1]\n",
" ndd_lookback = np.zeros_like(new_infections)\n",
" lookback_window = 3*SIR_SMOOTHING_STD\n",
" ndd_cutoff = 1e-8\n",
" ndd_lookback[lookback_window:] = normalized_daily_deaths[:-lookback_window]\n",
" beta_[np.logical_not(ndd_lookback > 1e-8)] = np.nan\n",
" \n",
" beta = np.nan * np.zeros_like(dead)\n",
" beta[:-1] = beta_\n",
" \n",
" # Undo normalization\n",
" susceptible *= state_population\n",
" infected *= state_population\n",
" recovered *= state_population\n",
" \n",
" return susceptible, infected, recovered, beta"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Fill the SIR and Beta dataframes with their inferred values\n",
"for st in US_STATE_ORDER:\n",
" susceptible_df[st], infected_df[st], recovered_df[st], beta_df[st] = infer_sir_and_beta(us_state_name=st)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"## Visualize the SIR and BETA for a specified US state\n",
"# Warning: some values near the ends may be \"missing\" because of smoothing\n",
"\n",
"state = \"California\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"susceptible_df[state].plot(figsize=(15,3), x='Date', title=\"(Inferred) Susceptible Population in {}\".format(state), grid=True);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"infected_df[state].plot(figsize=(15,3), x='Date', title=\"(Inferred) Infected Population in {}\".format(state), grid=True);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"recovered_df[state].plot(figsize=(15,3), x='Date', title=\"(Inferred) Recovered Population in {}\".format(state), grid=True);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"beta_df[state].plot(figsize=(15,3), x='Date', title=\"(Inferred) SIR Beta in {}\".format(state), grid=True);"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 5. COVID-19 Unemployment data\n",
"### Source: Bureau of Labor and Statistics\n",
"\n",
"https://www.bls.gov/lau/"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"monthly_unemployment_us = DatasetCovidUnemploymentUS(\n",
" data_dir=data_dir,\n",
" download_latest_data=DOWNLOAD_LATEST_DATA).data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sample_monthly_unemployment = monthly_unemployment_us['California']\n",
"unemp_year_keys = sorted(sample_monthly_unemployment.keys())\n",
"unemp_starting_month_key = sorted(sample_monthly_unemployment[unemp_year_keys[0]].keys())[0]\n",
"unemp_ending_month_key = sorted(sample_monthly_unemployment[unemp_year_keys[-1]].keys())[-1]\n",
"unemp_starting_date = datetime.strptime(\n",
" str(unemp_year_keys[0]) + '-' + str(unemp_ending_month_key+1) + '-1', DATE_FORMAT)\n",
"unemp_ending_date = datetime.strptime(\n",
" str(unemp_year_keys[-1]) + '-' + str(unemp_ending_month_key+1) + '-1', DATE_FORMAT) - timedelta(1)\n",
"\n",
"print(\"Unemployment data is available between {} and {}\".format(datetime.strftime(unemp_starting_date, DATE_FORMAT),\n",
" datetime.strftime(unemp_ending_date, DATE_FORMAT)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Convert this to a daily unemployment dataframe\n",
"\n",
"unemployment_df = pd.DataFrame(COMMON_DATE_INDEX, columns=['Date']).set_index('Date')\n",
"\n",
"for us_state_name in monthly_unemployment_us.keys():\n",
" unemployment_df[us_state_name] = [\n",
" monthly_unemployment_us[us_state_name][x.year].get(x.month, np.nan)\n",
" for x in unemployment_df.index\n",
" ]\n",
"unemployment_df = unemployment_df[US_STATE_ORDER]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"## Visualize the unemployment rate for a specified US state\n",
"# There is likely going to be some unemployment data missing at the tail end, \n",
"# as the unemployment data isn't updated as frequently as the other data.\n",
"\n",
"state = \"California\"\n",
"unemployment_df[state].plot(figsize=(15,5), x='Date', title=\"Unemployment for {} (%)\".format(state), grid=True);"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Unemployment rate -> unemployed (the number of unemployed people)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"us_state_to_pop_dict = {}\n",
"for us_state in US_STATE_ORDER:\n",
" us_state_to_pop_dict[us_state] = deaths_us_df[deaths_us_df.Province_State==us_state].Population.sum()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"unemployed_df = unemployment_df.multiply([us_state_to_pop_dict[col]/100.0 for col in unemployment_df.columns])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Saving"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Save some of the data processing constants for use within the environment"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model_constants_dict = {}\n",
"\n",
"model_constants_dict[\"DATE_FORMAT\"] = DATE_FORMAT\n",
"model_constants_dict[\"STRINGENCY_POLICY_KEY\"] = STRINGENCY_POLICY_KEY\n",
"model_constants_dict[\"NUM_STRINGENCY_LEVELS\"] = int(NUM_STRINGENCY_LEVELS)\n",
"model_constants_dict[\"SIR_SMOOTHING_STD\"] = SIR_SMOOTHING_STD\n",
"model_constants_dict[\"SIR_MORTALITY\"] = SIR_MORTALITY\n",
"model_constants_dict[\"SIR_GAMMA\"] = SIR_GAMMA\n",
"model_constants_dict[\"US_STATE_IDX_TO_STATE_NAME\"] = {\n",
" us_state_idx: us_state for us_state_idx, us_state in enumerate(US_STATE_ORDER)\n",
"}\n",
"model_constants_dict[\"US_STATE_POPULATION\"] = [int(us_state_to_pop_dict[us_state]) for us_state in US_STATE_ORDER]\n",
"model_constants_dict[\"US_POPULATION\"] = int(sum([us_state_to_pop_dict[us_state] for us_state in US_STATE_ORDER]))\n",
"\n",
"# 2019: https://data.worldbank.org/indicator/NY.GDP.PCAP.CD?locations=US&view=chart\n",
"model_constants_dict[\"GDP_PER_CAPITA\"] = 65300 # TODO: Load this in from model_constants.json.\n",
"\n",
"model_constants_filename = \"model_constants.json\"\n",
"with open(os.path.join(data_dir, model_constants_filename), \"w\") as fp: \n",
" json.dump(model_constants_dict, fp)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Save all the processed dataframes in order to use for model fitting notebook (fit_model_parameters.ipynb)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dataframes = {\n",
" \"policy\": policy_df,\n",
" \"subsidy\": subsidy_df,\n",
" \"deaths\": deaths_df,\n",
" \"vaccinated\": vaccinated_df,\n",
" \"smoothed_deaths\": smoothed_deaths_df,\n",
" \"susceptible\": susceptible_df,\n",
" \"infected\": infected_df,\n",
" \"recovered\": recovered_df,\n",
" \"beta\": beta_df,\n",
" \"unemployment\": unemployment_df,\n",
" \"unemployed\": unemployed_df,\n",
"}\n",
"\n",
"for k, df in dataframes.items():\n",
" if k == \"subsidy\": # This is at the USA level, not at the US states level\n",
" continue\n",
" assert (df.columns.to_list() == US_STATE_ORDER).all()\n",
"\n",
"with open(os.path.join(data_dir, 'dataframes.pkl'), 'wb') as F:\n",
" pickle.dump(dataframes, F)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Also save all the data as numpy arrays for use within the covid19 simulation environment"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"real_world_data = {}\n",
"for key in dataframes:\n",
" real_world_data[key] = dataframes[key].values\n",
" \n",
"# Save the real-world data as a .npz for use within the environment\n",
"np.savez(os.path.join(data_dir, \"real_world_data.npz\"), **real_world_data) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Finally, in order to use this gathered real-world data when you run the covid19 simulation, you will need to also"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 1. Run the \"fit_model_parameters.ipynb\" notebook with the base data directory specified below."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"BASE_DATA_DIR_PATH = '{}'\".format(BASE_DATA_DIR_PATH))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2. Set \"path_to_data_and_fitted_params\" in the env config also to the data directory below."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"path_to_data_and_fitted_params = '{}'\".format(data_dir))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
ai_economist/datasets/covid19_datasets/us_deaths.py
+54
View File
@@ -0,0 +1,54 @@
# Copyright (c) 2021, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root
# or https://opensource.org/licenses/BSD-3-Clause
import os
from io import BytesIO
import pandas as pd
import requests
class DatasetCovidDeathsUS:
"""
Class to load COVID-19 deaths data for the US.
Source: https://github.com/CSSEGISandData/COVID-19
Note: in this dataset, reporting deaths only started on the 22nd of January 2020,
Attributes:
df: Timeseries dataframe of confirmed COVID deaths for all the US states
"""
def __init__(self, data_dir="", download_latest_data=True):
if not os.path.exists(data_dir):
print(
"Creating a dynamic data directory to store "
"COVID-19 deaths data: {}".format(data_dir)
)
os.makedirs(data_dir)
filename = "daily_us_deaths.csv"
if download_latest_data or filename not in os.listdir(data_dir):
print(
"Fetching latest U.S. COVID-19 deaths data from John Hopkins, "
"and saving it in {}".format(data_dir)
)
req = requests.get(
"https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/"
"csse_covid_19_data/csse_covid_19_time_series/"
"time_series_covid19_deaths_US.csv"
)
self.df = pd.read_csv(BytesIO(req.content))
self.df.to_csv(
os.path.join(data_dir, filename)
) # Note: performs an overwrite
else:
print(
"Not fetching the latest U.S. COVID-19 deaths data from John Hopkins."
" Using whatever was saved earlier in {}!!".format(data_dir)
)
assert filename in os.listdir(data_dir)
self.df = pd.read_csv(os.path.join(data_dir, filename), low_memory=False)
ai_economist/datasets/covid19_datasets/us_policies.py
+122
View File
@@ -0,0 +1,122 @@
# Copyright (c) 2021, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root
# or https://opensource.org/licenses/BSD-3-Clause
import os
from datetime import datetime
from io import BytesIO
import numpy as np
import pandas as pd
import requests
class DatasetCovidPoliciesUS:
"""
Class to load COVID-19 government policies for the US states.
Source: https://github.com/OxCGRT/USA-covid-policy
Other references:
- Codebook: https://github.com/OxCGRT/covid-policy-tracker/blob/master/
documentation/codebook.md
- Index computation methodology: https://github.com/OxCGRT/covid-policy-tracker/
blob/master/documentation/index_methodology.md
Attributes:
df: Timeseries dataframe of state-wide policies
"""
def __init__(self, data_dir="", download_latest_data=True):
if not os.path.exists(data_dir):
print(
"Creating a dynamic data directory to store COVID-19 "
"policy tracking data: {}".format(data_dir)
)
os.makedirs(data_dir)
filename = "daily_us_policies.csv"
if download_latest_data or filename not in os.listdir(data_dir):
print(
"Fetching latest U.S. COVID-19 policies data from OxCGRT, "
"and saving it in {}".format(data_dir)
)
req = requests.get(
"https://raw.githubusercontent.com/OxCGRT/USA-covid-policy/master/"
"data/OxCGRT_US_latest.csv"
)
self.df = pd.read_csv(BytesIO(req.content), low_memory=False)
self.df["Date"] = self.df["Date"].apply(
lambda x: datetime.strptime(str(x), "%Y%m%d")
)
# Fetch only the state-wide policies
self.df = self.df.loc[self.df["Jurisdiction"] != "NAT_GOV"]
self.df.to_csv(
os.path.join(data_dir, filename)
) # Note: performs an overwrite
else:
print(
"Not fetching the latest U.S. COVID-19 policies data from OxCGRT. "
"Using whatever was saved earlier in {}!!".format(data_dir)
)
assert filename in os.listdir(data_dir)
self.df = pd.read_csv(os.path.join(data_dir, filename), low_memory=False)
def process_policy_data(
self, stringency_policy_key="StringencyIndex", num_stringency_levels=10
):
"""
Gather the relevant policy indicator frm the dataframe,
fill in the null values (if any),
and discretize/quantize the policy into num_stringency_levels.
Note: Possible values for stringency_policy_key are
["StringencyIndex", "Government response index",
"Containment and health index", "Economic Support index".]
Reference: https://github.com/OxCGRT/covid-policy-tracker/blob/master/
documentation/index_methodology.md
"""
def discretize(policies, num_indicator_levels=10):
"""
Discretize the policies (a Pandas series) into num_indicator_levels
"""
# Indices are normalized to be in [0, 100]
bins = np.linspace(0, 100, num_indicator_levels)
# Find left and right values of bin and find the nearer edge
bin_index = np.digitize(policies, bins, right=True)
bin_left_edges = bins[bin_index - 1]
bin_right_edges = bins[bin_index]
discretized_policies = bin_index + np.argmin(
np.stack(
(
np.abs(policies.values - bin_left_edges),
np.abs(policies.values - bin_right_edges),
)
),
axis=0,
)
return discretized_policies
# Gather just the relevant columns
policy_df = self.df[["RegionName", "Date", stringency_policy_key]].copy()
# Fill in null values via a "forward fill"
policy_df[stringency_policy_key].fillna(method="ffill", inplace=True)
# Discretize the stringency indices
discretized_stringency_policies = discretize(
policy_df[stringency_policy_key], num_indicator_levels=num_stringency_levels
)
policy_df.loc[:, stringency_policy_key] = discretized_stringency_policies
# Replace Washington DC by District of Columbia to keep consistent
# (with the other data sources)
policy_df = policy_df.replace("Washington DC", "District of Columbia")
policy_df = policy_df.sort_values(by=["RegionName", "Date"])
return policy_df
ai_economist/datasets/covid19_datasets/us_unemployment.py
+128
View File
@@ -0,0 +1,128 @@
# Copyright (c) 2021, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root
# or https://opensource.org/licenses/BSD-3-Clause
import bz2
import os
import pickle
import queue
import threading
import urllib.request as urllib2
import pandas as pd
from bs4 import BeautifulSoup
class DatasetCovidUnemploymentUS:
"""
Class to load COVID-19 unemployment data for the US states.
Source: https://www.bls.gov/lau/
"""
def __init__(self, data_dir="", download_latest_data=True):
if not os.path.exists(data_dir):
print(
"Creating a dynamic data directory to store COVID-19 "
"unemployment data: {}".format(data_dir)
)
os.makedirs(data_dir)
filename = "monthly_us_unemployment.bz2"
if download_latest_data or filename not in os.listdir(data_dir):
# Construct the U.S. state to FIPS code mapping
state_fips_df = pd.read_excel(
"https://www2.census.gov/programs-surveys/popest/geographies/2017/"
"state-geocodes-v2017.xlsx",
header=5,
)
# remove all statistical areas and cities
state_fips_df = state_fips_df.loc[state_fips_df["State (FIPS)"] != 0]
self.us_state_to_fips_dict = pd.Series(
state_fips_df["State (FIPS)"].values, index=state_fips_df.Name
).to_dict()
print(
"Fetching the U.S. unemployment data from "
"Bureau of Labor and Statistics, and saving it in {}".format(data_dir)
)
self.data = self.scrape_bls_data()
fp = bz2.BZ2File(os.path.join(data_dir, filename), "wb")
pickle.dump(self.data, fp)
fp.close()
else:
print(
"Not fetching the U.S. unemployment data from Bureau of Labor and"
" Statistics. Using whatever was saved earlier in {}!!".format(data_dir)
)
assert filename in os.listdir(data_dir)
with bz2.BZ2File(os.path.join(data_dir, filename), "rb") as fp:
self.data = pickle.load(fp)
fp.close()
# Scrape monthly unemployment from the Bureau of Labor Statistics website
def get_monthly_bls_unemployment_rates(self, state_fips):
with urllib2.urlopen(
"https://data.bls.gov/timeseries/LASST{:02d}0000000000003".format(
state_fips
)
) as response:
html_doc = response.read()
soup = BeautifulSoup(html_doc, "html.parser")
table = soup.find_all("table")[1]
table_rows = table.find_all("tr")
unemployment_dict = {}
mth2idx = {
"Jan": 1,
"Feb": 2,
"Mar": 3,
"Apr": 4,
"May": 5,
"Jun": 6,
"Jul": 7,
"Aug": 8,
"Sep": 9,
"Oct": 10,
"Nov": 11,
"Dec": 12,
}
for tr in table_rows[1:-1]:
td = tr.find_all("td")[-1]
unemp = float("".join([c for c in td.text if c.isdigit() or c == "."]))
th = tr.find_all("th")
year = int(th[0].text)
month = mth2idx[th[1].text]
if year not in unemployment_dict:
unemployment_dict[year] = {}
unemployment_dict[year][month] = unemp
return unemployment_dict
def scrape_bls_data(self):
def do_scrape(us_state, fips, queue_obj):
out = self.get_monthly_bls_unemployment_rates(fips)
queue_obj.put([us_state, out])
print("Getting BLS Data. This might take a minute...")
result = queue.Queue()
threads = [
threading.Thread(target=do_scrape, args=(us_state, fips, result))
for us_state, fips in self.us_state_to_fips_dict.items()
]
for t in threads:
t.start()
for t in threads:
t.join()
monthly_unemployment = {}
while not result.empty():
us_state, data = result.get()
monthly_unemployment[us_state] = data
return monthly_unemployment
ai_economist/datasets/covid19_datasets/us_vaccinations.py
+61
View File
@@ -0,0 +1,61 @@
# Copyright (c) 2021, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root
# or https://opensource.org/licenses/BSD-3-Clause
import os
from io import BytesIO
import pandas as pd
import requests
class DatasetCovidVaccinationsUS:
"""
Class to load COVID-19 vaccination data for the US.
Source: https://ourworldindata.org/covid-vaccinations
Attributes:
df: Timeseries dataframe of COVID vaccinations for all the US states
"""
def __init__(self, data_dir="", download_latest_data=True):
if not os.path.exists(data_dir):
print(
"Creating a dynamic data directory to store COVID-19 "
"vaccination data: {}".format(data_dir)
)
os.makedirs(data_dir)
filename = "daily_us_vaccinations.csv"
if download_latest_data or filename not in os.listdir(data_dir):
print(
"Fetching latest U.S. COVID-19 vaccination data from "
"Our World in Data, and saving it in {}".format(data_dir)
)
req = requests.get(
"https://raw.githubusercontent.com/owid/covid-19-data/master/"
"public/data/vaccinations/us_state_vaccinations.csv"
)
self.df = pd.read_csv(BytesIO(req.content))
# Rename New York State to New York for consistency with other datasets
self.df = self.df.replace("New York State", "New York")
# Interpolate missing values
self.df = self.df.interpolate(method="linear")
self.df.to_csv(
os.path.join(data_dir, filename)
) # Note: performs an overwrite
else:
print(
"Not fetching the latest U.S. COVID-19 deaths data from "
"Our World in Data. Using whatever was saved earlier in {}!!".format(
data_dir
)
)
assert filename in os.listdir(data_dir)
self.df = pd.read_csv(os.path.join(data_dir, filename), low_memory=False)