import io import os import ssl import time import json import requests import numpy as np import pandas as pd from datetime import datetime from bs4 import BeautifulSoup from multiprocessing import Pool import dateutil.parser as dparser from fake_useragent import UserAgent ssl._create_default_https_context = ssl._create_unverified_context # Main Economic Indicators: https://alfred.stlouisfed.org/release?rid=205 url = { "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?", "philfed": "https://www.philadelphiafed.org/surveys-and-data/real-time-data-research/", "chicagofed": "https://www.chicagofed.org/~/media/publications/", "OECD": "https://stats.oecd.org/sdmx-json/data/DP_LIVE/" } def date_transform(df, format_origin, format_after): return_list = [] for i in range(0, len(df)): return_list.append(datetime.strptime( df[i], format_origin).strftime(format_after)) return return_list def gdp_quarterly(startdate="1947-01-01", enddate="2021-01-01"): """ Full Name: Gross Domestic Product Description: Billions of Dollars, Quarterly, Seasonally Adjusted Annual Rate Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "GDP", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df.columns = ["Date", "GDP"] df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") df["GDP"] = df["GDP"].astype(float) return df def gdpc1_quarterly(startdate="1947-01-01", enddate="2021-01-01"): """ Full Name: Real Gross Domestic Product Description: Billions of Chained 2012 Dollars, Quarterly, Seasonally Adjusted Annual Rate Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "GDPC1", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) return df def oecd_gdp_monthly(startdate="1947-01-01", enddate="2021-01-01"): """ Full Name: Real Gross Domestic Product Description: Billions of Chained 2012 Dollars, Quarterly, Seasonally Adjusted Annual Rate Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USALORSGPNOSTSAM", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) return df def payems_monthly(startdate="1939-01-01", enddate="2021-01-01"): """ Full Name: All Employees, Total Nonfarm Description: Thousands of Persons,Seasonally Adjusted, Monthly Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "PAYEMS", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df.columns = ["Date", "Payems"] df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") df["Payems"] = df["Payems"].astype(float) return df def ppi(): tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=on&txtcolor=%23444444&ts=12&tts=12&width=968&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=PPIACO,PCUOMFGOMFG&scale=left,left&cosd=1913-01-01,1984-12-01&coed=2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643&link_values=false,false&line_style=solid,solid&mark_type=none,none&mw=3,3&lw=2,2&ost=-99999,-99999&oet=99999,99999&mma=0,0&fml=a,a&fq=Monthly,Monthly&fam=avg,avg&fgst=lin,lin&fgsnd=2020-02-01,2020-02-01&line_index=1,2&transformation=lin,lin&vintage_date=2021-06-10,2021-06-10&revision_date=2021-06-10,2021-06-10&nd=1913-01-01,1984-12-01" ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} r = requests.get(tmp_url, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) name_list = { "PPIACO": "Producer Price Index by Commodity: All Commodities", "PCUOMFGOMFG": "Producer Price Index by Industry: Total Manufacturing Industries" } df.replace(".", np.nan, inplace=True) df.columns = ["Date", "PPI_C", "PPI_I"] df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") df[["PPI_C", "PPI_I"]] = df[["PPI_C", "PPI_I"]].astype(float) return df def pmi(): t = time.time() res = requests.get( f"https://cdn.jin10.com/dc/reports/dc_usa_ism_pmi_all.js?v={str(int(round(t * 1000))), str(int(round(t * 1000)) + 90)}" ) json_data = json.loads( res.text[res.text.find("{"): res.text.rfind("}") + 1]) date_list = [item["date"] for item in json_data["list"]] value_list = [item["datas"]["美国ISM制造业PMI报告"] for item in json_data["list"]] value_df = pd.DataFrame(value_list) value_df.columns = json_data["kinds"] value_df.index = pd.to_datetime(date_list) temp_df = value_df["今值"] url = "https://datacenter-api.jin10.com/reports/list_v2" params = { "max_date": "", "category": "ec", "attr_id": "28", "_": str(int(round(t * 1000))), } headers = { "accept": "*/*", "accept-encoding": "gzip, deflate, br", "accept-language": "zh-CN,zh;q=0.9,en;q=0.8", "cache-control": "no-cache", "origin": "https://datacenter.jin10.com", "pragma": "no-cache", "referer": "https://datacenter.jin10.com/reportType/dc_usa_michigan_consumer_sentiment", "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-site", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36", "x-app-id": "rU6QIu7JHe2gOUeR", "x-csrf-token": "", "x-version": "1.0.0", } r = requests.get(url, params=params, headers=headers) temp_se = pd.DataFrame(r.json()["data"]["values"]).iloc[:, :2] temp_se.index = pd.to_datetime(temp_se.iloc[:, 0]) temp_se = temp_se.iloc[:, 1] temp_df = temp_df.append(temp_se) temp_df.dropna(inplace=True) temp_df.sort_index(inplace=True) temp_df = temp_df.reset_index() temp_df.drop_duplicates(subset="index", inplace=True) temp_df.set_index("index", inplace=True) temp_df = temp_df.squeeze() temp_df.index.name = None temp_df.name = "usa_ism_pmi" temp_df = temp_df.astype("float") PMI_I = pd.DataFrame() PMI_I["Date"] = pd.to_datetime(temp_df.index, format="%Y-%m-%d") PMI_I["ISM_PMI_I"] = np.array(temp_df).astype(float) t = time.time() res = requests.get( f"https://cdn.jin10.com/dc/reports/dc_usa_ism_non_pmi_all.js?v={str(int(round(t * 1000))), str(int(round(t * 1000)) + 90)}" ) json_data = json.loads( res.text[res.text.find("{"): res.text.rfind("}") + 1]) date_list = [item["date"] for item in json_data["list"]] value_list = [item["datas"]["美国ISM非制造业PMI报告"] for item in json_data["list"]] value_df = pd.DataFrame(value_list) value_df.columns = json_data["kinds"] value_df.index = pd.to_datetime(date_list) temp_df = value_df["今值"] url = "https://datacenter-api.jin10.com/reports/list_v2" params = { "max_date": "", "category": "ec", "attr_id": "29", "_": str(int(round(t * 1000))), } headers = { "accept": "*/*", "accept-encoding": "gzip, deflate, br", "accept-language": "zh-CN,zh;q=0.9,en;q=0.8", "cache-control": "no-cache", "origin": "https://datacenter.jin10.com", "pragma": "no-cache", "referer": "https://datacenter.jin10.com/reportType/dc_usa_michigan_consumer_sentiment", "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-site", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36", "x-app-id": "rU6QIu7JHe2gOUeR", "x-csrf-token": "", "x-version": "1.0.0", } r = requests.get(url, params=params, headers=headers) temp_se = pd.DataFrame(r.json()["data"]["values"]).iloc[:, :2] temp_se.index = pd.to_datetime(temp_se.iloc[:, 0]) temp_se = temp_se.iloc[:, 1] temp_df = temp_df.append(temp_se) temp_df.dropna(inplace=True) temp_df.sort_index(inplace=True) temp_df = temp_df.reset_index() temp_df.drop_duplicates(subset="index", inplace=True) temp_df.set_index("index", inplace=True) temp_df = temp_df.squeeze() temp_df.index.name = None temp_df.name = "usa_ism_non_pmi" temp_df = temp_df.astype("float") PMI_NI = pd.DataFrame() PMI_NI["Date"] = pd.to_datetime(temp_df.index, format="%Y-%m-%d") PMI_NI["ISM_PMI_NI"] = np.array(temp_df).astype(float) PMI = pd.merge_asof(PMI_I, PMI_NI, on="Date") return PMI def unrate(startdate="1948-01-01", enddate="2021-01-01"): """ Full Name: Unemployment Rate: Aged 15-64: All Persons for the United States Description: Percent, Seasonally Adjusted, Monthly, Quarterly and Annually Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "LRUN64TTUSM156S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "LRUN64TTUSQ156S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "LRUN64TTUSA156S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_monthly, df_quarterly, on="DATE", direction="backward") df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") df.columns = ["Date", "UR_Monthly", "UR_Quarterly", "UR_Annually"] return df def erate(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name: Employment Rate: Aged 25-54: All Persons for the United States Description: Percent,Seasonally Adjusted, Monthly, Quarterly and Annually Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "LREM25TTUSM156S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "LREM25TTUSQ156S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "LREM25TTUSA156S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_monthly, df_quarterly, on="DATE", direction="backward") df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") df.columns = ["Date", "ER_Monthly", "ER_Quarterly", "ER_Annually"] def pce_monthly(startdate="1959-01-01", enddate="2021-01-01"): """ Full Name: PCE Description: Percent, Monthly, Seasonally Adjusted Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "PCE", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) return df def cpi(startdate="1960-01-01", enddate="2021-01-01"): """ Full Name: Consumer Price Index: Total All Items for the United States Description: Percent, Monthly, Quarterly and Annually, Seasonally Adjusted Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "CPALTT01USM661S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "CPALTT01USQ661S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "CPALTT01USA661S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_monthly, df_quarterly, on="DATE", direction="backward") df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") df.columns = ["Date", "CPI_Monthly", "CPI_Quarterly", "CPI_Annually"] df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") df[["CPI_Monthly", "CPI_Quarterly", "CPI_Annually"]] = df[[ "CPI_Monthly", "CPI_Quarterly", "CPI_Annually"]].astype(float) return df def m1(startdate="1960-01-01", enddate="2021-01-01"): """ Full Name: Consumer Price Index: M3 for the United States Description: Growth Rate Previous Period, Monthly, Quarterly and Annually, Seasonally Adjusted Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "WM1NS", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_weekly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_weekly["DATE"] = pd.to_datetime(df_weekly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "MANMM101USM657S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "MANMM101USQ657S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "MANMM101USA657S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof(df_weekly, df_monthly, on="DATE", direction="backward") df = pd.merge_asof(df, df_quarterly, on="DATE", direction="backward") df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") df.columns = [ "Date", "M1_Weekly", "M1_Monthly", "M1_Quarterly", "M1_Annually"] return df def m2(startdate="1960-01-01", enddate="2021-01-01"): """ Full Name: M2 Money Stock Description: Seasonally Adjusted, Weekly, Monthly, Quarterly and Annually, Seasonally Adjusted Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "WM2NS", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_weekly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_weekly["DATE"] = pd.to_datetime(df_weekly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "M2SL", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") df = pd.merge_asof(df_weekly, df_monthly, on="DATE", direction="backward") df.columns = ["Date", "M2_Weekly", "M2_Monthly"] return df def m3(startdate="1960-01-01", enddate="2021-01-01"): """ Full Name: Consumer Price Index: M3 for the United States Description: Growth Rate Previous Period, Monthly, Quarterly and Annually, Seasonally Adjusted Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "MABMM301USM657S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "MABMM301USQ657S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "MABMM301USA657S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_monthly, df_quarterly, on="DATE", direction="backward") df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") df.columns = ["Date", "M3_Monthly", "M3_Quarterly", "M3_Annually"] return df def ltgby_10(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name: Long-Term Government Bond Yields: 10-year: Main (Including Benchmark) for the United States Description: Percent,Not Seasonally Adjusted, Monthly, Quarterly and Annually Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "IRLTLT01USM156N", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "IRLTLT01USQ156N", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "IRLTLT01USA156N", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_monthly, df_quarterly, on="DATE", direction="backward") df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") df.columns = ["Date", "ltgby_Monthly", "ltgby_Quarterly", "ltgby_Annually"] return df def gdp_ipd(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name: Long-Term Government Bond Yields: 10-year: Main (Including Benchmark) for the United States Description: Percent,Not Seasonally Adjusted, Monthly, Quarterly and Annually Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USAGDPDEFQISMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USAGDPDEFAISMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_quarterly, df_annually, on="DATE", direction="backward") df.columns = ["Date", "gdp_ipd_Quarterly", "gdp_ipd_Annually"] return df def cci(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name: Consumer Opinion Surveys: Confidence Indicators: Composite Indicators: OECD Indicator for the United States Description: Normalised (Normal=100), Seasonally Adjusted, Monthly Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "CSCICP03USM665S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df.columns = ["Date", "CCI_Monthly"] df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") return df def bci(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name: Business confidence index OECD Indicator for the United States Description: Normalised (Normal=100), Seasonally Adjusted, Monthly Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "BSCICP03USM665S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df.columns = ["Date", "BCI_Annually"] df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") return df def ibr_3(startdate="1965-01-01", enddate="2021-01-01"): """ Full Name: 3-Month or 90-day Rates and Yields: Interbank Rates for the United States Description: Percent, Not Seasonally Adjusted, Monthly and Quarterly """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "IR3TIB01USM156N", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "IR3TIB01USQ156N", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_quarterly, df_quarterly, on="DATE", direction="backward") df.columns = ["Date", "ibr3_Monthly", "ibr3_Quarterly"] def gfcf_3(startdate="1965-01-01", enddate="2021-01-01"): """ Full Name: Gross Fixed Capital Formation in United States Description: United States Dollars,Not Seasonally Adjusted, Quarterly and Annually """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USAGFCFQDSMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USAGFCFADSMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_quarterly, df_quarterly, on="DATE", direction="backward") df.columns = ["Date", "ibr3_Monthly", "ibr3_Annually"] return df def pfce(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name: Private Final Consumption Expenditure in United States """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USAPFCEQDSMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USAPFCEADSMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_quarterly, df_annually, on="DATE", direction="backward") df.columns = ["Date", "PFCE_Quarterly", "PFCE_Annually"] return df def tlp(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name: Early Estimate of Quarterly ULC Indicators: Total Labor Productivity for the United States Description: Growth Rate Previous Period,Seasonally Adjusted, Quarterly and YoY Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "ULQELP01USQ657S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_quarterly["DATE"] = pd.to_datetime( df_quarterly["DATE"], format="%Y-%m-%d") ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "ULQELP01USQ659S", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_quarterly, df_annually, on="DATE", direction="backward") df.columns = ["Date", "PFCE_Quarterly", "PFCE_Quarterly_YoY"] return df def rt(startdate="1955-01-01", enddate="2021-01-01"): """ Full Name:Total Retail Trade in United States Description: Monthly and Anually Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "USASARTMISMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") request_header = {"User-Agent": ua.random} request_params = { "id": "USASARTAISMEI", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_annually["DATE"] = pd.to_datetime( df_annually["DATE"], format="%Y-%m-%d") df = pd.merge_asof( df_monthly, df_annually, on="DATE", direction="backward") df.columns = ["Date", "RT_Quarterly", "RT_Annually"] return df def bir(startdate="2003-01-01", enddate="2021-01-01"): """ Full Name:Total Retail Trade in United States Description: Monthly and Anually Return: pd.DataFrame """ tmp_url = url["fred_econ"] ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} request_params = { "id": "T5YIE", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_5y = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_5y["DATE"] = pd.to_datetime(df_5y["DATE"], format="%Y-%m-%d") request_header = {"User-Agent": ua.random} request_params = { "id": "T10YIE", "cosd": "{}".format(startdate), "coed": "{}".format(enddate) } r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_10y = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df_10y["DATE"] = pd.to_datetime(df_10y["DATE"], format="%Y-%m-%d") df = pd.merge_asof(df_5y, df_10y, on="DATE", direction="backward") df.columns = ["Date", "BIR_5y", "BIR_10y"] return df def adsbci(): """ An index designed to track real business conditions at high observation frequency """ ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} tmp_url = url["philfed"] + "ads" r = requests.get(tmp_url, headers=request_header) file = open("ads_temp.xls", "wb") file.write(r.content) file.close() df = pd.read_excel("ads_temp.xls") df.columns = ["Date", "ADS_Index"] df['Date'] = pd.to_datetime(df["Date"], format="%Y:%m:%d") os.remove("ads_temp.xls") return df def pci(): """ Tracks the degree of political disagreement among U.S. politicians at the federal level, Monthly """ df = pd.read_excel( "https://www.philadelphiafed.org/-/media/frbp/assets/data-visualizations/partisan-conflict.xlsx") df["Date"] = df["Year"].astype(str) + df["Month"] df["Date"] = pd.to_datetime(df["Date"], format="%Y%B") df = df.drop(["Year", "Month"], axis=1) df = df[["Date", "Partisan Conflict"]] return df def inflation_nowcasting(): ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} tmp_url = "https://www.clevelandfed.org/~/media/files/charting/%20nowcast_quarter.json" r = requests.get(tmp_url, headers=request_header) tmp_df = pd.DataFrame(demjson.decode(r.text)) df = pd.DataFrame() for i in range(0, len(tmp_df)): date = tmp_df['chart'][i]['subcaption'][:4] + "/" + \ pd.DataFrame(tmp_df["dataset"][i][0]['data'])['tooltext'].str.extract(r"\b(0?[1-9]|1[0-2])/(0?[1-9]|[12][0-9]|3[01])\b")[0] + "/" + \ pd.DataFrame(tmp_df["dataset"][i][0]['data'])['tooltext'].str.extract( r"\b(0?[1-9]|1[0-2])/(0?[1-9]|[12][0-9]|3[01])\b")[1] CPI_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[0])["value"] C_CPI_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[1])["value"] PCE_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[2])["value"] C_PCE_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[3])["value"] A_CPI_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[4])["value"] A_C_CPI_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[5])["value"] A_PCE_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[6])["value"] A_C_PCE_I = pd.DataFrame( (pd.DataFrame(tmp_df["dataset"][i])['data'])[7])["value"] tmp_df2 = pd.DataFrame({"Date": date, "CPI_I": CPI_I, "C_CPI_I": C_CPI_I, "PCE_I": PCE_I, "C_PCE_I": C_PCE_I, "A_CPI_I": A_CPI_I, "A_C_CPI_I": A_C_CPI_I, "A_PCE_I": A_PCE_I, "A_C_PCE_I": A_C_PCE_I}) df = pd.concat([df, tmp_df2], axis=0) df.reset_index(drop=True, inplace=True) df.replace('', np.nan, inplace=True) return df def bbki(): tmp_url = url["chicagofed"] + "bbki/bbki-monthly-data-series-csv.csv" df = pd.read_csv(tmp_url) df["Date"] = date_transform(df["Date"], "%m/%d/%Y", "%Y-%m-%d") df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") return df def cfnai(): tmp_url = url["chicagofed"] + "cfnai/cfnai-data-series-csv.csv" df = pd.read_csv(tmp_url) df["Date"] = date_transform(df["Date"], "%Y/%m", "%Y-%m-%d") df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") return df def cfsbc(): tmp_url = url["chicagofed"] + "cfsbc/cfsbc-data-xlsx.xlsx" df = pd.read_excel(tmp_url) df["Date"] = date_transform(df["Date"], "%Y-%m", "%Y-%m-%d") df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") return df def nfci(): tmp_url = url["chicagofed"] + "nfci/decomposition-nfci-csv.csv" df = pd.read_csv(tmp_url) df.columns = ["Date", "NFCI", "Risk", "Credit", "Leverage"] df["Date"] = date_transform(df["Date"], "%Y/%m/%d", "%Y-%m-%d") df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") return df def Leading_Indicators_OECD(startdate="1950-01", enddate="2021-05"): # CLI tmp_url = url["OECD"] + "USA.CLI.AMPLITUD.LTRENDIDX.M/OECD" ua = UserAgent(verify_ssl=False) request_params = { "contentType": "csv", "detail": "code", "separator": "comma", "csv-lang": "en", "startPeriod": "{}".format(startdate), "endPeriod": "{}".format(enddate) } request_header = {"User-Agent": ua.random} r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_cli = pd.read_csv(io.StringIO( data_text.decode('utf-8')))[["TIME", "Value"]] df_cli.columns = ["Date", "US_OECD_CLI"] df_cli["Date"] = pd.to_datetime(df_cli["Date"], format="%Y-%m") df_cli["US_OECD_CLI"] = df_cli["US_OECD_CLI"].astype(float) # BCI tmp_url = url["OECD"] + "USA.BCI.AMPLITUD.LTRENDIDX.M/OECD" ua = UserAgent(verify_ssl=False) request_params = { "contentType": "csv", "detail": "code", "separator": "comma", "csv-lang": "en", "startPeriod": "{}".format(startdate), "endPeriod": "{}".format(enddate) } request_header = {"User-Agent": ua.random} r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_bci = pd.read_csv(io.StringIO( data_text.decode('utf-8')))[["TIME", "Value"]] df_bci.columns = ["Date", "US_OECD_BCI"] df_bci["Date"] = pd.to_datetime(df_bci["Date"], format="%Y-%m") df_bci["US_OECD_BCI"] = df_bci["US_OECD_BCI"].astype(float) # CCI tmp_url = url["OECD"] + "USA.CCI.AMPLITUD.LTRENDIDX.M/OECD" ua = UserAgent(verify_ssl=False) request_params = { "contentType": "csv", "detail": "code", "separator": "comma", "csv-lang": "en", "startPeriod": "{}".format(startdate), "endPeriod": "{}".format(enddate) } request_header = {"User-Agent": ua.random} r = requests.get(tmp_url, params=request_params, headers=request_header) data_text = r.content df_cci = pd.read_csv(io.StringIO( data_text.decode('utf-8')))[["TIME", "Value"]] df_cci.columns = ["Date", "US_OECD_CCI"] df_cci["Date"] = pd.to_datetime(df_cci["Date"], format="%Y-%m") df_cci["US_OECD_CCI"] = df_cci["US_OECD_CCI"].astype(float) df = pd.merge_asof(df_cli, df_bci, on="Date") df = pd.merge_asof(df, df_cci, on="Date") return df def US_EPU_Monthly(): df = pd.read_excel( "https://www.policyuncertainty.com/media/US_Policy_Uncertainty_Data.xlsx")[:-1] df['Date'] = pd.to_datetime(df['Year'].apply(str).str.cat( df['Month'].apply(int).apply(str), sep='-'), format='%Y-%m') df = df[["Date", "Three_Component_Index", "News_Based_Policy_Uncert_Index"]] return df def US_EPU_Daily(): df = pd.read_csv( "https://www.policyuncertainty.com/media/All_Daily_Policy_Data.csv")[:-1] df['Date'] = pd.to_datetime(df['year'].apply(str).str.cat(df['month'].apply( str), sep='-').apply(str).str.cat(df['day'].apply(str), sep='-'), format='%Y-%m-%d') df = df.drop(["year", "month", "day"], axis=1) return df