From 09bfcdffd5c7de8306bc32c0fea86121b3b984b8 Mon Sep 17 00:00:00 2001 From: TerenceLiu98 Date: Fri, 11 Jun 2021 11:07:52 +0800 Subject: [PATCH] v0.1.6 --- CEDA/__init__.py | 2 +- CEDA/macroecon/cn.py | 83 ++++++++++++++++++++++++++++++++++---------- CEDA/macroecon/eu.py | 75 ++++++++++++++++++++++++++++++--------- CEDA/macroecon/us.py | 81 +++++++++++++++++++++++++++++++++++++----- setup.py | 2 +- 5 files changed, 197 insertions(+), 46 deletions(-) diff --git a/CEDA/__init__.py b/CEDA/__init__.py index 8ae4140..bb522e8 100644 --- a/CEDA/__init__.py +++ b/CEDA/__init__.py @@ -1,7 +1,7 @@ import sys import os -__version__ = "0.1.5" +__version__ = "0.1.6" __author__ = "Terence Lau" diff --git a/CEDA/macroecon/cn.py b/CEDA/macroecon/cn.py index fb816a7..508f6e0 100644 --- a/CEDA/macroecon/cn.py +++ b/CEDA/macroecon/cn.py @@ -5,11 +5,13 @@ import requests import numpy as np import pandas as pd from fake_useragent import UserAgent + # TODO need add comments url = { "eastmoney": "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx", - "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?" + "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?", + "OECD": "https://stats.oecd.org/sdmx-json/data/DP_LIVE/" } # https://fred.stlouisfed.org/release/tables?rid=205&eid=712378 def gdp_quarterly(): @@ -1067,8 +1069,8 @@ def fl_monthly(): # Forex Loan df.columns = [ "Date", "Current_Month", - "YoY", - "MoM", + "YoY_Rate", + "MoM_Rate", "Accumulation" ] df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") @@ -1116,8 +1118,10 @@ def drr_monthly(): # Deposit Reserve Ratio ] df["Announcement Date"] = pd.to_datetime( df["Announcement Date"], format="%Y-%m-%d") + df["Date"] = df["Announcement Date"] df["Effective Date"] = pd.to_datetime( df["Effective Date"], format="%Y-%m-%d") + df[["SHIndex_Rate", "SZIndex_Rate"]] = df[["SHIndex_Rate", "SZIndex_Rate"]].astype(float) df[["Large_Financial_institution_Before", "Large_Financial_institution_After", "Large_Financial_institution_Adj_Rate", @@ -1172,7 +1176,7 @@ def interest_monthly(): # Interest df["Effective Date"] = pd.to_datetime( df["Effective Date"], format="%Y-%m-%d") df["Date"] = df["Announcement Date"] - df = df.replace("", np.nan).astype(float) + df = df.replace("", np.nan) df[["Deposit_Benchmark_Interest_Rate_Before", "Deposit_Benchmark_Interest_Rate_After", "Deposit_Benchmark_Interest_Rate_Adj_Rate", @@ -1214,24 +1218,65 @@ def gdc_daily(): # gasoline, Diesel and Crude Oil df = pd.to_datetime(df["Date"], format="%Y-%m-%d") return df -def Leading_Indicators_OECD(): - tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=CHNLOLITOAASTSAM,CHNLOLITONOSTSAM,CHNLOLITOTRSTSAM,CHNLORSGPNOSTSAM,CHNLORSGPRTSTSAM&scale=left,left,left,left,left&cosd=1992-05-01,1992-05-01,1992-05-01,1978-01-01,1978-01-01&coed=2021-03-01,2021-03-01,2021-02-01,2021-02-01,2021-02-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae&link_values=false,false,false,false,false&line_style=solid,solid,solid,solid,solid&mark_type=none,none,none,none,none&mw=3,3,3,3,3&lw=2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999&mma=0,0,0,0,0&fml=a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5&transformation=lin,lin,lin,lin,lin&vintage_date=2021-06-09,2021-06-09,2021-06-09,2021-06-09,2021-06-09&revision_date=2021-06-09,2021-06-09,2021-06-09,2021-06-09,2021-06-09&nd=1992-05-01,1992-05-01,1992-05-01,1978-01-01,1978-01-01" +def Leading_Indicators_OECD(startdate = "1950-01", enddate = "2021-05"): + # CLI + tmp_url = url["OECD"] + "CHN.CLI.AMPLITUD.LTRENDIDX.M/OECD" ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } request_header = {"User-Agent": ua.random} - r = requests.get(tmp_url, headers=request_header) + r = requests.get(tmp_url, params = request_params, headers=request_header) data_text = r.content - df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) - df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") - #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) - name_list = { - 'CHNLOLITOAASTSAM': "Leading Indicators OECD: Leading indicators: CLI: Amplitude adjusted for China", - 'CHNLOLITONOSTSAM': "Leading Indicators OECD: Leading indicators: CLI: Normalised for China", - 'CHNLOLITOTRSTSAM': "Leading Indicators OECD: Leading indicators: CLI: Trend restored for China", - 'CHNLORSGPNOSTSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Normalised for China", - 'CHNLORSGPRTSTSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Ratio to trend for China" - } - description = "Leading Indicators OECD, Monthly, Seasonally Adjusted" - return df, name_list, description + df_cli = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_cli.columns = ["Date", "CN_OECD_CLI"] + df_cli["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m") + df_cli["CN_OECD_CLI"] = df_cli["CN_OECD_CLI"].astype(float) + #BCI + tmp_url = url["OECD"] + "CHN.BCI.AMPLITUD.LTRENDIDX.M/OECD" + ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, params = request_params, headers=request_header) + data_text = r.content + df_bci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_bci.columns = ["Date", "CN_OECD_BCI"] + df_bci["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m") + df_bci["CN_OECD_BCI"] = df_bci["CN_OECD_BCI"].astype(float) + # CCI + tmp_url = url["OECD"] + "CHN.CCI.AMPLITUD.LTRENDIDX.M/OECD" + ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, params = request_params, headers=request_header) + data_text = r.content + df_cci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_cci.columns = ["Date", "CN_OECD_CCI"] + df_cci["Date"] = pd.to_datetime(df_cci["Date"], format = "%Y-%m") + df_cci["CN_OECD_CCI"] = df_cci["CN_OECD_CCI"].astype(float) + df = pd.merge_asof(df_cli, df_bci, on = "Date") + df = pd.merge_asof(df, df_cci, on = "Date") + + return df """ if __name__ == "__main__": diff --git a/CEDA/macroecon/eu.py b/CEDA/macroecon/eu.py index bcd3870..e1eb4fd 100644 --- a/CEDA/macroecon/eu.py +++ b/CEDA/macroecon/eu.py @@ -12,7 +12,8 @@ from pandas.core.reshape.merge import merge url = { "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?", "eurostat": "http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/", - "ecb": "https://sdw-wsrest.ecb.europa.eu/service/data/" + "ecb": "https://sdw-wsrest.ecb.europa.eu/service/data/", + "OECD": "https://stats.oecd.org/sdmx-json/data/DP_LIVE/" } def merge_data(data_1: pd.DataFrame, data_2: pd.DataFrame, col_name: str): @@ -82,25 +83,65 @@ def Balance_of_Payments_BPM6(): return df, name_list, description -def Leading_Indicators_OECD(): - tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19LORSGPNOSTSAM,EA19LOLITOTRGYSAM,EA19LOLITONOSTSAM,EA19LOLITOAASTSAM,EA19LORSGPORIXOBSAM,EA19LORSGPRTSTSAM,EA19LORSGPTDSTSAM&scale=left,left,left,left,left,left,left&cosd=1960-03-01,1966-12-01,1965-12-01,1965-12-01,1960-03-01,1960-03-01,1960-03-01&coed=2020-11-01,2020-11-01,2021-03-01,2021-03-01,2020-11-01,2020-11-01,2020-11-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1960-03-01,1966-12-01,1965-12-01,1965-12-01,1960-03-01,1960-03-01,1960-03-01" +def Leading_Indicators_OECD(startdate = "1950-01", enddate = "2021-05"): + # CLI + tmp_url = url["OECD"] + "EA19.CLI.AMPLITUD.LTRENDIDX.M/OECD" ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } request_header = {"User-Agent": ua.random} - r = requests.get(tmp_url, headers=request_header) + r = requests.get(tmp_url, params = request_params, headers=request_header) data_text = r.content - df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) - df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") - #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) - name_list = { - 'EA19CPALTT01GYQ': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Normalised for the Euro Area", - 'EA19LOLITOTRGYSAM': "Leading Indicators OECD: Leading indicators: CLI: Trend restored for the Euro Area", - 'EA19LOLITONOSTSAM': "Leading Indicators OECD: Leading indicators: CLI: Normalised for the Euro Area", - 'EA19LOLITOAASTSAM': "Leading Indicators OECD: Leading indicators: CLI: Amplitude adjusted for the Euro Area", - 'EA19LORSGPORIXOBSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Original series for the Euro Area", - 'EA19LORSGPRTSTSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Ratio to trend for the Euro Area", - 'EA19LORSGPTDSTSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Trend for the Euro Area"} - description = "Leading Indicators OECD, Monthly, Seasonally Adjusted" - return df, name_list, description + df_cli = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_cli.columns = ["Date", "EU_OECD_CLI"] + df_cli["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m") + df_cli["EU_OECD_CLI"] = df_cli["EU_OECD_CLI"].astype(float) + #BCI + tmp_url = url["OECD"] + "EA19.BCI.AMPLITUD.LTRENDIDX.M/OECD" + ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, params = request_params, headers=request_header) + data_text = r.content + df_bci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_bci.columns = ["Date", "EU_OECD_BCI"] + df_bci["Date"] = pd.to_datetime(df_bci["Date"], format = "%Y-%m") + df_bci["EU_OECD_BCI"] = df_bci["EU_OECD_BCI"].astype(float) + # CCI + tmp_url = url["OECD"] + "EA19.CCI.AMPLITUD.LTRENDIDX.M/OECD" + ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, params = request_params, headers=request_header) + data_text = r.content + df_cci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_cci.columns = ["Date", "EU_OECD_CCI"] + df_cci["Date"] = pd.to_datetime(df_cci["Date"], format = "%Y-%m") + df_cci["EU_OECD_CCI"] = df_cci["EU_OECD_CCI"].astype(float) + df = pd.merge_asof(df_cli, df_bci, on = "Date") + df = pd.merge_asof(df, df_cci, on = "Date") + + return df def Monetary_Aggregates_Monthly_Adj(): diff --git a/CEDA/macroecon/us.py b/CEDA/macroecon/us.py index acef1c5..4759ea3 100644 --- a/CEDA/macroecon/us.py +++ b/CEDA/macroecon/us.py @@ -13,7 +13,9 @@ from datetime import datetime url = { "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?", "philfed": "https://www.philadelphiafed.org/surveys-and-data/real-time-data-research/", - "chicagofed": "https://www.chicagofed.org/~/media/publications/"} + "chicagofed": "https://www.chicagofed.org/~/media/publications/", + "OECD": "https://stats.oecd.org/sdmx-json/data/DP_LIVE/" +} def date_transform(df, format_origin, format_after): return_list = [] @@ -469,7 +471,7 @@ def m1(startdate="1960-01-01", enddate="2021-01-01"): df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") df.columns = [ "Date", - "M1_weekly", + "M1_Weekly", "M1_Monthly", "M1_Quarterly", "M1_Annually"] @@ -672,6 +674,7 @@ def cci(startdate="1955-01-01", enddate="2021-01-01"): data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df.columns = ["Date", "CCI_Monthly"] + df["Date"] = pd.to_datetime(df["Date"], format = "%Y-%m-%d") return df @@ -693,6 +696,7 @@ def bci(startdate="1955-01-01", enddate="2021-01-01"): data_text = r.content df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) df.columns = ["Date", "BCI_Annually"] + df["Date"] = pd.to_datetime(df["Date"], format = "%Y-%m-%d") return df @@ -730,7 +734,7 @@ def ibr_3(startdate="1965-01-01", enddate="2021-01-01"): df_quarterly, on="DATE", direction="backward") - df.columns = ["Date", "ibr3_monthly", "ibr3_Quarterly"] + df.columns = ["Date", "ibr3_Monthly", "ibr3_Quarterly"] def gfcf_3(startdate="1965-01-01", enddate="2021-01-01"): @@ -768,7 +772,7 @@ def gfcf_3(startdate="1965-01-01", enddate="2021-01-01"): df_quarterly, on="DATE", direction="backward") - df.columns = ["Date", "ibr3_monthly", "ibr3_Annually"] + df.columns = ["Date", "ibr3_Monthly", "ibr3_Annually"] return df @@ -952,10 +956,7 @@ def pci(): return df -def inflation_noewcasting(): - """ - - """ +def inflation_nowcasting(): ua = UserAgent(verify_ssl=False) request_header = {"User-Agent": ua.random} tmp_url = "https://www.clevelandfed.org/~/media/files/charting/%20nowcast_quarter.json" @@ -1003,6 +1004,7 @@ def bbki(): tmp_url = url["chicagofed"] + "bbki/bbki-monthly-data-series-csv.csv" df = pd.read_csv(tmp_url) df["Date"] = date_transform(df["Date"], "%m/%d/%Y", "%Y-%m-%d") + df["Date"] = pd.to_datetime(df["Date"], format = "%Y-%m-%d") return df @@ -1010,6 +1012,7 @@ def cfnai(): tmp_url = url["chicagofed"] + "cfnai/cfnai-data-series-csv.csv" df = pd.read_csv(tmp_url) df["Date"] = date_transform(df["Date"], "%Y/%m", "%Y-%m-%d") + df["Date"] = pd.to_datetime(df["Date"], format = "%Y-%m-%d") return df @@ -1017,6 +1020,7 @@ def cfsbc(): tmp_url = url["chicagofed"] + "cfsbc/cfsbc-data-xlsx.xlsx" df = pd.read_excel(tmp_url) df["Date"] = date_transform(df["Date"], "%Y-%m", "%Y-%m-%d") + df["Date"] = pd.to_datetime(df["Date"], format = "%Y-%m-%d") return df @@ -1025,4 +1029,65 @@ def nfci(): df = pd.read_csv(tmp_url) df.columns = ["Date", "NFCI", "Risk", "Credit", "Leverage"] df["Date"] = date_transform(df["Date"], "%Y/%m/%d", "%Y-%m-%d") + df["Date"] = pd.to_datetime(df["Date"], format = "%Y-%m-%d") + return df + +def Leading_Indicators_OECD(startdate = "1950-01", enddate = "2021-05"): + # CLI + tmp_url = url["OECD"] + "USA.CLI.AMPLITUD.LTRENDIDX.M/OECD" + ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, params = request_params, headers=request_header) + data_text = r.content + df_cli = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_cli.columns = ["Date", "US_OECD_CLI"] + df_cli["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m") + df_cli["US_OECD_CLI"] = df_cli["US_OECD_CLI"].astype(float) + #BCI + tmp_url = url["OECD"] + "USA.BCI.AMPLITUD.LTRENDIDX.M/OECD" + ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, params = request_params, headers=request_header) + data_text = r.content + df_bci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_bci.columns = ["Date", "US_OECD_BCI"] + df_bci["Date"] = pd.to_datetime(df_bci["Date"], format = "%Y-%m") + df_bci["US_OECD_BCI"] = df_bci["US_OECD_BCI"].astype(float) + # CCI + tmp_url = url["OECD"] + "USA.CCI.AMPLITUD.LTRENDIDX.M/OECD" + ua = UserAgent(verify_ssl=False) + request_params = { + "contentType": "csv", + "detail": "code", + "separator": "comma", + "csv-lang": "en", + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, params = request_params, headers=request_header) + data_text = r.content + df_cci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] + df_cci.columns = ["Date", "US_OECD_CCI"] + df_cci["Date"] = pd.to_datetime(df_cci["Date"], format = "%Y-%m") + df_cci["US_OECD_CCI"] = df_cci["US_OECD_CCI"].astype(float) + df = pd.merge_asof(df_cli, df_bci, on = "Date") + df = pd.merge_asof(df, df_cci, on = "Date") + return df \ No newline at end of file diff --git a/setup.py b/setup.py index d7861cc..31a3673 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import os setup( name = "CEDApy", - version = "0.1.5", + version = "0.1.6", keywords = "quantitative economic data", long_description = open( os.path.join(