diff --git a/CEDA/economic/XHData.py b/CEDA/economic/XHData.py new file mode 100644 index 0000000..5b65126 --- /dev/null +++ b/CEDA/economic/XHData.py @@ -0,0 +1,223 @@ +import ast +import requests +import pandas as pd + +url = { + "CNFIN": "https://api.cnfin.com/roll/charts/" +} + +class XHData(object): + def __init__(self): + pass + + def download(self, id:int=None): + tmp_url = url["CNFIN"] + "getContent?ids={}".format(id) + r = requests.get(tmp_url) + if r.ok: + raw_data = r.json() + data = pd.DataFrame(ast.literal_eval(raw_data["data"]["list"][0]["content"])) + return data + else: + return ValueError("Something went wrong, try again later") + + + def GDP(self): + """ + quarterly + """ + data = self.download(id=12006) + data.columns = ["date", "data"] + return data + + def Household_Consumption_SPLY(self): + """ + quarterly + """ + data = self.download(id=12073) + data.columns = ["date", "data"] + return data + + def Household_Consumption(self): + """ + quarterly + """ + data = self.download(id=12074) + data.columns = ["date", "data"] + return data + + def Per_capita_Disposable_Income(self): + """ + quarterly + """ + data = self.download(id=12071) + data.columns = ["date", "data"] + return data + + def Urban_Average_Salary_Annual(self): + """ + quarterly + """ + data = self.download(id=12070) + data.columns = ["date", "data"] + return data + + def Urban_Uneployment_Rate(self): + """ + quarterly + """ + data = self.download(id=12069) + data.columns = ["date", "data"] + return data + + def Government_Bound_Return_Rate_10_Year(self): + """ + quarterly + """ + data = self.download(id=12068) + data.columns = ["date", "data"] + return data + + def Government_Bound_Return_Rate_3_Year(self): + """ + quarterly + """ + data = self.download(id=12067) + data.columns = ["date", "data"] + return data + + def Government_Bound_Return_Rate_1_Year(self): + """ + quarterly + """ + data = self.download(id=12066) + data.columns = ["date", "data"] + return data + + def LPR_1_Year(self): + """ + Monthly + """ + data = self.download(id=12065) + data.columns = ["date", "data"] + return data + + def SHIBOR_3_Month(self): + """ + Daily + """ + data = self.download(id=12064) + data.columns = ["date", "data"] + return data + + def SHIBOR_2_Week(self): + """ + Daily + """ + data = self.download(id=12063) + data.columns = ["date", "data"] + return data + + def SHIBOR_1_Day(self): + """ + Daily + """ + data = self.download(id=12063) + data.columns = ["date", "data"] + return data + + def Foreign_Exchange_Options(self): + data = self.download(id=12060) + data.columns = ["date", "data"] + return data + + def Foreign_Exchange_Swaps(self): + data = self.download(id=12059) + data.columns = ["date", "data"] + return data + + def Foreign_Exchange_Forward(self): + data = self.download(id=12058) + data.columns = ["date", "data"] + return data + + def Foreign_Exchange_Spot(self): + data = self.download(id=12057) + data.columns = ["date", "data"] + return data + + def Loan_to_Deposit(self): + data = self.download(id=12056) + data.columns = ["date", "data"] + return data + + def RMB_Deposits(self): + data = self.download(id=12055) + data.columns = ["date", "data"] + return data + + def RMB_Loan(self): + data = self.download(id=12054) + data.columns = ["date", "data"] + return data + + def M0_SPLY(self): + data = self.download(id=12053) + data.columns = ["date", "data"] + return data + + def M1_SPLY(self): + data = self.download(id=12052) + data.columns = ["date", "data"] + return data + + def M2_SPLY(self): + data = self.download(id=12051) + data.columns = ["date", "data"] + return data + + def M0(self): + data = self.download(id=12050) + data.columns = ["date", "data"] + return data + + def M1(self): + data = self.download(id=12049) + data.columns = ["date", "data"] + return data + + def M2(self): + data = self.download(id=12048) + data.columns = ["date", "data"] + return data + + def Total_Retail_Sales_of_Consumer_Goods_LP(self): + data = self.download(id=12047) + data.columns = ["date", "data"] + return data + + def Total_Retail_Sales_of_Consumer_Goods_SPLY(self): + data = self.download(id=12046) + data.columns = ["date", "data"] + return data + +""" +import json +import requests +from tqdm import tqdm + +urls, titles = [], [] +for i in tqdm(range(5000, 20000)): + url = "https://api.cnfin.com/roll/charts/getContent?ids={}".format(i) + r = requests.get(url) + if r.ok: + data = r.json() + if data["data"] == "图表数据不存在": + pass + else: + urls.append(url) + titles.append(json.loads(data["data"]["list"][0]["modelCode"])["title"]["text"]) +""" + + + + diff --git a/CEDA/economic/cn.py b/CEDA/economic/cn.py deleted file mode 100644 index 693fde9..0000000 --- a/CEDA/economic/cn.py +++ /dev/null @@ -1,1309 +0,0 @@ -import re -import io -import demjson -import requests -import numpy as np -import pandas as pd -from fake_useragent import UserAgent - -# TODO need add comments - -url = { - "eastmoney": "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx", - "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?", - "OECD": "https://stats.oecd.org/sdmx-json/data/DP_LIVE/" -} -# https://fred.stlouisfed.org/release/tables?rid=205&eid=712378 -def gdp_quarterly(): - """ - ABS: absolute value (per 100 million CNY) - YoY: year on year growth - Data source: http://data.eastmoney.com/cjsj/gdp.html - """ - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable7519513", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "20", - "_": "1622020352668" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Absolute_Value", - "YoY_Rate", - "Primary_Industry_ABS", - "Primary_Industry_YoY_Rate", - "Secondary_Industry_ABS", - "Secondary_Industry_YoY_Rate", - "Tertiary_Industry_ABS", - "Tertiary_Industry_YoY_Rate", - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df["Absolute_Value"] = df["Absolute_Value"].astype(float) - df["Secondary_Industry_ABS"] = df["Secondary_Industry_ABS"].astype(float) - df["Tertiary_Industry_ABS"] = df["Tertiary_Industry_ABS"].astype(float) - df["Absolute_Value"] = df["Absolute_Value"].astype(float) - df["YoY_Rate"] = df["YoY_Rate"].astype(float) / 100 - df["Secondary_Industry_YoY_Rate"] = df["Secondary_Industry_YoY_Rate"].astype( - float) / 100 - df["Tertiary_Industry_YoY_Rate"] = df["Tertiary_Industry_YoY_Rate"].astype( - float) / 100 - return df - - -def ppi_monthly(): - """ - ABS: absolute value (per 100 million CNY) - YoY: year on year growth - Accum: Accumulation - Data source: http://data.eastmoney.com/cjsj/ppi.html - """ - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable9051497", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "22", - "_": "1622047940401" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "Current_Month_YoY_Rate", - "Current_Month_Accum" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df["Current_Month"] = df["Current_Month"].astype(float) - df["Current_Month_YoY_Rate"] = df["Current_Month_YoY_Rate"].astype( - float) / 100 - df["Current_Month_Accum"] = df["Current_Month_Accum"].astype(float) - return df - - -def cpi_monthly(): - """ - Accum: Accumulation - YoY: year on year growth - MoM: month on month growth - Data source: http://data.eastmoney.com/cjsj/cpi.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable2790750", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "19", - "_": "1622020352668" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Notion_Monthly", - "Notion_YoY_Rate", - "Notion_MoM_Rate", - "Notion_Accum", - "Urban_Monthly", - "Urban_YoY_Rate", - "Urban_MoM_Rate", - "Urban_Accum", - "Rural_Monthly", - "Rural_YoY_Rate", - "Rural_MoM_Rate", - "Rural_Accum", - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Notion_Monthly", - "Notion_Accum", - "Urban_Monthly", - "Urban_Accum", - "Rural_Monthly", - "Rural_Accum"]] = df[["Notion_Monthly", - "Notion_Accum", - "Urban_Monthly", - "Urban_Accum", - "Rural_Monthly", - "Rural_Accum"]].astype(float) - df[["Notion_YoY_Rate", - "Notion_MoM_Rate", - "Urban_YoY_Rate", - "Urban_MoM_Rate", - "Rural_YoY_Rate", - "Rural_MoM_Rate"]] = df[["Notion_YoY_Rate", - "Notion_MoM_Rate", - "Urban_YoY_Rate", - "Urban_MoM_Rate", - "Rural_YoY_Rate", - "Rural_MoM_Rate"]].astype(float) / 100 - return df - - -def pmi_monthly(): - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/pmi.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable4515395", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "2", - "ps": "200", - "mkt": "21", - "_": "162202151821" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Man_Industry_Index", - "Man_Index_YoY_Rate", - "Non-Man_Industry_Index", - "Non-Man_Index_YoY_Rate", - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Man_Industry_Index", "Non-Man_Industry_Index"]] = \ - df[["Man_Industry_Index", "Non-Man_Industry_Index"]].astype(float) - df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]] = \ - df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]].astype(float) / 100 - return df - - -def fai_monthly(): # fix asset investment - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/gdzctz.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable607120", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "12", - "_": "1622021790947" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "YoY_Rate", - "MoM_Rate", - "Current_Year_Accum" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Current_Month", "Current_Year_Accum"]] = \ - df[["Current_Month", "Current_Year_Accum"]].astype(float) - df[["YoY_Rate", "MoM_Rate"]] = \ - df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100 - return df - - -def hi_old_monthly(): # house index old version (2008-2010) - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/house.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable1895714", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "10", - "_": "1622022794457" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Housing_Prosperity_Index", - "HPI_YoY_Rate", - "Land_Development_Area_Index", - "LDAI_YoY_Rate", - "Sales_Price_Index", - "SPI_YoY_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Housing_Prosperity_Index", - "Land_Development_Area_Index", - "Sales_Price_Index"]] = df[["Housing_Prosperity_Index", - "Land_Development_Area_Index", - "Sales_Price_Index"]].astype(float) - df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]] = \ - df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]].astype(float) / 100 - return df - -# mkt=1&stat=2&city1=%E5%B9%BF%E5%B7%9E&city2=%E4%B8%8A%E6%B5%B7 - - -# newly built commercial housing & second-hand commercial housing -def hi_new_monthly(city1: str, city2: str): - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/newhouse.html - """ - tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getnewhousechartdata?" - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - request_params_nbch_MoM = { - "mkt": "1", - "stat": "2", - "city1": "{}".format(city1), - "city2": "{}".format(city2) - } - request_params_shch_MoM = { - "mkt": "1", - "stat": "3", - "city1": "{}".format(city1), - "city2": "{}".format(city2) - } - r_nbch_MoM = requests.get( - tmp_url, - params=request_params_nbch_MoM, - headers=request_header) - r_shch_MoM = requests.get( - tmp_url, - params=request_params_shch_MoM, - headers=request_header) - data_text_nbch_MoM = r_nbch_MoM.text - data_text_shch_MoM = r_shch_MoM.text - data_json_nbch_MoM = demjson.decode(data_text_nbch_MoM) - data_json_shch_MoM = demjson.decode(data_text_shch_MoM) - date_nbch = data_json_nbch_MoM['chart']['series']['value'] - data1_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][0]['value'] - data2_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][1]['value'] - data1_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][0]['value'] - data2_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][1]['value'] - df_MoM = pd.DataFrame({"Date": date_nbch, - "City1_nbch_MoM": data1_nbch_MoM, - "City1_shch_MoM": data1_shch_MoM, - "City2_nbch_MoM": data2_nbch_MoM, - "City2_shch_MoM": data2_shch_MoM}) - df_MoM["Date"] = pd.to_datetime(df_MoM["Date"], format="%m/%d/%Y") - - request_params_nbch_YoY = { - "mkt": "2", - "stat": "2", - "city1": "{}".format(city1), - "city2": "{}".format(city2) - } - request_params_shch_YoY = { - "mkt": "2", - "stat": "3", - "city1": "{}".format(city1), - "city2": "{}".format(city2) - } - r_nbch_YoY = requests.get( - tmp_url, - params=request_params_nbch_YoY, - headers=request_header) - r_shch_YoY = requests.get( - tmp_url, - params=request_params_shch_YoY, - headers=request_header) - data_text_nbch_YoY = r_nbch_YoY.text - data_text_shch_YoY = r_shch_YoY.text - data_json_nbch_YoY = demjson.decode(data_text_nbch_YoY) - data_json_shch_YoY = demjson.decode(data_text_shch_YoY) - date_nbch = data_json_nbch_YoY['chart']['series']['value'] - data1_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][0]['value'] - data2_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][1]['value'] - data1_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][0]['value'] - data2_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][1]['value'] - df_YoY = pd.DataFrame({"Date": date_nbch, - "City1_nbch_YoY": data1_nbch_YoY, - "City1_shch_YoY": data1_shch_YoY, - "City2_nbch_YoY": data2_nbch_YoY, - "City2_shch_YoY": data2_shch_YoY}) - df_YoY["Date"] = pd.to_datetime(df_YoY["Date"], format="%m/%d/%Y") - df = df_YoY.merge(df_MoM, on="Date") - return df - - -def ci_eei_monthly(): # Climate Index & Entrepreneur Expectation Index - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/qyjqzs.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable7709842", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "8", - "_": "1622041485306" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Climate_Index", - "CI_YoY_Rate", - "CI_MoM_Rate", - "Entrepreneur_Expectation_Index", - "EEI_YoY_Rate", - "EEI_MoM_Rate" - ] - df.replace('', np.nan, inplace=True) - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Climate_Index", "Entrepreneur_Expectation_Index"]] = \ - df[["Climate_Index", "Entrepreneur_Expectation_Index"]].astype(float) - df[["CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]] = df[[ - "CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]].astype(float) / 100 - return df - - -def ig_monthly(): # Industry Growth - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/gyzjz.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable4577327", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "0", - "_": "1622042259898" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "IG_YoY_Rate", - "IG_Accum_Rate", - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["IG_YoY_Rate", "IG_Accum_Rate"]] = \ - df[["IG_YoY_Rate", "IG_Accum_Rate"]].astype(float) / 100 - return df - - -def cgpi_monthly(): # Corporate Goods Price Index - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/qyspjg.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable7184534", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "9", - "_": "1622042652353" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "General_Index", - "General_Index_YoY_Rate", - "Total_Index_MoM_Rate", - "Agricultural_Product", - "Agricultural_Product_YoY_Rate", - "Agricultural_Product_MoM_Rate", - "Mineral_Product", - "Mineral_Product_YoY_Rate", - "Mineral_Product_MoM_Rate", - "Coal_Oil_Electricity", - "Coal_Oil_Electricity_YoY_Rate", - "Coal_Oil_Electricity_MoM_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["General_Index", - "Agricultural_Product", - "Mineral_Product", - "Coal_Oil_Electricity"]] = df[["General_Index", - "Agricultural_Product", - "Mineral_Product", - "Coal_Oil_Electricity"]].astype(float) - df[["General_Index_YoY_Rate", - "Total_Index_MoM_Rate", - "Agricultural_Product_YoY_Rate", - "Agricultural_Product_MoM_Rate", - "Coal_Oil_Electricity_YoY_Rate", - "Coal_Oil_Electricity_MoM_Rate"]] = df[["General_Index_YoY_Rate", - "Total_Index_MoM_Rate", - "Agricultural_Product_YoY_Rate", - "Agricultural_Product_MoM_Rate", - "Coal_Oil_Electricity_YoY_Rate", - "Coal_Oil_Electricity_MoM_Rate"]].astype(float) / 100 - return df - - -def cci_csi_cei_monthly(): # Consumer Confidence Index & Consumer Satisfaction Index & Consumer Expectation Index - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/xfzxx.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable1243218", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "4", - "_": "1622043704818" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "CCI", - "CCI_YoY_Rate", - "CCI_MoM_Rate", - "CSI", - "CSI_YoY_Rate", - "CSI_MoM_Rate", - "CEI", - "CEI_YoY_Rate", - "CEI_MoM_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["CCI", "CSI", "CEI"]] = \ - df[["CCI", "CSI", "CEI"]].astype(float) - df[["CCI_YoY_Rate", "CCI_MoM_Rate", - "CSI_YoY_Rate", "CSI_MoM_Rate", - "CEI_YoY_Rate", "CEI_MoM_Rate"]] = \ - df[["CCI_YoY_Rate", "CCI_MoM_Rate", - "CSI_YoY_Rate", "CSI_MoM_Rate", - "CEI_YoY_Rate", "CEI_MoM_Rate"]].astype(float) / 100 - return df - - -def trscg_monthly(): # Total Retail Sales of Consumer Goods - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/xfp.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable3665821", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "5", - "_": "1622044011316" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "TRSCG_YoY_Rate", - "TRSCG_MoM_Rate", - "TRSCG_Accum", - "TRSCG_Accum_YoY_Rate" - ] - df.replace("", np.nan, inplace=True) - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Current_Month", "TRSCG_Accum"]] = \ - df[["Current_Month", "TRSCG_Accum"]].astype(float) - df[["TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]] = df[[ - "TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]].astype(float) / 100 - return df - - -def ms_monthly(): # monetary Supply - """ - Man: manufacturing - Non-Man: Non-manufacturing - Data Source: http://data.eastmoney.com/cjsj/hbgyl.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable3818891", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "11", - "_": "1622044292103" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "M2", - "M2_YoY_Rate", - "M2_MoM_Rate", - "M1", - "M1_YoY_Rate", - "M1_MoM_Rate", - "M0", - "M0_YoY_Rate", - "M0_MoM_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["M0", "M1", "M2"]] = \ - df[["M0", "M1", "M2"]].astype(float) - df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate", - "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]] = \ - df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate", - "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]].astype(float) / 100 - return df - - -def ie_monthly(): # Import & Export - """ - Data Source: http://data.eastmoney.com/cjsj/hgjck.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable3818891", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "1", - "_": "1622044292103" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month_Export", - "Current_Month_Export_YoY_Rate", - "Current_Month_Export_MoM_Rate", - "Current_Month_Import", - "Current_Month_Import_YoY_Rate", - "Current_Month_Import_MoM_Rate", - "Accumulation_Export", - "Accumulation_Export_YoY_Rate", - "Accumulation_Import", - "Accumulation_Import_YoY_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df.replace("", np.nan, inplace=True) - df[["Current_Month_Export", "Current_Month_Import", - "Accumulation_Export", "Accumulation_Import"]] = \ - df[["Current_Month_Export", "Current_Month_Import", - "Accumulation_Export", "Accumulation_Import"]].astype(float) - df[["Current_Month_Export_YoY_Rate", - "Current_Month_Export_MoM_Rate", - "Current_Month_Import_YoY_Rate", - "Current_Month_Import_MoM_Rate", - "Accumulation_Export_YoY_Rate", - "Accumulation_Import_YoY_Rate"]] = \ - df[["Current_Month_Export_YoY_Rate", - "Current_Month_Export_MoM_Rate", - "Current_Month_Import_YoY_Rate", - "Current_Month_Import_MoM_Rate", - "Accumulation_Export_YoY_Rate", - "Accumulation_Import_YoY_Rate"]].astype(float)/100 - return df - - -def stock_monthly(): # Stock Trading Statistics - """ - Data Source: http://data.eastmoney.com/cjsj/gpjytj.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "jQuery112308659690274138041_1622084599455", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "2", - "_": "1622084599456" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("(") + 1:-1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "SH_Total_Stock_issue", - "SZ_Total_Stock_Issue", - "SH_Total_Market_Capitalization", - "SZ_Total_Market_Capitalization", - "SH_Turnover", - "SZ_Turnover", - "SH_Volume", - "SZ_Volume", - "SH_Highest", - "SZ_Highest", - "SH_lowest", - "SZ_lowest" - ] - df.replace("", np.nan, inplace=True) - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[list(df.columns[1:])] = df[list(df.columns[1:])].astype(float) - return df - - -def fgr_monthly(): # Forex and Gold Reserve - """ - Data Source: http://data.eastmoney.com/cjsj/gpjytj.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "tatable6260802", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "16", - "_": "1622044863548" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Forex", - "Forex_YoY_Rate", - "Forex_MoM_Rate", - "Gold", - "Gold_YoY_Rate", - "Gold_MoM_Rate" - ] - df.replace("", np.nan, inplace=True) - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Forex", "Gold"]] = \ - df[["Forex", "Gold"]].astype(float) - df[["Forex_YoY_Rate", "Gold_YoY_Rate", - "Forex_MoM_Rate", "Gold_MoM_Rate"]] = \ - df[["Forex_YoY_Rate", "Gold_YoY_Rate", - "Forex_MoM_Rate", "Gold_MoM_Rate"]].astype(float) / 100 - return df - -def ctsf_monthly(): # Client Transaction Settlement Funds - """ - http://data.eastmoney.com/cjsj/banktransfer.html - """ - tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getbanktransferdata?" - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - request_params = { - "p": "1", - "ps": "200" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("["):-11]) - df = pd.DataFrame(data_json) - df.replace("", np.nan, inplace=True) - df["StartDate"] = pd.to_datetime(df["StartDate"], format="%Y-%m-%d") - df["EndDate"] = pd.to_datetime(df["EndDate"], format="%Y-%m-%d") - df[list(df.columns)[2:]] = df[list(df.columns)[2:]].astype(float) - return df - -# TODO: SPECIAL CASE - - -def sao_monthly(): # Stock Account Overview - """ - http://data.eastmoney.com/cjsj/gpkhsj.html - """ - tmp_url = "http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get?" - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - request_params = { - "callback": "datatable4006236", - "type": "GPKHData", - "js": "({data:[(x)],pages:(pc)})", - "st": "SDATE", - "sr": "-1", - "token": "894050c76af8597a853f5b408b759f5d", - "p": "1", - "ps": "2000", - "_": "1622079339035" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{") + 6: -14]) - df = pd.DataFrame(data_json[0]) - df.columns = [ - "Date", - "New_Investor", - "New_Investor_MoM_Rate", - "New_Investor_YoY_Rate", - "Active_Investor", - "Active_Investor_A_Share", - "Active_Investor_B_share", - "SHIndex_Close", - "SHIndex_Rate", - "SHSZ_Market_Capitalization", - "SHSZ_Average_Capitalization" - ] - df.replace("-", np.nan, inplace=True) - df.Date = pd.to_datetime(df.Date, format="%Y年%m月") - df[list(df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])] = df[list( - df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])].astype(float) - df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]] = \ - df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]].astype(float) / 100 - return df - - -def fdi_monthly(): # Foreign Direct Investment - """ - http://data.eastmoney.com/cjsj/fdi.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable1477466", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "15", - "_": "1622044863548" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "YoY_Rate", - "MoM_Rate", - "Accumulation", - "Accum_YoY_Rate" - ] - df.replace("", np.nan, inplace=True) - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Current_Month", "Accumulation"]] = \ - df[["Current_Month", "Accumulation"]].astype(float) - df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] = \ - df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100 - return df - - -def gr_monthly(): # Government Revenue - """ - http://data.eastmoney.com/cjsj/czsr.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable7840652", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "14", - "_": "1622080618625" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "YoY_Rate", - "MoM_Rate", - "Accumulation", - "Accum_YoY_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Current_Month", "Accumulation"]] = \ - df[["Current_Month", "Accumulation"]].astype(float) - df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] = \ - df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100 - return df - - -def ti_monthly(): # Tax Income - """ - http://data.eastmoney.com/cjsj/qgsssr.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable8280567", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "3", - "_": "1622080669713" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "YoY_Rate", - "MoM_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df = df.replace("", np.nan) - df[["Current_Month"]] = \ - df[["Current_Month"]].astype(float) - df[["YoY_Rate", "MoM_Rate"]] = \ - df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100 - return df - - -def nl_monthly(): # New Loan - """ - http://data.eastmoney.com/cjsj/xzxd.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable2533707", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "7", - "_": "1622080800162" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "YoY_Rate", - "MoM_Rate", - "Accumulation", - "Accum_YoY_Rate" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Current_Month", "Accumulation"]] = \ - df[["Current_Month", "Accumulation"]].astype(float) - df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] =\ - df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100 - return df - - -def dfclc_monthly(): # Deposit of Foreign Currency and Local Currency - """ - http://data.eastmoney.com/cjsj/wbck.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable2899877", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "18", - "_": "1622081057370" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "YoY_Rate", - "MoM_Rate", - "Accumulation" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Current_Month", "Accumulation"]] = \ - df[["Current_Month", "Accumulation"]].astype(float) - df[["YoY_Rate", "MoM_Rate"]] = \ - df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100 - return df - - -def fl_monthly(): # Forex Loan - """ - http://data.eastmoney.com/cjsj/whxd.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable636844", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "17", - "_": "1622081336038" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Date", - "Current_Month", - "YoY_Rate", - "MoM_Rate", - "Accumulation" - ] - df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") - df[["Current_Month", "Accumulation"]] = \ - df[["Current_Month", "Accumulation"]].astype(float) - df[["YoY_Rate", "MoM_Rate"]] = \ - df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100 - return df - - -def drr_monthly(): # Deposit Reserve Ratio - """ - http://data.eastmoney.com/cjsj/ckzbj.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable4285562", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "23", - "_": "1622081448882" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Announcement Date", - "Effective Date", - "Large_Financial_institution_Before", - "Large_Financial_institution_After", - "Large_Financial_institution_Adj_Rate", - "S&M_Financial_institution_Before", - "S&M_Financial_institution_After", - "S&M_Financial_institution_Adj_Rate", - "Comment", - "SHIndex_Rate", - "SZIndex_Rate" - ] - df["Announcement Date"] = pd.to_datetime( - df["Announcement Date"], format="%Y-%m-%d") - df["Date"] = df["Announcement Date"] - df["Effective Date"] = pd.to_datetime( - df["Effective Date"], format="%Y-%m-%d") - df[["SHIndex_Rate", "SZIndex_Rate"]] = df[["SHIndex_Rate", "SZIndex_Rate"]].astype(float) - df[["Large_Financial_institution_Before", - "Large_Financial_institution_After", - "Large_Financial_institution_Adj_Rate", - "S&M_Financial_institution_Before", - "S&M_Financial_institution_After", - "S&M_Financial_institution_Adj_Rate"]] = df[["Large_Financial_institution_Before", - "Large_Financial_institution_After", - "Large_Financial_institution_Adj_Rate", - "S&M_Financial_institution_Before", - "S&M_Financial_institution_After", - "S&M_Financial_institution_Adj_Rate"]].astype(float) / 100 - return df - - -def interest_monthly(): # Interest - """ - http://data.eastmoney.com/cjsj/yhll.html - """ - tmp_url = url["eastmoney"] - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - tmp_url = url["eastmoney"] - request_params = { - "cb": "datatable7591685", - "type": "GJZB", - "sty": "ZGZB", - "js": "({data:[(x)],pages:(pc)})", - "p": "1", - "ps": "200", - "mkt": "13", - "_": "1622081956464" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -1]) - df = pd.DataFrame([item.split(",") for item in data_json["data"]]) - df.columns = [ - "Announcement Date", - "Deposit_Benchmark_Interest_Rate_Before", - "Deposit_Benchmark_Interest_Rate_After", - "Deposit_Benchmark_Interest_Rate_Adj_Rate", - "Loan_Benchmark_Interest_Rate_Before", - "Loan_Benchmark_Interest_Rate_After", - "Loan_Benchmark_Interest_Rate_Adj_Rate", - "SHIndex_Rate", - "SZIndex_Rate", - "Effective Date" - ] - df["Announcement Date"] = pd.to_datetime( - df["Announcement Date"], format="%Y-%m-%d") - df["Date"] = df["Announcement Date"] - df["Effective Date"] = pd.to_datetime( - df["Effective Date"], format="%Y-%m-%d") - df["Date"] = df["Announcement Date"] - df = df.replace("", np.nan) - df[["Deposit_Benchmark_Interest_Rate_Before", - "Deposit_Benchmark_Interest_Rate_After", - "Deposit_Benchmark_Interest_Rate_Adj_Rate", - "Loan_Benchmark_Interest_Rate_Before", - "Loan_Benchmark_Interest_Rate_After", - "Loan_Benchmark_Interest_Rate_Adj_Rate"]] = df[["Deposit_Benchmark_Interest_Rate_Before", - "Deposit_Benchmark_Interest_Rate_After", - "Deposit_Benchmark_Interest_Rate_Adj_Rate", - "Loan_Benchmark_Interest_Rate_Before", - "Loan_Benchmark_Interest_Rate_After", - "Loan_Benchmark_Interest_Rate_Adj_Rate"]].astype(float) / 100 - return df - -def gdc_daily(): # gasoline, Diesel and Crude Oil - """ - http://data.eastmoney.com/cjsj/oil_default.html - """ - tmp_url = "http://datacenter-web.eastmoney.com/api/data/get?" - ua = UserAgent(verify_ssl=False) - request_header = {"User-Agent": ua.random} - request_params = { - "callback": "jQuery112302601302322321093_1622082348721", - "type": "RPTA_WEB_JY_HQ", - "sty": "ALL", - "st": "date", - "sr": "-1", - "token": "894050c76af8597a853f5b408b759f5d", - "p": "1", - "ps": "50000", - "source": "WEB", - "_": "1622082348722" - } - r = requests.get(tmp_url, params=request_params, headers=request_header) - data_text = r.text - data_json = demjson.decode(data_text[data_text.find("{"): -2]) - df = pd.DataFrame(data_json["result"]["data"]) - df.columns = ["Crude_Oil", "Date", "Gasoline", "Diesel"] - df = df[["Date", "Gasoline", "Diesel", "Crude_Oil"]] - df = pd.to_datetime(df["Date"], format="%Y-%m-%d") - return df - -def Leading_Indicators_OECD(startdate = "1950-01", enddate = "2021-05"): - # CLI - tmp_url = url["OECD"] + "CHN.CLI.AMPLITUD.LTRENDIDX.M/OECD" - ua = UserAgent(verify_ssl=False) - request_params = { - "contentType": "csv", - "detail": "code", - "separator": "comma", - "csv-lang": "en", - "startPeriod": "{}".format(startdate), - "endPeriod": "{}".format(enddate) - } - request_header = {"User-Agent": ua.random} - r = requests.get(tmp_url, params = request_params, headers=request_header) - data_text = r.content - df_cli = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] - df_cli.columns = ["Date", "CN_OECD_CLI"] - df_cli["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m") - df_cli["CN_OECD_CLI"] = df_cli["CN_OECD_CLI"].astype(float) - #BCI - tmp_url = url["OECD"] + "CHN.BCI.AMPLITUD.LTRENDIDX.M/OECD" - ua = UserAgent(verify_ssl=False) - request_params = { - "contentType": "csv", - "detail": "code", - "separator": "comma", - "csv-lang": "en", - "startPeriod": "{}".format(startdate), - "endPeriod": "{}".format(enddate) - } - request_header = {"User-Agent": ua.random} - r = requests.get(tmp_url, params = request_params, headers=request_header) - data_text = r.content - df_bci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] - df_bci.columns = ["Date", "CN_OECD_BCI"] - df_bci["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m") - df_bci["CN_OECD_BCI"] = df_bci["CN_OECD_BCI"].astype(float) - # CCI - tmp_url = url["OECD"] + "CHN.CCI.AMPLITUD.LTRENDIDX.M/OECD" - ua = UserAgent(verify_ssl=False) - request_params = { - "contentType": "csv", - "detail": "code", - "separator": "comma", - "csv-lang": "en", - "startPeriod": "{}".format(startdate), - "endPeriod": "{}".format(enddate) - } - request_header = {"User-Agent": ua.random} - r = requests.get(tmp_url, params = request_params, headers=request_header) - data_text = r.content - df_cci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]] - df_cci.columns = ["Date", "CN_OECD_CCI"] - df_cci["Date"] = pd.to_datetime(df_cci["Date"], format = "%Y-%m") - df_cci["CN_OECD_CCI"] = df_cci["CN_OECD_CCI"].astype(float) - df = pd.merge_asof(df_cli, df_bci, on = "Date") - df = pd.merge_asof(df, df_cci, on = "Date") - - return df - -def CN_EPU_Monthly(): - df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/cnepu_1_july_2021_updated.xlsx")[:-2] - df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S") - return df - -def CN_Hongkong_EPU(): - df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/hkepu_1_july_2021_updated.xlsx")[:-2] - df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S") - return df - -def CN_Macau_EPU(): - df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/mcepu_1_july_2021_updated.xlsx")[:-2] - df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S") - return df - -def CN_EPU_Daily(): - df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/cnepu_daily_7_july_2021_updated.xlsx")[:-2] - df = df[["Date", "CNEPU_Daily"]] - df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S") - return df - -def Policy_Specific_EPU(): - df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/cnepu_policy-specific_1_july_2021_updated.xlsx")[:-2] - df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S") - return df - -""" -if __name__ == "__main__": -""" \ No newline at end of file