TerenceLiu
/
CEDApy
mirror of https://github.com/TerenceLiu98/CEDApy.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
CEDApy/CEDA/economic/macro.py

589 lines
16 KiB
Python
Raw Blame History

import pandas as pd
import numpy as np
import re
import demjson
import requests
from fake_useragent import UserAgent
# TODO need add comments
url = {
"eastmoney": "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx"
}
def cn_gdp_quarter():
"""
ABS: absolute value (per 100 million CNY)
YoY: year on year growth
"""
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable7519513",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "20",
"_": "1622020352668"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Absolute_Value",
"YoY",
"Primary_Industry_ABS",
"Primary_Industry_YoY",
"Secondary_Industry_ABS",
"Secondary_Industry_YoY",
"Tertiary_Industry_ABS",
"Tertiary_Industry_YoY",
]
#df[(df['Date'] >= startdate) & (df['Date'] <= enddate)]
return df
def cn_ppi_monthly():
"""
ABS: absolute value (per 100 million CNY)
YoY: year on year growth
"""
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable9051497",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "22",
"_": "1622047940401"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Current_Month",
"Current_Month_YoY",
"Current_Month_Accum"
]
#df[(df['Date'] >= startdate) & (df['Date'] <= enddate)]
return df
def cn_cpi_monthly():
"""
Accum: Accumulation
YoY: year on year growth
MoM: month on month growth
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable2790750",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "19",
"_": "1622020352668"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Notion_Monthly",
"Notion_YoY",
"Notion_MoM",
"Notion_Accum",
"Urban_Monthly",
"Urban_YoY",
"Urban_MoM",
"Urban_Accum",
"Rural_Monthly",
"Rural_YoY",
"Rural_MoM",
"Rural_Accum",
]
return df
def cn_pmi_monthly():
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable4515395",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "2",
"ps": "200",
"mkt": "21",
"_": "162202151821"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
temp_df = pd.DataFrame([item.split(",") for item in data_json["data"]])
temp_df.columns = [
"Date",
"Man_Industry_Index",
"Man_Index_YoY",
"Non-Man_Industry_Index",
"Non-Man_Index_YoY",
]
return temp_df
def cn_fai_monthly(): # fix asset investment
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable607120",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "12",
"_": "1622021790947"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Current_Month",
"YoY",
"MoM",
"Current_Year_Accum"
]
return df
def cn_hi_old_monthly(): # house index old version (2008-2010)
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable1895714",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "10",
"_": "1622022794457"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Housing_Prosperity_Index",
"HPI_YoY",
"Land_Development_Area_Index",
"LDAI_YoY",
"Sales_Price_Index",
"SPI_YoY"
]
return df
def cn_ci_eei_monthly(): # Climate Index & Entrepreneur Expectation Index
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable7709842",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "8",
"_": "1622041485306"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Climate_Index",
"CI_YoY",
"CI_MoM",
"Entrepreneur_Expectation_Index",
"EEI_YoY",
"EEI_MoM"
]
return df
def cn_ig_monthly(): # Industry Growth
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable4577327",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "0",
"_": "1622042259898"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"IG_YoY",
"IG_Accum",
]
return df
def cn_cgpi_monthly(): # Corporate Goods Price Index
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable7184534",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "9",
"_": "1622042652353"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"General_Index",
"General_Index_YoY",
"Total_Index_MoM",
"Agricultural_Product",
"Agricultural_Product_YoY",
"Agricultural_PRoduct_MoM",
"Mineral_Product",
"Mineral_Product_YoY",
"Mineral_Product_MoM",
"Coal_Oil_Electricity",
"Coal_Oil_Electricity_YoY",
"Coal_Oil_Electricity_MoM"
]
return df
def cn_cci_csi_cei_monthly(): # Consumer Confidence Index & Consumer Satisfaction Index & Consumer Expectation Index
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable1243218",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "4",
"_": "1622043704818"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"CCI",
"CCI_YoY",
"CCI_MoM",
"CSI",
"CSI_YoY",
"CSI_MoM",
"CEI",
"CEI_YoY",
"CEI_MoM"
]
return df
def cn_trscg_monthly(): # Total Retail Sales of Consumer Goods
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable3665821",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "5",
"_": "1622044011316"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Current_Month",
"TRSCG_YoY",
"TRSCG_MoM",
"TRSCG_Accum",
"TRSCG_Accum_YoY"
]
return df
def cn_ms_monthly(): # monetary Supply
"""
Man: manufacturing
Non-Man: Non-manufacturing
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable3818891",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "11",
"_": "1622044292103"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"M2",
"M2_YoY",
"M2_MoM",
"M1",
"M1_YoY",
"M1_MoM",
"M0",
"M0_YoY",
"M0_MoM"
]
return df
def cn_ie_monthly(): # Import & Export
"""
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable3818891",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "1",
"_": "1622044292103"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Current_Month_Export",
"Current_Month_Export_YoY",
"Current_Month_Export_MoM",
"Current_Month_Import",
"Current_Month_Import_YoY",
"Current_Month_Import_MoM",
"Accumulation_Export",
"Accumulation_Export_YoY",
"Accumulation_Import",
"Accumulation_Import_YoY",
]
return df
def cn_ie_monthly(): # Import & Export
"""
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "datatable3818891",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "1",
"_": "1622044292103"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Current_Month_Export",
"Current_Month_Export_YoY",
"Current_Month_Export_MoM",
"Current_Month_Import",
"Current_Month_Import_YoY",
"Current_Month_Import_MoM",
"Accumulation_Export",
"Accumulation_Export_YoY",
"Accumulation_Import",
"Accumulation_Import_YoY",
]
return df
def cn_fgr_monthly(): # Forex and Gold Reserve
"""
"""
tmp_url = url["eastmoney"]
ua = UserAgent()
request_header = {"User-Agent": ua.random}
tmp_url = url["eastmoney"]
request_params = {
"cb": "atatable6260802",
"type": "GJZB",
"sty": "ZGZB",
"js": "({data:[(x)],pages:(pc)})",
"p": "1",
"ps": "200",
"mkt": "16",
"_": "1622044863548"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("{") : -1])
df = pd.DataFrame([item.split(",") for item in data_json["data"]])
df.columns = [
"Date",
"Forex",
"Forex_YoY",
"Forex_MoM",
"Gold",
"Gold_YoY",
"Gold_MoM"
]
return df
def cn_ctsf_monthly(): # Client Transaction Settlement Funds
"""
"""
tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getbanktransferdata?"
ua = UserAgent()
request_header = {"User-Agent": ua.random}
request_params = {
"p": "1",
"ps": "200"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("["):-11])
df = pd.DataFrame(data_json)
return df
# TODO: needs help (missing two tables)
def cn_sao_monthly(): # Stock Account Overview
"""
"""
tmp_url = "http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get?"
ua = UserAgent()
request_header = {"User-Agent": ua.random}
request_params = {
"callback": "jQuery1123014377091065513636_1622046865705",
"type": "GPKHData",
"st": "HdDate",
"sr": "-1",
"sty": "Chart",
"token": "894050c76af8597a853f5b408b759f5d",
"ps": "2000",
"_": "1622046865706"
}
r = requests.get(tmp_url, params = request_params, headers = request_header)
data_text = r.text
data_json = demjson.decode(data_text[data_text.find("(")+1:-1])
df = pd.DataFrame(data_json)
df.columns = [
"Date",
"New_Investor",
"Active_Investor",
"SHIndexClose"
]
df.Date = pd.to_datetime(df.Date, format = "%Y年%m月")
return df
"""
if __name__ == "__main__":
"""
Reference in New Issue View Git Blame Copy Permalink