Merge pull request #4 from TerenceLiu98/develop

Develop

CEDA/MacroEcon/cn.py

@@ -11,10 +11,12 @@ url = {
     "eastmoney": "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx"
 }
 
+
 def gdp_quarterly():
     """
     ABS: absolute value (per 100 million CNY)
     YoY: year on year growth
+    Data source: http://data.eastmoney.com/cjsj/gdp.html
     """
     ua = UserAgent()
     request_header = {"User-Agent": ua.random}
@@ -36,21 +38,33 @@ def gdp_quarterly():
     df.columns = [
         "Date",
         "Absolute_Value",
-        "YoY",
+        "YoY_Rate",
         "Primary_Industry_ABS",
-        "Primary_Industry_YoY",
+        "Primary_Industry_YoY_Rate",
         "Secondary_Industry_ABS",
-        "Secondary_Industry_YoY",
+        "Secondary_Industry_YoY_Rate",
         "Tertiary_Industry_ABS",
-        "Tertiary_Industry_YoY",
+        "Tertiary_Industry_YoY_Rate",
     ]
-    #df[(df['Date'] >= startdate) & (df['Date'] <= enddate)]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df["Absolute_Value"] = df["Absolute_Value"].astype(float)
+    df["Secondary_Industry_ABS"] = df["Secondary_Industry_ABS"].astype(float)
+    df["Tertiary_Industry_ABS"] = df["Tertiary_Industry_ABS"].astype(float)
+    df["Absolute_Value"] = df["Absolute_Value"].astype(float)
+    df["YoY_Rate"] = df["YoY_Rate"].astype(float) / 100
+    df["Secondary_Industry_YoY_Rate"] = df["Secondary_Industry_YoY_Rate"].astype(
+        float) / 100
+    df["Tertiary_Industry_YoY_Rate"] = df["Tertiary_Industry_YoY_Rate"].astype(
+        float) / 100
     return df
 
+
 def ppi_monthly():
     """
     ABS: absolute value (per 100 million CNY)
     YoY: year on year growth
+    Accum: Accumulation
+    Data source: http://data.eastmoney.com/cjsj/ppi.html
     """
     ua = UserAgent()
     request_header = {"User-Agent": ua.random}
@@ -72,17 +86,23 @@ def ppi_monthly():
     df.columns = [
         "Date",
         "Current_Month",
-        "Current_Month_YoY",
+        "Current_Month_YoY_Rate",
         "Current_Month_Accum"
     ]
-    #df[(df['Date'] >= startdate) & (df['Date'] <= enddate)]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df["Current_Month"] = df["Current_Month"].astype(float)
+    df["Current_Month_YoY_Rate"] = df["Current_Month_YoY_Rate"].astype(
+        float) / 100
+    df["Current_Month_Accum"] = df["Current_Month_Accum"].astype(float)
     return df
 
+
 def cpi_monthly():
     """
     Accum: Accumulation
     YoY: year on year growth
     MoM: month on month growth
+    Data source: http://data.eastmoney.com/cjsj/cpi.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -105,24 +125,49 @@ def cpi_monthly():
     df.columns = [
         "Date",
         "Notion_Monthly",
-        "Notion_YoY",
+        "Notion_YoY_Rate",
-        "Notion_MoM",
+        "Notion_MoM_Rate",
         "Notion_Accum",
         "Urban_Monthly",
-        "Urban_YoY",
+        "Urban_YoY_Rate",
-        "Urban_MoM",
+        "Urban_MoM_Rate",
         "Urban_Accum",
         "Rural_Monthly",
-        "Rural_YoY",
+        "Rural_YoY_Rate",
-        "Rural_MoM",
+        "Rural_MoM_Rate",
         "Rural_Accum",
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Notion_Monthly",
+        "Notion_Accum",
+        "Urban_Monthly",
+        "Urban_Accum",
+        "Rural_Monthly",
+        "Rural_Accum"]] = df[["Notion_Monthly",
+                              "Notion_Accum",
+                              "Urban_Monthly",
+                              "Urban_Accum",
+                              "Rural_Monthly",
+                              "Rural_Accum"]].astype(float)
+    df[["Notion_YoY_Rate",
+        "Notion_MoM_Rate",
+        "Urban_YoY_Rate",
+        "Urban_MoM_Rate",
+        "Rural_YoY_Rate",
+        "Rural_MoM_Rate"]] = df[["Notion_YoY_Rate",
+                                 "Notion_MoM_Rate",
+                                 "Urban_YoY_Rate",
+                                 "Urban_MoM_Rate",
+                                 "Rural_YoY_Rate",
+                                 "Rural_MoM_Rate"]].astype(float) / 100
     return df
 
+
 def pmi_monthly():
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/pmi.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -141,20 +186,27 @@ def pmi_monthly():
     r = requests.get(tmp_url, params=request_params, headers=request_header)
     data_text = r.text
     data_json = demjson.decode(data_text[data_text.find("{"): -1])
-    temp_df = pd.DataFrame([item.split(",") for item in data_json["data"]])
+    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
-    temp_df.columns = [
+    df.columns = [
         "Date",
         "Man_Industry_Index",
-        "Man_Index_YoY",
+        "Man_Index_YoY_Rate",
         "Non-Man_Industry_Index",
-        "Non-Man_Index_YoY",
+        "Non-Man_Index_YoY_Rate",
     ]
-    return temp_df
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Man_Industry_Index", "Non-Man_Industry_Index"]] = \
+        df[["Man_Industry_Index", "Non-Man_Industry_Index"]].astype(float)
+    df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]] = \
+        df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]].astype(float) / 100
+    return df
+
 
 def fai_monthly():  # fix asset investment
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/gdzctz.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -177,16 +229,23 @@ def fai_monthly(): # fix asset investment
     df.columns = [
         "Date",
         "Current_Month",
-        "YoY",
+        "YoY_Rate",
-        "MoM",
+        "MoM_Rate",
         "Current_Year_Accum"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month", "Current_Year_Accum"]] = \
+        df[["Current_Month", "Current_Year_Accum"]].astype(float)
+    df[["YoY_Rate", "MoM_Rate"]] = \
+        df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100
     return df
 
+
 def hi_old_monthly():  # house index old version (2008-2010)
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/house.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -209,58 +268,115 @@ def hi_old_monthly(): # house index old version (2008-2010)
     df.columns = [
         "Date",
         "Housing_Prosperity_Index",
-        "HPI_YoY",
+        "HPI_YoY_Rate",
         "Land_Development_Area_Index",
-        "LDAI_YoY",
+        "LDAI_YoY_Rate",
         "Sales_Price_Index",
-        "SPI_YoY"
+        "SPI_YoY_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Housing_Prosperity_Index",
+        "Land_Development_Area_Index",
+        "Sales_Price_Index"]] = df[["Housing_Prosperity_Index",
+                                    "Land_Development_Area_Index",
+                                    "Sales_Price_Index"]].astype(float)
+    df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]] = \
+        df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]].astype(float) / 100
     return df
 
 # mkt=1&stat=2&city1=%E5%B9%BF%E5%B7%9E&city2=%E4%B8%8A%E6%B5%B7
 
-def hi_new_monthly(city1:str, city2:str): # newly built commercial housing &  second-hand commercial housing
+
+# newly built commercial housing &  second-hand commercial housing
+def hi_new_monthly(city1: str, city2: str):
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/newhouse.html
     """
     tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getnewhousechartdata?"
     ua = UserAgent()
     request_header = {"User-Agent": ua.random}
-    request_params_nbch = {
+    request_params_nbch_MoM = {
         "mkt": "1",
         "stat": "2",
         "city1": "{}".format(city1),
         "city2": "{}".format(city2)
     }
-    request_params_shch = {
+    request_params_shch_MoM = {
         "mkt": "1",
         "stat": "3",
         "city1": "{}".format(city1),
         "city2": "{}".format(city2)
     }
-    r_nbch = requests.get(tmp_url, params = request_params_nbch, headers = request_header)
+    r_nbch_MoM = requests.get(
-    r_shch = requests.get(tmp_url, params = request_params_shch, headers = request_header)
+        tmp_url,
-    data_text_nbch = r_nbch.text
+        params=request_params_nbch_MoM,
-    data_text_shch = r_shch.text
+        headers=request_header)
-    data_json_nbch = demjson.decode(data_text_nbch)
+    r_shch_MoM = requests.get(
-    data_json_shch = demjson.decode(data_text_shch)
+        tmp_url,
-    date_nbch = data_json_nbch['chart']['series']['value']
+        params=request_params_shch_MoM,
-    data1_nbch = data_json_nbch['chart']['graphs']['graph'][0]['value']
+        headers=request_header)
-    data2_nbch = data_json_nbch['chart']['graphs']['graph'][1]['value']
+    data_text_nbch_MoM = r_nbch_MoM.text
-    data1_shch = data_json_shch['chart']['graphs']['graph'][0]['value']
+    data_text_shch_MoM = r_shch_MoM.text
-    data2_shch = data_json_shch['chart']['graphs']['graph'][1]['value']
+    data_json_nbch_MoM = demjson.decode(data_text_nbch_MoM)
-    df = pd.DataFrame({"Date": date_nbch, 
+    data_json_shch_MoM = demjson.decode(data_text_shch_MoM)
-                       "City1":data1_nbch, 
+    date_nbch = data_json_nbch_MoM['chart']['series']['value']
-                       "City2":data2_nbch,
+    data1_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][0]['value']
-                       "City1":data1_shch, 
+    data2_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][1]['value']
-                       "City2":data2_shch})
+    data1_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][0]['value']
+    data2_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][1]['value']
+    df_MoM = pd.DataFrame({"Date": date_nbch,
+                           "City1_nbch_MoM": data1_nbch_MoM,
+                           "City1_shch_MoM": data1_shch_MoM,
+                           "City2_nbch_MoM": data2_nbch_MoM,
+                           "City2_shch_MoM": data2_shch_MoM})
+    df_MoM["Date"] = pd.to_datetime(df_MoM["Date"], format="%m/%d/%Y")
+
+    request_params_nbch_YoY = {
+        "mkt": "2",
+        "stat": "2",
+        "city1": "{}".format(city1),
+        "city2": "{}".format(city2)
+    }
+    request_params_shch_YoY = {
+        "mkt": "2",
+        "stat": "3",
+        "city1": "{}".format(city1),
+        "city2": "{}".format(city2)
+    }
+    r_nbch_YoY = requests.get(
+        tmp_url,
+        params=request_params_nbch_YoY,
+        headers=request_header)
+    r_shch_YoY = requests.get(
+        tmp_url,
+        params=request_params_shch_YoY,
+        headers=request_header)
+    data_text_nbch_YoY = r_nbch_YoY.text
+    data_text_shch_YoY = r_shch_YoY.text
+    data_json_nbch_YoY = demjson.decode(data_text_nbch_YoY)
+    data_json_shch_YoY = demjson.decode(data_text_shch_YoY)
+    date_nbch = data_json_nbch_YoY['chart']['series']['value']
+    data1_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][0]['value']
+    data2_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][1]['value']
+    data1_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][0]['value']
+    data2_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][1]['value']
+    df_YoY = pd.DataFrame({"Date": date_nbch,
+                           "City1_nbch_YoY": data1_nbch_YoY,
+                           "City1_shch_YoY": data1_shch_YoY,
+                           "City2_nbch_YoY": data2_nbch_YoY,
+                           "City2_shch_YoY": data2_shch_YoY})
+    df_YoY["Date"] = pd.to_datetime(df_YoY["Date"], format="%m/%d/%Y")
+    df = df_YoY.merge(df_MoM, on="Date")
     return df
 
+
 def ci_eei_monthly():  # Climate Index & Entrepreneur Expectation Index
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/qyjqzs.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -283,18 +399,26 @@ def ci_eei_monthly(): # Climate Index & Entrepreneur Expectation Index
     df.columns = [
         "Date",
         "Climate_Index",
-        "CI_YoY",
+        "CI_YoY_Rate",
-        "CI_MoM",
+        "CI_MoM_Rate",
         "Entrepreneur_Expectation_Index",
-        "EEI_YoY",
+        "EEI_YoY_Rate",
-        "EEI_MoM"
+        "EEI_MoM_Rate"
     ]
+    df.replace('', np.nan, inplace=True)
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Climate_Index", "Entrepreneur_Expectation_Index"]] = \
+        df[["Climate_Index", "Entrepreneur_Expectation_Index"]].astype(float)
+    df[["CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]] = df[[
+        "CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]].astype(float) / 100
     return df
 
+
 def ig_monthly():  # Industry Growth
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/gyzjz.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -316,15 +440,20 @@ def ig_monthly(): # Industry Growth
     df = pd.DataFrame([item.split(",") for item in data_json["data"]])
     df.columns = [
         "Date",
-        "IG_YoY",
+        "IG_YoY_Rate",
-        "IG_Accum",
+        "IG_Accum_Rate",
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["IG_YoY_Rate", "IG_Accum_Rate"]] = \
+        df[["IG_YoY_Rate", "IG_Accum_Rate"]].astype(float) / 100
     return df
 
+
 def cgpi_monthly():  # Corporate Goods Price Index
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/qyspjg.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -347,24 +476,45 @@ def cgpi_monthly(): # Corporate Goods Price Index
     df.columns = [
         "Date",
         "General_Index",
-        "General_Index_YoY",
+        "General_Index_YoY_Rate",
-        "Total_Index_MoM",
+        "Total_Index_MoM_Rate",
         "Agricultural_Product",
-        "Agricultural_Product_YoY",
+        "Agricultural_Product_YoY_Rate",
-        "Agricultural_PRoduct_MoM",
+        "Agricultural_Product_MoM_Rate",
         "Mineral_Product",
-        "Mineral_Product_YoY",
+        "Mineral_Product_YoY_Rate",
-        "Mineral_Product_MoM",
+        "Mineral_Product_MoM_Rate",
         "Coal_Oil_Electricity",
-        "Coal_Oil_Electricity_YoY",
+        "Coal_Oil_Electricity_YoY_Rate",
-        "Coal_Oil_Electricity_MoM"
+        "Coal_Oil_Electricity_MoM_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["General_Index",
+        "Agricultural_Product",
+        "Mineral_Product",
+        "Coal_Oil_Electricity"]] = df[["General_Index",
+                                       "Agricultural_Product",
+                                       "Mineral_Product",
+                                       "Coal_Oil_Electricity"]].astype(float)
+    df[["General_Index_YoY_Rate",
+        "Total_Index_MoM_Rate",
+        "Agricultural_Product_YoY_Rate",
+        "Agricultural_Product_MoM_Rate",
+        "Coal_Oil_Electricity_YoY_Rate",
+        "Coal_Oil_Electricity_MoM_Rate"]] = df[["General_Index_YoY_Rate",
+                                                "Total_Index_MoM_Rate",
+                                                "Agricultural_Product_YoY_Rate",
+                                                "Agricultural_Product_MoM_Rate",
+                                                "Coal_Oil_Electricity_YoY_Rate",
+                                                "Coal_Oil_Electricity_MoM_Rate"]].astype(float) / 100
     return df
 
+
 def cci_csi_cei_monthly():  # Consumer Confidence Index & Consumer Satisfaction Index & Consumer Expectation Index
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/xfzxx.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -387,21 +537,32 @@ def cci_csi_cei_monthly(): # Consumer Confidence Index & Consumer Satisfaction I
     df.columns = [
         "Date",
         "CCI",
-        "CCI_YoY",
+        "CCI_YoY_Rate",
-        "CCI_MoM",
+        "CCI_MoM_Rate",
         "CSI",
-        "CSI_YoY",
+        "CSI_YoY_Rate",
-        "CSI_MoM",
+        "CSI_MoM_Rate",
         "CEI",
-        "CEI_YoY",
+        "CEI_YoY_Rate",
-        "CEI_MoM"
+        "CEI_MoM_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["CCI", "CSI", "CEI"]] = \
+        df[["CCI", "CSI", "CEI"]].astype(float)
+    df[["CCI_YoY_Rate", "CCI_MoM_Rate",
+        "CSI_YoY_Rate", "CSI_MoM_Rate",
+        "CEI_YoY_Rate", "CEI_MoM_Rate"]] = \
+        df[["CCI_YoY_Rate", "CCI_MoM_Rate",
+            "CSI_YoY_Rate", "CSI_MoM_Rate",
+            "CEI_YoY_Rate", "CEI_MoM_Rate"]].astype(float) / 100
     return df
 
+
 def trscg_monthly():  # Total Retail Sales of Consumer Goods
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/xfp.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -424,17 +585,25 @@ def trscg_monthly(): # Total Retail Sales of Consumer Goods
     df.columns = [
         "Date",
         "Current_Month",
-        "TRSCG_YoY",
+        "TRSCG_YoY_Rate",
-        "TRSCG_MoM",
+        "TRSCG_MoM_Rate",
         "TRSCG_Accum",
-        "TRSCG_Accum_YoY" 
+        "TRSCG_Accum_YoY_Rate"
     ]
+    df.replace("", np.nan, inplace=True)
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month", "TRSCG_Accum"]] = \
+        df[["Current_Month", "TRSCG_Accum"]].astype(float)
+    df[["TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]] = df[[
+        "TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]].astype(float) / 100
     return df
 
+
 def ms_monthly():  # monetary Supply
     """
     Man: manufacturing
     Non-Man: Non-manufacturing
+    Data Source: http://data.eastmoney.com/cjsj/hbgyl.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -457,20 +626,28 @@ def ms_monthly(): # monetary Supply
     df.columns = [
         "Date",
         "M2",
-        "M2_YoY",
+        "M2_YoY_Rate",
-        "M2_MoM",
+        "M2_MoM_Rate",
         "M1",
-        "M1_YoY",
+        "M1_YoY_Rate",
-        "M1_MoM",
+        "M1_MoM_Rate",
         "M0",
-        "M0_YoY",
+        "M0_YoY_Rate",
-        "M0_MoM"
+        "M0_MoM_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["M0", "M1", "M2"]] = \
+        df[["M0", "M1", "M2"]].astype(float)
+    df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate",
+        "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]] = \
+        df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate",
+            "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]].astype(float) / 100
     return df
 
+
 def ie_monthly():  # Import & Export
     """
-
+    Data Source: http://data.eastmoney.com/cjsj/hgjck.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -493,22 +670,38 @@ def ie_monthly(): # Import & Export
     df.columns = [
         "Date",
         "Current_Month_Export",
-        "Current_Month_Export_YoY",
+        "Current_Month_Export_YoY_Rate",
-        "Current_Month_Export_MoM",
+        "Current_Month_Export_MoM_Rate",
         "Current_Month_Import",
-        "Current_Month_Import_YoY",
+        "Current_Month_Import_YoY_Rate",
-        "Current_Month_Import_MoM",
+        "Current_Month_Import_MoM_Rate",
         "Accumulation_Export",
-        "Accumulation_Export_YoY",
+        "Accumulation_Export_YoY_Rate",
         "Accumulation_Import",
-        "Accumulation_Import_YoY"
+        "Accumulation_Import_YoY_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month_Export", "Current_Month_Import",
+        "Accumulation_Export", "Accumulation_Import"]] = \
+        df[["Current_Month_Export", "Current_Month_Import",
+            "Accumulation_Export", "Accumulation_Import"]].astype(float)
+    df[["Current_Month_Export_YoY_Rate",
+        "Current_Month_Export_MoM_Rate",
+        "Current_Month_Import_YoY_Rate",
+        "Current_Month_Import_MoM_Rate",
+        "Accumulation_Export_YoY_Rate",
+        "Accumulation_Export_MoM_Rate"]] = df[["Current_Month_Export_YoY_Rate",
+                                               "Current_Month_Export_MoM_Rate",
+                                               "Current_Month_Import_YoY_Rate",
+                                               "Current_Month_Import_MoM_Rate",
+                                               "Accumulation_Export_YoY_Rate",
+                                               "Accumulation_Export_MoM_Rate"]].astype(float) / 100
     return df
 
 
 def stock_monthly():  # Import & Export
     """
-&type=GJZB&sty=ZGZB&js=(%5B(x)%5D)&p=1&ps=200&mkt=2&_=1622084599456
+    Data Source: http://data.eastmoney.com/cjsj/gpjytj.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -543,11 +736,15 @@ def stock_monthly(): # Import & Export
         "SH_lowest",
         "SZ_lowest"
     ]
+    df.replace("", np.nan, inplace=True)
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[list(df.columns[1:])] = df[list(df.columns[1:])].astype(float)
     return df
 
+
 def fgr_monthly():  # Forex and Gold Reserve
     """
-
+    Data Source: http://data.eastmoney.com/cjsj/gpjytj.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -570,17 +767,27 @@ def fgr_monthly(): # Forex and Gold Reserve
     df.columns = [
         "Date",
         "Forex",
-        "Forex_YoY",
+        "Forex_YoY_Rate",
-        "Forex_MoM",
+        "Forex_MoM_Rate",
         "Gold",
-        "Gold_YoY",
+        "Gold_YoY_Rate",
-        "Gold_MoM"
+        "Gold_MoM_Rate"
     ]
+    df.replace("", np.nan, inplace=True)
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Forex", "Gold"]] = \
+        df["Forex", "Gold"].astype(float)
+    df[["Forex_YoY_Rate", "Gold_YoY_Rate",
+        "Forex_MoM_Rate", "Gold_MoM_Rate"]] = \
+        df["Forex_YoY_Rate", "Gold_YoY_Rate",
+           "Forex_MoM_Rate", "Gold_MoM_Rate"].astype(float) / 100
     return df
 # TODO: SPECIAL CASE
+
+
 def ctsf_monthly():  # Client Transaction Settlement Funds
     """
-
+    http://data.eastmoney.com/cjsj/banktransfer.html
     """
     tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getbanktransferdata?"
     ua = UserAgent()
@@ -593,12 +800,18 @@ def ctsf_monthly(): # Client Transaction Settlement Funds
     data_text = r.text
     data_json = demjson.decode(data_text[data_text.find("["):-11])
     df = pd.DataFrame(data_json)
+    df.replace("", np.nan, inplace=True)
+    df["StartDate"] = pd.to_datetime(df["StartDate"], format="%Y-%m-%d")
+    df["EndDate"] = pd.to_datetime(df["EndDate"], format="%Y-%m-%d")
+    df[list(df.columns)[2:]] = df[list(df.columns)[2:]].astype(float)
     return df
 
 # TODO: SPECIAL CASE
+
+
 def sao_monthly():  # Stock Account Overview
     """
-
+    http://data.eastmoney.com/cjsj/gpkhsj.html
     """
     tmp_url = "http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get?"
     ua = UserAgent()
@@ -621,8 +834,8 @@ def sao_monthly(): # Stock Account Overview
     df.columns = [
         "Date",
         "New_Investor",
-        "New_Investor_MoM",
+        "New_Investor_MoM_Rate",
-        "New_Investor_YoY",
+        "New_Investor_YoY_Rate",
         "Active_Investor",
         "Active_Investor_A_Share",
         "Active_Investor_B_share",
@@ -631,12 +844,18 @@ def sao_monthly(): # Stock Account Overview
         "SHSZ_Market_Capitalization",
         "SHSZ_Average_Capitalization"
     ]
+    df.replace("-", np.nan, inplace=True)
     df.Date = pd.to_datetime(df.Date, format="%Y年%m月")
+    df[list(df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])] = df[list(
+        df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])].astype(float)
+    df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]] = \
+        df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]].astype(float) / 100
     return df
 
+
 def fdi_monthly():  # Foreign Direct Investment
     """
-
+    http://data.eastmoney.com/cjsj/fdi.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -659,16 +878,23 @@ def fdi_monthly(): # Foreign Direct Investment
     df.columns = [
         "Date",
         "Current_Month",
-        "YoY",
+        "YoY_Rate",
-        "MoM",
+        "MoM_Rate",
         "Accumulation",
-        "Accum_YoY"
+        "Accum_YoY_Rate"
     ]
+    df.replace("", np.nan, inplace=True)
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month", "Accumulation"]] = \
+        df[["Current_Month", "Accumulation"]].astype(float)
+    df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] = \
+        df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100
     return df
 
+
 def gr_monthly():  # Government Revenue
     """
-
+    http://data.eastmoney.com/cjsj/czsr.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -691,16 +917,22 @@ def gr_monthly(): # Government Revenue
     df.columns = [
         "Date",
         "Current_Month",
-        "YoY",
+        "YoY_Rate",
-        "MoM",
+        "MoM_Rate",
         "Accumulation",
-        "Accum_YoY"
+        "Accum_YoY_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month", "Accumulation"]] = \
+        df[["Current_Month", "Accumulation"]].astype(float)
+    df[["YoY_Rate", "MoM_rate", "Accum_YoY_Rate"]] = \
+        df[["YoY_Rate", "MoM_rate", "Accum_YoY_Rate"]].astype(float) / 100
     return df
 
+
 def ti_monthly():  # Tax Income
     """
-
+    http://data.eastmoney.com/cjsj/qgsssr.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -723,15 +955,20 @@ def ti_monthly(): # Tax Income
     df.columns = [
         "Date",
         "Current_Month",
-        "YoY",
+        "YoY_Rate",
-        "MoM"
+        "MoM_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month"]] = \
+        df[["Current_Month"]].astype(float)
+    df[["YoY_Rate", "MoM_rate"]] = \
+        df[["YoY_Rate", "MoM_rate"]].astype(float) / 100
     return df
 
 
 def nl_monthly():  # New Loan
     """
-
+    http://data.eastmoney.com/cjsj/xzxd.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -754,16 +991,22 @@ def nl_monthly(): # New Loan
     df.columns = [
         "Date",
         "Current_Month",
-        "YoY",
+        "YoY_Rate",
-        "MoM",
+        "MoM_Rate",
         "Accumulation",
-        "Accum_YoY"
+        "Accum_YoY_Rate"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month", "Accumulation"]] = \
+        df[["Current_Month", "Accumulation"]].astype(float)
+    df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] =\
+        df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100
     return df
 
+
 def dfclc_monthly():  # Deposit of Foreign Currency and Local Currency
     """
-
+    http://data.eastmoney.com/cjsj/wbck.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -786,15 +1029,21 @@ def dfclc_monthly(): # Deposit of Foreign Currency and Local Currency
     df.columns = [
         "Date",
         "Current_Month",
-        "YoY",
+        "YoY_Rate",
-        "MoM",
+        "MoM_Rate",
         "Accumulation"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month", "Accumulation"]] = \
+        df[["Current_Month", "Accumulation"]].astype(float)
+    df[["YoY_Rate", "MoM_Rate"]] = \
+        df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100
     return df
 
+
 def fl_monthly():  # Forex Loan
     """
-
+    http://data.eastmoney.com/cjsj/whxd.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -821,11 +1070,17 @@ def fl_monthly(): # Forex Loan
         "MoM",
         "Accumulation"
     ]
+    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
+    df[["Current_Month", "Accumulation"]] = \
+        df[["Current_Month", "Accumulation"]].astype(float)
+    df[["YoY_Rate", "MoM_Rate"]] = \
+        df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100
     return df
 
+
 def drr_monthly():  # Deposit Reserve Ratio
     """
-
+    http://data.eastmoney.com/cjsj/ckzbj.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -858,11 +1113,27 @@ def drr_monthly(): # Deposit Reserve Ratio
         "SHIndex_Rate",
         "SZIndex_Rate"
     ]
+    df["Announcement Date"] = pd.to_datetime(
+        df["Announcement Date"], format="%Y-%m-%d")
+    df["Effective Date"] = pd.to_datetime(
+        df["Effective Date"], format="%Y-%m-%d")
+    df[["Large_Financial_institution_Before",
+        "Large_Financial_institution_After",
+        "Large_Financial_institution_Adj_Rate",
+        "S&M_Financial_institution_Before",
+        "S&M_Financial_institution_After",
+        "S&M_Financial_institution_Adj_Rate"]] = df[["Large_Financial_institution_Before",
+                                                     "Large_Financial_institution_After",
+                                                     "Large_Financial_institution_Adj_Rate",
+                                                     "S&M_Financial_institution_Before",
+                                                     "S&M_Financial_institution_After",
+                                                     "S&M_Financial_institution_Adj_Rate"]].astype(float) / 100
     return df
 
+
 def interest_monthly():  # Interest
     """
-
+    http://data.eastmoney.com/cjsj/yhll.html
     """
     tmp_url = url["eastmoney"]
     ua = UserAgent()
@@ -906,11 +1177,15 @@ def interest_monthly(): # Interest
         "SHIndex_Rate",
         "SZIndex_Rate"
     ]]
+    df[list(df.columns)] = df[list(df.columns)].astype(float) / 100
     return df
 
 # TODO: SPECIAL CASE
+
+
 def gdc_daily():  # gasoline, Diesel and Crude Oil
     """
+    http://data.eastmoney.com/cjsj/oil_default.html
     """
     tmp_url = "http://datacenter-web.eastmoney.com/api/data/get?"
     ua = UserAgent()
@@ -933,8 +1208,10 @@ def gdc_daily(): # gasoline, Diesel and Crude Oil
     df = pd.DataFrame(data_json["result"]["data"])
     df.columns = ["Crude_Oil", "Date", "Gasoline", "Diesel"]
     df = df[["Date", "Gasoline", "Diesel", "Crude_Oil"]]
+    df = pd.to_datetime(df["Date"], format="%Y-%m-%d")
     return df
 
+
 """
 if __name__ == "__main__":
 """

CEDA/MacroEcon/eu.py

@@ -0,0 +1,74 @@
+import pandas as pd
+import numpy as np
+import io
+import demjson
+import requests
+from fake_useragent import UserAgent
+
+
+url = {
+    "eurostat": "http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/",
+    "ecb": "https://sdw-wsrest.ecb.europa.eu/service/data/"
+}
+
+class ecb_data(object):
+    def __init__(self, url=url["ecb"]):
+        self.url = url
+
+    def codebook(self):
+        return "please follow the ECB's codebook: https://sdw.ecb.europa.eu/browse.do?node=9691101"
+
+    def get_data(self,
+                 datacode="ICP", 
+                 key="M.U2.N.000000.4.ANR", 
+                 startdate="2000-01-01", 
+                 enddate="2020-01-01"):
+        """
+        """
+        tmp_url = self.url + "{}/".format(datacode) + "{}".format(key)
+        ua = UserAgent()
+        request_header = {"User-Agent": ua.random, 'Accept': 'text/csv'}
+        request_params = {
+            "startPeriod": "{}".format(startdate),
+            "endPeriod": "{}".format(enddate)
+        }
+        r = requests.get(tmp_url, params = request_params, headers = request_header)
+        data_text = r.content
+        df = pd.read_csv(io.StringIO(data_text.decode('utf-8')))
+        return df
+
+class eurostat_data(object):
+    def __init__(self, url=url["eurostat"]):
+        self.url = url
+
+    def codebook(self):
+        return "please follow the EuroStat's codebook: \nhttps://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&dir=dic"
+        
+    def get_data(self,
+                 datasetcode="nama_10_gdp", 
+                 precision="1",
+                 unit="CP_MEUR",
+                 na_item="B1GQ",
+                 time="2020"):
+        """
+        """
+        tmp_url = self.url + "{}".format(datasetcode)
+        ua = UserAgent()
+        request_header = {"User-Agent": ua.random, 'Accept': 'text/csv'}
+        request_params = {
+            "precision": "{}".format(precision),
+            "unit": "{}".format(unit),
+            "na_item": "{}".format(na_item),
+            "time": "{}".format(time)
+        }
+        r = requests.get(tmp_url, params = request_params, headers = request_header)
+        data_text = r.text
+        data_json = demjson.decode(data_text)
+        value = data_json['value']
+        abb = data_json['dimension']['geo']['category']['index']
+        abb = {abb[k]:k for k in abb}
+        geo = data_json['dimension']['geo']['category']['label']
+        geo_list = [abb[int(k)] for k in list(value.keys())]
+        geo = [geo[k] for k in geo_list]
+        df = pd.DataFrame({"Geo":geo, "{}".format(na_item): list(value.values())})
+        return df

CEDA/MacroEcon/us.py

@@ -3,10 +3,14 @@ import numpy as np
 import requests
 from fake_useragent import UserAgent
 import io
+import os 
+import demjson
 
 # Main Economic Indicators: https://alfred.stlouisfed.org/release?rid=205
 url = {
-    "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?"
+    "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?",
+    "philfed": "https://www.philadelphiafed.org/surveys-and-data/real-time-data-research/",
+    "chicagofed": "https://www.chicagofed.org/~/media/publications/"
 }
 
 def gdp_quarterly(startdate="1947-01-01", enddate="2021-01-01"):
@@ -682,3 +686,94 @@ def bir(startdate="2003-01-01", enddate="2021-01-01"):
     df = pd.merge_asof(df_5y, df_10y, on = "DATE", direction = "backward")
     df.columns = ["Date", "BIR_5y", "BIR_10y"]
     return df
+
+def adsbci():
+    """
+    An index designed to track real business conditions at high observation frequency
+    """
+    ua = UserAgent()
+    request_header = {"User-Agent": ua.random}
+    tmp_url = url["philfed"] + "ads"
+    r = requests.get(tmp_url, headers = request_header)
+    file = open("ads_temp.xls", "wb")
+    file.write(r.content)
+    file.close()
+    df = pd.read_excel("ads_temp.xls")
+    df.columns = ["Date", "ADS_Index"]
+    df['Date'] = pd.to_datetime(df["Date"], format="%Y:%m:%d")
+    os.remove("ads_temp.xls")
+    return df
+
+def pci():
+    """
+    Tracks the degree of political disagreement among U.S. politicians at the federal level, Monthly
+    """
+    df = pd.read_excel("https://www.philadelphiafed.org/-/media/frbp/assets/data-visualizations/partisan-conflict.xlsx")
+    df["Date"] = df["Year"].astype(str) + df["Month"]
+    df["Date"] = pd.to_datetime(df["Date"], format = "%Y%B")
+    df = df.drop(["Year", "Month"], axis=1)
+    df = df[["Date", "Partisan Conflict"]]
+    return df
+
+def inflation_noewcasting():
+    """
+    
+    """
+    ua = UserAgent()
+    request_header = {"User-Agent": ua.random}
+    tmp_url = "https://www.clevelandfed.org/~/media/files/charting/%20nowcast_quarter.json"
+
+    r = requests.get(tmp_url, headers = request_header)
+    tmp_df = pd.DataFrame(demjson.decode(r.text))
+    df = pd.DataFrame()
+    for i in range(0, len(tmp_df)):
+        date = tmp_df['chart'][i]['subcaption'][:4] + "/"  + \
+            pd.DataFrame(tmp_df["dataset"][i][0]['data'])['tooltext'].str.extract(r"\b(0?[1-9]|1[0-2])/(0?[1-9]|[12][0-9]|3[01])\b")[0] + "/"  + \
+            pd.DataFrame(tmp_df["dataset"][i][0]['data'])['tooltext'].str.extract(r"\b(0?[1-9]|1[0-2])/(0?[1-9]|[12][0-9]|3[01])\b")[1]
+        CPI_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[0])["value"]
+        C_CPI_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[1])["value"]
+        PCE_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[2])["value"]
+        C_PCE_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[3])["value"]
+        A_CPI_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[4])["value"]
+        A_C_CPI_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[5])["value"]
+        A_PCE_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[6])["value"]
+        A_C_PCE_I = pd.DataFrame((pd.DataFrame(tmp_df["dataset"][i])['data'])[7])["value"]
+        tmp_df2 = pd.DataFrame({"date": date,
+                                "CPI_I": CPI_I,
+                                "C_CPI_I": C_CPI_I,
+                                "PCE_I": PCE_I,
+                                "C_PCE_I": C_PCE_I,
+                                "A_CPI_I": A_CPI_I,
+                                "A_C_CPI_I": A_C_CPI_I,
+                                "A_PCE_I": A_PCE_I,
+                                "A_C_PCE_I": A_C_PCE_I})
+        df = pd.concat([df,tmp_df2], axis=0)
+        df.reset_index(drop=True, inplace=True)
+    
+    df.replace('', np.nan, inplace = True)
+    return df
+
+def bbki():
+    tmp_url = url["chicagofed"] + "bbki/bbki-monthly-data-series-csv.csv"
+    df = pd.read_csv(tmp_url)
+    return df
+
+def cfnai():
+    tmp_url = url["chicagofed"] + "cfnai/cfnai-data-series-csv.csv"
+    df = pd.read_csv(tmp_url)
+    return df
+
+def cfsbc():
+    tmp_url = url["chicagofed"] + "cfsbc-activity-index-csv.csv"
+    df = pd.read_csv(tmp_url)
+    return df
+
+def nfci():
+    tmp_url = url["chicagofed"] + "nfci/decomposition-nfci-csv.csv"
+    df = pd.read_csv(tmp_url)
+    return df
+
+def nfci():
+    tmp_url = url["chicagofed"] + "nfci/decomposition-anfci-csv.csv"
+    df = pd.read_csv(tmp_url)
+    return df

CEDA/Market/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# time: 05/29/2021 UTC+8
+# author: terencelau
+# email: t_lau@uicstat.com

CEDA/Market/market.py

@@ -47,9 +47,18 @@ def market_data(
 
 if __name__ == "__main__":
     data = market_data(instrument="eurusd",
+<<<<<<< HEAD
                        startdate="2020-01-01",
                        enddate="2021-01-01",
                        timeframe="d1",
                        pricetype="bid",
                        volume=True,
                        flat=True)
+=======
+                        startdate="2020-01-01",
+                        enddate="2021-01-01",
+                        timeframe="d1",
+                        pricetype="bid",
+                        volume=True,
+                        flat=True)
+>>>>>>> master

CEDA/__init__.py

@@ -1,29 +1 @@
-from CEDA.MacroEcon.cn import (
+from CEDA import * 
-    gdp_quarterly,
-    ppi_monthly,
-    cpi_monthly,
-    pmi_monthly,
-    fai_monthly,
-    hi_old_monthly,
-    hi_new_monthly,
-    ci_eei_monthly,
-    ig_monthly,
-    cgpi_monthly,
-    cci_csi_cei_monthly,
-    trscg_monthly,
-    ms_monthly,
-    ie_monthly,
-    stock_monthly,
-    fgr_monthly,
-    ctsf_monthly,
-    sao_monthly,
-    fdi_monthly,
-    gr_monthly,
-    ti_monthly,
-    nl_monthly,
-    dfclc_monthly,
-    fl_monthly,
-    drr_monthly,
-    interest_monthly,
-    gdc_daily
-)

CEDA/config.py

@@ -0,0 +1,15 @@
+import requests
+
+def config(http:str, https:str, auth:bool, user:str, passwd:str):
+    if auth == False:
+        proxies = {
+            "http": "{}".format(http),
+            "https": "{}".format(https)
+        }
+        return proxies
+    if auth == True:
+        proxies = {
+            "http": "http://{}:{}@{}".format(user, passwd, http),
+            "https": "https://{}:{}@{}".format(user, passwd, https),
+        }
+        return proxies

setup.py

@@ -2,7 +2,11 @@ from setuptools import setup, find_packages
 import os
 setup(
     name = "CEDApy",
-    version = "1.0.1",
+<<<<<<< HEAD
+    version = "1.0.3",
+=======
+    version = "1.0.3",
+>>>>>>> master
     keywords = "quantitative economic data",
     long_description = open(
     os.path.join(