From dabafeb2dee5553d074bef19c412b944bf0ee0be Mon Sep 17 00:00:00 2001 From: TerenceLiu98 Date: Wed, 9 Jun 2021 10:29:35 +0800 Subject: [PATCH] captitalization --- CEDA/macroecon/__init__.py | 4 + CEDA/macroecon/cn.py | 1217 ++++++++++++++++++++++++++++++++++++ CEDA/macroecon/eu.py | 908 +++++++++++++++++++++++++++ CEDA/macroecon/us.py | 884 ++++++++++++++++++++++++++ CEDA/market/__init__.py | 4 + CEDA/market/duka.py | 50 ++ CEDA/market/marketwatch.py | 252 ++++++++ 7 files changed, 3319 insertions(+) create mode 100644 CEDA/macroecon/__init__.py create mode 100644 CEDA/macroecon/cn.py create mode 100644 CEDA/macroecon/eu.py create mode 100644 CEDA/macroecon/us.py create mode 100644 CEDA/market/__init__.py create mode 100644 CEDA/market/duka.py create mode 100644 CEDA/market/marketwatch.py diff --git a/CEDA/macroecon/__init__.py b/CEDA/macroecon/__init__.py new file mode 100644 index 0000000..8ad896a --- /dev/null +++ b/CEDA/macroecon/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# time: 05/25/2021 UTC+8 +# author: terencelau +# email: t_lau@uicstat.com diff --git a/CEDA/macroecon/cn.py b/CEDA/macroecon/cn.py new file mode 100644 index 0000000..8a8884a --- /dev/null +++ b/CEDA/macroecon/cn.py @@ -0,0 +1,1217 @@ +import pandas as pd +import numpy as np +import re +import demjson +import requests +from fake_useragent import UserAgent + +# TODO need add comments + +url = { + "eastmoney": "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx" +} + + +def gdp_quarterly(): + """ + ABS: absolute value (per 100 million CNY) + YoY: year on year growth + Data source: http://data.eastmoney.com/cjsj/gdp.html + """ + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable7519513", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "20", + "_": "1622020352668" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Absolute_Value", + "YoY_Rate", + "Primary_Industry_ABS", + "Primary_Industry_YoY_Rate", + "Secondary_Industry_ABS", + "Secondary_Industry_YoY_Rate", + "Tertiary_Industry_ABS", + "Tertiary_Industry_YoY_Rate", + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df["Absolute_Value"] = df["Absolute_Value"].astype(float) + df["Secondary_Industry_ABS"] = df["Secondary_Industry_ABS"].astype(float) + df["Tertiary_Industry_ABS"] = df["Tertiary_Industry_ABS"].astype(float) + df["Absolute_Value"] = df["Absolute_Value"].astype(float) + df["YoY_Rate"] = df["YoY_Rate"].astype(float) / 100 + df["Secondary_Industry_YoY_Rate"] = df["Secondary_Industry_YoY_Rate"].astype( + float) / 100 + df["Tertiary_Industry_YoY_Rate"] = df["Tertiary_Industry_YoY_Rate"].astype( + float) / 100 + return df + + +def ppi_monthly(): + """ + ABS: absolute value (per 100 million CNY) + YoY: year on year growth + Accum: Accumulation + Data source: http://data.eastmoney.com/cjsj/ppi.html + """ + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable9051497", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "22", + "_": "1622047940401" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "Current_Month_YoY_Rate", + "Current_Month_Accum" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df["Current_Month"] = df["Current_Month"].astype(float) + df["Current_Month_YoY_Rate"] = df["Current_Month_YoY_Rate"].astype( + float) / 100 + df["Current_Month_Accum"] = df["Current_Month_Accum"].astype(float) + return df + + +def cpi_monthly(): + """ + Accum: Accumulation + YoY: year on year growth + MoM: month on month growth + Data source: http://data.eastmoney.com/cjsj/cpi.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable2790750", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "19", + "_": "1622020352668" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Notion_Monthly", + "Notion_YoY_Rate", + "Notion_MoM_Rate", + "Notion_Accum", + "Urban_Monthly", + "Urban_YoY_Rate", + "Urban_MoM_Rate", + "Urban_Accum", + "Rural_Monthly", + "Rural_YoY_Rate", + "Rural_MoM_Rate", + "Rural_Accum", + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Notion_Monthly", + "Notion_Accum", + "Urban_Monthly", + "Urban_Accum", + "Rural_Monthly", + "Rural_Accum"]] = df[["Notion_Monthly", + "Notion_Accum", + "Urban_Monthly", + "Urban_Accum", + "Rural_Monthly", + "Rural_Accum"]].astype(float) + df[["Notion_YoY_Rate", + "Notion_MoM_Rate", + "Urban_YoY_Rate", + "Urban_MoM_Rate", + "Rural_YoY_Rate", + "Rural_MoM_Rate"]] = df[["Notion_YoY_Rate", + "Notion_MoM_Rate", + "Urban_YoY_Rate", + "Urban_MoM_Rate", + "Rural_YoY_Rate", + "Rural_MoM_Rate"]].astype(float) / 100 + return df + + +def pmi_monthly(): + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/pmi.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable4515395", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "2", + "ps": "200", + "mkt": "21", + "_": "162202151821" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Man_Industry_Index", + "Man_Index_YoY_Rate", + "Non-Man_Industry_Index", + "Non-Man_Index_YoY_Rate", + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Man_Industry_Index", "Non-Man_Industry_Index"]] = \ + df[["Man_Industry_Index", "Non-Man_Industry_Index"]].astype(float) + df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]] = \ + df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]].astype(float) / 100 + return df + + +def fai_monthly(): # fix asset investment + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/gdzctz.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable607120", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "12", + "_": "1622021790947" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "YoY_Rate", + "MoM_Rate", + "Current_Year_Accum" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month", "Current_Year_Accum"]] = \ + df[["Current_Month", "Current_Year_Accum"]].astype(float) + df[["YoY_Rate", "MoM_Rate"]] = \ + df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100 + return df + + +def hi_old_monthly(): # house index old version (2008-2010) + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/house.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable1895714", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "10", + "_": "1622022794457" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Housing_Prosperity_Index", + "HPI_YoY_Rate", + "Land_Development_Area_Index", + "LDAI_YoY_Rate", + "Sales_Price_Index", + "SPI_YoY_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Housing_Prosperity_Index", + "Land_Development_Area_Index", + "Sales_Price_Index"]] = df[["Housing_Prosperity_Index", + "Land_Development_Area_Index", + "Sales_Price_Index"]].astype(float) + df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]] = \ + df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]].astype(float) / 100 + return df + +# mkt=1&stat=2&city1=%E5%B9%BF%E5%B7%9E&city2=%E4%B8%8A%E6%B5%B7 + + +# newly built commercial housing & second-hand commercial housing +def hi_new_monthly(city1: str, city2: str): + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/newhouse.html + """ + tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getnewhousechartdata?" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params_nbch_MoM = { + "mkt": "1", + "stat": "2", + "city1": "{}".format(city1), + "city2": "{}".format(city2) + } + request_params_shch_MoM = { + "mkt": "1", + "stat": "3", + "city1": "{}".format(city1), + "city2": "{}".format(city2) + } + r_nbch_MoM = requests.get( + tmp_url, + params=request_params_nbch_MoM, + headers=request_header) + r_shch_MoM = requests.get( + tmp_url, + params=request_params_shch_MoM, + headers=request_header) + data_text_nbch_MoM = r_nbch_MoM.text + data_text_shch_MoM = r_shch_MoM.text + data_json_nbch_MoM = demjson.decode(data_text_nbch_MoM) + data_json_shch_MoM = demjson.decode(data_text_shch_MoM) + date_nbch = data_json_nbch_MoM['chart']['series']['value'] + data1_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][0]['value'] + data2_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][1]['value'] + data1_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][0]['value'] + data2_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][1]['value'] + df_MoM = pd.DataFrame({"Date": date_nbch, + "City1_nbch_MoM": data1_nbch_MoM, + "City1_shch_MoM": data1_shch_MoM, + "City2_nbch_MoM": data2_nbch_MoM, + "City2_shch_MoM": data2_shch_MoM}) + df_MoM["Date"] = pd.to_datetime(df_MoM["Date"], format="%m/%d/%Y") + + request_params_nbch_YoY = { + "mkt": "2", + "stat": "2", + "city1": "{}".format(city1), + "city2": "{}".format(city2) + } + request_params_shch_YoY = { + "mkt": "2", + "stat": "3", + "city1": "{}".format(city1), + "city2": "{}".format(city2) + } + r_nbch_YoY = requests.get( + tmp_url, + params=request_params_nbch_YoY, + headers=request_header) + r_shch_YoY = requests.get( + tmp_url, + params=request_params_shch_YoY, + headers=request_header) + data_text_nbch_YoY = r_nbch_YoY.text + data_text_shch_YoY = r_shch_YoY.text + data_json_nbch_YoY = demjson.decode(data_text_nbch_YoY) + data_json_shch_YoY = demjson.decode(data_text_shch_YoY) + date_nbch = data_json_nbch_YoY['chart']['series']['value'] + data1_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][0]['value'] + data2_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][1]['value'] + data1_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][0]['value'] + data2_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][1]['value'] + df_YoY = pd.DataFrame({"Date": date_nbch, + "City1_nbch_YoY": data1_nbch_YoY, + "City1_shch_YoY": data1_shch_YoY, + "City2_nbch_YoY": data2_nbch_YoY, + "City2_shch_YoY": data2_shch_YoY}) + df_YoY["Date"] = pd.to_datetime(df_YoY["Date"], format="%m/%d/%Y") + df = df_YoY.merge(df_MoM, on="Date") + return df + + +def ci_eei_monthly(): # Climate Index & Entrepreneur Expectation Index + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/qyjqzs.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable7709842", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "8", + "_": "1622041485306" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Climate_Index", + "CI_YoY_Rate", + "CI_MoM_Rate", + "Entrepreneur_Expectation_Index", + "EEI_YoY_Rate", + "EEI_MoM_Rate" + ] + df.replace('', np.nan, inplace=True) + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Climate_Index", "Entrepreneur_Expectation_Index"]] = \ + df[["Climate_Index", "Entrepreneur_Expectation_Index"]].astype(float) + df[["CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]] = df[[ + "CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]].astype(float) / 100 + return df + + +def ig_monthly(): # Industry Growth + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/gyzjz.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable4577327", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "0", + "_": "1622042259898" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "IG_YoY_Rate", + "IG_Accum_Rate", + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["IG_YoY_Rate", "IG_Accum_Rate"]] = \ + df[["IG_YoY_Rate", "IG_Accum_Rate"]].astype(float) / 100 + return df + + +def cgpi_monthly(): # Corporate Goods Price Index + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/qyspjg.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable7184534", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "9", + "_": "1622042652353" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "General_Index", + "General_Index_YoY_Rate", + "Total_Index_MoM_Rate", + "Agricultural_Product", + "Agricultural_Product_YoY_Rate", + "Agricultural_Product_MoM_Rate", + "Mineral_Product", + "Mineral_Product_YoY_Rate", + "Mineral_Product_MoM_Rate", + "Coal_Oil_Electricity", + "Coal_Oil_Electricity_YoY_Rate", + "Coal_Oil_Electricity_MoM_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["General_Index", + "Agricultural_Product", + "Mineral_Product", + "Coal_Oil_Electricity"]] = df[["General_Index", + "Agricultural_Product", + "Mineral_Product", + "Coal_Oil_Electricity"]].astype(float) + df[["General_Index_YoY_Rate", + "Total_Index_MoM_Rate", + "Agricultural_Product_YoY_Rate", + "Agricultural_Product_MoM_Rate", + "Coal_Oil_Electricity_YoY_Rate", + "Coal_Oil_Electricity_MoM_Rate"]] = df[["General_Index_YoY_Rate", + "Total_Index_MoM_Rate", + "Agricultural_Product_YoY_Rate", + "Agricultural_Product_MoM_Rate", + "Coal_Oil_Electricity_YoY_Rate", + "Coal_Oil_Electricity_MoM_Rate"]].astype(float) / 100 + return df + + +def cci_csi_cei_monthly(): # Consumer Confidence Index & Consumer Satisfaction Index & Consumer Expectation Index + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/xfzxx.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable1243218", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "4", + "_": "1622043704818" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "CCI", + "CCI_YoY_Rate", + "CCI_MoM_Rate", + "CSI", + "CSI_YoY_Rate", + "CSI_MoM_Rate", + "CEI", + "CEI_YoY_Rate", + "CEI_MoM_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["CCI", "CSI", "CEI"]] = \ + df[["CCI", "CSI", "CEI"]].astype(float) + df[["CCI_YoY_Rate", "CCI_MoM_Rate", + "CSI_YoY_Rate", "CSI_MoM_Rate", + "CEI_YoY_Rate", "CEI_MoM_Rate"]] = \ + df[["CCI_YoY_Rate", "CCI_MoM_Rate", + "CSI_YoY_Rate", "CSI_MoM_Rate", + "CEI_YoY_Rate", "CEI_MoM_Rate"]].astype(float) / 100 + return df + + +def trscg_monthly(): # Total Retail Sales of Consumer Goods + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/xfp.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable3665821", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "5", + "_": "1622044011316" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "TRSCG_YoY_Rate", + "TRSCG_MoM_Rate", + "TRSCG_Accum", + "TRSCG_Accum_YoY_Rate" + ] + df.replace("", np.nan, inplace=True) + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month", "TRSCG_Accum"]] = \ + df[["Current_Month", "TRSCG_Accum"]].astype(float) + df[["TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]] = df[[ + "TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]].astype(float) / 100 + return df + + +def ms_monthly(): # monetary Supply + """ + Man: manufacturing + Non-Man: Non-manufacturing + Data Source: http://data.eastmoney.com/cjsj/hbgyl.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable3818891", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "11", + "_": "1622044292103" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "M2", + "M2_YoY_Rate", + "M2_MoM_Rate", + "M1", + "M1_YoY_Rate", + "M1_MoM_Rate", + "M0", + "M0_YoY_Rate", + "M0_MoM_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["M0", "M1", "M2"]] = \ + df[["M0", "M1", "M2"]].astype(float) + df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate", + "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]] = \ + df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate", + "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]].astype(float) / 100 + return df + + +def ie_monthly(): # Import & Export + """ + Data Source: http://data.eastmoney.com/cjsj/hgjck.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable3818891", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "1", + "_": "1622044292103" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month_Export", + "Current_Month_Export_YoY_Rate", + "Current_Month_Export_MoM_Rate", + "Current_Month_Import", + "Current_Month_Import_YoY_Rate", + "Current_Month_Import_MoM_Rate", + "Accumulation_Export", + "Accumulation_Export_YoY_Rate", + "Accumulation_Import", + "Accumulation_Import_YoY_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month_Export", "Current_Month_Import", + "Accumulation_Export", "Accumulation_Import"]] = \ + df[["Current_Month_Export", "Current_Month_Import", + "Accumulation_Export", "Accumulation_Import"]].astype(float) + df[["Current_Month_Export_YoY_Rate", + "Current_Month_Export_MoM_Rate", + "Current_Month_Import_YoY_Rate", + "Current_Month_Import_MoM_Rate", + "Accumulation_Export_YoY_Rate", + "Accumulation_Export_MoM_Rate"]] = df[["Current_Month_Export_YoY_Rate", + "Current_Month_Export_MoM_Rate", + "Current_Month_Import_YoY_Rate", + "Current_Month_Import_MoM_Rate", + "Accumulation_Export_YoY_Rate", + "Accumulation_Export_MoM_Rate"]].astype(float) / 100 + return df + + +def stock_monthly(): # Import & Export + """ + Data Source: http://data.eastmoney.com/cjsj/gpjytj.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "jQuery112308659690274138041_1622084599455", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "2", + "_": "1622084599456" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("(") + 1:-1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "SH_Total_Stock_issue", + "SZ_Total_Stock_Issue", + "SH_Total_Market_Capitalization", + "SZ_Total_Market_Capitalization", + "SH_Turnover", + "SZ_Turnover", + "SH_Volume", + "SZ_Volume", + "SH_Highest", + "SZ_Highest", + "SH_lowest", + "SZ_lowest" + ] + df.replace("", np.nan, inplace=True) + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[list(df.columns[1:])] = df[list(df.columns[1:])].astype(float) + return df + + +def fgr_monthly(): # Forex and Gold Reserve + """ + Data Source: http://data.eastmoney.com/cjsj/gpjytj.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "tatable6260802", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "16", + "_": "1622044863548" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Forex", + "Forex_YoY_Rate", + "Forex_MoM_Rate", + "Gold", + "Gold_YoY_Rate", + "Gold_MoM_Rate" + ] + df.replace("", np.nan, inplace=True) + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Forex", "Gold"]] = \ + df["Forex", "Gold"].astype(float) + df[["Forex_YoY_Rate", "Gold_YoY_Rate", + "Forex_MoM_Rate", "Gold_MoM_Rate"]] = \ + df["Forex_YoY_Rate", "Gold_YoY_Rate", + "Forex_MoM_Rate", "Gold_MoM_Rate"].astype(float) / 100 + return df +# TODO: SPECIAL CASE + + +def ctsf_monthly(): # Client Transaction Settlement Funds + """ + http://data.eastmoney.com/cjsj/banktransfer.html + """ + tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getbanktransferdata?" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "p": "1", + "ps": "200" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("["):-11]) + df = pd.DataFrame(data_json) + df.replace("", np.nan, inplace=True) + df["StartDate"] = pd.to_datetime(df["StartDate"], format="%Y-%m-%d") + df["EndDate"] = pd.to_datetime(df["EndDate"], format="%Y-%m-%d") + df[list(df.columns)[2:]] = df[list(df.columns)[2:]].astype(float) + return df + +# TODO: SPECIAL CASE + + +def sao_monthly(): # Stock Account Overview + """ + http://data.eastmoney.com/cjsj/gpkhsj.html + """ + tmp_url = "http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get?" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "callback": "datatable4006236", + "type": "GPKHData", + "js": "({data:[(x)],pages:(pc)})", + "st": "SDATE", + "sr": "-1", + "token": "894050c76af8597a853f5b408b759f5d", + "p": "1", + "ps": "2000", + "_": "1622079339035" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{") + 6: -14]) + df = pd.DataFrame(data_json[0]) + df.columns = [ + "Date", + "New_Investor", + "New_Investor_MoM_Rate", + "New_Investor_YoY_Rate", + "Active_Investor", + "Active_Investor_A_Share", + "Active_Investor_B_share", + "SHIndex_Close", + "SHIndex_Rate", + "SHSZ_Market_Capitalization", + "SHSZ_Average_Capitalization" + ] + df.replace("-", np.nan, inplace=True) + df.Date = pd.to_datetime(df.Date, format="%Y年%m月") + df[list(df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])] = df[list( + df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])].astype(float) + df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]] = \ + df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]].astype(float) / 100 + return df + + +def fdi_monthly(): # Foreign Direct Investment + """ + http://data.eastmoney.com/cjsj/fdi.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable1477466", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "15", + "_": "1622044863548" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "YoY_Rate", + "MoM_Rate", + "Accumulation", + "Accum_YoY_Rate" + ] + df.replace("", np.nan, inplace=True) + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month", "Accumulation"]] = \ + df[["Current_Month", "Accumulation"]].astype(float) + df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] = \ + df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100 + return df + + +def gr_monthly(): # Government Revenue + """ + http://data.eastmoney.com/cjsj/czsr.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable7840652", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "14", + "_": "1622080618625" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "YoY_Rate", + "MoM_Rate", + "Accumulation", + "Accum_YoY_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month", "Accumulation"]] = \ + df[["Current_Month", "Accumulation"]].astype(float) + df[["YoY_Rate", "MoM_rate", "Accum_YoY_Rate"]] = \ + df[["YoY_Rate", "MoM_rate", "Accum_YoY_Rate"]].astype(float) / 100 + return df + + +def ti_monthly(): # Tax Income + """ + http://data.eastmoney.com/cjsj/qgsssr.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable8280567", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "3", + "_": "1622080669713" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "YoY_Rate", + "MoM_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month"]] = \ + df[["Current_Month"]].astype(float) + df[["YoY_Rate", "MoM_rate"]] = \ + df[["YoY_Rate", "MoM_rate"]].astype(float) / 100 + return df + + +def nl_monthly(): # New Loan + """ + http://data.eastmoney.com/cjsj/xzxd.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable2533707", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "7", + "_": "1622080800162" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "YoY_Rate", + "MoM_Rate", + "Accumulation", + "Accum_YoY_Rate" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month", "Accumulation"]] = \ + df[["Current_Month", "Accumulation"]].astype(float) + df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] =\ + df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100 + return df + + +def dfclc_monthly(): # Deposit of Foreign Currency and Local Currency + """ + http://data.eastmoney.com/cjsj/wbck.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable2899877", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "18", + "_": "1622081057370" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "YoY_Rate", + "MoM_Rate", + "Accumulation" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month", "Accumulation"]] = \ + df[["Current_Month", "Accumulation"]].astype(float) + df[["YoY_Rate", "MoM_Rate"]] = \ + df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100 + return df + + +def fl_monthly(): # Forex Loan + """ + http://data.eastmoney.com/cjsj/whxd.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable636844", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "17", + "_": "1622081336038" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Date", + "Current_Month", + "YoY", + "MoM", + "Accumulation" + ] + df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d") + df[["Current_Month", "Accumulation"]] = \ + df[["Current_Month", "Accumulation"]].astype(float) + df[["YoY_Rate", "MoM_Rate"]] = \ + df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100 + return df + + +def drr_monthly(): # Deposit Reserve Ratio + """ + http://data.eastmoney.com/cjsj/ckzbj.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable4285562", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "23", + "_": "1622081448882" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Announcement Date", + "Effective Date", + "Large_Financial_institution_Before", + "Large_Financial_institution_After", + "Large_Financial_institution_Adj_Rate", + "S&M_Financial_institution_Before", + "S&M_Financial_institution_After", + "S&M_Financial_institution_Adj_Rate", + "Comment", + "SHIndex_Rate", + "SZIndex_Rate" + ] + df["Announcement Date"] = pd.to_datetime( + df["Announcement Date"], format="%Y-%m-%d") + df["Effective Date"] = pd.to_datetime( + df["Effective Date"], format="%Y-%m-%d") + df[["Large_Financial_institution_Before", + "Large_Financial_institution_After", + "Large_Financial_institution_Adj_Rate", + "S&M_Financial_institution_Before", + "S&M_Financial_institution_After", + "S&M_Financial_institution_Adj_Rate"]] = df[["Large_Financial_institution_Before", + "Large_Financial_institution_After", + "Large_Financial_institution_Adj_Rate", + "S&M_Financial_institution_Before", + "S&M_Financial_institution_After", + "S&M_Financial_institution_Adj_Rate"]].astype(float) / 100 + return df + + +def interest_monthly(): # Interest + """ + http://data.eastmoney.com/cjsj/yhll.html + """ + tmp_url = url["eastmoney"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["eastmoney"] + request_params = { + "cb": "datatable7591685", + "type": "GJZB", + "sty": "ZGZB", + "js": "({data:[(x)],pages:(pc)})", + "p": "1", + "ps": "200", + "mkt": "13", + "_": "1622081956464" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -1]) + df = pd.DataFrame([item.split(",") for item in data_json["data"]]) + df.columns = [ + "Announcement Date", + "Deposit_Benchmark_Interest_Rate_Before", + "Deposit_Benchmark_Interest_Rate_After", + "Deposit_Benchmark_Interest_Rate_Adj_Rate", + "Loan_Benchmark_Interest_Rate_Before", + "Loan_Benchmark_Interest_Rate_After", + "Loan_Benchmark_Interest_Rate_Adj_Rate", + "SHIndex_Rate", + "SZIndex_Rate", + "Effective Date" + ] + df = df[[ + "Announcement Date", + "Effective Date", + "Deposit_Benchmark_Interest_Rate_Before", + "Deposit_Benchmark_Interest_Rate_After", + "Deposit_Benchmark_Interest_Rate_Adj_Rate", + "Loan_Benchmark_Interest_Rate_Before", + "Loan_Benchmark_Interest_Rate_After", + "Loan_Benchmark_Interest_Rate_Adj_Rate", + "SHIndex_Rate", + "SZIndex_Rate" + ]] + df[list(df.columns)] = df[list(df.columns)].astype(float) / 100 + return df + +# TODO: SPECIAL CASE + + +def gdc_daily(): # gasoline, Diesel and Crude Oil + """ + http://data.eastmoney.com/cjsj/oil_default.html + """ + tmp_url = "http://datacenter-web.eastmoney.com/api/data/get?" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "callback": "jQuery112302601302322321093_1622082348721", + "type": "RPTA_WEB_JY_HQ", + "sty": "ALL", + "st": "date", + "sr": "-1", + "token": "894050c76af8597a853f5b408b759f5d", + "p": "1", + "ps": "50000", + "source": "WEB", + "_": "1622082348722" + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text[data_text.find("{"): -2]) + df = pd.DataFrame(data_json["result"]["data"]) + df.columns = ["Crude_Oil", "Date", "Gasoline", "Diesel"] + df = df[["Date", "Gasoline", "Diesel", "Crude_Oil"]] + df = pd.to_datetime(df["Date"], format="%Y-%m-%d") + return df + + +""" +if __name__ == "__main__": +""" diff --git a/CEDA/macroecon/eu.py b/CEDA/macroecon/eu.py new file mode 100644 index 0000000..1b90988 --- /dev/null +++ b/CEDA/macroecon/eu.py @@ -0,0 +1,908 @@ +import io +import os +import demjson +import requests +import numpy as np +import pandas as pd +from fake_useragent import UserAgent +from pandas.core.frame import DataFrame +from pandas.core.reshape.merge import merge + +# Main Economic Indicators: https://alfred.stlouisfed.org/release?rid=205 +url = { + "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?", + "eurostat": "http://ec.europa.eu/eurostat/wdds/rest/data/v2.1/json/en/", + "ecb": "https://sdw-wsrest.ecb.europa.eu/service/data/" +} + + +def merge_data(data_1: pd.DataFrame, data_2: pd.DataFrame, col_name: str): + data = pd.merge_asof(data_1, data_2, on=col_name) + return data + + +def National_Account(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=NAEXCP04EZQ189S,NAEXCP02EZQ189S,NAEXCP01EZQ189S,NAEXCP06EZQ189S,NAEXCP07EZQ189S,NAEXCP03EZQ189S,NAGIGP01EZQ661S,NAEXKP06EZQ659S,NAEXKP04EZQ659S,NAEXKP01EZQ652S,NAEXKP07EZQ652S,NAEXKP03EZQ659S&scale=left,left,left,left,left,left,left,left,left,left,left,left&cosd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1996-01-01,1996-01-01,1995-01-01,1995-01-01,1996-01-01&coed=2020-10-01,2020-10-01,2020-10-01,2020-10-01,2020-10-01,2020-10-01,2020-10-01,2020-10-01,2020-10-01,2021-01-01,2020-10-01,2020-10-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92,%2391e8e1,%238d4653,%238085e8&link_values=false,false,false,false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9,10,11,12&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1996-01-01,1996-01-01,1995-01-01,1995-01-01,1996-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'NAEXCP04EZQ189S': "Gross Domestic Product by Expenditure in Current Prices: Gross Fixed Capital Formation for the Euro Area", + 'NAEXCP02EZQ189S': "Gross Domestic Product by Expenditure in Current Prices: Private Final Consumption Expenditure for the Euro Area", + 'NAEXCP01EZQ189S': "Gross Domestic Product by Expenditure in Current Prices: Total Gross Domestic Product for the Euro Area", + 'NAEXCP06EZQ189S': "Gross Domestic Product by Expenditure in Current Prices: Exports of Goods and Services for the Euro Area", + 'NAEXCP07EZQ189S': "Gross Domestic Product by Expenditure in Current Prices: Less Imports of Goods and Services for the Euro Area", + 'NAEXCP03EZQ189S': "Gross Domestic Product by Expenditure in Current Prices: Government Final Consumption Expenditure for the Euro Area", + 'NAGIGP01EZQ661S': "Gross Domestic Product Deflator for the Euro Area", + 'NAEXKP06EZQ659S': "Gross Domestic Product by Expenditure in Constant Prices: Exports of Goods and Services for the Euro Area", + 'NAEXKP04EZQ659S': "Gross Domestic Product by Expenditure in Constant Prices: Gross Fixed Capital Formation for the Euro Area", + 'NAEXKP01EZQ652S': "Gross Domestic Product by Expenditure in Constant Prices: Total Gross Domestic Product for the Euro Area", + 'NAEXKP07EZQ652S': "Gross Domestic Product by Expenditure in Constant Prices: Less: Imports of Goods and Services for the Euro Area", + 'NAEXKP03EZQ659S': "Gross Domestic Product by Expenditure in Constant Prices: Government Final Consumption Expenditure for the Euro Area"} + description = "National Accounts, Quarterly, Seasonally, Adjusted" + return df, name_list, description + + +def International_Trade(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=XTEXVA01EZQ188S,XTIMVA01EZQ188S,EA19XTNTVA01STSAQ&scale=left,left,left&cosd=1995-01-01,1995-01-01,1995-01-01&coed=2020-10-01,2020-10-01,2017-04-01&line_color=%234572a7,%23aa4643,%2389a54e&link_values=false,false,false&line_style=solid,solid,solid&mark_type=none,none,none&mw=3,3,3&lw=2,2,2&ost=-99999,-99999,-99999&oet=99999,99999,99999&mma=0,0,0&fml=a,a,a&fq=Quarterly,Quarterly,Quarterly&fam=avg,avg,avg&fgst=lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2017-04-01&line_index=1,2,3&transformation=lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07&nd=1995-01-01,1995-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'XTEXVA01EZQ188S': "Exports: Value Goods for the Euro Area", + 'XTIMVA01EZQ188SS': "Imports: Value Goods for the Euro Area", + 'EA19XTNTVA01STSAQ': "International Trade: Net trade: Value (goods): Total for the Euro Area"} + description = "International Trade, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Balance_of_Payments_BPM6(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19B6BLTT02STSAQ,EA19B6DBSE02STSAQ,EA19B6DBSE03STSAQ,EA19B6CRSE03STSAQ,EA19B6CRSE02STSAQ&scale=left,left,left,left,left&cosd=1999-01-01,1999-01-01,1999-01-01,1999-01-01,1999-01-01&coed=2020-10-01,2020-10-01,2020-10-01,2020-10-01,2020-10-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae&link_values=false,false,false,false,false&line_style=solid,solid,solid,solid,solid&mark_type=none,none,none,none,none&mw=3,3,3,3,3&lw=2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999&mma=0,0,0,0,0&fml=a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5&transformation=lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1999-01-01,1999-01-01,1999-01-01,1999-01-01,1999-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19B6BLTT02STSAQ': "Balance of payments BPM6: Current account Debits: Services: Total Debits as % of Current account for the Euro Area", + 'EA19B6DBSE02STSAQ': "Balance of payments BPM6: Current account Debits: Services: Total Debits as % of Current account for the Euro Area", + 'EA19B6DBSE03STSAQ': "Balance of payments BPM6: Current account Debits: Services: Total Debits as % of Goods and Services for the Euro Area", + 'EA19B6CRSE03STSAQ': "Balance of payments BPM6: Current account Credits: Services: Total Credits as % of Goods and Services for Euro Area", + 'EA19B6CRSE02STSAQ': "Balance of payments BPM6: Current account Credits: Services: Total Credits as % of Current account for Euro Area"} + description = "Balanced of payments BPM6, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Learning_Indicators_OECD(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19LORSGPNOSTSAM,EA19LOLITOTRGYSAM,EA19LOLITONOSTSAM,EA19LOLITOAASTSAM,EA19LORSGPORIXOBSAM,EA19LORSGPRTSTSAM,EA19LORSGPTDSTSAM&scale=left,left,left,left,left,left,left&cosd=1960-03-01,1966-12-01,1965-12-01,1965-12-01,1960-03-01,1960-03-01,1960-03-01&coed=2020-11-01,2020-11-01,2021-03-01,2021-03-01,2020-11-01,2020-11-01,2020-11-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1960-03-01,1966-12-01,1965-12-01,1965-12-01,1960-03-01,1960-03-01,1960-03-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19CPALTT01GYQ': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Normalised for the Euro Area", + 'EA19LOLITOTRGYSAM': "Leading Indicators OECD: Leading indicators: CLI: Trend restored for the Euro Area", + 'EA19LOLITONOSTSAM': "Leading Indicators OECD: Leading indicators: CLI: Normalised for the Euro Area", + 'EA19LOLITOAASTSAM': "Leading Indicators OECD: Leading indicators: CLI: Amplitude adjusted for the Euro Area", + 'EA19LORSGPORIXOBSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Original series for the Euro Area", + 'EA19LORSGPRTSTSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Ratio to trend for the Euro Area", + 'EA19LORSGPTDSTSAM': "Leading Indicators OECD: Reference series: Gross Domestic Product (GDP): Trend for the Euro Area"} + description = "Leading Indicators OECD, Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Monetary_Aggregates_Monthly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19MABMM301GYSAM,EA19MANMM101IXOBSAM&scale=left,left&cosd=1971-01-01,1970-01-01&coed=2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643&link_values=false,false&line_style=solid,solid&mark_type=none,none&mw=3,3&lw=2,2&ost=-99999,-99999&oet=99999,99999&mma=0,0&fml=a,a&fq=Monthly,Monthly&fam=avg,avg&fgst=lin,lin&fgsnd=2020-02-01,2020-02-01&line_index=1,2&transformation=lin,lin&vintage_date=2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07&nd=1971-01-01,1970-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = {'EA19MABMM301GYSAM': "Monetary aggregates and their components: Broad money and components: M3: M3 for the Euro Area", + 'EA19MANMM101IXOBSAM': "Monetary aggregates and their components: Narrow money and components: M1 and components: M1 for the Euro Area"} + description = "Monetary aggregates and their components, Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Monetary_Aggregates_Quarterly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=MABMM301EZQ189S,MANMM101EZQ189S&scale=left,left&cosd=1970-01-01,1970-01-01&coed=2021-01-01,2021-01-01&line_color=%234572a7,%23aa4643&link_values=false,false&line_style=solid,solid&mark_type=none,none&mw=3,3&lw=2,2&ost=-99999,-99999&oet=99999,99999&mma=0,0&fml=a,a&fq=Quarterly,Quarterly&fam=avg,avg&fgst=lin,lin&fgsnd=2020-02-01,2020-02-01&line_index=1,2&transformation=lin,lin&vintage_date=2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07&nd=1970-01-01,1970-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'MABMM301EZQ189S': "M3 for the Euro Area", + 'MANMM101EZQ189S': "M1 for the Euro Area" + } + description = "Monetary aggregates and their components, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Currency_Conversion_Quarterly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=CCEUSP02EZQ655N,CCUSMA02EZQ618N,CCUSSP01EZQ650N,CCRETT02EZQ661N,CCRETT01EZQ661N&scale=left,left,left,left,left&cosd=1999-01-01,1979-01-01,1999-01-01,1970-01-01,1970-01-01&coed=2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae&link_values=false,false,false,false,false&line_style=solid,solid,solid,solid,solid&mark_type=none,none,none,none,none&mw=3,3,3,3,3&lw=2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999&mma=0,0,0,0,0&fml=a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5&transformation=lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1999-01-01,1979-01-01,1999-01-01,1970-01-01,1970-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'CCEUSP02EZQ655N': "National Currency to Euro Spot Exchange Rate for the Euro Area", + 'CCUSMA02EZQ618N': "National Currency to US Dollar Exchange Rate: Average of Daily Rates for the Euro Area", + 'CCUSSP01EZQ650N': "US Dollar to National Currency Spot Exchange Rate for the Euro Area", + 'CCRETT02EZQ661N': "Real Effective Exchange Rates Based on Manufacturing Unit Labor Cost for the Euro Area", + 'CCRETT01EZQ661N': "Real Effective Exchange Rates Based on Manufacturing Consumer Price Index for the Euro Area"} + description = "Currency Conversions, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Currency_Conversion_Monthly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=CCRETT01EZM661N,CCUSMA02EZM659N,CCUSSP01EZM650N,CCEUSP02EZM655N&scale=left,left,left,left&cosd=1970-01-01,1991-01-01,1999-01-01,1999-01-01&coed=2021-04-01,2021-04-01,2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b&link_values=false,false,false,false&line_style=solid,solid,solid,solid&mark_type=none,none,none,none&mw=3,3,3,3&lw=2,2,2,2&ost=-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999&mma=0,0,0,0&fml=a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg&fgst=lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4&transformation=lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1970-01-01,1991-01-01,1999-01-01,1999-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'CCRETT01EZM661N': "Real Effective Exchange Rates Based on Manufacturing Consumer Price Index for the Euro Area", + 'CCUSMA02EZM659N': "National Currency to US Dollar Exchange Rate: Average of Daily Rates for the Euro Area", + 'CCUSSP01EZM650N': "US Dollar to National Currency Spot Exchange Rate for the Euro Area", + 'CCEUSP02EZM655N': "National Currency to Euro Spot Exchange Rate for the Euro Area"} + description = "Currency Conversions, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def Interest_Rates_Quarterly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=IRLTLT01EZQ156N,IR3TIB01EZQ156N,IRSTCI01EZQ156N&scale=left,left,left&cosd=1970-01-01,1994-01-01,1994-01-01&coed=2021-01-01,2021-01-01,2021-01-01&line_color=%234572a7,%23aa4643,%2389a54e&link_values=false,false,false&line_style=solid,solid,solid&mark_type=none,none,none&mw=3,3,3&lw=2,2,2&ost=-99999,-99999,-99999&oet=99999,99999,99999&mma=0,0,0&fml=a,a,a&fq=Quarterly,Quarterly,Quarterly&fam=avg,avg,avg&fgst=lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3&transformation=lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07&nd=1970-01-01,1994-01-01,1994-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'IRLTLT01EZQ156N': "Long-Term Government Bond Yields: 10-year: Main (Including Benchmark) for the Euro Area", + 'IR3TIB01EZQ156N': "3-Month or 90-day Rates and Yields: Interbank Rates for the Euro Area", + 'IRSTCI01EZQ156N': "Immediate Rates: Less than 24 Hours: Call Money/Interbank Rate for the Euro Area"} + description = "Interest Rates, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Interest_Rates_Monthly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=IRLTLT01EZM156N,IR3TIB01EZM156N,IRSTCI01EZM156N&scale=left,left,left&cosd=1970-01-01,1994-01-01,1994-01-01&coed=2021-04-01,2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643,%2389a54e&link_values=false,false,false&line_style=solid,solid,solid&mark_type=none,none,none&mw=3,3,3&lw=2,2,2&ost=-99999,-99999,-99999&oet=99999,99999,99999&mma=0,0,0&fml=a,a,a&fq=Monthly,Monthly,Monthly&fam=avg,avg,avg&fgst=lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3&transformation=lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07&nd=1970-01-01,1994-01-01,1994-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'IRLTLT01EZM156N': "Long-Term Government Bond Yields: 10-year: Main (Including Benchmark) for the Euro Area", + 'IR3TIB01EZM156N': "3-Month or 90-day Rates and Yields: Interbank Rates for the Euro Area", + 'IRSTCI01EZM156N': "Immediate Rates: Less than 24 Hours: Call Money/Interbank Rate for the Euro Area"} + description = "Interest Rates, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def Share_Prices_Quarterly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=SPASTT01EZQ661N&scale=left&cosd=1987-01-01&coed=2021-01-01&line_color=%234572a7&link_values=false&line_style=solid&mark_type=none&mw=3&lw=2&ost=-99999&oet=99999&mma=0&fml=a&fq=Quarterly&fam=avg&fgst=lin&fgsnd=2020-02-01&line_index=1&transformation=lin&vintage_date=2021-06-07&revision_date=2021-06-07&nd=1987-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'SPASTT01EZQ661N': "Total Share Prices for All Shares for the Euro Area"} + description = "Share Prices, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Share_Prices_Monthly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=SPASTT01EZM661N&scale=left&cosd=1986-12-01&coed=2021-04-01&line_color=%234572a7&link_values=false&line_style=solid&mark_type=none&mw=3&lw=2&ost=-99999&oet=99999&mma=0&fml=a&fq=Monthly&fam=avg&fgst=lin&fgsnd=2020-02-01&line_index=1&transformation=lin&vintage_date=2021-06-07&revision_date=2021-06-07&nd=1986-12-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'SPASTT01EZM661N': "Total Share Prices for All Shares for the Euro Area"} + description = "Share Prices, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def CPI_Monthly(startdate="1970-01-01", enddate="2021-01-01"): + """ + """ + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=CPHPTT01EZM661N,EA19CPHP0401IXOBM,EA19CPHP0403IXOBM,EA19CPHP0404IXOBM,EA19CPHP0405IXOBM,EA19CPHP0500IXOBM,EA19CPHP0600IXOBM,EA19CPHP0700IXOBM,EA19CPHP0702IXOBM,EA19CPHP0800IXOBM,EA19CPHP0900IXOBM,CPHPEN01EZM661N&scale=left,left,left,left,left,left,left,left,left,left,left,left&cosd=1990-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01&coed=2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92,%2391e8e1,%238d4653,%238085e8&link_values=false,false,false,false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9,10,11,12&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1990-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01,1996-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + "CPHPTT01EZM661N": "CPI:Harmonized Prices: Total All Items for the Euro Area", + "EA19CPHP0401IXOBM": "CPI:Harmonised_Price:Housing, water, electricity, gas and other fuels (COICOP 04): Actual rentals for housing for the Euro Area", + "EA19CPHP0403IXOBM": "CPI:Harmonised_Price:Housing, water, electricity, gas and other fuels (COICOP 04): Maintenance & repairs of the dwellings for the Euro Area", + "EA19CPHP0404IXOBM": "CPI:Harmonised_Price:Housing, water, electricity, gas and other fuels (COICOP 04): Water supply and miscellaneous services relating to the dwelling for the Euro Area", + "EA19CPHP0405IXOBM": "CPI:Harmonised_Price:Housing, water, electricity, gas and other fuels (COICOP 04): Electricity, gas and other fuels for the Euro Area", + "EA19CPHP0500IXOBM": "CPI:Harmonised_Price:Furnishings, household equip. and routine household maintenance (COICOP 05): Total for the Euro Area ", + "EA19CPHP0600IXOBM": "CPI:Harmonised_Price:Health (COICOP 06): Total for the Euro Area", + "EA19CPHP0700IXOBM": "CPI:Harmonised_Price:Transport (COICOP 07): Total for the Euro Area", + "EA19CPHP0702IXOBM": "CPI:Harmonised_Price:Transport (COICOP 07): Fuels and lubricants for personal transport equipment for the Euro Area", + "EA19CPHP0800IXOBM": "CPI:Harmonised_Price:Communication (COICOP 08): Total for the Euro Area", + "EA19CPHP0900IXOBM": "CPI:Harmonised_Price:Recreation and culture (COICOP 09): Total for the Euro Area", + "CPHPEN01EZM661N": "CPI:Harmonized Prices: Total Energy for the Euro Area"} + description = "Consumer Price Index, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def CPI_Quarterly(): + """ + """ + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19CPALTT01GYQ,EA19CPGRLE01GYQ,EA19CPGREN01GYQ,EA19CPHP0401IXOBQ&scale=left,left,left,left&cosd=1991-01-01,1997-01-01,1997-01-01,1996-01-01&coed=2021-01-01,2021-01-01,2021-01-01,2021-01-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b&link_values=false,false,false,false&line_style=solid,solid,solid,solid&mark_type=none,none,none,none&mw=3,3,3,3&lw=2,2,2,2&ost=-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999&mma=0,0,0,0&fml=a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg&fgst=lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4&transformation=lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1991-01-01,1997-01-01,1997-01-01,1996-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19CPALTT01GYQ': "CPI:All items:Total:Total for the Euro Area", + 'EA19CPGRLE01GYQ': "CPI:OECD Groups:All items non-food non-energy:Total for the Euro Area", + 'EA19CPGREN01GYQ': "CPI:OECD Groups:Energy (Fuel, electricity & gasoline):Total for the Euro Area", + 'EA19CPHP0401IXOBQ': "CPI:Harmonised prices:Housing, water, electricity, gas and other fuels (COICOP 04):Actual rentals for housing for the Euro Area"} + description = "Consumer Price Index, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def PPI_Monthly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=PIEAMP02EZM659N,PIEAMP01EZM661N,PIEATI01EZM661N,PIEATI02EZM661N,PITGND02EZM661N,PITGND01EZM661N,PITGIG01EZM661N,PITGIG02EZM661N,PIEAFD02EZM661N,PITGCG02EZM661N,PITGCG01EZM661N,PITGCD01EZM661N&scale=left,left,left,left,left,left,left,left,left,left,left,left&cosd=1996-01-01,2000-01-01,2000-01-01,2000-01-01,1995-01-01,2000-01-01,2000-01-01,1995-01-01,1995-01-01,1995-01-01,2000-01-01,2000-01-01&coed=2021-03-01,2021-02-01,2021-02-01,2021-03-01,2021-03-01,2021-02-01,2021-02-01,2021-03-01,2021-03-01,2021-03-01,2021-02-01,2021-02-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92,%2391e8e1,%238d4653,%238085e8&link_values=false,false,false,false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9,10,11,12&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1996-01-01,2000-01-01,2000-01-01,2000-01-01,1995-01-01,2000-01-01,2000-01-01,1995-01-01,1995-01-01,1995-01-01,2000-01-01,2000-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'PIEAMP02EZM659N': "Producer Prices Index: Economic Activities: Domestic Manufacturing for the Euro Area", + "PIEAMP01EZM661N": "Producer Prices Index: Economic Activities: Total Manufacturing for the Euro Area", + "PIEATI01EZM661N": "Producer Prices Index: Economic Activities: Total Industrial Activities for the Euro Area", + "PIEATI02EZM661N": "Producer Prices Index: Economic Activities: Domestic Industrial Activities for the Euro Area", + "PITGND02EZM661N": "Producer Prices Index: Domestic Nondurable Consumer Goods for the Euro Area", + "PITGND01EZM661N": "Producer Prices Index: Total Nondurable Consumer Goods for the Euro Area", + "PITGIG01EZM661N": "Producer Prices Index: Total Intermediate Goods for the Euro Area", + "PITGIG02EZM661N": "Producer Prices Index: Domestic Intermediate Goods for the Euro Area", + "PIEAFD02EZM661N": "Producer Prices Index: Economic Activities: Domestic Manufacture of Food Products for the Euro Area", + "PITGCG02EZM661N": "Producer Prices Index: Domestic Consumer Goods for the Euro Area", + "PITGCG01EZM661N": "Producer Prices Index: Total Consumer Goods for the Euro Area", + "PITGCD01EZM661N": "Producer Prices Index: Total Durable Consumer Goods for the Euro Area"} + description = "Producer Prices Index, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def PPI_Quarterly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=PIEAFD01EZQ661N,PIEAEN02EZQ661N,PIEAEN01EZQ661N,PITGND02EZQ661N,PITGND01EZQ661N,PITGIG01EZQ661N,PITGIG02EZQ661N,PIEAFD02EZQ661N,PITGCD02EZQ661N,PITGCD01EZQ661N,PITGVG01EZQ661N,PITGVG02EZQ661N&scale=left,left,left,left,left,left,left,left,left,left,left,left&cosd=2000-01-01,2000-01-01,2000-01-01,1995-01-01,2000-01-01,2000-01-01,1995-01-01,1995-01-01,2000-01-01,2000-01-01,2000-01-01,1995-01-01&coed=2020-10-01,2021-01-01,2020-10-01,2021-01-01,2020-10-01,2020-10-01,2021-01-01,2021-01-01,2021-01-01,2020-10-01,2020-10-01,2021-01-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92,%2391e8e1,%238d4653,%238085e8&link_values=false,false,false,false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9,10,11,12&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=2000-01-01,2000-01-01,2000-01-01,1995-01-01,2000-01-01,2000-01-01,1995-01-01,1995-01-01,2000-01-01,2000-01-01,2000-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'PIEAFD01EZQ661N': "Producer Prices Index: Economic Activities: Total Manufacture of Food Products for the Euro Area", + "PIEAEN02EZQ661N": "Producer Prices Index: Economic Activities: Domestic Energy for the Euro Area", + "PIEAEN01EZQ661N": "Producer Prices Index: Economic Activities: Total Energy for the Euro Area", + "PITGND02EZQ661N": "Producer Prices Index: Domestic Nondurable Consumer Goods for the Euro Area", + "PITGND01EZQ661N": "Producer Prices Index: Total Nondurable Consumer Goods for the Euro Area", + "PITGIG01EZQ661N": "Producer Prices Index: Total Intermediate Goods for the Euro Area", + "PITGIG02EZQ661N": "Producer Prices Index: Domestic Intermediate Goods for the Euro Area", + "PIEAFD02EZQ661N": "Producer Prices Index: Economic Activities: Domestic Manufacture of Food Products for the Euro Area", + "PITGCD02EZQ661N": "Producer Prices Index: Domestic Durable Consumer Goods for the Euro Area", + "PITGCD01EZQ661N": "Producer Prices Index: Total Durable Consumer Goods for the Euro Area", + "PITGVG01EZQ661N": "Producer Prices Index: Investments Goods: Total for the Euro Area", + "PITGVG02EZQ661N": "Producer Prices Index: Domestic Investments Goods for the Euro Area"} + description = "Producer Prices Index, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Business_Tendency_Surveys_Construction(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19BCBUTE02STSAM,BCOBLV02EZM460S,BCEMFT02EZM460S,BCCICP02EZM460S,BCSPFT02EZM460S&scale=left,left,left,left,left&cosd=1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01&coed=2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae&link_values=false,false,false,false,false&line_style=solid,solid,solid,solid,solid&mark_type=none,none,none,none,none&mw=3,3,3,3,3&lw=2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999&mma=0,0,0,0,0&fml=a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5&transformation=lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19BCBUTE02STSAM': "Business tendency surveys (construction): Business situation - Activity: Tendency: National indicator for the Euro Area", + 'BCOBLV02EZM460S': "Business Tendency Surveys for Construction: Order Books: Level: European Commission Indicator for the Euro Area", + 'BCEMFT02EZM460S': "Business Tendency Surveys for Construction: Employment: Future Tendency: European Commission and National Indicators for the Euro Area", + 'BCCICP02EZM460S': "Business Tendency Surveys for Construction: Confidence Indicators: Composite Indicators: European Commission and National Indicators for the Euro Area", + 'BCSPFT02EZM460S': "Business Tendency Surveys for Construction: Selling Prices: Future Tendency: European Commission Indicator for the Euro Area"} + description = "Business tendency surveys (construction), Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Business_Tendency_Surveys_Services(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19BVBUTE02STSAM,BVCICP02EZM460S,BVEMTE02EZM460S,BVEMFT02EZM460S,BVDEFT02EZM460S,BVDETE02EZM460S&scale=left,left,left,left,left,left&cosd=1995-04-01,1995-04-01,1995-04-01,1996-10-01,1995-04-01,1995-04-01&coed=2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d&link_values=false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none&mw=3,3,3,3,3,3&lw=2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0&fml=a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6&transformation=lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1995-04-01,1995-04-01,1995-04-01,1996-10-01,1995-04-01,1995-04-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19BVBUTE02STSAM': "Business tendency surveys (services): Business situation - Activity: Tendency: National indicator for Euro Area", + 'BVCICP02EZM460S': "Business Tendency Surveys for Services: Confidence Indicators: Composite Indicators: European Commission and National Indicators for the Euro Area", + 'BVEMTE02EZM460S': "Business Tendency Surveys for Services: Employment: Tendency: European Commission Indicator for the Euro Area", + 'BVEMFT02EZM460S': "Business Tendency Surveys for Services: Employment: Future Tendency: European Commission and National Indicators for the Euro Area", + 'BVDEFT02EZM460S': "Business Tendency Surveys for Services: Demand Evolution: Future Tendency: European Commission Indicator for the Euro Area", + 'BVDETE02EZM460S': "Business Tendency Surveys for Services: Demand Evolution: Tendency: European Commission Indicator for the Euro Area"} + description = "Business tendency surveys (services), Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Business_Tendency_Surveys_Manufacturing_Quarterly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=BSCURT02EZQ160S,BSOITE02EZQ460S&scale=left,left&cosd=1985-01-01,1985-01-01&coed=2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643&link_values=false,false&line_style=solid,solid&mark_type=none,none&mw=3,3&lw=2,2&ost=-99999,-99999&oet=99999,99999&mma=0,0&fml=a,a&fq=Quarterly,Quarterly&fam=avg,avg&fgst=lin,lin&fgsnd=2020-02-01,2020-02-01&line_index=1,2&transformation=lin,lin&vintage_date=2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07&nd=1985-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'BSCURT02EZQ160S': "Business Tendency Surveys for Manufacturing: Capacity Utilization: Rate of Capacity Utilization: European Commission and National Indicators for the Euro Area", + 'BSOITE02EZQ460S': "Business Tendency Surveys for Manufacturing: Orders Inflow: Tendency: European Commission Indicator for the Euro Area"} + description = "Business tendency surveys (manufacturing), Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Business_Tendency_Surveys_Manufacturing_Monthly(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=BSSPFT02EZM460S,BSOBLV02EZM460S,BSEMFT02EZM460S,BSFGLV02EZM460S,BSXRLV02EZM086S,BSCICP02EZM460S,BSPRTE02EZM460S,BSPRFT02EZM460S&scale=left,left,left,left,left,left,left,left&cosd=1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01&coed=2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c&link_values=false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8&transformation=lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'BSSPFT02EZM460S': "Business Tendency Surveys for Manufacturing: Selling Prices: Future Tendency: European Commission Indicator for the Euro Area", + 'BSOBLV02EZM460S': "Business Tendency Surveys for Manufacturing: Order Books: Level: European Commission and National Indicators for the Euro Area", + 'BSEMFT02EZM460S': "Business Tendency Surveys for Manufacturing: Employment: Future Tendency: European Commission and National Indicators for the Euro Area", + 'BSFGLV02EZM460S': "Business Tendency Surveys for Manufacturing: Finished Goods Stocks: Level: European Commission and National Indicators for the Euro Area", + 'BSXRLV02EZM086S': "Business Tendency Surveys for Manufacturing: Export Order Books or Demand: Level: European Commission Indicator for the Euro Area", + 'BSCICP02EZM460S': "Business Tendency Surveys for Manufacturing: Confidence Indicators: Composite Indicators: European Commission and National Indicators for the Euro Area", + 'BSPRTE02EZM460S': "Business Tendency Surveys for Manufacturing: Production: Tendency: European Commission and National Indicators for the Euro Area", + 'BSPRFT02EZM460S': "Business Tendency Surveys for Manufacturing: Production: Future Tendency: European Commission and National Indicators for the Euro Area"} + description = "Business tendency surveys (manufacturing), Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Business_Tendency_Surveys_Retail_Trade(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19BREMFT02STSAM,EA19BRODFT02STSAM,EA19BRVSLV02STSAM,EA19BRCICP02STSAM,EA19BRBUFT02STSAM,EA19BRBUTE02STSAM&scale=left,left,left,left,left,left&cosd=1985-04-01,1985-02-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01&coed=2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d&link_values=false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none&mw=3,3,3,3,3,3&lw=2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0&fml=a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6&transformation=lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1985-04-01,1985-02-01,1985-01-01,1985-01-01,1985-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19BREMFT02STSAM': "Business tendency surveys (retail trade): Employment: Future tendency: National indicator for the Euro Area", + 'EA19BRODFT02STSAM': "Business tendency surveys (retail trade): Order intentions or Demand: Future tendency: National indicator for the Euro Area", + 'EA19BRVSLV02STSAM': "Business tendency surveys (retail trade): Volume of stocks: Level: National indicator for the Euro Area", + 'EA19BRCICP02STSAM': "Business tendency surveys (retail trade): Confidence indicators: Composite indicators: National indicator for the Euro Area", + 'EA19BRBUFT02STSAM': "Business tendency surveys (retail trade): Business situation - Activity: Future tendency: National indicator for Euro Area", + 'EA19BRBUTE02STSAM': "Business tendency surveys (retail trade): Business situation - Activity: Tendency: National indicator for Euro Area"} + description = "Business tendency surveys (retail trade), Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Labor_Compoenstiveion_Quarterly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LCEAMN01EZQ661S,LCEAPR01EZQ661S&scale=left,left&cosd=1971-01-01,1996-01-01&coed=2020-10-01,2020-10-01&line_color=%234572a7,%23aa4643&link_values=false,false&line_style=solid,solid&mark_type=none,none&mw=3,3&lw=2,2&ost=-99999,-99999&oet=99999,99999&mma=0,0&fml=a,a&fq=Quarterly,Quarterly&fam=avg,avg&fgst=lin,lin&fgsnd=2020-02-01,2020-02-01&line_index=1,2&transformation=lin,lin&vintage_date=2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07&nd=1971-01-01,1996-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LCEAMN01EZQ661S': "Hourly Earnings: Manufacturing for the Euro Area", + 'LCEAPR01EZQ661S': "Hourly Earnings: Private Sector for the Euro Area" + } + description = "Labor Compensation, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Labor_Compoenstiveion_Quarterly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LCEAMN01EZQ661S,LCEAPR01EZQ661S&scale=left,left&cosd=1971-01-01,1996-01-01&coed=2020-10-01,2020-10-01&line_color=%234572a7,%23aa4643&link_values=false,false&line_style=solid,solid&mark_type=none,none&mw=3,3&lw=2,2&ost=-99999,-99999&oet=99999,99999&mma=0,0&fml=a,a&fq=Quarterly,Quarterly&fam=avg,avg&fgst=lin,lin&fgsnd=2020-02-01,2020-02-01&line_index=1,2&transformation=lin,lin&vintage_date=2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07&nd=1971-01-01,1996-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LCEAMN01EZQ661N': "Hourly Earnings: Manufacturing for the Euro Area", + 'LCEAPR01EZQ661N': "Hourly Earnings: Private Sector for the Euro Area" + } + description = "Labor Compensation, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Unit_Labor_costs(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=ULQECU01EZQ661S,ULQEUL01EZQ659S,ULQELP01EZQ661S&scale=left,left,left&cosd=1995-01-01,1996-01-01,1995-01-01&coed=2020-10-01,2020-10-01,2020-10-01&line_color=%234572a7,%23aa4643,%2389a54e&link_values=false,false,false&line_style=solid,solid,solid&mark_type=none,none,none&mw=3,3,3&lw=2,2,2&ost=-99999,-99999,-99999&oet=99999,99999,99999&mma=0,0,0&fml=a,a,a&fq=Quarterly,Quarterly,Quarterly&fam=avg,avg,avg&fgst=lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3&transformation=lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07&nd=1995-01-01,1996-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'ULQECU01EZQ661S': "Early Estimate of Quarterly ULC Indicators: Total Labor Compensation per Unit of Labor Input for the Euro Area", + 'ULQEUL01EZQ659S': "Early Estimate of Quarterly ULC Indicators: Total for the Euro Area", + 'ULQELP01EZQ661S': "Early Estimate of Quarterly ULC Indicators: Total Labor Productivity for the Euro Area"} + description = "Unit Labor Costs, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Labor_Force_Survey_Rates_Quarterly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LRHU24TTEZQ156N,LRHU24FEEZQ156N,LRHU24MAEZQ156N,LRHUADMAEZQ156N,LRHUADTTEZQ156N,LRHUADFEEZQ156N,LRHUTTFEEZQ156N,LRHUTTTTEZQ156N,LRHUTTMAEZQ156N&scale=left,left,left,left,left,left,left,left,left&cosd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1993-01-01,1993-01-01,1993-01-01&coed=2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92&link_values=false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1993-01-01,1993-01-01,1993-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LRHU24TTEZQ156N': "Harmonized Unemployment: Aged 15-24: All Persons for the Euro Area", + 'LRHU24FEEZQ156N': "Harmonized Unemployment: Aged 15-24: Females for the Euro Area", + 'LRHU24MAEZQ156N': "Harmonized Unemployment: Aged 15-24: Males for the Euro Area", + 'LRHUADMAEZQ156N': "Harmonized Unemployment: Aged 25 and Over: Males for the Euro Area", + 'LRHUADTTEZQ156N': "Harmonized Unemployment: Aged 25 and Over: All Persons for the Euro Area", + 'LRHUADFEEZQ156N': "Harmonized Unemployment: Aged 25 and Over: Females for the Euro Area", + 'LRHUTTFEEZQ156N': "Harmonized Unemployment: Total: Females for the Euro Area", + 'LRHUTTTTEZQ156N': "Harmonized Unemployment Rate: Total: All Persons for the Euro Area", + 'LRHUTTMAEZQ156N': "Harmonized Unemployment: Total: Males for the Euro Area"} + description = "Labor Force Survey - quarterly rates, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Labor_Force_Survey_Rates_Quarterly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LRHU24MAEZQ156S,LRHU24TTEZQ156S,LRHU24FEEZQ156S,LRHUADFEEZQ156S,LRHUADMAEZQ156S,LRHUADTTEZQ156S,LRHUTTTTEZQ156S,LRHUTTMAEZQ156S,LRHUTTFEEZQ156S&scale=left,left,left,left,left,left,left,left,left&cosd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1990-07-01,1990-07-01,1990-07-01&coed=2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01,2021-01-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92&link_values=false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1990-07-01,1990-07-01,1990-07-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LRHU24MAEZQ156S': "Harmonized Unemployment: Aged 15-24: Males for the Euro Area", + 'LRHU24TTEZQ156S': "Harmonized Unemployment: Aged 15-24: All Persons for the Euro Area", + 'LRHU24FEEZQ156S': "Harmonized Unemployment: Aged 15-24: Females for the Euro Area", + 'LRHUADFEEZQ156S': "Harmonized Unemployment: Aged 25 and Over: Females for the Euro Area", + 'LRHUADMAEZQ156S': "Harmonized Unemployment: Aged 25 and Over: Males for the Euro Area", + 'LRHUADTTEZQ156S': "Harmonized Unemployment: Aged 25 and Over: All Persons for the Euro Area", + 'LRHUTTTTEZQ156S': "Harmonized Unemployment Rate: Total: All Persons for the Euro Area", + 'LRHUTTMAEZQ156S': "Harmonized Unemployment: Total: Males for the Euro Area", + 'LRHUTTFEEZQ156S': "Harmonized Unemployment: Total: Females for the Euro Area"} + description = "Labor Force Survey - quarterly rates, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Labor_Force_Survey_Rates_Monthly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LRHUTTFEEZM156N,LRHUTTMAEZM156N,LRHUTTTTEZM156N,LRHUADTTEZM156N,LRHUADMAEZM156N,LRHUADFEEZM156N,LRHU24FEEZM156N,LRHU24MAEZM156N,LRHU24TTEZM156N&scale=left,left,left,left,left,left,left,left,left&cosd=1993-01-01,1993-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01&coed=2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92&link_values=false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1993-01-01,1993-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LRHUTTFEEZM156N': "Harmonized Unemployment: Total: Females for the Euro Area", + 'LRHUTTMAEZM156N': "Harmonized Unemployment: Total: Males for the Euro Area", + 'LRHUTTTTEZM156N': "Harmonized Unemployment Rate: Total: All Persons for the Euro Area", + 'LRHUADTTEZM156N': "Harmonized Unemployment: Aged 25 and Over: All Persons for the Euro Area", + 'LRHUADMAEZM156N': "Harmonized Unemployment: Aged 25 and Over: Males for the Euro Area", + 'LRHUADFEEZM156N': "Harmonized Unemployment: Aged 25 and Over: Females for the Euro Area", + 'LRHU24FEEZM156N': "Harmonized Unemployment: Aged 15-24: Females for the Euro Area", + 'LRHU24MAEZM156N': "Harmonized Unemployment: Aged 15-24: Males for the Euro Area", + 'LRHU24TTEZM156N': "Harmonized Unemployment: Aged 15-24: All Persons for the Euro Area"} + description = "Labor Force Survey - quarterly rates, Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Labor_Force_Survey_Level_Quarterly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LRHUTTFEEZM156N,LRHUTTMAEZM156N,LRHUTTTTEZM156N,LRHUADTTEZM156N,LRHUADMAEZM156N,LRHUADFEEZM156N,LRHU24FEEZM156N,LRHU24MAEZM156N,LRHU24TTEZM156N&scale=left,left,left,left,left,left,left,left,left&cosd=1993-01-01,1993-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01&coed=2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92&link_values=false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1993-01-01,1993-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LFHU24FEEZQ647N': "Harmonized Unemployment: Aged 15-24: Females for the Euro Area", + 'LFHU24TTEZQ647N': "Harmonized Unemployment: Aged 15-24: All Persons for the Euro Area", + 'LFHU24MAEZQ647N': "Harmonized Unemployment: Aged 15-24: Males for the Euro Area", + 'LFHUADTTEZQ647N': "Harmonized Unemployment: Aged 25 and Over: All Persons for the Euro Area", + 'LFHUADMAEZQ647N': "Harmonized Unemployment: Aged 25 and Over: Males for the Euro Area", + 'LFHUADFEEZQ647N': "Harmonized Unemployment: Aged 25 and Over: Females for the Euro Area", + 'LFHUTTMAEZQ647N': "Total Harmonized Unemployment: Males for the Euro Area", + 'LFHUTTFEEZQ647N': "Total Harmonized Unemployment: Females for the Euro Area", + 'LFHUTTTTEZQ647N': "Total Harmonized Unemployment: All Persons for the Euro Area"} + description = "Labor Force Survey - quarterly levels, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Labor_Force_Survey_Level_Quarterly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LRHUTTFEEZM156N,LRHUTTMAEZM156N,LRHUTTTTEZM156N,LRHUADTTEZM156N,LRHUADMAEZM156N,LRHUADFEEZM156N,LRHU24FEEZM156N,LRHU24MAEZM156N,LRHU24TTEZM156N&scale=left,left,left,left,left,left,left,left,left&cosd=1993-01-01,1993-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01&coed=2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92&link_values=false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1993-01-01,1993-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LFHU24TTEZQ647S': "Harmonized Unemployment: Aged 15-24: All Persons for the Euro Area", + 'LFHU24MAEZQ647S': "Harmonized Unemployment: Aged 15-24: Males for the Euro Area", + 'LFHU24FEEZQ647S': "Harmonized Unemployment: Aged 15-24: Females for the Euro Area", + 'LFHUTTFEEZQ647S': "Total Harmonized Unemployment: Females for the Euro Area", + 'LFHUTTTTEZQ647S': "Total Harmonized Unemployment: All Persons for the Euro Area", + 'LFHUTTMAEZQ647S': "Total Harmonized Unemployment: Males for the Euro Area", + 'LFHUADMAEZQ647S': "Harmonized Unemployment: Aged 25 and Over: Males for the Euro Area", + 'LFHUADFEEZQ647S': "Harmonized Unemployment: Aged 25 and Over: Females for the Euro Area", + 'LFHUADTTEZQ647S': "Harmonized Unemployment: Aged 25 and Over: All Persons for the Euro Area"} + description = "Labor Force Survey - quarterly levels, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Labor_Force_Survey_Level_Monthly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LFHU24FEEZM647S,LFHU24TTEZM647S,LFHU24MAEZM647S,LFHUADFEEZM647S,LFHUADTTEZM647S,LFHUADMAEZM647S,LFHUTTTTEZM647S,LFHUTTMAEZM647S,LFHUTTFEEZM647S&scale=left,left,left,left,left,left,left,left,left&cosd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01&coed=2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92&link_values=false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LFHU24FEEZM647S': "Harmonized Unemployment: Aged 15-24: Females for the Euro Area", + 'LFHU24TTEZM647S': "Harmonized Unemployment: Aged 15-24: All Persons for the Euro Area", + 'LFHU24MAEZM647S': "Harmonized Unemployment: Aged 15-24: Males for the Euro Area", + 'LFHUADFEEZM647S': "Harmonized Unemployment: Aged 25 and Over: Females for the Euro Area", + 'LFHUADTTEZM647S': "Harmonized Unemployment: Aged 25 and Over: All Persons for the Euro Area", + 'LFHUADMAEZM647S': "Harmonized Unemployment: Aged 25 and Over: Males for the Euro Area", + 'LFHUTTTTEZM647S': "Total Harmonized Unemployment: All Persons for the Euro Area", + 'LFHUTTMAEZM647S': "Total Harmonized Unemployment: Males for the Euro Area", + 'LFHUTTFEEZM647S': "Total Harmonized Unemployment: Females for the Euro Area"} + description = "Labor Force Survey - quarterly levels, Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Labor_Force_Survey_Level_Monthly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=LFHU24MAEZM647N,LFHU24FEEZM647N,LFHU24TTEZM647N,LFHUADMAEZM647N,LFHUADFEEZM647N,LFHUADTTEZM647N,LFHUTTFEEZM647N,LFHUTTTTEZM647N,LFHUTTMAEZM647N&scale=left,left,left,left,left,left,left,left,left&cosd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01&coed=2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01,2021-03-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c,%23b5ca92&link_values=false,false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7,8,9&transformation=lin,lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'LFHU24MAEZM647N': "Harmonized Unemployment: Aged 15-24: Males for the Euro Area", + 'LFHU24FEEZM647N': "Harmonized Unemployment: Aged 15-24: Females for the Euro Area", + 'LFHU24TTEZM647N': "Harmonized Unemployment: Aged 15-24: All Persons for the Euro Area", + 'LFHUADMAEZM647N': "Harmonized Unemployment: Aged 25 and Over: Males for the Euro Area", + 'LFHUADFEEZM647N': "Harmonized Unemployment: Aged 25 and Over: Females for the Euro Area", + 'LFHUADTTEZM647N': "Harmonized Unemployment: Aged 25 and Over: All Persons for the Euro Area", + 'LFHUTTFEEZM647N': "Total Harmonized Unemployment: Females for the Euro Area", + 'LFHUTTTTEZM647N': "Total Harmonized Unemployment: All Persons for the Euro Area", + 'LFHUTTMAEZM647N': "Total Harmonized Unemployment: Males for the Euro Area"} + description = "Labor Force Survey - quarterly levels, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def Production_Monthly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19PRINTO01GYSAM,EA19PRMNCG03IXOBSAM,EA19PRMNCG02IXOBSAM,EA19PRMNVG01IXOBSAM,EA19PRMNTO01IXOBSAM,EA19PRMNIG01IXOBSAM,EA19PRCNTO01IXOBSAM&scale=left,left,left,left,left,left,left&cosd=1976-07-01,1985-01-01,1990-01-01,1985-01-01,1980-01-01,1985-01-01,1985-01-01&coed=2021-02-01,2017-12-01,2018-12-01,2018-12-01,2021-02-01,2018-12-01,2021-02-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2017-12-01,2018-12-01,2018-12-01,2020-02-01,2018-12-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1976-07-01,1985-01-01,1990-01-01,1985-01-01,1980-01-01,1985-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19PRINTO01GYSAM': "Production: Industry: Total industry: Total industry excluding construction for the Euro Area", + 'EA19PRMNCG03IXOBSAM': "Production: Manufacturing: Consumer goods: Non durable goods for the Euro Area", + 'EA19PRMNCG02IXOBSAM': "Production: Manufacturing: Consumer goods: Durable goods for the Euro Area", + 'EA19PRMNVG01IXOBSAM': "Production: Manufacturing: Investment goods: Total for the Euro Area", + 'EA19PRMNTO01IXOBSAM': "Production: Manufacturing: Total manufacturing: Total manufacturing for the Euro Area", + 'EA19PRMNIG01IXOBSAM': "Production: Manufacturing: Intermediate goods: Total for the Euro Area", + 'EA19PRCNTO01IXOBSAM': "Production: Construction: Total construction: Total for the Euro Area"} + description = "Production, Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Production_Quarterly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=PRINTO01EZQ659S,PRMNVG01EZQ661S,PRMNCG02EZQ661S,PRMNCG03EZQ661S,PRMNTO01EZQ661S,PRMNIG01EZQ661S,PRCNTO01EZQ661S&scale=left,left,left,left,left,left,left&cosd=1976-07-01,1985-01-01,1990-01-01,1985-01-01,1980-01-01,1985-01-01,1985-01-01&coed=2020-10-01,2018-10-01,2018-10-01,2017-10-01,2020-10-01,2018-10-01,2020-10-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2018-10-01,2018-10-01,2017-10-01,2020-02-01,2018-10-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1976-07-01,1985-01-01,1990-01-01,1985-01-01,1980-01-01,1985-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'PRINTO01EZQ659S': "Total Industry Production Excluding Construction for the Euro Area", + 'PRMNVG01EZQ661S': "Total Production of Investment Goods for Manufacturing for the Euro Area", + 'PRMNCG02EZQ661S': "Production of Durable Consumer Goods for Manufacturing for the Euro Area", + 'PRMNCG03EZQ661S': "Production of Nondurable Consumer Goods for Manufacturing for the Euro Area", + 'PRMNTO01EZQ661S': "Total Manufacturing Production for the Euro Area", + 'PRMNIG01EZQ661S': "Total Production of Intermediate Goods for Manufacturing for the Euro Area", + 'PRCNTO01EZQ661S': "Total Construction for the Euro Area"} + description = "Production, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def Production_Monthly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19PRMNIG01IXOBM,EA19PRMNTO01IXOBM,EA19PRMNCG02IXOBM,EA19PRMNCG03IXOBM,EA19PRMNVG01IXOBM,EA19PRCNTO01IXOBM,EA19PRINTO01IXOBM&scale=left,left,left,left,left,left,left&cosd=1985-01-01,1980-01-01,1990-01-01,1985-01-01,1985-01-01,1985-01-01,1980-01-01&coed=2018-12-01,2021-02-01,2018-12-01,2018-12-01,2018-12-01,2021-02-01,2021-02-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2018-12-01,2020-02-01,2018-12-01,2018-12-01,2018-12-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1985-01-01,1980-01-01,1990-01-01,1985-01-01,1985-01-01,1985-01-01,1980-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19PRMNIG01IXOBM': "Production: Manufacturing: Intermediate goods: Total for the Euro Area", + 'EA19PRMNTO01IXOBM': "Production: Manufacturing: Total manufacturing: Total manufacturing for the Euro Area", + 'EA19PRMNCG02IXOBM': "Production: Manufacturing: Consumer goods: Durable goods for the Euro Area", + 'EA19PRMNCG03IXOBM': "Production: Manufacturing: Consumer goods: Non durable goods for the Euro Area", + 'EA19PRMNVG01IXOBM': "Production: Manufacturing: Investment goods: Total for the Euro Area", + 'EA19PRCNTO01IXOBM': "Production: Construction: Total construction: Total for the Euro Area", + 'EA19PRINTO01IXOBM': "Production: Industry: Total industry: Total industry excluding construction for the Euro Area"} + description = "Production, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def Production_Quarterly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=PRMNCG03EZQ661N,PRMNCG02EZQ661N,PRMNVG01EZQ661N,PRMNIG01EZQ661N,PRMNTO01EZQ661N,PRINTO01EZQ661N,PRCNTO01EZQ661N&scale=left,left,left,left,left,left,left&cosd=1985-01-01,1990-01-01,1985-01-01,1985-01-01,1980-01-01,1980-01-01,1985-01-01&coed=2018-10-01,2018-10-01,2018-10-01,2018-10-01,2020-10-01,2020-10-01,2020-10-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2018-10-01,2018-10-01,2018-10-01,2018-10-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1985-01-01,1990-01-01,1985-01-01,1985-01-01,1980-01-01,1980-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'PRMNCG03EZQ661N': "Production of Nondurable Consumer Goods for Manufacturing for the Euro Area", + 'PRMNCG02EZQ661N': "Production of Durable Consumer Goods for Manufacturing for the Euro Area", + 'PRMNVG01EZQ661N': "Total Production of Investment Goods for Manufacturing for the Euro Area", + 'PRMNIG01EZQ661N': "Total Production of Intermediate Goods for Manufacturing for the Euro Area", + 'PRMNTO01EZQ661N': "Total Manufacturing Production for the Euro Area", + 'PRINTO01EZQ661N': "Total Industry Production Excluding Construction for the Euro Area", + 'PRCNTO01EZQ661N': "Total Construction for the Euro Area"} + description = "Production, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Sales_Monthly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19SLMNTO02IXOBSAM,EA19SLMNIG02IXOBSAM,EA19SLMNCD02IXOBSAM,EA19SLMNCN02IXOBSAM,EA19SLMNVG02IXOBSAM,EA19SLRTTO01IXOBSAM,EA19SLRTTO02IXOBSAM,EA19SLRTCR03IXOBSAM&scale=left,left,left,left,left,left,left,left&cosd=1980-01-01,1990-01-01,1993-01-01,1995-01-01,1980-01-01,1995-01-01,1995-01-01,1970-01-01&coed=2021-02-01,2018-12-01,2018-12-01,2018-12-01,2018-12-01,2021-02-01,2021-02-01,2018-12-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c&link_values=false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2018-12-01,2018-12-01,2018-12-01,2018-12-01,2020-02-01,2020-02-01,2018-12-01&line_index=1,2,3,4,5,6,7,8&transformation=lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1980-01-01,1990-01-01,1993-01-01,1995-01-01,1980-01-01,1995-01-01,1995-01-01,1970-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19SLMNTO02IXOBSAM': "Sales: Manufacturing: Total manufacturing: Value for the Euro Area", + 'EA19SLMNIG02IXOBSAM': "Sales: Manufacturing: Intermediate goods: Value for the Euro Area", + 'EA19SLMNCD02IXOBSAM': "Sales: Manufacturing: Consumer goods durable: Value for the Euro Area", + 'EA19SLMNCN02IXOBSAM': "Sales: Manufacturing: Consumer goods non durable: Value for the Euro Area", + 'EA19SLMNVG02IXOBSAM': "Sales: Manufacturing: Investment goods: Value for the Euro Area", + 'EA19SLRTTO01IXOBSAM': "Sales: Retail trade: Total retail trade: Volume for the Euro Area", + 'EA19SLRTTO02IXOBSAM': "Sales: Retail trade: Total retail trade: Value for the Euro Area", + 'EA19SLRTCR03IXOBSAM': "Sales: Retail trade: Car registration: Passenger cars for the Euro Area"} + description = "Sales, Monthly, Seasonally Adjusted" + return df, name_list, description + + +def Sales_Quarterly_Adj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=SLMNTO02EZQ661S,SLMNVG02EZQ661S,SLMNCD02EZQ661S,SLMNCN02EZQ661S,SLMNIG02EZQ661S,SLRTTO02EZQ661S,SLRTTO01EZQ659S,SLRTCR03EZQ661S&scale=left,left,left,left,left,left,left,left&cosd=1980-01-01,1980-01-01,1993-01-01,1995-01-01,1990-01-01,1995-01-01,1996-01-01,1970-01-01&coed=2020-10-01,2018-10-01,2018-10-01,2018-10-01,2018-10-01,2020-10-01,2020-10-01,2018-10-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd,%23a47d7c&link_values=false,false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin,lin&fgsnd=2020-02-01,2018-10-01,2018-10-01,2018-10-01,2018-10-01,2020-02-01,2020-02-01,2018-10-01&line_index=1,2,3,4,5,6,7,8&transformation=lin,lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1980-01-01,1980-01-01,1993-01-01,1995-01-01,1990-01-01,1995-01-01,1996-01-01,1970-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'SLMNTO02EZQ661S': "Sales Value of Total Manufactured Goods for the Euro Area", + 'SLMNVG02EZQ661S': "Sales Value of Manufactured Investment Goods for the Euro Area", + 'SLMNCD02EZQ661S': "Sales Value of Manufactured Durable Consumer Goods for the Euro Area", + 'SLMNCN02EZQ661S': "Sales Value of Manufactured Nondurable Consumer Goods for the Euro Area", + 'SLMNIG02EZQ661S': "Sales Value of Manufactured Intermediate Goods for the Euro Area", + 'SLRTTO02EZQ661S': "Value of Total Retail Trade sales for the Euro Areaa", + 'SLRTTO01EZQ659S': "Volume of Total Retail Trade sales for the Euro Area", + 'SLRTCR03EZQ661S': "Retail Trade Sales: Passenger Car Registrations for the Euro Area"} + description = "Sales, Quarterly, Seasonally Adjusted" + return df, name_list, description + + +def Sales_Monthly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=EA19SLMNIG02IXOBM,EA19SLRTTO02IXOBM,EA19SLMNCD02IXOBM,EA19SLMNCN02IXOBM,EA19SLMNTO02IXOBM,EA19SLRTCR03IXOBM,EA19SLRTTO01IXOBM&scale=left,left,left,left,left,left,left&cosd=1990-01-01,1995-01-01,1993-01-01,1995-01-01,1980-01-01,1985-01-01,1995-01-01&coed=2018-12-01,2021-02-01,2018-12-01,2018-12-01,2021-02-01,2021-03-01,2021-02-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Monthly,Monthly,Monthly,Monthly,Monthly,Monthly,Monthly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2018-12-01,2020-02-01,2018-12-01,2018-12-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1990-01-01,1995-01-01,1993-01-01,1995-01-01,1980-01-01,1985-01-01,1995-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'EA19SLMNIG02IXOBM': "Sales: Manufacturing: Intermediate goods: Value for the Euro Area", + 'EA19SLRTTO02IXOBM': "Sales: Retail trade: Total retail trade: Value for the Euro Area", + 'EA19SLMNCD02IXOBM': "Sales: Manufacturing: Consumer goods durable: Value for the Euro Area", + 'EA19SLMNCN02IXOBM': "Sales: Manufacturing: Consumer goods non durable: Value for the Euro Area", + 'EA19SLMNTO02IXOBM': "Sales: Manufacturing: Total manufacturing: Value for the Euro Area", + 'EA19SLRTCR03IXOBM': "Sales: Retail trade: Car registration: Passenger cars for the Euro Area", + 'EA19SLRTTO01IXOBM': "Sales: Retail trade: Total retail trade: Volume for the Euro Area"} + description = "Sales, Monthly, Not Seasonally Adjusted" + return df, name_list, description + + +def Sales_Quarterly_NAdj(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=SLMNIG02EZQ661N,SLMNTO02EZQ661N,SLMNCD02EZQ661N,SLMNCN02EZQ661N,SLRTTO01EZQ661N,SLRTTO02EZQ661N,SLRTCR03EZQ661N&scale=left,left,left,left,left,left,left&cosd=1990-01-01,1980-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1985-01-01&coed=2018-10-01,2020-10-01,2018-10-01,2018-10-01,2020-10-01,2020-10-01,2021-01-01&line_color=%234572a7,%23aa4643,%2389a54e,%2380699b,%233d96ae,%23db843d,%2392a8cd&link_values=false,false,false,false,false,false,false&line_style=solid,solid,solid,solid,solid,solid,solid&mark_type=none,none,none,none,none,none,none&mw=3,3,3,3,3,3,3&lw=2,2,2,2,2,2,2&ost=-99999,-99999,-99999,-99999,-99999,-99999,-99999&oet=99999,99999,99999,99999,99999,99999,99999&mma=0,0,0,0,0,0,0&fml=a,a,a,a,a,a,a&fq=Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly,Quarterly&fam=avg,avg,avg,avg,avg,avg,avg&fgst=lin,lin,lin,lin,lin,lin,lin&fgsnd=2018-10-01,2020-02-01,2018-10-01,2018-10-01,2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3,4,5,6,7&transformation=lin,lin,lin,lin,lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07,2021-06-07&nd=1990-01-01,1980-01-01,1993-01-01,1995-01-01,1995-01-01,1995-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'SLMNIG02EZQ661N': "Sales Value of Manufactured Intermediate Goods for the Euro Area", + 'SLMNTO02EZQ661N': "Sales Value of Total Manufactured Goods for the Euro Area", + 'SLMNCD02EZQ661N': "Sales Value of Manufactured Durable Consumer Goods for the Euro Area", + 'SLMNCN02EZQ661N': "Sales Value of Manufactured Nondurable Consumer Goods for the Euro Area", + 'SLRTTO01EZQ661N': "Volume of Total Retail Trade sales for the Euro Area", + 'SLRTTO02EZQ661N': "Value of Total Retail Trade sales for the Euro Area", + 'SLRTCR03EZQ661N': "Retail Trade Sales: Passenger Car Registrations for the Euro Area"} + description = "Sales, Quarterly, Not Seasonally Adjusted" + return df, name_list, description + + +def Cumsumer_Opinion_Survey(): + tmp_url = url["fred_econ"] + "bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=748&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=CSCICP02EZM460S,CSESFT02EZM460S,CSINFT02EZM460S&scale=left,left,left&cosd=1973-01-01,1985-01-01,1985-01-01&coed=2021-04-01,2021-04-01,2021-04-01&line_color=%234572a7,%23aa4643,%2389a54e&link_values=false,false,false&line_style=solid,solid,solid&mark_type=none,none,none&mw=3,3,3&lw=2,2,2&ost=-99999,-99999,-99999&oet=99999,99999,99999&mma=0,0,0&fml=a,a,a&fq=Monthly,Monthly,Monthly&fam=avg,avg,avg&fgst=lin,lin,lin&fgsnd=2020-02-01,2020-02-01,2020-02-01&line_index=1,2,3&transformation=lin,lin,lin&vintage_date=2021-06-07,2021-06-07,2021-06-07&revision_date=2021-06-07,2021-06-07,2021-06-07&nd=1973-01-01,1985-01-01,1985-01-01" + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + r = requests.get(tmp_url, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df["DATE"] = pd.to_datetime(df["DATE"], format="%Y-%m-%d") + #df = df[list(df.columns[1:])].replace(".", np.nan).astype(float) + name_list = { + 'CSCICP02EZM460S': "Consumer Opinion Surveys: Confidence Indicators: Composite Indicators: European Commission and National Indicators for the Euro Area", + 'CSESFT02EZM460S': "Consumer Opinion Surveys: Economic Situation: Future Tendency: European Commission Indicator for the Euro Area", + 'CSINFT02EZM460S': "Consumer Opinion Surveys: Consumer Prices: Future Tendency of Inflation: European Commission and National Indicators for the Euro Area"} + description = "Consumer opinion surveys, Monthly, Seasonally Adjusted" + return df, name_list, description + + +class ecb_data(object): + def __init__(self, url=url["ecb"]): + self.url = url + + def codebook(self): + return "please follow the ECB's codebook: https://sdw.ecb.europa.eu/browse.do?node=9691101" + + def get_data(self, + datacode="ICP", + key="M.U2.N.000000.4.ANR", + startdate="2000-01-01", + enddate="2020-01-01"): + """ + """ + tmp_url = self.url + "{}/".format(datacode) + "{}".format(key) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random, 'Accept': 'text/csv'} + request_params = { + "startPeriod": "{}".format(startdate), + "endPeriod": "{}".format(enddate) + } + r = requests.get( + tmp_url, + params=request_params, + headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + return df + + +class eurostat_data(object): + def __init__(self, url=url["eurostat"]): + self.url = url + + def codebook(self): + return "please follow the EuroStat's codebook: \nhttps://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&dir=dic" + + def get_data(self, + datasetcode="nama_10_gdp", + precision="1", + unit="CP_MEUR", + na_item="B1GQ", + time="2020"): + """ + """ + tmp_url = self.url + "{}".format(datasetcode) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random, 'Accept': 'text/csv'} + request_params = { + "precision": "{}".format(precision), + "unit": "{}".format(unit), + "na_item": "{}".format(na_item), + "time": "{}".format(time) + } + r = requests.get( + tmp_url, + params=request_params, + headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text) + value = data_json['value'] + abb = data_json['dimension']['geo']['category']['index'] + abb = {abb[k]: k for k in abb} + geo = data_json['dimension']['geo']['category']['label'] + geo_list = [abb[int(k)] for k in list(value.keys())] + geo = [geo[k] for k in geo_list] + df = pd.DataFrame( + {"Geo": geo, "{}".format(na_item): list(value.values())}) + return df + + +if __name__ == "__main__": + data, name_list = CPI_monthly() diff --git a/CEDA/macroecon/us.py b/CEDA/macroecon/us.py new file mode 100644 index 0000000..96cbb6d --- /dev/null +++ b/CEDA/macroecon/us.py @@ -0,0 +1,884 @@ +import pandas as pd +import numpy as np +import requests +from fake_useragent import UserAgent +import io +import os +import demjson + +# Main Economic Indicators: https://alfred.stlouisfed.org/release?rid=205 +url = { + "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?", + "philfed": "https://www.philadelphiafed.org/surveys-and-data/real-time-data-research/", + "chicagofed": "https://www.chicagofed.org/~/media/publications/"} + + +def gdp_quarterly(startdate="1947-01-01", enddate="2021-01-01"): + """ + Full Name: Gross Domestic Product + Description: Billions of Dollars, Quarterly, Seasonally Adjusted Annual Rate + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "GDP", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + return df + + +def gdpc1_quarterly(startdate="1947-01-01", enddate="2021-01-01"): + """ + Full Name: Real Gross Domestic Product + Description: Billions of Chained 2012 Dollars, Quarterly, Seasonally Adjusted Annual Rate + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "GDPC1", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + return df + + +def oecd_gdp_monthly(startdate="1947-01-01", enddate="2021-01-01"): + """ + Full Name: Real Gross Domestic Product + Description: Billions of Chained 2012 Dollars, Quarterly, Seasonally Adjusted Annual Rate + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USALORSGPNOSTSAM", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + return df + + +def payems_monthly(startdate="1939-01-01", enddate="2021-01-01"): + """ + Full Name: All Employees, Total Nonfarm + Description: Thousands of Persons,Seasonally Adjusted, Monthly + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "PAYEMS", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + return df + + +def unrate(startdate="1948-01-01", enddate="2021-01-01"): + """ + Full Name: Unemployment Rate: Aged 15-64: All Persons for the United States + Description: Percent, Seasonally Adjusted, Monthly, Quarterly and Annually + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "LRUN64TTUSM156S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "LRUN64TTUSQ156S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "LRUN64TTUSA156S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_monthly, + df_quarterly, + on="DATE", + direction="backward") + df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") + df.columns = ["Date", "UR_Monthly", "UR_Quarterly", "UR_Annually"] + return df + + +def erate(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name: Employment Rate: Aged 25-54: All Persons for the United States + Description: Percent,Seasonally Adjusted, Monthly, Quarterly and Annually + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "LREM25TTUSM156S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "LREM25TTUSQ156S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "LREM25TTUSA156S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_monthly, + df_quarterly, + on="DATE", + direction="backward") + df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") + df.columns = ["Date", "ER_Monthly", "ER_Quarterly", "ER_Annually"] + + +def pce_monthly(startdate="1959-01-01", enddate="2021-01-01"): + """ + Full Name: PCE + Description: Percent, Monthly, Seasonally Adjusted + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "PCE", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + return df + + +def cpi(startdate="1960-01-01", enddate="2021-01-01"): + """ + Full Name: Consumer Price Index: Total All Items for the United States + Description: Percent, Monthly, Quarterly and Annually, Seasonally Adjusted + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "CPALTT01USM661S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "CPALTT01USQ661S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "CPALTT01USA661S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_monthly, + df_quarterly, + on="DATE", + direction="backward") + df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") + df.columns = ["Date", "CPI_Monthly", "CPI_Quarterly", "CPI_Annually"] + return df + + +def m1(startdate="1960-01-01", enddate="2021-01-01"): + """ + Full Name: Consumer Price Index: M3 for the United States + Description: Growth Rate Previous Period, Monthly, Quarterly and Annually, Seasonally Adjusted + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "WM1NS", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_weekly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_weekly["DATE"] = pd.to_datetime(df_weekly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "MANMM101USM657S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "MANMM101USQ657S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "MANMM101USA657S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof(df_weekly, df_monthly, on="DATE", direction="backward") + df = pd.merge_asof(df, df_quarterly, on="DATE", direction="backward") + df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") + df.columns = [ + "Date", + "M1_weekly", + "M1_Monthly", + "M1_Quarterly", + "M1_Annually"] + return df + + +def m2(startdate="1960-01-01", enddate="2021-01-01"): + """ + Full Name: M2 Money Stock + Description: Seasonally Adjusted, Weekly, Monthly, Quarterly and Annually, Seasonally Adjusted + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "WM2NS", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_weekly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_weekly["DATE"] = pd.to_datetime(df_weekly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "M2SL", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + df = pd.merge_asof(df_weekly, df_monthly, on="DATE", direction="backward") + df.columns = ["Date", "M2_Weekly", "M2_Monthly"] + + +def m3(startdate="1960-01-01", enddate="2021-01-01"): + """ + Full Name: Consumer Price Index: M3 for the United States + Description: Growth Rate Previous Period, Monthly, Quarterly and Annually, Seasonally Adjusted + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "MABMM301USM657S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "MABMM301USQ657S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "MABMM301USA657S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_monthly, + df_quarterly, + on="DATE", + direction="backward") + df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") + df.columns = ["Date", "M3_Monthly", "M3_Quarterly", "M3_Annually"] + return df + + +def ltgby_10(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name: Long-Term Government Bond Yields: 10-year: Main (Including Benchmark) for the United States + Description: Percent,Not Seasonally Adjusted, Monthly, Quarterly and Annually + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "IRLTLT01USM156N", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "IRLTLT01USQ156N", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "IRLTLT01USA156N", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_monthly, + df_quarterly, + on="DATE", + direction="backward") + df = pd.merge_asof(df, df_annually, on="DATE", direction="backward") + df.columns = ["Date", "ltgby_Monthly", "ltgby_Quarterly", "ltgby_Annually"] + return df + + +def gdp_ipd(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name: Long-Term Government Bond Yields: 10-year: Main (Including Benchmark) for the United States + Description: Percent,Not Seasonally Adjusted, Monthly, Quarterly and Annually + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USAGDPDEFQISMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USAGDPDEFAISMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_quarterly, + df_annually, + on="DATE", + direction="backward") + df.columns = ["Date", "gdp_ipd_Quarterly", "gdp_ipd_Annually"] + return df + + +def cci(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name: Consumer Opinion Surveys: Confidence Indicators: Composite Indicators: OECD Indicator for the United States + Description: Normalised (Normal=100), Seasonally Adjusted, Monthly + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "CSCICP03USM665S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df.columns = ["Date", "CCI_Monthly"] + return df + + +def bci(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name: Consumer Opinion Surveys: Confidence Indicators: Composite Indicators: OECD Indicator for the United States + Description: Normalised (Normal=100), Seasonally Adjusted, Monthly + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "BSCICP03USM665S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df.columns = ["Date", "BCI_Annually"] + return df + + +def ibr_3(startdate="1965-01-01", enddate="2021-01-01"): + """ + Full Name: 3-Month or 90-day Rates and Yields: Interbank Rates for the United States + Description: Percent, Not Seasonally Adjusted, Monthly and Quarterly + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "IR3TIB01USM156N", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "IR3TIB01USQ156N", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_quarterly, + df_quarterly, + on="DATE", + direction="backward") + df.columns = ["Date", "ibr3_monthly", "ibr3_Quarterly"] + + +def gfcf_3(startdate="1965-01-01", enddate="2021-01-01"): + """ + Full Name: Gross Fixed Capital Formation in United States + Description: United States Dollars,Not Seasonally Adjusted, Quarterly and Annually + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USAGFCFQDSMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USAGFCFADSMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_quarterly, + df_quarterly, + on="DATE", + direction="backward") + df.columns = ["Date", "ibr3_monthly", "ibr3_Annually"] + + +def pfce(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name: Employment Rate: Aged 25-54: All Persons for the United States + Description: Percent,Seasonally Adjusted, Monthly, Quarterly and Annually + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USAPFCEQDSMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USAPFCEADSMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_quarterly, + df_annually, + on="DATE", + direction="backward") + df.columns = ["Date", "PFCE_Quarterly", "PFCE_Annually"] + + +def tlp(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name: Early Estimate of Quarterly ULC Indicators: Total Labor Productivity for the United States + Description: Growth Rate Previous Period,Seasonally Adjusted, Quarterly and YoY + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "ULQELP01USQ657S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_quarterly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_quarterly["DATE"] = pd.to_datetime( + df_quarterly["DATE"], format="%Y-%m-%d") + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "ULQELP01USQ659S", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_quarterly, + df_annually, + on="DATE", + direction="backward") + df.columns = ["Date", "PFCE_Quarterly", "PFCE_Quarterly_YoY"] + + +def rt(startdate="1955-01-01", enddate="2021-01-01"): + """ + Full Name:Total Retail Trade in United States + Description: Monthly and Anually + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USASARTMISMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_monthly = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_monthly["DATE"] = pd.to_datetime(df_monthly["DATE"], format="%Y-%m-%d") + request_header = {"User-Agent": ua.random} + request_params = { + "id": "USASARTAISMEI", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_annually = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_annually["DATE"] = pd.to_datetime( + df_annually["DATE"], format="%Y-%m-%d") + df = pd.merge_asof( + df_monthly, + df_annually, + on="DATE", + direction="backward") + df.columns = ["Date", "RT_Quarterly", "RT_Annually"] + return df + + +def bir(startdate="2003-01-01", enddate="2021-01-01"): + """ + Full Name:Total Retail Trade in United States + Description: Monthly and Anually + Return: pd.DataFrame + """ + tmp_url = url["fred_econ"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "id": "T5YIE", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_5y = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_5y["DATE"] = pd.to_datetime(df_5y["DATE"], format="%Y-%m-%d") + request_header = {"User-Agent": ua.random} + request_params = { + "id": "T10YIE", + "cosd": "{}".format(startdate), + "coed": "{}".format(enddate) + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.content + df_10y = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + df_10y["DATE"] = pd.to_datetime(df_10y["DATE"], format="%Y-%m-%d") + df = pd.merge_asof(df_5y, df_10y, on="DATE", direction="backward") + df.columns = ["Date", "BIR_5y", "BIR_10y"] + return df + + +def adsbci(): + """ + An index designed to track real business conditions at high observation frequency + """ + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = url["philfed"] + "ads" + r = requests.get(tmp_url, headers=request_header) + file = open("ads_temp.xls", "wb") + file.write(r.content) + file.close() + df = pd.read_excel("ads_temp.xls") + df.columns = ["Date", "ADS_Index"] + df['Date'] = pd.to_datetime(df["Date"], format="%Y:%m:%d") + os.remove("ads_temp.xls") + return df + + +def pci(): + """ + Tracks the degree of political disagreement among U.S. politicians at the federal level, Monthly + """ + df = pd.read_excel( + "https://www.philadelphiafed.org/-/media/frbp/assets/data-visualizations/partisan-conflict.xlsx") + df["Date"] = df["Year"].astype(str) + df["Month"] + df["Date"] = pd.to_datetime(df["Date"], format="%Y%B") + df = df.drop(["Year", "Month"], axis=1) + df = df[["Date", "Partisan Conflict"]] + return df + + +def inflation_noewcasting(): + """ + + """ + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + tmp_url = "https://www.clevelandfed.org/~/media/files/charting/%20nowcast_quarter.json" + + r = requests.get(tmp_url, headers=request_header) + tmp_df = pd.DataFrame(demjson.decode(r.text)) + df = pd.DataFrame() + for i in range(0, len(tmp_df)): + date = tmp_df['chart'][i]['subcaption'][:4] + "/" + \ + pd.DataFrame(tmp_df["dataset"][i][0]['data'])['tooltext'].str.extract(r"\b(0?[1-9]|1[0-2])/(0?[1-9]|[12][0-9]|3[01])\b")[0] + "/" + \ + pd.DataFrame(tmp_df["dataset"][i][0]['data'])['tooltext'].str.extract(r"\b(0?[1-9]|1[0-2])/(0?[1-9]|[12][0-9]|3[01])\b")[1] + CPI_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[0])["value"] + C_CPI_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[1])["value"] + PCE_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[2])["value"] + C_PCE_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[3])["value"] + A_CPI_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[4])["value"] + A_C_CPI_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[5])["value"] + A_PCE_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[6])["value"] + A_C_PCE_I = pd.DataFrame( + (pd.DataFrame(tmp_df["dataset"][i])['data'])[7])["value"] + tmp_df2 = pd.DataFrame({"date": date, + "CPI_I": CPI_I, + "C_CPI_I": C_CPI_I, + "PCE_I": PCE_I, + "C_PCE_I": C_PCE_I, + "A_CPI_I": A_CPI_I, + "A_C_CPI_I": A_C_CPI_I, + "A_PCE_I": A_PCE_I, + "A_C_PCE_I": A_C_PCE_I}) + df = pd.concat([df, tmp_df2], axis=0) + df.reset_index(drop=True, inplace=True) + + df.replace('', np.nan, inplace=True) + return df + + +def bbki(): + tmp_url = url["chicagofed"] + "bbki/bbki-monthly-data-series-csv.csv" + df = pd.read_csv(tmp_url) + return df + + +def cfnai(): + tmp_url = url["chicagofed"] + "cfnai/cfnai-data-series-csv.csv" + df = pd.read_csv(tmp_url) + return df + + +def cfsbc(): + tmp_url = url["chicagofed"] + "cfsbc-activity-index-csv.csv" + df = pd.read_csv(tmp_url) + return df + + +def nfci(): + tmp_url = url["chicagofed"] + "nfci/decomposition-nfci-csv.csv" + df = pd.read_csv(tmp_url) + return df + + +def nfci(): + tmp_url = url["chicagofed"] + "nfci/decomposition-anfci-csv.csv" + df = pd.read_csv(tmp_url) + return df diff --git a/CEDA/market/__init__.py b/CEDA/market/__init__.py new file mode 100644 index 0000000..89ed024 --- /dev/null +++ b/CEDA/market/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# time: 05/29/2021 UTC+8 +# author: terencelau +# email: t_lau@uicstat.com diff --git a/CEDA/market/duka.py b/CEDA/market/duka.py new file mode 100644 index 0000000..6296c0b --- /dev/null +++ b/CEDA/market/duka.py @@ -0,0 +1,50 @@ +import re +import io +import requests +import demjson +import pandas as pd +from bs4 import BeautifulSoup +from datetime import datetime +from urllib.parse import quote, urlencode +from fake_useragent import UserAgent + +url = { + "dukascopy": "http://data.uicstat.com/api_1.0" +} + + +def dukascopy( + instrument: str, + startdate: str, + enddate: str, + timeframe: str, + pricetype: str, + volume: bool, + flat: bool): + tmp_url = url["dukascopy"] + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = { + "instrument": "{}".format(instrument), + "startdate": "{}".format(startdate), + "enddate": "{}".format(enddate), + "timeframe": "{}".format(timeframe), + "pricetype": "{}".format(pricetype), + "volume": "{}".format(volume), + "flat": "{}".format(flat) + + } + r = requests.get(tmp_url, params=request_params, headers=request_header) + data_text = r.text + data_json = demjson.decode(data_text) + df = pd.DataFrame(data_json['result']) + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms') + df.columns = [ + "Date", + "Open", + "High", + "Low", + "Close", + "Volume" + ] + return df diff --git a/CEDA/market/marketwatch.py b/CEDA/market/marketwatch.py new file mode 100644 index 0000000..2a4ad28 --- /dev/null +++ b/CEDA/market/marketwatch.py @@ -0,0 +1,252 @@ +import re +import io +import requests +import demjson +import pandas as pd +from bs4 import BeautifulSoup +from datetime import datetime +from urllib.parse import quote, urlencode +from fake_useragent import UserAgent + +url = { + "moneywatch": "https://www.marketwatch.com/investing/" +} + + +def forex(instrument="eurusd", startdate="2019-01-01", enddate="2021-01-01"): + startdate = datetime.strptime(startdate, "%Y-%m-%d").strftime("%m/%d/%y") + enddate = datetime.strptime(enddate, "%Y-%m-%d").strftime("%m/%d/%y") + df = pd.DataFrame() + + def _FX(instrument="eurusd", startdate="01/01/2020", enddate="01/01/2021"): + """ + https://www.marketwatch.com/investing/ + """ + tmp_url = url["moneywatch"] + \ + "currency/{}/downloaddatapartial".format(instrument) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = urlencode({ + "startdate": r"{}".format(startdate), + "enddate": r"{}".format(enddate), + "daterange": "d30", + "frequency": "p1d", + "csvdownload": "true", + "downloadpartial": "false", + "newdates": "false"}, quote_via=quote) + r = requests.get(tmp_url, params=request_params.replace( + "%2F", "/").replace("%20", " ").replace("%3A", ":"), headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + Date = [] + for i in range(0, len(df)): + Date.append(datetime.strptime(df["Date"][i], "%m/%d/%Y")) + + df["Date"] = Date + return df + + for i in range(int(startdate[6:10]), int(enddate[6:10])): + if i == int(startdate[6:10]): + tmp_startdate = startdate + else: + tmp_startdate = "01/01/" + str(i) + " 00:00:00" + if (i + 1) == int(enddate[6:10]): + tmp_enddate = enddate + else: + tmp_enddate = "01/01/" + str(i + 1) + " 00:00:00" + + tmp_df = _FX( + instrument=instrument, + startdate=tmp_startdate, + enddate=tmp_enddate) + if i == int(startdate[6:10]): + df = tmp_df + else: + df = pd.concat([tmp_df, df], axis=0) + + df = df.reset_index(drop=True) + return df + + +def index(instrument="vix", startdate="2019-01-01", enddate="2021-01-01"): + startdate = datetime.strptime(startdate, "%Y-%m-%d").strftime("%m/%d/%y") + enddate = datetime.strptime(enddate, "%Y-%m-%d").strftime("%m/%d/%y") + df = pd.DataFrame() + + def _index(instrument="vix", startdate="01/01/2020", enddate="01/01/2021"): + """ + https://www.marketwatch.com/investing/ + """ + tmp_url = url["moneywatch"] + \ + "index/{}/downloaddatapartial".format(instrument) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = urlencode({ + "startdate": r"{}".format(startdate), + "enddate": r"{}".format(enddate), + "daterange": "d30", + "frequency": "p1d", + "csvdownload": "true", + "downloadpartial": "false", + "newdates": "false"}, quote_via=quote) + r = requests.get(tmp_url, params=request_params.replace( + "%2F", "/").replace("%20", " ").replace("%3A", ":"), headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + Date = [] + for i in range(0, len(df)): + Date.append(datetime.strptime(df["Date"][i], "%m/%d/%Y")) + + df["Date"] = Date + return df + + for i in range(int(startdate[6:10]), int(enddate[6:10])): + if i == int(startdate[6:10]): + tmp_startdate = startdate + else: + tmp_startdate = "01/01/" + str(i) + " 00:00:00" + if (i + 1) == int(enddate[6:10]): + tmp_enddate = enddate + else: + tmp_enddate = "01/01/" + str(i + 1) + " 00:00:00" + + tmp_df = _index( + instrument=instrument, + startdate=tmp_startdate, + enddate=tmp_enddate) + if i == int(startdate[6:10]): + df = tmp_df + else: + df = pd.concat([tmp_df, df], axis=0) + + df = df.reset_index(drop=True) + return df + + +def crypto(instrument="btcusd", startdate="2019-01-01", enddate="2021-01-01"): + startdate = datetime.strptime(startdate, "%Y-%m-%d").strftime("%m/%d/%y") + enddate = datetime.strptime(enddate, "%Y-%m-%d").strftime("%m/%d/%y") + df = pd.DataFrame() + + def _crypto( + instrument="btcusd", + startdate="01/01/2020", + enddate="01/01/2021"): + """ + https://www.marketwatch.com/investing/ + """ + tmp_url = url["moneywatch"] + \ + "cryptocurrency/{}/downloaddatapartial".format(instrument) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = urlencode({ + "startdate": r"{}".format(startdate), + "enddate": r"{}".format(enddate), + "daterange": "d30", + "frequency": "p1d", + "csvdownload": "true", + "downloadpartial": "false", + "newdates": "false"}, quote_via=quote) + r = requests.get(tmp_url, params=request_params.replace( + "%2F", "/").replace("%20", " ").replace("%3A", ":"), headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + Date = [] + for i in range(0, len(df)): + Date.append(datetime.strptime(df["Date"][i], "%m/%d/%Y")) + + df["Date"] = Date + return df + + for i in range(int(startdate[6:10]), int(enddate[6:10])): + if i == int(startdate[6:10]): + tmp_startdate = startdate + else: + tmp_startdate = "01/01/" + str(i) + " 00:00:00" + if (i + 1) == int(enddate[6:10]): + tmp_enddate = enddate + else: + tmp_enddate = "01/01/" + str(i + 1) + " 00:00:00" + + tmp_df = _crypto( + instrument=instrument, + startdate=tmp_startdate, + enddate=tmp_enddate) + if i == int(startdate[6:10]): + df = tmp_df + else: + df = pd.concat([tmp_df, df], axis=0) + + df = df.reset_index(drop=True) + return df + + +def stock( + countrycode="cn", + instrument="601988", + startdate="2019-01-01", + enddate="2021-01-01"): + startdate = datetime.strptime(startdate, "%Y-%m-%d").strftime("%m/%d/%y") + enddate = datetime.strptime(enddate, "%Y-%m-%d").strftime("%m/%d/%y") + df = pd.DataFrame() + + def _stock( + countrycode="cn", + instrument="601988", + startdate="01/01/2020", + enddate="01/01/2021"): + """ + https://www.marketwatch.com/investing/ + """ + tmp_url = url["moneywatch"] + \ + "stock/{}/downloaddatapartial".format(instrument) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = urlencode({ + "startdate": r"{}".format(startdate), + "enddate": r"{}".format(enddate), + "daterange": "d30", + "frequency": "p1d", + "csvdownload": "true", + "downloadpartial": "false", + "newdates": "false", + "countrycode": "{}".format(countrycode)}, quote_via=quote) + r = requests.get(tmp_url, params=request_params.replace( + "%2F", "/").replace("%20", " ").replace("%3A", ":"), headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + Date = [] + for i in range(0, len(df)): + Date.append(datetime.strptime(df["Date"][i], "%m/%d/%Y")) + + df["Date"] = Date + return df + + for i in range(int(startdate[6:10]), int(enddate[6:10])): + if i == int(startdate[6:10]): + tmp_startdate = startdate + else: + tmp_startdate = "01/01/" + str(i) + " 00:00:00" + if (i + 1) == int(enddate[6:10]): + tmp_enddate = enddate + else: + tmp_enddate = "01/01/" + str(i + 1) + " 00:00:00" + + if countrycode == "us": + countrycode = "" + tmp_df = _stock( + countrycode=countrycode, + instrument=instrument, + startdate=tmp_startdate, + enddate=tmp_enddate) + if i == int(startdate[6:10]): + df = tmp_df + else: + df = pd.concat([tmp_df, df], axis=0) + + df = df.reset_index(drop=True) + return df + + +if __name__ == "__main__": + data = FX()