From f47db3fd096f3832c6c1387636cb4a0f420d9988 Mon Sep 17 00:00:00 2001 From: TerenceLiu98 Date: Mon, 7 Jun 2021 15:28:29 +0800 Subject: [PATCH] v0.1.2 --- CEDA/Market/marketwatch.py | 109 +++++++++++++++++++++++++++++++++++-- CEDA/__init__.py | 12 ++-- CEDA/source.md | 7 +++ README.md | 2 +- setup.py | 2 +- 5 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 CEDA/source.md diff --git a/CEDA/Market/marketwatch.py b/CEDA/Market/marketwatch.py index f12f6db..0640981 100644 --- a/CEDA/Market/marketwatch.py +++ b/CEDA/Market/marketwatch.py @@ -61,12 +61,12 @@ def forex(instrument = "eurusd", startdate = "2019-01-01", enddate = "2021-01-01 df = df.reset_index(drop = True) return df -def index(instrument = "eurusd", startdate = "2019-01-01", enddate = "2021-01-01"): +def index(instrument = "vix", startdate = "2019-01-01", enddate = "2021-01-01"): startdate = datetime.strptime(startdate, "%Y-%m-%d").strftime("%m/%d/%y") enddate = datetime.strptime(enddate, "%Y-%m-%d").strftime("%m/%d/%y") df = pd.DataFrame() - def _FX(instrument = "eurusd", startdate = "01/01/2020", enddate = "01/01/2021"): + def _index(instrument = "vix", startdate = "01/01/2020", enddate = "01/01/2021"): """ https://www.marketwatch.com/investing/ """ @@ -101,7 +101,7 @@ def index(instrument = "eurusd", startdate = "2019-01-01", enddate = "2021-01-01 else: tmp_enddate = "01/01/" + str(i+1) + " 00:00:00" - tmp_df = _FX(instrument=instrument, startdate = tmp_startdate, enddate = tmp_enddate) + tmp_df = _index(instrument=instrument, startdate = tmp_startdate, enddate = tmp_enddate) if i == int(startdate[6:10]): df = tmp_df else: @@ -110,8 +110,109 @@ def index(instrument = "eurusd", startdate = "2019-01-01", enddate = "2021-01-01 df = df.reset_index(drop = True) return df +def crypto(instrument = "btcusd", startdate = "2019-01-01", enddate = "2021-01-01"): + startdate = datetime.strptime(startdate, "%Y-%m-%d").strftime("%m/%d/%y") + enddate = datetime.strptime(enddate, "%Y-%m-%d").strftime("%m/%d/%y") + df = pd.DataFrame() + def _crypto(instrument = "btcusd", startdate = "01/01/2020", enddate = "01/01/2021"): + """ + https://www.marketwatch.com/investing/ + """ + tmp_url = url["moneywatch"] + "cryptocurrency/{}/downloaddatapartial".format(instrument) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = urlencode({ + "startdate": r"{}".format(startdate), + "enddate": r"{}".format(enddate), + "daterange": "d30", + "frequency": "p1d", + "csvdownload": "true", + "downloadpartial": "false", + "newdates": "false"}, quote_via= quote) + r = requests.get(tmp_url, params=request_params.replace("%2F", "/").replace("%20", " ").replace("%3A", ":"), headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + Date = [] + for i in range(0, len(df)): + Date.append(datetime.strptime(df["Date"][i], "%m/%d/%Y")) + + df["Date"] = Date + return df + for i in range(int(startdate[6:10]), int(enddate[6:10])): + if i == int(startdate[6:10]): + tmp_startdate = startdate + else: + tmp_startdate = "01/01/" + str(i) + " 00:00:00" + if (i+1) == int(enddate[6:10]): + tmp_enddate = enddate + else: + tmp_enddate = "01/01/" + str(i+1) + " 00:00:00" + + tmp_df = _crypto(instrument=instrument, startdate = tmp_startdate, enddate = tmp_enddate) + if i == int(startdate[6:10]): + df = tmp_df + else: + df = pd.concat([tmp_df, df], axis=0) + + df = df.reset_index(drop = True) + return df + +def stock(countrycode = "cn", instrument = "601988", startdate = "2019-01-01", enddate = "2021-01-01"): + startdate = datetime.strptime(startdate, "%Y-%m-%d").strftime("%m/%d/%y") + enddate = datetime.strptime(enddate, "%Y-%m-%d").strftime("%m/%d/%y") + df = pd.DataFrame() + + def _stock(countrycode = "cn", instrument = "601988", startdate = "01/01/2020", enddate = "01/01/2021"): + """ + https://www.marketwatch.com/investing/ + """ + tmp_url = url["moneywatch"] + "stock/{}/downloaddatapartial".format(instrument) + ua = UserAgent(verify_ssl=False) + request_header = {"User-Agent": ua.random} + request_params = urlencode({ + "startdate": r"{}".format(startdate), + "enddate": r"{}".format(enddate), + "daterange": "d30", + "frequency": "p1d", + "csvdownload": "true", + "downloadpartial": "false", + "newdates": "false", + "countrycode": "{}".format(countrycode)}, quote_via= quote) + r = requests.get(tmp_url, params=request_params.replace("%2F", "/").replace("%20", " ").replace("%3A", ":"), headers=request_header) + data_text = r.content + df = pd.read_csv(io.StringIO(data_text.decode('utf-8'))) + Date = [] + for i in range(0, len(df)): + Date.append(datetime.strptime(df["Date"][i], "%m/%d/%Y")) + + df["Date"] = Date + return df + + for i in range(int(startdate[6:10]), int(enddate[6:10])): + if i == int(startdate[6:10]): + tmp_startdate = startdate + else: + tmp_startdate = "01/01/" + str(i) + " 00:00:00" + if (i+1) == int(enddate[6:10]): + tmp_enddate = enddate + else: + tmp_enddate = "01/01/" + str(i+1) + " 00:00:00" + + if countrycode == "us": + countrycode = "" + tmp_df = _stock(countrycode = countrycode, instrument=instrument, startdate = tmp_startdate, enddate = tmp_enddate) + if i == int(startdate[6:10]): + df = tmp_df + else: + df = pd.concat([tmp_df, df], axis=0) + + df = df.reset_index(drop = True) + return df + +#https://www.marketwatch.com/investing/stock/googl/downloaddatapartial?startdate=05/05/2021 00:00:00&enddate=06/04/2021 23:59:59&daterange=d30&frequency=p1d&csvdownload=true&downloadpartial=false&newdates=false if __name__ == "__main__": - data = FX() \ No newline at end of file + data = FX() + diff --git a/CEDA/__init__.py b/CEDA/__init__.py index 57b000a..202b9fd 100644 --- a/CEDA/__init__.py +++ b/CEDA/__init__.py @@ -1,12 +1,16 @@ -__version__ = "0.1.1" -__author__ = "TerenceLau" - import sys +import os + +__version__ = "0.1.2" +__author__ = "Terence Lau" + +__data_source__ = open(os.path.join( + os.path.dirname(__file__), + 'source.md')).read() if sys.version_info < (3, 6): print(f"CEDA {__version__} requires Python 3.6+") sys.exit(1) - del sys from CEDA import * \ No newline at end of file diff --git a/CEDA/source.md b/CEDA/source.md new file mode 100644 index 0000000..374b54b --- /dev/null +++ b/CEDA/source.md @@ -0,0 +1,7 @@ +(https://www.eastmoney.com) +(https://fred.stlouisfed.org/) +(https://www.chicagofed.org/) +(https://www.philadelphiafed.org/) +(https://ec.europa.eu/eurostat/cache/infographs/economy/desktop/index.html) +(https://www.ecb.europa.eu) +(https://www.marketwatch.com/) \ No newline at end of file diff --git a/README.md b/README.md index ebe9c29..e59e4bc 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ python -m pip install CEDApy * Thanks [Philadelphia Federal Reserve Bank](https://www.philadelphiafed.org/) * Thanks [eurostat Economic Indicators](https://ec.europa.eu/eurostat/cache/infographs/economy/desktop/index.html) * Thanks [Europen Central Bank](https://www.ecb.europa.eu) - +* Thanks [MarketWatch](https://www.marketwatch.com/) ## If you want to cite... diff --git a/setup.py b/setup.py index 3485930..5fa053d 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import os setup( name = "CEDApy", - version = "0.1.1", + version = "0.1.2", keywords = "quantitative economic data", long_description = open( os.path.join(