From c2393f48c342e04f3ffc3e1594ac1e8b53f0c1d7 Mon Sep 17 00:00:00 2001 From: Tom Hunter Date: Sat, 11 Feb 2017 10:38:28 +0000 Subject: [PATCH 1/4] Oanda Bar Price Handler Adding price handler to supply candle bars from the oanda rest API Need to add OANDA_API_ACCESS_TOKEN to qstrader.yml --- examples/display_prices_backtest_oanda.py | 100 +++++++++++++ qstrader/price_handler/oanda.py | 166 ++++++++++++++++++++++ qstrader/strategy/display.py | 5 +- tests/test_price_handler_oanda.py | 56 ++++++++ 4 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 examples/display_prices_backtest_oanda.py create mode 100644 qstrader/price_handler/oanda.py create mode 100644 tests/test_price_handler_oanda.py diff --git a/examples/display_prices_backtest_oanda.py b/examples/display_prices_backtest_oanda.py new file mode 100644 index 00000000..864a5daa --- /dev/null +++ b/examples/display_prices_backtest_oanda.py @@ -0,0 +1,100 @@ +import click +from click_datetime import Datetime + +from qstrader import settings +from qstrader.compat import queue +from qstrader.price_parser import PriceParser +from qstrader.price_handler.oanda import OandaBarPriceHandler +from qstrader.strategy import DisplayStrategy +from qstrader.position_sizer.fixed import FixedPositionSizer +from qstrader.risk_manager.example import ExampleRiskManager +from qstrader.portfolio_handler import PortfolioHandler +from qstrader.compliance.example import ExampleCompliance +from qstrader.execution_handler.ib_simulated import IBSimulatedExecutionHandler +from qstrader.statistics.simple import SimpleStatistics +from qstrader.trading_session.backtest import Backtest + + +def run(config, testing, tickers, granularity, start_date, end_date, + daily_alignment, alignment_timezone, filename, n, n_window): + + # Set up variables needed for backtest + events_queue = queue.Queue() + initial_equity = PriceParser.parse(500000.00) + + server = 'api-fxpractice.oanda.com' + bearer_token = config.OANDA_API_ACCESS_TOKEN + + instrument = tickers[0] + warmup_bar_count = 0 + + price_handler = OandaBarPriceHandler( + instrument, granularity, + start_date, end_date, + daily_alignment, alignment_timezone, + warmup_bar_count, + server, bearer_token, + events_queue + ) + + # Use the Display Strategy + strategy = DisplayStrategy(n=n, n_window=n_window) + + # Use an example Position Sizer + position_sizer = FixedPositionSizer() + + # Use an example Risk Manager + risk_manager = ExampleRiskManager() + + # Use the default Portfolio Handler + portfolio_handler = PortfolioHandler( + initial_equity, events_queue, price_handler, + position_sizer, risk_manager + ) + + # Use the ExampleCompliance component + compliance = ExampleCompliance(config) + + # Use a simulated IB Execution Handler + execution_handler = IBSimulatedExecutionHandler( + events_queue, price_handler, compliance + ) + + # Use the default Statistics + statistics = SimpleStatistics(config, portfolio_handler) + + # Set up the backtest + backtest = Backtest( + price_handler, strategy, + portfolio_handler, execution_handler, + position_sizer, risk_manager, + statistics, initial_equity + ) + results = backtest.simulate_trading(testing=testing) + statistics.save(filename) + return results + + +@click.command() +@click.option('--config', default=settings.DEFAULT_CONFIG_FILENAME, help='Config filename') +@click.option('--testing/--no-testing', default=False, help='Enable testing mode') +@click.option('--tickers', default='EUR_USD', help='Instrument') +@click.option('--granularity', default='D') +@click.option('--start_date', default='2016-01-01', type=Datetime(format='%Y-%m-%d')) +@click.option('--end_date', default='2016-12-31', type=Datetime(format='%Y-%m-%d')) +@click.option('--daily_alignment', default='0') +@click.option('--alignment_timezone', default='Europe/Paris') +@click.option('--filename', default='', help='Pickle (.pkl) statistics filename') +@click.option('--n', default=100, help='Display prices every n price events') +@click.option('--n_window', default=5, help='Display n_window prices') +def main( + config, testing, tickers, granularity, start_date, end_date, + daily_alignment, alignment_timezone, filename, n, n_window): + tickers = tickers.split(",") + config = settings.from_file(config, testing) + run(config, testing, tickers, granularity, start_date, end_date, + daily_alignment, alignment_timezone, filename, n, n_window) + + +if __name__ == "__main__": + main() diff --git a/qstrader/price_handler/oanda.py b/qstrader/price_handler/oanda.py new file mode 100644 index 00000000..ac706aa9 --- /dev/null +++ b/qstrader/price_handler/oanda.py @@ -0,0 +1,166 @@ +from .base import AbstractBarPriceHandler +from ..event import BarEvent + +import requests +from collections import deque +from urllib import parse +from datetime import datetime, timedelta + +oanda_request_date_format_string = '%Y-%m-%dT%H:%M:%SZ' +oanda_RFC3339_format = '%Y-%m-%dT%H:%M:%S.000000Z' + + +class OandaBarPriceHandler(AbstractBarPriceHandler): + """ + OandaBarPriceHandler.. + """ + def __init__(self, instrument, granularity, + start: datetime, end: datetime, + daily_alignment=0, alignment_timezone=None, + warmup_bar_count=0, + server=None, bearer_token=None, + events_queue=None): + if len(instrument) == 6: + self.instrument = instrument[:3] + "_" + instrument[3:] + else: + self.instrument = instrument + self.granularity = granularity + self.start_date = start + self.end_date = end + self.daily_alignment = daily_alignment + self.alignment_timezone = alignment_timezone + self.warmup_bar_count = warmup_bar_count + # self.warmup_bar_counter = warmup_bar_count + + self.server = server + self.bearer_token = f"Bearer {bearer_token}" + self.request_headers = { + 'Authorization': self.bearer_token, + 'Connection': 'Keep-Alive', + 'Accept-Encoding': 'gzip, deflate', + 'Content-type': 'application/x-www-form-urlencoded' + } + + self.events_queue = events_queue + self.continue_backtest = True + + self.next_start_date = start + self.candle_queue = deque() + self.last_candle_time = '' + self.candle_timespan = timedelta( + seconds=self._granularity_to_seconds() + ) + + if warmup_bar_count > 0: + # request warmup items (note: max 5000) + + start_string = self.start_date.strftime( + oanda_request_date_format_string + ) + + url = ( + f"https://{self.server}/v1/candles" + f"?instrument={parse.quote_plus(self.instrument)}" + f"&granularity={self.granularity}" + f"&count={self.warmup_bar_count}" + f"&end=" + f"{parse.quote_plus(start_string)}" # grab bars up to the + # start date + f"&candleFormat=midpoint" + f"&dailyAlignment={self.daily_alignment}" + f"&alignmentTimezone=" + f"{parse.quote_plus(self.alignment_timezone)}" + ) + response_json = requests.get(url, headers=self.request_headers) + self.candle_queue.extend(response_json.json()['candles']) + + def _granularity_to_seconds(self): + if self.granularity == 'D': + return 86400 # Seconds in a day + return None + + def _create_event(self, candle): + return BarEvent( + ticker=self.instrument, + time=candle['time'], + period=self._granularity_to_seconds(), + open_price=candle['openMid'], + high_price=candle['highMid'], + low_price=candle['lowMid'], + close_price=candle['closeMid'], + volume=candle['volume'] + ) + + def _pop_candle_onto_event_queue(self): + if len(self.candle_queue) > 0: + candle = self.candle_queue.popleft() + bar_event = self._create_event(candle) + self.events_queue.put(bar_event) + else: + self.events_queue.put(None) + + def _fetch_more_candles(self): + + start_string = self.next_start_date.strftime( + oanda_request_date_format_string + ) + + url = ( + f"https://{self.server}/v1/candles" + f"?instrument={parse.quote_plus(self.instrument)}" + f"&granularity={self.granularity}" + f"&count=5000" + f"&start={parse.quote_plus(start_string)}" + f"&candleFormat=midpoint" + f"&dailyAlignment={self.daily_alignment}" + f"&alignmentTimezone=" + f"{parse.quote_plus(self.alignment_timezone)}" + ) + + response_json = requests.get(url, headers=self.request_headers) + response_dict = response_json.json() + + # filter out incomplete and already queued candles + candles = list(filter( + lambda x: + x['complete'] and + x['time'] > self.last_candle_time and + x['time'] < self.end_date.strftime(oanda_RFC3339_format), + response_dict['candles'] + )) + + if len(candles) > 0: + self.candle_queue.extend(candles) + self.last_candle_time = candles[-1]['time'] + self.next_start_date = datetime.strptime( + candles[-1]['time'], + oanda_RFC3339_format + ) + self.candle_timespan + else: + # self.events_queue.put(None) + + # either we have to wait for a new candle to become available (live + # scenario) or we have to jump forward over a gap in candles (e.g. + # a weekend, back test scenario) + + if self.next_start_date + self.candle_timespan > datetime.utcnow(): + return + + self.next_start_date += self.candle_timespan * 5000 + if self.next_start_date > self.end_date: + self.continue_backtest = False + + def stream_next(self): + """ + Place the next BarEvent onto the event queue. + """ + if len(self.candle_queue) > 0: + self._pop_candle_onto_event_queue() + else: + if (self.next_start_date > datetime.now() or + self.next_start_date > self.end_date): + self.continue_backtest = False + return + + self._fetch_more_candles() + self._pop_candle_onto_event_queue() diff --git a/qstrader/strategy/display.py b/qstrader/strategy/display.py index ca918c02..5186a1fb 100644 --- a/qstrader/strategy/display.py +++ b/qstrader/strategy/display.py @@ -30,7 +30,10 @@ def calculate_signals(self, event): event.high_price = PriceParser.display(event.high_price) event.low_price = PriceParser.display(event.low_price) event.close_price = PriceParser.display(event.close_price) - event.adj_close_price = PriceParser.display(event.adj_close_price) + if event.adj_close_price is not None: + event.adj_close_price = PriceParser.display( + event.adj_close_price + ) else: # event.type == EventType.TICK event.bid = PriceParser.display(event.bid) event.ask = PriceParser.display(event.ask) diff --git a/tests/test_price_handler_oanda.py b/tests/test_price_handler_oanda.py new file mode 100644 index 00000000..acf37fb4 --- /dev/null +++ b/tests/test_price_handler_oanda.py @@ -0,0 +1,56 @@ +import unittest +import os +import datetime +import sys + +from qstrader.compat import queue +from qstrader.price_handler.oanda import OandaBarPriceHandler + + +class TestOandaBarPriceHandler(unittest.TestCase): + def test_warmup(self): + OANDA_API_ACCESS_TOKEN = os.environ.get('OANDA_API_ACCESS_TOKEN', None) + events_queue = queue.Queue() + oanda_bar_price_handler = OandaBarPriceHandler( + instrument='EURUSD', granularity='D', + start=datetime.date(2016, 1, 1), + end=datetime.date(2016, 12, 31), + daily_alignment=0, + alignment_timezone='Europe/Paris', + warmup_bar_count=1, + server='api-fxpractice.oanda.com', + bearer_token=OANDA_API_ACCESS_TOKEN, + events_queue=events_queue + ) + + oanda_bar_price_handler.stream_next() + + event = events_queue.get(False) + self.assertEqual(event.open_price, 1.09266) + + def test_no_warmup(self): + OANDA_API_ACCESS_TOKEN = os.environ.get('OANDA_API_ACCESS_TOKEN', None) + events_queue = queue.Queue() + + oanda_bar_price_handler = OandaBarPriceHandler( + instrument='EURUSD', granularity='D', + start=datetime.datetime(2016, 1, 1), + end=datetime.datetime(2016, 12, 31), + daily_alignment=0, + alignment_timezone='Europe/Paris', + warmup_bar_count=0, + server='api-fxpractice.oanda.com', + bearer_token=OANDA_API_ACCESS_TOKEN, + events_queue=events_queue + ) + + oanda_bar_price_handler.stream_next() + self.assertEqual(len(oanda_bar_price_handler.candle_queue), 309) + event = events_queue.get(False) + self.assertEqual(event.open_price, 1.08743) + + # def test_continue_backtest_set(self): + + +if __name__ == "__main__": + unittest.main() From ce244945984a3e85f53d87bfbbb1c4cd64131b80 Mon Sep 17 00:00:00 2001 From: Tom Hunter Date: Sat, 11 Feb 2017 12:36:29 +0000 Subject: [PATCH 2/4] Removing Python 3.6 string substitution syntax To fix Travis CI build --- qstrader/price_handler/oanda.py | 41 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/qstrader/price_handler/oanda.py b/qstrader/price_handler/oanda.py index ac706aa9..e3669850 100644 --- a/qstrader/price_handler/oanda.py +++ b/qstrader/price_handler/oanda.py @@ -33,7 +33,7 @@ def __init__(self, instrument, granularity, # self.warmup_bar_counter = warmup_bar_count self.server = server - self.bearer_token = f"Bearer {bearer_token}" + self.bearer_token = "Bearer %s" % bearer_token self.request_headers = { 'Authorization': self.bearer_token, 'Connection': 'Keep-Alive', @@ -59,17 +59,16 @@ def __init__(self, instrument, granularity, ) url = ( - f"https://{self.server}/v1/candles" - f"?instrument={parse.quote_plus(self.instrument)}" - f"&granularity={self.granularity}" - f"&count={self.warmup_bar_count}" - f"&end=" - f"{parse.quote_plus(start_string)}" # grab bars up to the - # start date - f"&candleFormat=midpoint" - f"&dailyAlignment={self.daily_alignment}" - f"&alignmentTimezone=" - f"{parse.quote_plus(self.alignment_timezone)}" + "https://" + self.server + "/v1/candles" + + "?instrument=" + parse.quote_plus(self.instrument) + + "&granularity=" + self.granularity + + "&count=" + self.warmup_bar_count + + # grab bars up to the start date + "&end=" + parse.quote_plus(start_string) + + "&candleFormat=midpoint" + + "&dailyAlignment=" + self.daily_alignment + + "&alignmentTimezone=" + + parse.quote_plus(self.alignment_timezone) ) response_json = requests.get(url, headers=self.request_headers) self.candle_queue.extend(response_json.json()['candles']) @@ -106,15 +105,15 @@ def _fetch_more_candles(self): ) url = ( - f"https://{self.server}/v1/candles" - f"?instrument={parse.quote_plus(self.instrument)}" - f"&granularity={self.granularity}" - f"&count=5000" - f"&start={parse.quote_plus(start_string)}" - f"&candleFormat=midpoint" - f"&dailyAlignment={self.daily_alignment}" - f"&alignmentTimezone=" - f"{parse.quote_plus(self.alignment_timezone)}" + "https://" + self.server + "/v1/candles" + + "?instrument=" + parse.quote_plus(self.instrument) + + "&granularity=" + self.granularity + + "&count=5000" + "&start=" + parse.quote_plus(start_string) + + "&candleFormat=midpoint" + "&dailyAlignment=" + self.daily_alignment + + "&alignmentTimezone=" + + parse.quote_plus(self.alignment_timezone) ) response_json = requests.get(url, headers=self.request_headers) From d648e35a72238f98a2f0772b3839f4d43551f7b1 Mon Sep 17 00:00:00 2001 From: Tom Hunter Date: Sun, 12 Feb 2017 13:10:37 +0000 Subject: [PATCH 3/4] Fixing string concatenation --- qstrader/price_handler/oanda.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/qstrader/price_handler/oanda.py b/qstrader/price_handler/oanda.py index e3669850..3341ae3c 100644 --- a/qstrader/price_handler/oanda.py +++ b/qstrader/price_handler/oanda.py @@ -3,7 +3,7 @@ import requests from collections import deque -from urllib import parse +import six.moves.urllib as urllib from datetime import datetime, timedelta oanda_request_date_format_string = '%Y-%m-%dT%H:%M:%SZ' @@ -15,7 +15,7 @@ class OandaBarPriceHandler(AbstractBarPriceHandler): OandaBarPriceHandler.. """ def __init__(self, instrument, granularity, - start: datetime, end: datetime, + start, end, daily_alignment=0, alignment_timezone=None, warmup_bar_count=0, server=None, bearer_token=None, @@ -60,15 +60,15 @@ def __init__(self, instrument, granularity, url = ( "https://" + self.server + "/v1/candles" + - "?instrument=" + parse.quote_plus(self.instrument) + + "?instrument=" + urllib.parse.quote_plus(self.instrument) + "&granularity=" + self.granularity + - "&count=" + self.warmup_bar_count + + "&count={}".format(self.warmup_bar_count) + # grab bars up to the start date - "&end=" + parse.quote_plus(start_string) + + "&end=" + urllib.parse.quote_plus(start_string) + "&candleFormat=midpoint" + - "&dailyAlignment=" + self.daily_alignment + + "&dailyAlignment={}".format(self.daily_alignment) + "&alignmentTimezone=" + - parse.quote_plus(self.alignment_timezone) + urllib.parse.quote_plus(self.alignment_timezone) ) response_json = requests.get(url, headers=self.request_headers) self.candle_queue.extend(response_json.json()['candles']) @@ -106,14 +106,14 @@ def _fetch_more_candles(self): url = ( "https://" + self.server + "/v1/candles" + - "?instrument=" + parse.quote_plus(self.instrument) + + "?instrument=" + urllib.parse.quote_plus(self.instrument) + "&granularity=" + self.granularity + "&count=5000" - "&start=" + parse.quote_plus(start_string) + + "&start=" + urllib.parse.quote_plus(start_string) + "&candleFormat=midpoint" - "&dailyAlignment=" + self.daily_alignment + + "&dailyAlignment=" + str(self.daily_alignment) + "&alignmentTimezone=" + - parse.quote_plus(self.alignment_timezone) + urllib.parse.quote_plus(self.alignment_timezone) ) response_json = requests.get(url, headers=self.request_headers) From cf4a081ae629fa48d82d05d640a192b54fc4a68a Mon Sep 17 00:00:00 2001 From: Tom Hunter Date: Sun, 12 Feb 2017 13:11:21 +0000 Subject: [PATCH 4/4] Removing redundant 'import sys' --- tests/test_price_handler_oanda.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_price_handler_oanda.py b/tests/test_price_handler_oanda.py index acf37fb4..3af1fbbe 100644 --- a/tests/test_price_handler_oanda.py +++ b/tests/test_price_handler_oanda.py @@ -1,7 +1,6 @@ import unittest import os import datetime -import sys from qstrader.compat import queue from qstrader.price_handler.oanda import OandaBarPriceHandler @@ -49,8 +48,6 @@ def test_no_warmup(self): event = events_queue.get(False) self.assertEqual(event.open_price, 1.08743) - # def test_continue_backtest_set(self): - if __name__ == "__main__": unittest.main()