From 3b0a99b7389dbf504f5641108dc1d006d53ccbf4 Mon Sep 17 00:00:00 2001 From: Ram Date: Mon, 2 Mar 2020 14:07:01 +0100 Subject: [PATCH] Adding code --- MTtest.py | 492 +++++++++++++++++++++++++++++++ docs/modules.rst | 7 + docs/mql5_zmq_backtrader.rst | 62 ++++ environment.yml | 20 ++ mql5_zmq_backtrader/adapter.py | 0 mql5_zmq_backtrader/mt5broker.py | 2 +- mql5_zmq_backtrader/mt5data.py | 2 +- mql5_zmq_backtrader/mt5store.py | 24 +- requirements_dev.txt | 1 + 9 files changed, 596 insertions(+), 14 deletions(-) create mode 100644 MTtest.py create mode 100644 docs/modules.rst create mode 100644 docs/mql5_zmq_backtrader.rst create mode 100644 environment.yml mode change 100755 => 100644 mql5_zmq_backtrader/adapter.py mode change 100755 => 100644 mql5_zmq_backtrader/mt5broker.py mode change 100755 => 100644 mql5_zmq_backtrader/mt5data.py mode change 100755 => 100644 mql5_zmq_backtrader/mt5store.py diff --git a/MTtest.py b/MTtest.py new file mode 100644 index 0000000..3b35ed2 --- /dev/null +++ b/MTtest.py @@ -0,0 +1,492 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import argparse +import datetime +import sys + +# The above could be sent to an independent module +import backtrader as bt +from backtrader.utils import flushfile # win32 quick stdout flushing + +from mql5_zmq_backtrader.mt5data import MTraderData +from mql5_zmq_backtrader.mt5store import MTraderStore + +StoreCls = MTraderStore +DataCls = MTraderData + + +class TestStrategy(bt.Strategy): + params = dict( + smaperiod=5, + trade=False, + stake=0.1, + exectype=bt.Order.Market, + stopafter=0, + valid=None, + cancel=0, + donotcounter=False, + sell=False, + usebracket=False, + ) + + def __init__(self): + # To control operation entries + self.orderid = list() + self.order = None + + self.counttostop = 0 + self.datastatus = 0 + + self.last_pos = None + self.last_value = 0 + + # Create SMA on 2nd data + self.sma = bt.indicators.MovAv.SMA(self.data, period=self.p.smaperiod) + + print('--------------------------------------------------') + print('Strategy Created') + print('--------------------------------------------------') + + def notify_fund(self, cash, value, fundvalue, shares): + if value != self.last_value: + print(cash, value, fundvalue, shares) + self.last_value = value + + def notify_data(self, data, status, *args, **kwargs): + print('*' * 5, 'DATA NOTIF:', data._getstatusname(status), *args) + if status == data.LIVE: + self.counttostop = self.p.stopafter + self.datastatus = 1 + + def notify_store(self, msg, *args, **kwargs): + print('*' * 5, 'STORE NOTIF:', msg) + + def notify_order(self, order): + if order.status in [order.Completed, order.Cancelled, order.Rejected]: + self.order = None + + print('{}: Order ref: {} / Type {} / Status {}'.format( + self.data.datetime.date(0), + order.ref, 'Buy' * order.isbuy() or 'Sell', + order.getstatusname())) + + def notify_trade(self, trade): + print('-' * 50, 'TRADE BEGIN', datetime.datetime.now()) + print(trade) + print('-' * 50, 'TRADE END') + + def prenext(self): + self.next(frompre=True) + + def next(self, frompre=False): + # Visualize data0 + txt = list() + txt.append(str(self.getdatanames()[0])) + txt.append('%04d' % len(self.data0)) + dtfmt = '%Y-%m-%dT%H:%M:%S.%f' + txt.append('{:f}'.format(self.data.datetime[0])) + txt.append('%s' % self.data.datetime.datetime(0).strftime(dtfmt)) + txt.append('{:f}'.format(self.data.open[0])) + txt.append('{:f}'.format(self.data.high[0])) + txt.append('{:f}'.format(self.data.low[0])) + txt.append('{:f}'.format(self.data.close[0])) + txt.append('{:6d}'.format(int(self.data.volume[0]))) + txt.append('{:d}'.format(int(self.data.openinterest[0]))) + txt.append('{:f}'.format(self.sma[0])) + print(', '.join(txt)) + + # Visualize data1 + if len(self.datas) > 1 and len(self.data1): + txt = list() + txt.append(str(self.getdatanames()[1])) + txt.append('%04d' % len(self.data1)) + dtfmt = '%Y-%m-%dT%H:%M:%S.%f' + txt.append('{}'.format(self.data1.datetime[0])) + txt.append('%s' % self.data1.datetime.datetime(0).strftime(dtfmt)) + txt.append('{}'.format(self.data1.open[0])) + txt.append('{}'.format(self.data1.high[0])) + txt.append('{}'.format(self.data1.low[0])) + txt.append('{}'.format(self.data1.close[0])) + txt.append('{}'.format(self.data1.volume[0])) + txt.append('{}'.format(self.data1.openinterest[0])) + txt.append('{}'.format(float('NaN'))) + print(', '.join(txt)) + + if self.counttostop: # stop after x live lines + self.counttostop -= 1 + if not self.counttostop: + self.env.runstop() + return + + if not self.p.trade: + print('No trading. Parameter trade:', self.p.trade) + return + + if self.datastatus and not self.position and len(self.orderid) < 1: + if not self.p.usebracket: + if not self.p.sell: + # price = round(self.data0.close[0] * 0.90, 2) + price = self.data0.close[0] - 5 + self.order = self.buy(size=self.p.stake, + exectype=self.p.exectype, + price=price, + valid=self.p.valid, + magic=31416) + else: + # price = round(self.data0.close[0] * 1.10, 4) + price = self.data0.close[0] - 0.05 + self.order = self.sell(size=self.p.stake, + exectype=self.p.exectype, + price=price, + valid=self.p.valid) + + else: + print('USING BRACKET') + price = self.data0.close[0] - 0.05 + self.order, _, _ = self.buy_bracket(size=self.p.stake, + exectype=bt.Order.Market, + price=price, + stopprice=price - 0.10, + limitprice=price + 0.10, + valid=self.p.valid) + + self.orderid.append(self.order) + elif self.position and not self.p.donotcounter: + if self.order is None: + if not self.p.sell: + self.order = self.sell(size=self.p.stake // 2, + exectype=bt.Order.Market, + price=self.data0.close[0]) + else: + self.order = self.buy(size=self.p.stake // 2, + exectype=bt.Order.Market, + price=self.data0.close[0]) + + self.orderid.append(self.order) + + elif self.order is not None and self.p.cancel: + if self.datastatus > self.p.cancel: + self.cancel(self.order) + + if self.datastatus: + self.datastatus += 1 + + def start(self): + + header = ['Datetime', 'Open', 'High', 'Low', 'Close', 'Volume', + 'OpenInterest', 'SMA'] + print(', '.join(header)) + + self.done = False + + +def runstrategy(): + args = parse_args() + + # Create a cerebro + cerebro = bt.Cerebro() + + storekwargs = dict( + # ram key_id=args.keyid, + # ram secret_key=args.secretkey, + # ram paper=not args.live, + host="192.168.0.71", + ) + + store = StoreCls(**storekwargs) + + broker = store.getbroker() # MTraderBroker + cerebro.setbroker(broker) + + timeframe = bt.TimeFrame.TFrame(args.timeframe) + # Manage data1 parameters + tf1 = args.timeframe1 + tf1 = bt.TimeFrame.TFrame(tf1) if tf1 is not None else timeframe + cp1 = args.compression1 + cp1 = cp1 if cp1 is not None else args.compression + + if args.resample or args.replay: + datatf = datatf1 = bt.TimeFrame.Ticks + datacomp = datacomp1 = 1 + else: + datatf = timeframe + datacomp = args.compression + datatf1 = tf1 + datacomp1 = cp1 + + fromdate = None + if args.fromdate: + dtformat = '%Y-%m-%d' + ('T%H:%M:%S' * ('T' in args.fromdate)) + fromdate = datetime.datetime.strptime(args.fromdate, dtformat) + + DataFactory = store.getdata # MTraderData + + datakwargs = dict( + timeframe=datatf, compression=datacomp, + qcheck=args.qcheck, + historical=args.historical, + fromdate=fromdate, + bidask=args.bidask, + useask=args.useask, + backfill_start=not args.no_backfill_start, + backfill=not args.no_backfill, + tz=args.timezone + ) + + # if args.no_store and not args.broker: # neither store nor broker + # datakwargs.update(storekwargs) # pass the store args over the data + + data0 = DataFactory(dataname=args.data0, **datakwargs) + + data1 = None + if args.data1 is not None: + if args.data1 != args.data0: + datakwargs['timeframe'] = datatf1 + datakwargs['compression'] = datacomp1 + data1 = DataFactory(dataname=args.data1, **datakwargs) + #ram + print(datakwargs) + else: + data1 = data0 + + rekwargs = dict( + timeframe=timeframe, + compression=args.compression, + bar2edge=not args.no_bar2edge, + adjbartime=not args.no_adjbartime, + rightedge=not args.no_rightedge, + takelate=not args.no_takelate, + ) + + if args.replay: + cerebro.replaydata(data0, **rekwargs) + + if data1 is not None: + rekwargs['timeframe'] = tf1 + rekwargs['compression'] = cp1 + cerebro.replaydata(data1, **rekwargs) + + elif args.resample: + cerebro.resampledata(data0, **rekwargs) + + if data1 is not None: + rekwargs['timeframe'] = tf1 + rekwargs['compression'] = cp1 + cerebro.resampledata(data1, **rekwargs) + + else: + cerebro.adddata(data0) + if data1 is not None: + cerebro.adddata(data1) + + if args.valid is None: + valid = None + else: + valid = datetime.timedelta(seconds=args.valid) + # Add the strategy + cerebro.addstrategy(TestStrategy, + smaperiod=args.smaperiod, + trade=args.trade, + exectype=bt.Order.ExecType(args.exectype), + stake=args.stake, + stopafter=args.stopafter, + valid=valid, + cancel=args.cancel, + donotcounter=args.donotcounter, + sell=args.sell, + usebracket=args.usebracket) + + # Live data ... avoid long data accumulation by switching to "exactbars" + cerebro.run(exactbars=args.exactbars) + if args.exactbars < 1: # plotting is possible + if args.plot: + pkwargs = dict(style='line') + if args.plot is not True: # evals to True but is not True + npkwargs = eval('dict(' + args.plot + ')') # args were passed + pkwargs.update(npkwargs) + + cerebro.plot(**pkwargs) + + +def parse_args(pargs=None): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description='Test Metatrader integration') + + parser.add_argument('--exactbars', default=1, type=int, + required=False, action='store', + help='exactbars level, use 0/-1/-2 to enable plotting') + + parser.add_argument('--stopafter', default=0, type=int, + required=False, action='store', + help='Stop after x lines of LIVE data') + + parser.add_argument('--debug', + required=False, action='store_true', + help='Display all info received from source') + + parser.add_argument('--keyid', default=None, + required=False, action='store', + help='MT5 API key id') + + parser.add_argument('--secretkey', default=None, + required=False, action='store', + help='MT5 API secret key') + + parser.add_argument('--live', default=None, + required=False, action='store', + help='Go to live server rather than paper') + + parser.add_argument('--qcheck', default=0.5, type=float, + required=False, action='store', + help=('Timeout for periodic ' + 'notification/resampling/replaying check')) + + parser.add_argument('--data0', default=None, + required=True, action='store', + help='data 0 into the system') + + parser.add_argument('--data1', default=None, + required=False, action='store', + help='data 1 into the system') + + parser.add_argument('--timezone', default=None, + required=False, action='store', + help='timezone to get time output into (pytz names)') + + parser.add_argument('--bidask', default=None, + required=False, action='store_true', + help='Use bidask ... if False use midpoint') + + parser.add_argument('--useask', default=None, + required=False, action='store_true', + help='Use the "ask" of bidask prices/streaming') + + parser.add_argument('--no-backfill_start', + required=False, action='store_true', + help='Disable backfilling at the start') + + parser.add_argument('--no-backfill', + required=False, action='store_true', + help='Disable backfilling after a disconnection') + + parser.add_argument('--historical', + required=False, action='store_true', + help='do only historical download') + + parser.add_argument('--fromdate', + required=True, action='store', + help=('Starting date for historical download ' + 'with format: YYYY-MM-DD[THH:MM:SS]')) + + parser.add_argument('--smaperiod', default=5, type=int, + required=False, action='store', + help='Period to apply to the Simple Moving Average') + + pgroup = parser.add_mutually_exclusive_group(required=False) + + pgroup.add_argument('--replay', + required=False, action='store_true', + help='replay to chosen timeframe') + + pgroup.add_argument('--resample', + required=False, action='store_true', + help='resample to chosen timeframe') + + parser.add_argument('--timeframe', default='Minutes', + choices=bt.TimeFrame.Names, + required=False, action='store', + help='TimeFrame for Resample/Replay') + + parser.add_argument('--compression', default=1, type=int, + required=False, action='store', + help='Compression for Resample/Replay') + + parser.add_argument('--timeframe1', default=None, + choices=bt.TimeFrame.Names[4], + required=False, action='store', + help='TimeFrame for Resample/Replay - Data1') + + parser.add_argument('--compression1', default=None, type=int, + required=False, action='store', + help='Compression for Resample/Replay - Data1') + + parser.add_argument('--no-takelate', + required=False, action='store_true', + help=('resample/replay, do not accept late samples')) + + parser.add_argument('--no-bar2edge', + required=False, action='store_true', + help='no bar2edge for resample/replay') + + parser.add_argument('--no-adjbartime', + required=False, action='store_true', + help='no adjbartime for resample/replay') + + parser.add_argument('--no-rightedge', + required=False, action='store_true', + help='no rightedge for resample/replay') + + parser.add_argument('--trade', + required=False, action='store_true', + help='Do Sample Buy/Sell operations') + + parser.add_argument('--sell', + required=False, action='store_true', + help='Start by selling') + + parser.add_argument('--usebracket', + required=False, action='store_true', + help='Test buy_bracket') + + parser.add_argument('--donotcounter', + required=False, action='store_true', + help='Do not counter the 1st operation') + + parser.add_argument('--exectype', default=bt.Order.ExecTypes[0], + choices=bt.Order.ExecTypes, + required=False, action='store', + help='Execution to Use when opening position') + + parser.add_argument('--stake', default=0.1, type=float, + required=False, action='store', + help='Stake to use in buy operations') + + parser.add_argument('--valid', default=None, type=float, + required=False, action='store', + help='Seconds to keep the order alive (0 means DAY)') + + parser.add_argument('--cancel', default=0, type=int, + required=False, action='store', + help=('Cancel a buy order after n bars in operation,' + ' to be combined with orders like Limit')) + + # Plot options + parser.add_argument('--plot', '-p', nargs='?', required=False, + metavar='kwargs', const=True, + help=('Plot the read data applying any kwargs passed\n' + '\n' + 'For example (escape the quotes if needed):\n' + '\n' + ' --plot style="candle" (to plot candles)\n')) + + if pargs is not None: + return parser.parse_args(pargs) + + return parser.parse_args() + + +if __name__ == '__main__': + start_date = datetime.datetime.now() - datetime.timedelta(minutes=500) + sys.argv = [ + 'MTtest2.py', + '--data0', 'BTCEUR', + '--timeframe', 'Minutes', + '--compression', '1', + '--data1', 'BTCUSD', + '--timeframe1', 'Minutes', + '--compression1', '1', + '--fromdate', start_date.strftime("%Y-%m-%dT%H:%M:%S"), + '--trade'] + runstrategy() diff --git a/docs/modules.rst b/docs/modules.rst new file mode 100644 index 0000000..24ebb50 --- /dev/null +++ b/docs/modules.rst @@ -0,0 +1,7 @@ +mql5_zmq_backtrader +=================== + +.. toctree:: + :maxdepth: 4 + + mql5_zmq_backtrader diff --git a/docs/mql5_zmq_backtrader.rst b/docs/mql5_zmq_backtrader.rst new file mode 100644 index 0000000..65471c0 --- /dev/null +++ b/docs/mql5_zmq_backtrader.rst @@ -0,0 +1,62 @@ +mql5\_zmq\_backtrader package +============================= + +Submodules +---------- + +mql5\_zmq\_backtrader.adapter module +------------------------------------ + +.. automodule:: mql5_zmq_backtrader.adapter + :members: + :undoc-members: + :show-inheritance: + +mql5\_zmq\_backtrader.cli module +-------------------------------- + +.. automodule:: mql5_zmq_backtrader.cli + :members: + :undoc-members: + :show-inheritance: + +mql5\_zmq\_backtrader.mql5\_zmq\_backtrader module +-------------------------------------------------- + +.. automodule:: mql5_zmq_backtrader.mql5_zmq_backtrader + :members: + :undoc-members: + :show-inheritance: + +mql5\_zmq\_backtrader.mt5broker module +-------------------------------------- + +.. automodule:: mql5_zmq_backtrader.mt5broker + :members: + :undoc-members: + :show-inheritance: + +mql5\_zmq\_backtrader.mt5data module +------------------------------------ + +.. automodule:: mql5_zmq_backtrader.mt5data + :members: + :undoc-members: + :show-inheritance: + +mql5\_zmq\_backtrader.mt5store module +------------------------------------- + +.. automodule:: mql5_zmq_backtrader.mt5store + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: mql5_zmq_backtrader + :members: + :undoc-members: + :show-inheritance: diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..fbd2cce --- /dev/null +++ b/environment.yml @@ -0,0 +1,20 @@ +# run: conda env create --file environment.yml +name: backtrader + +dependencies: + - python=3.5 + - pip: + - pip==19.2.3 + - bump2version==0.5.11 + - wheel==0.33.6 + - watchdog==0.9.0 + - flake8==3.7.8 + - tox==3.14.0 + - coverage==4.5.4 + - Sphinx==1.8.5 + - twine==1.14.0 + - Click==7.0 + - backtrader>=1.9.74.123 + - pylint>=2.4.4 + - pyzmq==19.0.0 + diff --git a/mql5_zmq_backtrader/adapter.py b/mql5_zmq_backtrader/adapter.py old mode 100755 new mode 100644 diff --git a/mql5_zmq_backtrader/mt5broker.py b/mql5_zmq_backtrader/mt5broker.py old mode 100755 new mode 100644 index 293fcc6..5231078 --- a/mql5_zmq_backtrader/mt5broker.py +++ b/mql5_zmq_backtrader/mt5broker.py @@ -8,7 +8,7 @@ from backtrader.utils.py3 import with_metaclass from backtrader.comminfo import CommInfoBase from backtrader.position import Position -from mt5 import mt5store +from mql5_zmq_backtrader import mt5store class MTraderCommInfo(CommInfoBase): diff --git a/mql5_zmq_backtrader/mt5data.py b/mql5_zmq_backtrader/mt5data.py old mode 100755 new mode 100644 index 89ec9f3..dcecff7 --- a/mql5_zmq_backtrader/mt5data.py +++ b/mql5_zmq_backtrader/mt5data.py @@ -7,7 +7,7 @@ from backtrader.feed import DataBase from backtrader import date2num, num2date from backtrader.utils.py3 import queue, with_metaclass -from mt5 import mt5store +from mql5_zmq_backtrader import mt5store class MetaMTraderData(DataBase.__class__): diff --git a/mql5_zmq_backtrader/mt5store.py b/mql5_zmq_backtrader/mt5store.py old mode 100755 new mode 100644 index 500f39d..d75bd8a --- a/mql5_zmq_backtrader/mt5store.py +++ b/mql5_zmq_backtrader/mt5store.py @@ -6,7 +6,7 @@ import collections from datetime import datetime import threading -from mt5.adapter import PositionAdapter, OrderAdapter, BalanceAdapter +from mql5_zmq_backtrader.adapter import PositionAdapter, OrderAdapter, BalanceAdapter import backtrader as bt from backtrader.metabase import MetaParams @@ -81,14 +81,14 @@ class MTraderAPI: self.data_socket.connect( 'tcp://{}:{}'.format(self.HOST, self.DATA_PORT)) except zmq.ZMQError: - raise zmq.ZMQBindError("Binding ports ERROR") + raise zmq.ZMQBindError("E: Binding ports ERROR") def _send_request(self, data: dict) -> None: """Send request to server via ZeroMQ System socket Lazy Pirate implementation. """ # ram Caller's name - print("Caller 2 ", sys._getframe(2).f_code.co_name) + print("I: Caller 2 ", sys._getframe(2).f_code.co_name) try: # ram sequence = 0 @@ -137,7 +137,7 @@ class MTraderAPI: # ram self.context.term() except zmq.ZMQError: - raise zmq.NotDone("Sending request ERROR") + raise zmq.NotDone("E: Sending request ERROR") def _pull_reply(self): # Get reply from server via Data socket with timeout @@ -148,7 +148,7 @@ class MTraderAPI: except zmq.Again as e: return None except zmq.ZMQError as e: - logger.debug("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e)) + logger.debug("W: Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e)) return msg def live_socket(self, context=None): @@ -158,7 +158,7 @@ class MTraderAPI: socket = context.socket(zmq.PULL) socket.connect('tcp://{}:{}'.format(self.HOST, self.LIVE_PORT)) except zmq.ZMQError: - raise zmq.ZMQBindError("Live port connection ERROR") + raise zmq.ZMQBindError("E: Live port connection ERROR") return socket def streaming_socket(self, context=None): @@ -168,7 +168,7 @@ class MTraderAPI: socket = context.socket(zmq.PULL) socket.connect('tcp://{}:{}'.format(self.HOST, self.EVENTS_PORT)) except zmq.ZMQError: - raise zmq.ZMQBindError("Data port connection ERROR") + raise zmq.ZMQBindError("E: Data port connection ERROR") return socket def construct_and_send(self, **kwargs) -> dict: @@ -198,7 +198,7 @@ class MTraderAPI: if key in request: request[key] = value else: - raise KeyError('Unknown key in **kwargs ERROR') + raise KeyError('E: Unknown key in **kwargs ERROR') # send dict to server self._send_request(request) @@ -368,7 +368,7 @@ class MTraderStore(with_metaclass(MetaSingleton, object)): def get_granularity(self, timeframe, compression): granularity = self._GRANULARITIES.get((timeframe, compression), None) if granularity is None: - raise ValueError("Metatrader 5 doesn't support frame %s with compression %s" % + raise ValueError("W: Metatrader 5 doesn't support frame %s with compression %s" % (bt.TimeFrame.getname(timeframe), compression)) return granularity @@ -420,7 +420,7 @@ class MTraderStore(with_metaclass(MetaSingleton, object)): try: transaction = socket.recv_json() except zmq.ZMQError: - raise zmq.NotDone("Streaming data ERROR") + raise zmq.NotDone("E: Streaming data ERROR") self._transaction(transaction) @@ -441,7 +441,7 @@ class MTraderStore(with_metaclass(MetaSingleton, object)): side = 'buy' if order.isbuy() else 'sell' order_type = self._ORDEREXECS.get((order.exectype, side), None) if order_type is None: - raise ValueError("Wrong order type: %s or side: %s" % + raise ValueError("W: Wrong order type: %s or side: %s" % (order.exectype, side)) okwargs['actionType'] = order_type @@ -604,7 +604,7 @@ class MTraderStore(with_metaclass(MetaSingleton, object)): def check_account(self) -> None: """Get MetaTrader 5 account settings""" # ram Caller's name - print("Caller 3 ", sys._getframe(2).f_code.co_name) + print("I: Caller 3 ", sys._getframe(2).f_code.co_name) conf = self.oapi.construct_and_send(action="ACCOUNT") diff --git a/requirements_dev.txt b/requirements_dev.txt index 3439c45..ea90164 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -8,3 +8,4 @@ coverage==4.5.4 Sphinx==1.8.5 twine==1.14.0 Click==7.0 +backtrader>=1.9.74.123