From 759fc9cf265587984c3adeca6aed7df6da13091a Mon Sep 17 00:00:00 2001 From: Ichinga Samuel Date: Sun, 22 Sep 2024 20:00:43 +0100 Subject: [PATCH] testdata --- src/aiomql/contrib/backtester/__init__.py | 2 + src/aiomql/contrib/backtester/check1.py | 95 ++++------ src/aiomql/contrib/backtester/get_data.py | 2 +- src/aiomql/contrib/backtester/test_account.py | 41 ++--- src/aiomql/contrib/backtester/test_data.py | 172 ++++++++++-------- src/aiomql/contrib/backtester/types.py | 45 +++++ src/aiomql/contrib/backtester/utils.py | 24 +++ src/aiomql/core/meta_trader.py | 11 +- 8 files changed, 225 insertions(+), 167 deletions(-) create mode 100644 src/aiomql/contrib/backtester/types.py create mode 100644 src/aiomql/contrib/backtester/utils.py diff --git a/src/aiomql/contrib/backtester/__init__.py b/src/aiomql/contrib/backtester/__init__.py index ebfecad..86805d8 100644 --- a/src/aiomql/contrib/backtester/__init__.py +++ b/src/aiomql/contrib/backtester/__init__.py @@ -4,3 +4,5 @@ from .get_data import GetData from .test_strategy import TestStrategy from .event_manager import EventManager from .strategy_tester import StrategyTester, SingleStrategyTester +from .test_account import TestAccount +from .types import TradingData, PositionsManager, OrdersManager diff --git a/src/aiomql/contrib/backtester/check1.py b/src/aiomql/contrib/backtester/check1.py index ac9d26c..cba4363 100644 --- a/src/aiomql/contrib/backtester/check1.py +++ b/src/aiomql/contrib/backtester/check1.py @@ -1,75 +1,42 @@ -import asyncio -import inspect -from functools import cache, lru_cache, cached_property, wraps, partial +from functools import cached_property +class Data: + def __init__(self): + self.a = 1 + self.b = 2 -# from - -def async_cache(fun): - - @wraps(fun) - async def wrapper(*args, **kwargs): - print(wrapper.cache) - key = (args, frozenset(kwargs.items())) - async with wrapper.lock: - if key not in wrapper.cache: - print('not in cache') - wrapper.cache[key] = await fun(*args, **kwargs) - return wrapper.cache[key] - - wrapper.lock = asyncio.Lock() - wrapper.cache = {} - return wrapper - - -class Test: +class TData: + _a: int + _b: int + _dat: dict def __init__(self): - self.rr = 0 + self._data = Data() - def __repr__(self): - return f'{self.__class__.__name__}(...)' + def __getattr__(self, item): + if val := self.__annotations__.get(item): + return getattr(self._data, item, val()) - @async_cache - async def check(self, a, b): - an = a + b - self.rr - return an + @property + def a(self): + return self._a -# async def main(a, b): -# t = Test() -# -# @async_cache -# def check(_a, _b): -# an = _a + _b -# print('check', _a, _b, t.rr) -# return an -# -# return check(a, b) + @a.setter + def a(self, val): + self._a = val -t = Test() -y = asyncio.run(t.check(1, 2)) -t.rr = 6 -y1 = asyncio.run(t.check(1, 2)) -t.rr = 7 -y2 = asyncio.run(t.check(1, 2)) -print(y, y1, y2) + @property + def dat(self): + return self._dat -@async_cache -async def func(a, b): - an = a + b - print('func') - return an + @dat.setter + def dat(self, key, val): + self._dat |= val -@async_cache -async def func1(e, c=6, d=6): - an = e - c + d - print('func1') - return an + @dat.deleter + def dat(self): + self._dat = {} -# asyncio.run(func(1, 2)) -# asyncio.run(func1(1, d=8)) -# asyncio.run(func(1, 2)) -# asyncio.run(func(1, 3)) -# asyncio.run(func1(1, d=6)) -# asyncio.run(func1(1, d=6)) -# asyncio.run(func1(1, d=7)) +f = TData() +# f.data = {'a': 1} +print(f.a) diff --git a/src/aiomql/contrib/backtester/get_data.py b/src/aiomql/contrib/backtester/get_data.py index b3c0571..4fa35b7 100644 --- a/src/aiomql/contrib/backtester/get_data.py +++ b/src/aiomql/contrib/backtester/get_data.py @@ -225,7 +225,7 @@ class GetData: res = pd.DataFrame(res) res.drop_duplicates(subset=['time'], keep='last', inplace=True) res.set_index('time', inplace=True, drop=False) - res = res.reindex(self.span) # change method back to 'nearest' + res = res.reindex(self.span) # fill in missing values with NaN self.data.prices[symbol] = res @backoff_decorator diff --git a/src/aiomql/contrib/backtester/test_account.py b/src/aiomql/contrib/backtester/test_account.py index f3b3dbb..cb0be52 100644 --- a/src/aiomql/contrib/backtester/test_account.py +++ b/src/aiomql/contrib/backtester/test_account.py @@ -1,52 +1,45 @@ -from dataclasses import dataclass, asdict, field, fields +from dataclasses import dataclass from typing import ClassVar from ...core.constants import AccountTradeMode, AccountMarginMode, AccountStopOutMode -from MetaTrader5 import AccountInfo - -__match_args__ = SymbolInfo @dataclass -class AccountInfo: +class TestAccount: login: int = 0 - server: str = '' trade_mode: AccountTradeMode = AccountTradeMode.DEMO - balance: float = 0 leverage: float = 0 + limit_orders: float = 0 + margin_so_mode: AccountStopOutMode = AccountStopOutMode.PERCENT + trade_allowed: bool = True + trade_expert: bool = True + margin_mode: AccountMarginMode = AccountMarginMode.EXCHANGE + currency_digits: int = 2 + fifo_close: bool = False + balance: float = 0 + credit: float = 0 profit: float = 0 equity: float = 0 - credit: float = 0 margin: float = 0 - margin_level: float = 0 margin_free: float = 0 - margin_mode: AccountMarginMode = AccountMarginMode.EXCHANGE - margin_so_mode: AccountStopOutMode = AccountStopOutMode.PERCENT + margin_level: float = 0 margin_so_call: float = 0 margin_so_so: float = 0 margin_initial: float = 0 margin_maintenance: float = 0 - fifo_close: bool = False - limit_orders: float = 0 - currency: str = "USD" - trade_allowed: bool = True - trade_expert: bool = True - currency_digits: int = 2 assets: float = 0 liabilities: float = 0 commission_blocked: float = 0 name: str = '' + server: str = '' + currency: str = "USD" company: str = '' - _fields: list[ClassVar[str]] = field(default_factory=list) + __match_args__: ClassVar[tuple] def asdict(self): - res = {key: getattr(self, key) for key in __match_args__} + res = {key: getattr(self, key) for key in self.__match_args__} return res def set_attrs(self, **kwargs): - [setattr(self, k, v) for k, v in kwargs.items() if k in self.fields] - - @property - def fields(self): - return self._fields or [name for f in fields(self) if (name := f.name) != '_fields'] + [setattr(self, k, v) for k, v in kwargs.items() if k in self.__match_args__] diff --git a/src/aiomql/contrib/backtester/test_data.py b/src/aiomql/contrib/backtester/test_data.py index 872798c..d3d11dd 100644 --- a/src/aiomql/contrib/backtester/test_data.py +++ b/src/aiomql/contrib/backtester/test_data.py @@ -17,10 +17,12 @@ from MetaTrader5 import (Tick, SymbolInfo, AccountInfo, TradeOrder, TradePositio from ...core.meta_trader import MetaTrader from ...core.constants import TimeFrame, CopyTicks, OrderType, TradeAction, AccountStopOutMode from ...core.config import Config -from .get_data import Data, GetData -from .test_account import AccountInfo as Account from ...utils import round_down, round_up, error_handler, error_handler_sync, async_cache +from .get_data import Data, GetData +from .test_account import TestAccount +from .types import PositionsManager, OrdersManager + tz = pytz.timezone('Etc/UTC') Cursor = namedtuple('Cursor', ['index', 'time']) @@ -31,10 +33,7 @@ class TestData: def __init__(self, data: Data = None, speed: int = 1, start: float | datetime = 0, end: float | datetime = 0): self._data = data or Data() - self._account: Account = Account(**self._data.account) - self.prices: dict[str, DataFrame] = self._data.prices - self.ticks: dict[str, DataFrame] = self._data.ticks - self.rates: dict[str, dict[str, DataFrame]] = self._data.rates + self._account: TestAccount = TestAccount(**self._data.account) span_start = (int(start.timestamp()) if isinstance(start, datetime) else int(start)) or self._data.span.start span_end = (int(end.timestamp()) if isinstance(end, datetime) else int(end)) or self._data.span.stop self.span: range = range(span_start, span_end, speed) @@ -50,7 +49,7 @@ class TestData: self.mt5 = MetaTrader() self.iter = zip_longest(self.range, self.span) self.cursor: Cursor = Cursor(index=self.range.start, time=self.span.start) - self.config = Config() + self.config = Config(test_data=self) self._data.name = self._data.name or f"{datetime.fromtimestamp(span_start):%d-%m-%y}_{datetime.fromtimestamp(span_end):%d-%m-%y}" self.fh = open(f'{self.config.test_data_dir}/data.json', 'a') @@ -95,13 +94,6 @@ class TestData: def get_dtype(self, df: DataFrame) -> list[tuple[str, str]]: return [(c, t) for c, t in zip(df.columns, df.dtypes)] - @async_cache - async def get_price_tick(self, symbol, time: int) -> Tick | None: - if self.config.use_terminal_for_backtesting: - tick = await self.mt5.copy_ticks_from(symbol, time, 1, CopyTicks.ALL) - return Tick(tick[-1]) if tick else None - return self.prices[symbol].loc[self.cursor.time] - async def tracker(self): pos_tasks = [self.check_position(ticket) for ticket in self.open_positions] await asyncio.gather(*pos_tasks) @@ -127,6 +119,14 @@ class TestData: except Exception as err: print(err) + @async_cache + async def get_price_tick(self, symbol, time: int) -> Tick | None: + if self.config.use_terminal_for_backtesting: + tick = await self.mt5.copy_ticks_from(symbol, time, 1, CopyTicks.ALL) + return Tick(tick[-1]) if tick else None + tick = self.prices[symbol].loc[self.cursor.time] + return Tick(tick) + @error_handler async def check_order(self, ticket: int): order = self.open_orders[ticket] @@ -194,7 +194,7 @@ class TestData: if self._account.margin == 0: self._account.margin_level = 0 else: - mode = self._account.margin_mode + mode = self._account.margin_so_mode level = self._account.equity / self._account.margin * 100 self._account.margin_level = level if mode == AccountStopOutMode.PERCENT else self._account.margin_free @@ -202,25 +202,40 @@ class TestData: self.update_account(gain=amount) def withdraw(self, amount: float): + assert amount <= self._account.balance, 'Insufficient funds' self.update_account(gain=-amount) @error_handler - async def setup_account(self): + async def setup_account(self, **kwargs): + default = {'profit': self._account.profit, 'margin': self._account.margin, 'equity': self._account.equity, + 'margin_free': self._account.margin_free, 'margin_level': self._account.margin_level, + 'balance': self._account.balance, + **{k: v for k, v in kwargs.items() if k in self._account.__match_args__}} + if self.config.use_terminal_for_backtesting: - acc = self._account - default = {'profit': acc.profit, 'margin': acc.margin, 'equity': acc.equity, 'margin_free': acc.margin_free, - 'margin_level': acc.margin_level, 'balance': acc.balance} - acc = await self.mt5.account_info() - acc = acc._asdict() | default - self._account.set_attrs(**acc) + acc_info = await self.mt5.account_info() + default = {**acc_info._asdict(), **default} + + self._account.set_attrs(**default) + self.update_account() + + @cached_property + def prices(self) -> dict[str, DataFrame]: + return self._data.prices + + @cached_property + def ticks(self) -> dict[str, DataFrame]: + return self._data.ticks + + @property + def rates(self) -> dict[str, dict[str, DataFrame]]: + return self._data.rates @cached_property def symbols(self) -> dict[str, SymbolInfo]: - ma = SymbolInfo.__match_args__ symbols = {} for symbol, info in self._data.symbols.items(): - sym = {key: info.get(key) for key in ma} - symbols[symbol] = SymbolInfo(sym) + symbols[symbol] = SymbolInfo((info.get(key) for key in SymbolInfo.__match_args__)) return symbols @error_handler @@ -283,59 +298,71 @@ class TestData: @error_handler async def order_check(self, request: dict) -> OrderCheckResult: - action, symbol, volume = request.get('action'), request.get('symbol'), request.get('volume') - price = request.get('price') - ocr = {'retcode': 0, 'balance': 0, 'profit': 0, 'margin': 0, 'equity': 0, 'margin_free': 0, - 'margin_level': 0, 'comment': 'Done', request: TradeRequest(request)} - - # check margin and confirm order can go through - margin = 0 - if all([action, symbol, volume, price]): - margin = await self.order_calc_margin(action, symbol, volume, price) - acc = self._account - equity = acc.equity - used_margin = acc.margin + margin - free_margin = acc.margin_free - margin - - if used_margin == 0: - margin_level = 0 - else: - level = equity / used_margin * 100 - margin_level = level if acc.margin_mode == AccountStopOutMode.PERCENT else free_margin - - if self.mt5.config.use_terminal_for_backtesting: - ocr_t = await self.mt5.order_check(request) - # return order check result if invalid stops level are detected or bad request - if ocr_t.retcode in (10016, 10013, 10014): - return ocr_t + ocr = {'retcode': 10013, 'balance': 0, 'profit': 0, 'margin': 0, 'equity': 0, 'margin_free': 0, + 'margin_level': 0, 'comment': 'Invalid request', + 'request': TradeRequest(request.get(k, (0 if k != 'comment' else 0)) for k in TradeRequest.__match_args__)} - sym = self.symbols[symbol] - tsl = sym.trade_stops_level - sl, tp = request.get('sl', 0), request.get('tp', 0) + action, symbol, volume = request.get('action'), request.get('symbol'), request.get('volume') + + price, order_type = request.get('price'), request.get('type') + if price is None and (action is TradeAction.DEAL and order_type in (OrderType.BUY, OrderType.SELL)): + ocr['comment'] = 'Market is closed' + ocr['retcode'] = 10018 + return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__)) + + # check margin and confirm order can go through + if order_type in (OrderType.BUY, OrderType.SELL): + margin = await self.order_calc_margin(action, symbol, volume, price) + if margin is None: + return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__)) + + used_margin = self._account.margin + margin + free_margin = self._account.margin_free - margin + + level = self._account.equity / used_margin * 100 if used_margin else float('inf') + margin_level = level if self._account.margin_so_mode == AccountStopOutMode.PERCENT else free_margin + + ocr.update({'margin_level': margin_level, 'margin': margin, 'margin_free': free_margin}) + + # check if the account has enough money + if margin_level < self._account.margin_so_call: + ocr['retcode'] = 10019 + ocr['comment'] = 'No money' + return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__)) # check if the stops level is valid + sym = await self.get_symbol_info(symbol) + tsl = sym.trade_stops_level + sym.spread + sl, tp = request.get('sl', 0), request.get('tp', 0) + if tp or sl: + current_price = price + if action == TradeAction.SLTP: + pos = self.open_positions.get(request.get('position')) # ToDo: use positions manager + sym = pos.symbol + current_price = await self.get_price_tick(sym, self.cursor.time) min_sl = min(sl, tp) - dsl = abs(price - min_sl) / sym.point - if dsl < tsl: + dsl = abs(current_price - min_sl) / sym.point + if int(dsl) < int(tsl): ocr['retcode'] = 10016 ocr['comment'] = 'Invalid stops' - return OrderCheckResult(ocr) + return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__)) - # check if the account has enough money - if margin_level < acc.margin_so_call: - ocr['retcode'] = 10019 - ocr['comment'] = 'No money' + if self.mt5.config.use_terminal_for_backtesting: + ocr_t = await self.mt5.order_check(request) + if ocr_t.retcode in (10013, 10014): + return ocr_t + else: + # check volume + if volume < sym.volume_min or volume > sym.volume_max: + ocr['retcode'] = 10014 + ocr['comment'] = 'Invalid volume' + return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__)) - # check volume - if volume < sym.volume_min or volume > sym.volume_max: - ocr['retcode'] = 10014 - ocr['comment'] = 'Invalid volume' + ocr.update({'balance': self._account.balance, 'profit': self._account.profit, 'equity': self._account.equity, + 'comment': 'Done', 'retcode': 0}) - ocr.update({'balance': acc.balance, 'profit': acc.profit, 'margin': used_margin, 'equity': equity, - 'margin_free': free_margin, 'margin_level': margin_level}) - - return OrderCheckResult(ocr) + return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__)) @error_handler async def get_terminal_info(self) -> TerminalInfo: @@ -378,14 +405,13 @@ class TestData: async def get_symbol_info(self, symbol: str) -> SymbolInfo: if self.config.use_terminal_for_backtesting: info = await self.mt5.symbol_info(symbol) - return info + else: + info = self.symbols[symbol] - info = self.symbols[symbol] tick = await self.get_symbol_info_tick(symbol) - info = info._asdict() - info |= {'bid': tick.bid, 'bidhigh': tick.bid, 'bidlow': tick.bid, 'ask': tick.ask, + info = info._asdict() | {'bid': tick.bid, 'bidhigh': tick.bid, 'bidlow': tick.bid, 'ask': tick.ask, 'askhigh': tick.ask, 'asklow': tick.bid, 'last': tick.last, 'volume_real': tick.volume_real} - return SymbolInfo(info) + return SymbolInfo((info.get(key) for key in SymbolInfo.__match_args__)) @error_handler async def get_rates_from(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float, count: int) -> np.ndarray: diff --git a/src/aiomql/contrib/backtester/types.py b/src/aiomql/contrib/backtester/types.py new file mode 100644 index 0000000..29c01d2 --- /dev/null +++ b/src/aiomql/contrib/backtester/types.py @@ -0,0 +1,45 @@ +from MetaTrader5 import TradePosition, TradeOrder, TradeDeal + + +class TradingData: + _data: dict + _open_items: set[int] + + def __getitem__(self, item): + return self._data[item] + + def __setitem__(self, key, value: TradePosition | TradeOrder | TradeDeal): + self._open_items.add(value.ticket) + self._data[key] = value + + def __delitem__(self, key): + del self._data[key] + self._open_items.discard(key) + + def __contains__(self, item): + return item in self._open_items + + def __iter__(self): + return iter(self._data) + + def __len__(self): + return len(self._data) + + def get(self, key, default=None): + return self._data.get(key, default) if key in self._open_items else default + + def pop(self, key, default=None): + self._open_items.discard(key) + return self._data.pop(key, default) + + +class PositionsManager(TradingData): + def __init__(self, open_items: set[int] = None, data: dict = None): + self._open_items = open_items or set() + self._data = data or {} + + +class OrdersManager(TradingData): + def __init__(self, open_items: set[int] = None, data: dict = None): + self._open_items = open_items or set() + self._data = data or {} diff --git a/src/aiomql/contrib/backtester/utils.py b/src/aiomql/contrib/backtester/utils.py new file mode 100644 index 0000000..7bbc2c1 --- /dev/null +++ b/src/aiomql/contrib/backtester/utils.py @@ -0,0 +1,24 @@ +from MetaTrader5 import TradePosition, TradeOrder, TradeDeal +# from .get_data import Data + +class LiveDesc: + """A Descriptor for live trading data""" + def __set_name__(self, owner, name): + self.access_name = name + + def __get__(self, instance, owner): + return instance.__dict__.get(self.access_name, {}) + + def __set__(self, instance, value: tuple[int, str]): + prop = instance.__dict__.setdefault(self.access_name, {}) + prop[value[0]] = value[1] + + +class Data: + pos = LiveDesc() + ords = LiveDesc() + + +dd = Data() +dd.pos = (1, 'EURUSD') +print(dd.pos) diff --git a/src/aiomql/core/meta_trader.py b/src/aiomql/core/meta_trader.py index 6bea4f8..3c7a6fa 100644 --- a/src/aiomql/core/meta_trader.py +++ b/src/aiomql/core/meta_trader.py @@ -4,6 +4,7 @@ from logging import getLogger from typing import Callable import MetaTrader5 +import numpy as np from MetaTrader5 import BookInfo, SymbolInfo, AccountInfo, Tick, TerminalInfo, TradeOrder, TradeDeal, \ TradePosition, OrderSendResult, OrderCheckResult @@ -215,33 +216,33 @@ class MetaTrader(metaclass=BaseMeta): res = await self._handler(api) return res - async def copy_rates_from(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float, count: int): + async def copy_rates_from(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float, count: int) -> np.ndarray | None: api = {'func': self._copy_rates_from, 'args': (symbol, timeframe, date_from, count), 'error_msg': f'Error in obtaining rates for {symbol}'} res = await self._handler(api) return res - async def copy_rates_from_pos(self, symbol: str, timeframe: TimeFrame, start_pos: int, count: int): + async def copy_rates_from_pos(self, symbol: str, timeframe: TimeFrame, start_pos: int, count: int) -> np.ndarray | None: api = {'func': self._copy_rates_from_pos, 'args': (symbol, timeframe, start_pos, count), 'error_msg': f'Error in obtaining rates for {symbol}'} res = await self._handler(api) return res async def copy_rates_range(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float, - date_to: datetime | float): + date_to: datetime | float) -> np.ndarray | None: api = {'func': self._copy_rates_range, 'args': (symbol, timeframe, date_from, date_to), 'error_msg': f'Error in obtaining rates for {symbol}'} res = await self._handler(api) return res - async def copy_ticks_from(self, symbol: str, date_from: datetime | float, count: int, flags: CopyTicks): + async def copy_ticks_from(self, symbol: str, date_from: datetime | float, count: int, flags: CopyTicks) -> np.ndarray | None: api = {'func': self._copy_ticks_from, 'args': (symbol, date_from, count, flags), 'error_msg': f'Error in obtaining ticks for {symbol}'} res = await self._handler(api) return res async def copy_ticks_range(self, symbol: str, date_from: datetime | float, date_to: datetime | float, - flags: CopyTicks): + flags: CopyTicks) -> np.ndarray | None: api = {'func': self._copy_ticks_range, 'args': (symbol, date_from, date_to, flags), 'error_msg': f'Error in obtaining ticks for {symbol}'} res = await self._handler(api)