testdata

src/aiomql/contrib/backtester/__init__.py

@@ -4,3 +4,5 @@ from .get_data import GetData
 from .test_strategy import TestStrategy
 from .event_manager import EventManager
 from .strategy_tester import StrategyTester, SingleStrategyTester
+from .test_account import TestAccount
+from .types import TradingData, PositionsManager, OrdersManager

src/aiomql/contrib/backtester/check1.py

@@ -1,75 +1,42 @@
-import asyncio
+from functools import cached_property
-import inspect
+class Data:
-from functools import cache, lru_cache, cached_property, wraps, partial
+    def __init__(self):
+        self.a = 1
+        self.b = 2
 
-# from
+class TData:
-
+    _a: int
-def async_cache(fun):
+    _b: int
-
+    _dat: dict
-    @wraps(fun)
-    async def wrapper(*args, **kwargs):
-        print(wrapper.cache)
-        key = (args, frozenset(kwargs.items()))
-        async with wrapper.lock:
-            if key not in wrapper.cache:
-                print('not in cache')
-                wrapper.cache[key] = await fun(*args, **kwargs)
-        return wrapper.cache[key]
-
-    wrapper.lock = asyncio.Lock()
-    wrapper.cache = {}
-    return wrapper
-
-
-class Test:
 
     def __init__(self):
-        self.rr = 0
+        self._data = Data()
 
-    def __repr__(self):
+    def __getattr__(self, item):
-        return f'{self.__class__.__name__}(...)'
+        if val := self.__annotations__.get(item):
+            return getattr(self._data, item, val())
 
-    @async_cache
+    @property
-    async def check(self, a, b):
+    def a(self):
-        an = a + b - self.rr
+        return self._a
-        return an
 
-# async def main(a, b):
+    @a.setter
-#     t = Test()
+    def a(self, val):
-#
+        self._a = val
-#     @async_cache
-#     def check(_a, _b):
-#         an = _a + _b
-#         print('check', _a, _b, t.rr)
-#         return an
-#
-#     return check(a, b)
 
-t = Test()
+    @property
-y = asyncio.run(t.check(1, 2))
+    def dat(self):
-t.rr = 6
+        return self._dat
-y1 = asyncio.run(t.check(1, 2))
-t.rr = 7
-y2 = asyncio.run(t.check(1, 2))
-print(y, y1, y2)
 
-@async_cache
+    @dat.setter
-async def func(a, b):
+    def dat(self, key, val):
-    an = a + b
+        self._dat |= val
-    print('func')
-    return an
 
-@async_cache
+    @dat.deleter
-async def func1(e, c=6, d=6):
+    def dat(self):
-    an = e - c + d
+        self._dat = {}
-    print('func1')
-    return an
 
 
-# asyncio.run(func(1, 2))
+f = TData()
-# asyncio.run(func1(1, d=8))
+# f.data = {'a': 1}
-# asyncio.run(func(1, 2))
+print(f.a)
-# asyncio.run(func(1, 3))
-# asyncio.run(func1(1, d=6))
-# asyncio.run(func1(1, d=6))
-# asyncio.run(func1(1, d=7))

src/aiomql/contrib/backtester/get_data.py

@@ -225,7 +225,7 @@ class GetData:
         res = pd.DataFrame(res)
         res.drop_duplicates(subset=['time'], keep='last', inplace=True)
         res.set_index('time', inplace=True, drop=False)
-        res = res.reindex(self.span) # change method back to 'nearest'
+        res = res.reindex(self.span) # fill in missing values with NaN
         self.data.prices[symbol] = res
 
     @backoff_decorator

src/aiomql/contrib/backtester/test_account.py

@@ -1,52 +1,45 @@
-from dataclasses import dataclass, asdict, field, fields
+from dataclasses import dataclass
 from typing import ClassVar
 
 from ...core.constants import AccountTradeMode, AccountMarginMode, AccountStopOutMode
-from MetaTrader5 import AccountInfo
-
-__match_args__ = SymbolInfo
 
 
 @dataclass
-class AccountInfo:
+class TestAccount:
     login: int = 0
-    server: str = ''
     trade_mode: AccountTradeMode = AccountTradeMode.DEMO
-    balance: float = 0
     leverage: float = 0
+    limit_orders: float = 0
+    margin_so_mode: AccountStopOutMode = AccountStopOutMode.PERCENT
+    trade_allowed: bool = True
+    trade_expert: bool = True
+    margin_mode: AccountMarginMode = AccountMarginMode.EXCHANGE
+    currency_digits: int = 2
+    fifo_close: bool = False
+    balance: float = 0
+    credit: float = 0
     profit: float = 0
     equity: float = 0
-    credit: float = 0
     margin: float = 0
-    margin_level: float = 0
     margin_free: float = 0
-    margin_mode: AccountMarginMode = AccountMarginMode.EXCHANGE
+    margin_level: float = 0
-    margin_so_mode: AccountStopOutMode = AccountStopOutMode.PERCENT
     margin_so_call: float = 0
     margin_so_so: float = 0
     margin_initial: float = 0
     margin_maintenance: float = 0
-    fifo_close: bool = False
-    limit_orders: float = 0
-    currency: str = "USD"
-    trade_allowed: bool = True
-    trade_expert: bool = True
-    currency_digits: int = 2
     assets: float = 0
     liabilities: float = 0
     commission_blocked: float = 0
     name: str = ''
+    server: str = ''
+    currency: str = "USD"
     company: str = ''
 
-    _fields: list[ClassVar[str]] = field(default_factory=list)
+    __match_args__: ClassVar[tuple]
 
     def asdict(self):
-        res = {key: getattr(self, key) for key in __match_args__}
+        res = {key: getattr(self, key) for key in self.__match_args__}
         return res
 
     def set_attrs(self, **kwargs):
-        [setattr(self, k, v) for k, v in kwargs.items() if k in self.fields]
+        [setattr(self, k, v) for k, v in kwargs.items() if k in self.__match_args__]
-
-    @property
-    def fields(self):
-        return self._fields or [name for f in fields(self) if (name := f.name) != '_fields']

src/aiomql/contrib/backtester/test_data.py

@@ -17,10 +17,12 @@ from MetaTrader5 import (Tick, SymbolInfo, AccountInfo, TradeOrder, TradePositio
 from ...core.meta_trader import MetaTrader
 from ...core.constants import TimeFrame, CopyTicks, OrderType, TradeAction, AccountStopOutMode
 from ...core.config import Config
-from .get_data import Data, GetData
-from .test_account import AccountInfo as Account
 from ...utils import round_down, round_up, error_handler, error_handler_sync, async_cache
 
+from .get_data import Data, GetData
+from .test_account import TestAccount
+from .types import PositionsManager, OrdersManager
+
 tz = pytz.timezone('Etc/UTC')
 Cursor = namedtuple('Cursor', ['index', 'time'])
 
@@ -31,10 +33,7 @@ class TestData:
     
     def __init__(self, data: Data = None, speed: int = 1, start: float | datetime = 0, end: float | datetime = 0):
         self._data = data or Data()
-        self._account: Account = Account(**self._data.account)
+        self._account: TestAccount = TestAccount(**self._data.account)
-        self.prices: dict[str, DataFrame] = self._data.prices
-        self.ticks: dict[str, DataFrame] = self._data.ticks
-        self.rates: dict[str, dict[str, DataFrame]] = self._data.rates
         span_start = (int(start.timestamp()) if isinstance(start, datetime) else int(start)) or self._data.span.start
         span_end = (int(end.timestamp()) if isinstance(end, datetime) else int(end)) or self._data.span.stop
         self.span: range = range(span_start, span_end, speed)
@@ -50,7 +49,7 @@ class TestData:
         self.mt5 = MetaTrader()
         self.iter = zip_longest(self.range, self.span)
         self.cursor: Cursor = Cursor(index=self.range.start, time=self.span.start)
-        self.config = Config()
+        self.config = Config(test_data=self)
         self._data.name = self._data.name or f"{datetime.fromtimestamp(span_start):%d-%m-%y}_{datetime.fromtimestamp(span_end):%d-%m-%y}"
         self.fh = open(f'{self.config.test_data_dir}/data.json', 'a')
 
@@ -95,13 +94,6 @@ class TestData:
     def get_dtype(self, df: DataFrame) -> list[tuple[str, str]]:
         return [(c, t) for c, t in zip(df.columns, df.dtypes)]
 
-    @async_cache
-    async def get_price_tick(self, symbol, time: int) -> Tick | None:
-        if self.config.use_terminal_for_backtesting:
-            tick = await self.mt5.copy_ticks_from(symbol, time, 1, CopyTicks.ALL)
-            return Tick(tick[-1]) if tick else None
-        return self.prices[symbol].loc[self.cursor.time]
-
     async def tracker(self):
         pos_tasks = [self.check_position(ticket) for ticket in self.open_positions]
         await asyncio.gather(*pos_tasks)
@@ -127,6 +119,14 @@ class TestData:
         except Exception as err:
             print(err)
 
+    @async_cache
+    async def get_price_tick(self, symbol, time: int) -> Tick | None:
+        if self.config.use_terminal_for_backtesting:
+            tick = await self.mt5.copy_ticks_from(symbol, time, 1, CopyTicks.ALL)
+            return Tick(tick[-1]) if tick else None
+        tick = self.prices[symbol].loc[self.cursor.time]
+        return Tick(tick)
+
     @error_handler
     async def check_order(self, ticket: int):
         order = self.open_orders[ticket]
@@ -194,7 +194,7 @@ class TestData:
         if self._account.margin == 0:
             self._account.margin_level = 0
         else:
-            mode = self._account.margin_mode
+            mode = self._account.margin_so_mode
             level = self._account.equity / self._account.margin * 100
             self._account.margin_level = level if mode == AccountStopOutMode.PERCENT else self._account.margin_free
 
@@ -202,25 +202,40 @@ class TestData:
         self.update_account(gain=amount)
 
     def withdraw(self, amount: float):
+        assert amount <= self._account.balance, 'Insufficient funds'
         self.update_account(gain=-amount)
 
     @error_handler
-    async def setup_account(self):
+    async def setup_account(self, **kwargs):
+        default = {'profit': self._account.profit, 'margin': self._account.margin, 'equity': self._account.equity,
+                   'margin_free': self._account.margin_free, 'margin_level': self._account.margin_level,
+                   'balance': self._account.balance,
+                   **{k: v for k, v in kwargs.items() if k in self._account.__match_args__}}
+
         if self.config.use_terminal_for_backtesting:
-            acc = self._account
+            acc_info = await self.mt5.account_info()
-            default = {'profit': acc.profit, 'margin': acc.margin, 'equity': acc.equity, 'margin_free': acc.margin_free,
+            default = {**acc_info._asdict(), **default}
-                       'margin_level': acc.margin_level, 'balance': acc.balance}
+
-            acc = await self.mt5.account_info()
+        self._account.set_attrs(**default)
-            acc = acc._asdict() | default
+        self.update_account()
-            self._account.set_attrs(**acc)
+
+    @cached_property
+    def prices(self) -> dict[str, DataFrame]:
+        return self._data.prices
+
+    @cached_property
+    def ticks(self) -> dict[str, DataFrame]:
+        return self._data.ticks
+
+    @property
+    def rates(self) -> dict[str, dict[str, DataFrame]]:
+        return self._data.rates
 
     @cached_property
     def symbols(self) -> dict[str, SymbolInfo]:
-        ma = SymbolInfo.__match_args__
         symbols = {}
         for symbol, info in self._data.symbols.items():
-            sym = {key: info.get(key) for key in ma}
+            symbols[symbol] = SymbolInfo((info.get(key) for key in SymbolInfo.__match_args__))
-            symbols[symbol] = SymbolInfo(sym)
         return symbols
 
     @error_handler
@@ -283,59 +298,71 @@ class TestData:
 
     @error_handler
     async def order_check(self, request: dict) -> OrderCheckResult:
+        ocr = {'retcode': 10013, 'balance': 0, 'profit': 0, 'margin': 0, 'equity': 0, 'margin_free': 0,
+               'margin_level': 0, 'comment': 'Invalid request',
+               'request': TradeRequest(request.get(k, (0 if k != 'comment' else 0)) for k in TradeRequest.__match_args__)}
+
         action, symbol, volume = request.get('action'), request.get('symbol'), request.get('volume')
-        price = request.get('price')
+
-        ocr = {'retcode': 0, 'balance': 0, 'profit': 0, 'margin': 0, 'equity': 0, 'margin_free': 0,
+        price, order_type = request.get('price'), request.get('type')
-               'margin_level': 0, 'comment': 'Done', request: TradeRequest(request)}
+        if price is None and (action is TradeAction.DEAL and order_type in (OrderType.BUY, OrderType.SELL)):
+            ocr['comment'] = 'Market is closed'
+            ocr['retcode'] = 10018
+            return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__))
 
         # check margin and confirm order can go through
-        margin = 0 
+        if order_type in (OrderType.BUY, OrderType.SELL):
-        if all([action, symbol, volume, price]):
             margin = await self.order_calc_margin(action, symbol, volume, price)
-        acc = self._account
+            if margin is None:
-        equity = acc.equity
+                return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__))
-        used_margin = acc.margin + margin
-        free_margin = acc.margin_free - margin
 
-        if used_margin == 0:
+            used_margin = self._account.margin + margin
-            margin_level = 0
+            free_margin = self._account.margin_free - margin
-        else:
+
-            level = equity / used_margin * 100
+            level = self._account.equity / used_margin * 100 if used_margin else float('inf')
-            margin_level = level if acc.margin_mode == AccountStopOutMode.PERCENT else free_margin
+            margin_level = level if self._account.margin_so_mode == AccountStopOutMode.PERCENT else free_margin
+
+            ocr.update({'margin_level': margin_level, 'margin': margin, 'margin_free': free_margin})
+
+            # check if the account has enough money
+            if margin_level < self._account.margin_so_call:
+                ocr['retcode'] = 10019
+                ocr['comment'] = 'No money'
+                return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__))
+
+        # check if the stops level is valid
+        sym = await self.get_symbol_info(symbol)
+        tsl = sym.trade_stops_level + sym.spread
+        sl, tp = request.get('sl', 0), request.get('tp', 0)
+
+        if tp or sl:
+            current_price = price
+            if action == TradeAction.SLTP:
+                pos = self.open_positions.get(request.get('position')) # ToDo: use positions manager
+                sym = pos.symbol
+                current_price = await self.get_price_tick(sym, self.cursor.time)
+            min_sl = min(sl, tp)
+            dsl = abs(current_price - min_sl) / sym.point
+            if int(dsl) < int(tsl):
+                ocr['retcode'] = 10016
+                ocr['comment'] = 'Invalid stops'
+                return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__))
 
         if self.mt5.config.use_terminal_for_backtesting:
             ocr_t = await self.mt5.order_check(request)
-            # return order check result if invalid stops level are detected or bad request
+            if ocr_t.retcode in (10013, 10014):
-            if ocr_t.retcode in (10016, 10013, 10014):
                 return ocr_t
+        else:
+            # check volume
+            if volume < sym.volume_min or volume > sym.volume_max:
+                ocr['retcode'] = 10014
+                ocr['comment'] = 'Invalid volume'
+                return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__))
 
-        sym = self.symbols[symbol]
+        ocr.update({'balance': self._account.balance, 'profit': self._account.profit, 'equity': self._account.equity,
-        tsl = sym.trade_stops_level
+                    'comment': 'Done', 'retcode': 0})
-        sl, tp = request.get('sl', 0), request.get('tp', 0)
 
-        # check if the stops level is valid
+        return OrderCheckResult((ocr.get(k, 0) for k in OrderCheckResult.__match_args__))
-        if tp or sl:
-            min_sl = min(sl, tp)
-            dsl = abs(price - min_sl) / sym.point
-            if dsl < tsl:
-                ocr['retcode'] = 10016
-                ocr['comment'] = 'Invalid stops'
-                return OrderCheckResult(ocr)
-
-        # check if the account has enough money
-        if margin_level < acc.margin_so_call:
-            ocr['retcode'] = 10019
-            ocr['comment'] = 'No money'
-
-        # check volume
-        if volume < sym.volume_min or volume > sym.volume_max:
-            ocr['retcode'] = 10014
-            ocr['comment'] = 'Invalid volume'
-
-        ocr.update({'balance': acc.balance, 'profit': acc.profit, 'margin': used_margin, 'equity': equity,
-                    'margin_free': free_margin, 'margin_level': margin_level})
-
-        return OrderCheckResult(ocr)
 
     @error_handler
     async def get_terminal_info(self) -> TerminalInfo:
@@ -378,14 +405,13 @@ class TestData:
     async def get_symbol_info(self, symbol: str) -> SymbolInfo:
         if self.config.use_terminal_for_backtesting:
             info = await self.mt5.symbol_info(symbol)
-            return info
+        else:
+            info = self.symbols[symbol]
 
-        info = self.symbols[symbol]
         tick = await self.get_symbol_info_tick(symbol)
-        info = info._asdict()
+        info = info._asdict() | {'bid': tick.bid, 'bidhigh': tick.bid, 'bidlow': tick.bid, 'ask': tick.ask,
-        info |= {'bid': tick.bid, 'bidhigh': tick.bid, 'bidlow': tick.bid, 'ask': tick.ask,
                  'askhigh': tick.ask, 'asklow': tick.bid, 'last': tick.last, 'volume_real': tick.volume_real}
-        return SymbolInfo(info)
+        return SymbolInfo((info.get(key) for key in SymbolInfo.__match_args__))
 
     @error_handler
     async def get_rates_from(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float, count: int) -> np.ndarray:

src/aiomql/contrib/backtester/types.py

@@ -0,0 +1,45 @@
+from MetaTrader5 import TradePosition, TradeOrder, TradeDeal
+
+
+class TradingData:
+    _data: dict
+    _open_items: set[int]
+
+    def __getitem__(self, item):
+        return self._data[item]
+
+    def __setitem__(self, key, value: TradePosition | TradeOrder | TradeDeal):
+        self._open_items.add(value.ticket)
+        self._data[key] = value
+
+    def __delitem__(self, key):
+        del self._data[key]
+        self._open_items.discard(key)
+
+    def __contains__(self, item):
+        return item in self._open_items
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __len__(self):
+        return len(self._data)
+
+    def get(self, key, default=None):
+        return self._data.get(key, default) if key in self._open_items else default
+
+    def pop(self, key, default=None):
+        self._open_items.discard(key)
+        return self._data.pop(key, default)
+
+
+class PositionsManager(TradingData):
+    def __init__(self, open_items: set[int] = None, data: dict = None):
+        self._open_items = open_items or set()
+        self._data = data or {}
+
+
+class OrdersManager(TradingData):
+    def __init__(self, open_items: set[int] = None, data: dict = None):
+        self._open_items = open_items or set()
+        self._data = data or {}

src/aiomql/contrib/backtester/utils.py

@@ -0,0 +1,24 @@
+from MetaTrader5 import TradePosition, TradeOrder, TradeDeal
+# from .get_data import Data
+
+class LiveDesc:
+    """A Descriptor for live trading data"""
+    def __set_name__(self, owner, name):
+        self.access_name = name
+
+    def __get__(self, instance, owner):
+        return instance.__dict__.get(self.access_name, {})
+
+    def __set__(self, instance, value: tuple[int, str]):
+        prop = instance.__dict__.setdefault(self.access_name, {})
+        prop[value[0]] = value[1]
+
+
+class Data:
+    pos = LiveDesc()
+    ords = LiveDesc()
+
+
+dd = Data()
+dd.pos = (1, 'EURUSD')
+print(dd.pos)

src/aiomql/core/meta_trader.py

@@ -4,6 +4,7 @@ from logging import getLogger
 from typing import Callable
 
 import MetaTrader5
+import numpy as np
 from MetaTrader5 import BookInfo, SymbolInfo, AccountInfo, Tick, TerminalInfo, TradeOrder, TradeDeal, \
     TradePosition, OrderSendResult, OrderCheckResult
 
@@ -215,33 +216,33 @@ class MetaTrader(metaclass=BaseMeta):
         res = await self._handler(api)
         return res
 
-    async def copy_rates_from(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float, count: int):
+    async def copy_rates_from(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float, count: int) -> np.ndarray | None:
         api = {'func': self._copy_rates_from, 'args': (symbol, timeframe, date_from, count),
                'error_msg': f'Error in obtaining rates for {symbol}'}
         res = await self._handler(api)
         return res
 
-    async def copy_rates_from_pos(self, symbol: str, timeframe: TimeFrame, start_pos: int, count: int):
+    async def copy_rates_from_pos(self, symbol: str, timeframe: TimeFrame, start_pos: int, count: int) -> np.ndarray | None:
         api = {'func': self._copy_rates_from_pos, 'args': (symbol, timeframe, start_pos, count),
                'error_msg': f'Error in obtaining rates for {symbol}'}
         res = await self._handler(api)
         return res
 
     async def copy_rates_range(self, symbol: str, timeframe: TimeFrame, date_from: datetime | float,
-                               date_to: datetime | float):
+                               date_to: datetime | float) -> np.ndarray | None:
         api = {'func': self._copy_rates_range, 'args': (symbol, timeframe, date_from, date_to),
                'error_msg': f'Error in obtaining rates for {symbol}'}
         res = await self._handler(api)
         return res
 
-    async def copy_ticks_from(self, symbol: str, date_from: datetime | float, count: int, flags: CopyTicks):
+    async def copy_ticks_from(self, symbol: str, date_from: datetime | float, count: int, flags: CopyTicks) -> np.ndarray | None:
         api = {'func': self._copy_ticks_from, 'args': (symbol, date_from, count, flags),
                'error_msg': f'Error in obtaining ticks for {symbol}'}
         res = await self._handler(api)
         return res
 
     async def copy_ticks_range(self, symbol: str, date_from: datetime | float, date_to: datetime | float,
-                               flags: CopyTicks):
+                               flags: CopyTicks) -> np.ndarray | None:
         api = {'func': self._copy_ticks_range, 'args': (symbol, date_from, date_to, flags),
                'error_msg': f'Error in obtaining ticks for {symbol}'}
         res = await self._handler(api)