testdata

src/aiomql/core/models.py

@@ -4,7 +4,7 @@ from .constants import BookType, TradeAction, OrderType, OrderTime, OrderFilling
     DealReason, SymbolChartMode, SymbolTradeMode, SymbolCalcMode, SymbolOptionMode, SymbolOrderGTCMode, \
     SymbolOptionRight, \
     SymbolTradeExecution, SymbolSwapMode, DayOfWeek, AccountTradeMode, AccountStopOutMode, AccountMarginMode, \
-    OrderReason
+    OrderReason, TickFlag
 
 from .base import Base
 
@@ -334,9 +334,9 @@ class SymbolInfo(Base):
     path: str
 
     def __init__(self, **kwargs):
-        if 'name' not in kwargs:
+        if name := kwargs.pop('name', None):
             raise AttributeError('Symbol Object Must be initialized with a name')
-        self.name = kwargs.pop('name')
+        self.name = name
         super().__init__(**kwargs)
 
     def __repr__(self):
@@ -351,6 +351,28 @@ class SymbolInfo(Base):
     def __hash__(self):
         return hash(self.name)
 
+class TickInfo(Base):
+    """Price Tick of a Financial Instrument.
+
+    Attributes:
+        time (int): Time of the last prices update for the symbol
+        bid (float): Current Bid price
+        ask (float): Current Ask price
+        last (float): Price of the last deal (Last)
+        volume (float): Volume for the current Last price
+        time_msc (int): Time of the last prices update for the symbol in milliseconds
+        flags (TickFlag): Tick flags
+        volume_real (float): Volume for the current Last price
+        Index (int): Custom attribute representing the position of the tick in a sequence.
+    """
+    time: float
+    bid: float
+    ask: float
+    last: float
+    volume: float
+    time_msc: float
+    flags: TickFlag
+    volume_real: float
 
 class BookInfo(Base):
     """Book Information Class.

src/aiomql/lib/backtester/get_data.py

@@ -1,5 +1,6 @@
+from typing import TypedDict
 import pickle
-import random
+import lzma
 from datetime import datetime
 from logging import getLogger
 import asyncio
@@ -7,22 +8,31 @@ import asyncio
 import pytz
 from MetaTrader5 import Tick, SymbolInfo
 import pandas as pd
+from pandas import DataFrame
 
 from ...core.meta_trader import MetaTrader
 from ...core.config import Config
 from ...core.errors import Error
 from ...core.constants import TimeFrame, CopyTicks, OrderType
 from ...core.models import (AccountInfo, SymbolInfo, BookInfo, TradeOrder, OrderCheckResult, OrderSendResult,
-                            TradePosition, TradeDeal)
+                            TradePosition, TradeDeal, TickInfo)
 from ...utils import backoff_decorator
 
 logger = getLogger(__name__)
 
+class Data(TypedDict):
+    account: AccountInfo
+    symbols: dict[str, SymbolInfo]
+    prices: DataFrame
+    ticks: DataFrame
+    rates: DataFrame
+    span: range
+
 
 class GetData:
-    config = Config()
+    config: Config = Config()
 
-    def __init__(self, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str],
+    def __init__(self, *, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str],
                  interval: int = 60, name: str = '', tz: str = 'Etc/UTC'):
         """"""
         super().__init__()
@@ -38,55 +48,71 @@ class GetData:
         self.span = range(start := int(self.start.timestamp()), diff + start)
         self.mt5 = MetaTrader()
 
-    @classmethod
+    async def get_data(self) -> Data:
-    def load_data(cls, name: str = '') -> dict:
-        """"""
-        name = name
-        file = open(f'{cls.config.root}/data/{name}', 'rb')
-        data = pickle.load(file)
-        file.close()
-        return data
-
-    async def get_test_data(self) -> dict:
         """"""
         data = {}
         rates, ticks, prices, symbols, account = await asyncio.gather(self.get_symbols_rates(), self.get_symbols_ticks(),
                                                                 self.get_symbols_prices(), self.get_symbols_info(),
-                                                                      self.get_account_info())
+                                                    self.get_account_info())
+
         data['rates'] = rates
         data['ticks'] = ticks
         data['prices'] = prices
         data['symbols'] = symbols
         data['account'] = account
+        data['range'] = self.span
 
-        return data
+        return Data(**data)
 
-    async def get_and_save_data(self) -> None:
+    async def pickle_data(self) -> None:
         """"""
-        data = await self.get_test_data()
+        data = await self.get_data()
         fh = open(f'{self.config.root}/data/{self.name}', 'wb')
         pickle.dump(data, fh)
         fh.close()
+    
+    async def compress_data(self):
+        """"""
+        data = await self.get_data()
+        bdata = pickle.dumps(data)
+        name = self.name + 'xz' 
+        with lzma.open(name, 'w') as fh:
+            fh.write(bdata)
 
-    async def get_symbols_info(self):
+    @classmethod
+    def load_data(cls, name: str, compressed=False) -> dict:
+        """"""
+        fo = open(f'{cls.config.root}/data/{name}', 'rb')
+        data = fo.read()
+
+        if compressed:
+            data = lzma.decompress(data)
+        else:
+            data = pickle.loads(data)
+
+        fo.close()
+
+        return data
+
+    async def get_symbols_info(self) -> dict[str, SymbolInfo]:
         """"""
         tasks = [self.get_symbol_info(symbol) for symbol in self.symbols]
         res = await asyncio.gather(*tasks)
-        return {symbol: info for symbol, info in res}
+        return {symbol: SymbolInfo(**info) for symbol, info in res}
 
-    async def get_symbols_ticks(self):
+    async def get_symbols_ticks(self) -> dict[str, DataFrame]:
         """"""
         tasks = [self.get_symbol_ticks(symbol) for symbol in self.symbols]
         res = await asyncio.gather(*tasks)
-        return {symbol: ticks for symbol, ticks in res}
+        return {symbol: tick for symbol, tick in res}
 
-    async def get_symbols_prices(self):
+    async def get_symbols_prices(self) -> dict[str, DataFrame]:
         """"""
         tasks = [self.get_symbol_prices(symbol) for symbol in self.symbols]
         res = await asyncio.gather(*tasks)
         return {symbol: prices for symbol, prices in res}
 
-    async def get_symbols_rates(self):
+    async def get_symbols_rates(self) -> dict[str: dict[TimeFrame: DataFrame]]:
         """"""
         tasks = [self.get_symbol_rates(symbol, timeframe) for symbol in self.symbols for timeframe in self.timeframes]
         res = await asyncio.gather(*tasks)
@@ -96,19 +122,19 @@ class GetData:
         return data
 
     @backoff_decorator(max_retries=5)
-    async def get_account_info(self) -> AccountInfo | None:
+    async def get_account_info(self) -> dict:
         """"""
         res = await self.mt5.account_info()
         return res._asdict()
 
     @backoff_decorator(max_retries=5)
-    async def get_symbol_info(self, symbol: str):
+    async def get_symbol_info(self, symbol: str) -> tuple[str, dict]:
         """"""
         res = await self.mt5.symbol_info(symbol)
         return symbol, res._asdict()
 
     @backoff_decorator(max_retries=5)
-    async def get_symbol_ticks(self, symbol: str):
+    async def get_symbol_ticks(self, symbol: str) -> tuple[str, DataFrame]:
         """"""
         res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
         res = pd.DataFrame(res)
@@ -117,7 +143,7 @@ class GetData:
         return symbol, res
 
     @backoff_decorator(max_retries=5)
-    async def get_symbol_prices(self, symbol: str):
+    async def get_symbol_prices(self, symbol: str) -> tuple[str, DataFrame]:
         """"""
         res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
         res = pd.DataFrame(res)
@@ -127,7 +153,7 @@ class GetData:
         return symbol, res
 
     @backoff_decorator(max_retries=5)
-    async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame):
+    async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame) -> tuple[str, TimeFrame, DataFrame]:
         """"""
         res = await self.mt5.copy_rates_range(symbol, timeframe, self.start, self.end)
         res = pd.DataFrame(res)

src/aiomql/lib/backtester/meta_tester.py

@@ -2,165 +2,47 @@ import pickle
 from datetime import datetime
 from logging import getLogger
 import asyncio
+import re
 
 import pytz
 from MetaTrader5 import Tick, SymbolInfo
 import pandas as pd
 
+from ... import TestData
 from ...core.meta_trader import MetaTrader
 from ...core.config import Config
 from ...core.errors import Error
 from ...core.constants import TimeFrame, CopyTicks, OrderType
 from ...core.models import (AccountInfo, SymbolInfo, BookInfo, TradeOrder, OrderCheckResult, OrderSendResult,
                             TradePosition, TradeDeal)
-
+from ...utils import backoff_decorator
+from .test_data import TestData
 logger = getLogger(__name__)
 
 
 class MetaTester(MetaTrader):
+    """A class for testing trading strategies in the MetaTrader 5 terminal. A subclass of MetaTrader."""
 
-    def __init__(self, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str],
+    def __init__(self, data: TestData):
-                 interval: int = 60, name: str = '', tz: str = 'Etc/UTC'):
-        """"""
         super().__init__()
-        self.tz = pytz.timezone(tz)
+        self.error = None
-        self.start = start.replace(tzinfo=self.tz)
+        self.data = data
-        self.end = end.replace(tzinfo=self.tz)
-        self.interval = interval
-        self.symbols = symbols
-        self.timeframes = timeframes
-        self.counter = 0
-        self.name = name or f"{start:%d-%m-%y}_{end:%d-%m-%y}"
-        diff = int((self.end - self.start).total_seconds())
-        self.span = range(start := int(self.start.timestamp()), diff + start)
 
-    def __iter__(self):
+    async def account_info(self) -> AccountInfo:
-        yield
-
-    async def get_test_data(self) -> dict:
         """"""
-        data = {}
+        res = self.data.account
-        rates, ticks, prices = await asyncio.gather(self.get_symbols_rates(), self.get_symbols_ticks(),
-                                                                self.get_symbols_prices())
-
-        data['rates'] = rates
-        data['ticks'] = ticks
-        data['prices'] = prices
-        data['symbols'] = await self.get_symbols_info()
-        data['account'] = self.get_account_info()
-
-        return data
-
-    async def get_and_save_data(self) -> None:
-        """"""
-        data = await self.get_test_data()
-        fh = open(f'{self.config.root}/data/{self.name}', 'wb')
-        data_file = pickle.dump(data, fh)
-        # data_file.update(data)
-        # data_file.sync()
-        # data_file.close()
-        fh.close()
-
-    def load_data(self, name: str = '') -> dict:
-        """"""
-        name = name or self.name
-        data_file = shelve.open(f'{self.config.root}/data/{name}', writeback=True)
-        data = dict(data_file)
-        data_file.close()
-        return data
-
-    async def get_symbols_info(self):
-        """"""
-        tasks = [self.get_symbol_info(symbol) for symbol in self.symbols]
-        res = await asyncio.gather(*tasks)
-        return {symbol: info for symbol, info in res}
-
-    async def get_symbols_ticks(self):
-        """"""
-        tasks = [self.get_symbol_ticks(symbol) for symbol in self.symbols]
-        res = await asyncio.gather(*tasks)
-        return {symbol: ticks for symbol, ticks in res}
-
-    async def get_account_info(self):
-        """"""
-        res = await super().account_info()
-        return res._asdict()
-
-    async def get_symbols_prices(self):
-        """"""
-        tasks = [self.get_symbol_prices(symbol) for symbol in self.symbols]
-        res = await asyncio.gather(*tasks)
-        return {symbol: prices for symbol, prices in res}
-
-    async def get_symbols_rates(self):
-        """"""
-        tasks = [self.get_symbol_rates(symbol, timeframe) for symbol in self.symbols for timeframe in self.timeframes]
-        res = await asyncio.gather(*tasks)
-        data = {}
-        for symbol, timeframe, rates in res:
-            data.setdefault(symbol, {}).setdefault(timeframe.name, rates)
-        return data
-
-    async def get_symbol_info(self, symbol: str):
-        """"""
-        res = await super().symbol_info(symbol)
-        return symbol, res._asdict()
-
-    async def get_symbol_ticks(self, symbol: str):
-        """"""
-        res = await super().copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
-        # res = pd.DataFrame(res)
-        # res.drop_duplicates(subset=['time'], keep='last', inplace=True)
-        # res.set_index('time', inplace=True, drop=False)
-        return symbol, res
-
-    async def get_symbol_prices(self, symbol: str):
-        """"""
-        res = await super().copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
-        # res = pd.DataFrame(res)
-        # res.drop_duplicates(subset=['time'], keep='last', inplace=True)
-        # res.set_index('time', inplace=True, drop=False)
-        # res = res.reindex(self.span, method='nearest')
-        return symbol, res
-
-    async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame):
-        """"""
-        res = await super().copy_rates_range(symbol, timeframe, self.start, self.end)
-        # res = pd.DataFrame(res)
-        # res.drop_duplicates(subset=['time'], keep='last', inplace=True)
-        # res.set_index('time', inplace=True, drop=False)
-        return symbol, timeframe, res
-
-    async def account_info(self) -> AccountInfo | None:
-        """"""
-        res = await asyncio.to_thread(self._account_info)
-        if res is None:
-            err = await self.last_error()
-            self.error = Error(*err)
-            logger.warning(f'Error in obtaining account information.{self.error.description}')
         return res
 
     async def symbols_total(self) -> int:
-        return await asyncio.to_thread(self._symbols_total)
+        return len(self.data.symbols)
 
-    async def symbols_get(self, group: str = "") -> tuple[SymbolInfo] | None:
+    async def symbols_get(self, group: str = "") -> tuple[SymbolInfo]:
-        kwargs = {'group': group} if group else {}
+        """"""
-        res = await asyncio.to_thread(self._symbols_get, **kwargs)
+        symbols = self.data.symbols.values()
-        if res is None:
+        return tuple(symbols)
-            err = await self.last_error()
-            self.error = Error(*err)
-            logger.warning(f'Error in obtaining symbols.{self.error.description}')
-            return res
-        return res
 
     async def symbol_info(self, symbol: str) -> SymbolInfo | None:
-        res = await asyncio.to_thread(self._symbol_info, symbol)
+        return self.data.symbols.get(symbol)
-        if res is None:
-            err = await self.last_error()
-            self.error = Error(*err)
-            logger.warning(f'Error in obtaining information for {symbol}.{self.error.description}')
-            return res
-        return res
 
     async def symbol_info_tick(self, symbol: str) -> Tick | None:
         res = await asyncio.to_thread(self._symbol_info_tick, symbol)

src/aiomql/lib/backtester/test_data.py

@@ -1,23 +1,14 @@
-from typing import Literal
 from ...core.models import AccountInfo, SymbolInfo
-from ...core.constants import TimeFrame
+from .get_data import Data, GetData
-from ...core.meta_trader import MetaTrader
-# from ...account import Account
 
 
 class TestData:
-
+    def __init__(self, data: Data):
-    def __init__(self, data):
         self._data = data
-
+        self.account = data['account']
-    def __getitem__(self, item: tuple[Literal['ticks', 'rates'], SymbolInfo, TimeFrame]):
+        self.symbols = data['symbols']
-        type_, symbol, time_frame = item
+        self.prices = data['prices']
-        if type_ == 'ticks':
+        self.ticks = data['ticks']
-            return self._data[type_][symbol.name]
+        self.rates = data['rates']
-        return self._data[type_][symbol.name][time_frame.name]
+        self.span = data['span']
-
+        self.cursor = 0
-
-
-# res = pd.DataFrame(res)
-# res.drop_duplicates(subset=['time'], keep='last', inplace=True)
-# res.set_index('time', inplace=True, drop=False)

src/aiomql/ticks.py

@@ -32,7 +32,7 @@ class Tick:
     last: float
     volume: float
     time_msc: float
-    flags: float
+    flags: TickFlag
     volume_real: float
     Index: int
 

src/aiomql/utils.py

@@ -60,4 +60,5 @@ def backoff_decorator(func=None, *, max_retries: int = 3, retries: int = 0, dela
             delay += 1
             retries += 1
             return await wrapper(*args, **kwargs)
+        
     return wrapper