diff --git a/src/aiomql/core/models.py b/src/aiomql/core/models.py index e75b7ec..4b4282c 100644 --- a/src/aiomql/core/models.py +++ b/src/aiomql/core/models.py @@ -4,7 +4,7 @@ from .constants import BookType, TradeAction, OrderType, OrderTime, OrderFilling DealReason, SymbolChartMode, SymbolTradeMode, SymbolCalcMode, SymbolOptionMode, SymbolOrderGTCMode, \ SymbolOptionRight, \ SymbolTradeExecution, SymbolSwapMode, DayOfWeek, AccountTradeMode, AccountStopOutMode, AccountMarginMode, \ - OrderReason + OrderReason, TickFlag from .base import Base @@ -334,9 +334,9 @@ class SymbolInfo(Base): path: str def __init__(self, **kwargs): - if 'name' not in kwargs: + if name := kwargs.pop('name', None): raise AttributeError('Symbol Object Must be initialized with a name') - self.name = kwargs.pop('name') + self.name = name super().__init__(**kwargs) def __repr__(self): @@ -351,6 +351,28 @@ class SymbolInfo(Base): def __hash__(self): return hash(self.name) +class TickInfo(Base): + """Price Tick of a Financial Instrument. + + Attributes: + time (int): Time of the last prices update for the symbol + bid (float): Current Bid price + ask (float): Current Ask price + last (float): Price of the last deal (Last) + volume (float): Volume for the current Last price + time_msc (int): Time of the last prices update for the symbol in milliseconds + flags (TickFlag): Tick flags + volume_real (float): Volume for the current Last price + Index (int): Custom attribute representing the position of the tick in a sequence. + """ + time: float + bid: float + ask: float + last: float + volume: float + time_msc: float + flags: TickFlag + volume_real: float class BookInfo(Base): """Book Information Class. diff --git a/src/aiomql/lib/backtester/get_data.py b/src/aiomql/lib/backtester/get_data.py index aea0b4d..02735b1 100644 --- a/src/aiomql/lib/backtester/get_data.py +++ b/src/aiomql/lib/backtester/get_data.py @@ -1,5 +1,6 @@ +from typing import TypedDict import pickle -import random +import lzma from datetime import datetime from logging import getLogger import asyncio @@ -7,22 +8,31 @@ import asyncio import pytz from MetaTrader5 import Tick, SymbolInfo import pandas as pd +from pandas import DataFrame from ...core.meta_trader import MetaTrader from ...core.config import Config from ...core.errors import Error from ...core.constants import TimeFrame, CopyTicks, OrderType from ...core.models import (AccountInfo, SymbolInfo, BookInfo, TradeOrder, OrderCheckResult, OrderSendResult, - TradePosition, TradeDeal) + TradePosition, TradeDeal, TickInfo) from ...utils import backoff_decorator logger = getLogger(__name__) +class Data(TypedDict): + account: AccountInfo + symbols: dict[str, SymbolInfo] + prices: DataFrame + ticks: DataFrame + rates: DataFrame + span: range + class GetData: - config = Config() + config: Config = Config() - def __init__(self, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str], + def __init__(self, *, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str], interval: int = 60, name: str = '', tz: str = 'Etc/UTC'): """""" super().__init__() @@ -38,55 +48,71 @@ class GetData: self.span = range(start := int(self.start.timestamp()), diff + start) self.mt5 = MetaTrader() - @classmethod - def load_data(cls, name: str = '') -> dict: - """""" - name = name - file = open(f'{cls.config.root}/data/{name}', 'rb') - data = pickle.load(file) - file.close() - return data - - async def get_test_data(self) -> dict: + async def get_data(self) -> Data: """""" data = {} rates, ticks, prices, symbols, account = await asyncio.gather(self.get_symbols_rates(), self.get_symbols_ticks(), self.get_symbols_prices(), self.get_symbols_info(), - self.get_account_info()) + self.get_account_info()) + data['rates'] = rates data['ticks'] = ticks data['prices'] = prices data['symbols'] = symbols data['account'] = account + data['range'] = self.span - return data + return Data(**data) - async def get_and_save_data(self) -> None: + async def pickle_data(self) -> None: """""" - data = await self.get_test_data() + data = await self.get_data() fh = open(f'{self.config.root}/data/{self.name}', 'wb') pickle.dump(data, fh) fh.close() + + async def compress_data(self): + """""" + data = await self.get_data() + bdata = pickle.dumps(data) + name = self.name + 'xz' + with lzma.open(name, 'w') as fh: + fh.write(bdata) - async def get_symbols_info(self): + @classmethod + def load_data(cls, name: str, compressed=False) -> dict: + """""" + fo = open(f'{cls.config.root}/data/{name}', 'rb') + data = fo.read() + + if compressed: + data = lzma.decompress(data) + else: + data = pickle.loads(data) + + fo.close() + + return data + + async def get_symbols_info(self) -> dict[str, SymbolInfo]: """""" tasks = [self.get_symbol_info(symbol) for symbol in self.symbols] res = await asyncio.gather(*tasks) - return {symbol: info for symbol, info in res} + return {symbol: SymbolInfo(**info) for symbol, info in res} - async def get_symbols_ticks(self): + async def get_symbols_ticks(self) -> dict[str, DataFrame]: """""" tasks = [self.get_symbol_ticks(symbol) for symbol in self.symbols] res = await asyncio.gather(*tasks) - return {symbol: ticks for symbol, ticks in res} + return {symbol: tick for symbol, tick in res} - async def get_symbols_prices(self): + async def get_symbols_prices(self) -> dict[str, DataFrame]: """""" tasks = [self.get_symbol_prices(symbol) for symbol in self.symbols] res = await asyncio.gather(*tasks) return {symbol: prices for symbol, prices in res} - async def get_symbols_rates(self): + async def get_symbols_rates(self) -> dict[str: dict[TimeFrame: DataFrame]]: """""" tasks = [self.get_symbol_rates(symbol, timeframe) for symbol in self.symbols for timeframe in self.timeframes] res = await asyncio.gather(*tasks) @@ -96,19 +122,19 @@ class GetData: return data @backoff_decorator(max_retries=5) - async def get_account_info(self) -> AccountInfo | None: + async def get_account_info(self) -> dict: """""" res = await self.mt5.account_info() return res._asdict() @backoff_decorator(max_retries=5) - async def get_symbol_info(self, symbol: str): + async def get_symbol_info(self, symbol: str) -> tuple[str, dict]: """""" res = await self.mt5.symbol_info(symbol) return symbol, res._asdict() @backoff_decorator(max_retries=5) - async def get_symbol_ticks(self, symbol: str): + async def get_symbol_ticks(self, symbol: str) -> tuple[str, DataFrame]: """""" res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL) res = pd.DataFrame(res) @@ -117,7 +143,7 @@ class GetData: return symbol, res @backoff_decorator(max_retries=5) - async def get_symbol_prices(self, symbol: str): + async def get_symbol_prices(self, symbol: str) -> tuple[str, DataFrame]: """""" res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL) res = pd.DataFrame(res) @@ -127,7 +153,7 @@ class GetData: return symbol, res @backoff_decorator(max_retries=5) - async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame): + async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame) -> tuple[str, TimeFrame, DataFrame]: """""" res = await self.mt5.copy_rates_range(symbol, timeframe, self.start, self.end) res = pd.DataFrame(res) diff --git a/src/aiomql/lib/backtester/meta_tester.py b/src/aiomql/lib/backtester/meta_tester.py index b7a95c0..0521701 100644 --- a/src/aiomql/lib/backtester/meta_tester.py +++ b/src/aiomql/lib/backtester/meta_tester.py @@ -2,165 +2,47 @@ import pickle from datetime import datetime from logging import getLogger import asyncio +import re import pytz from MetaTrader5 import Tick, SymbolInfo import pandas as pd +from ... import TestData from ...core.meta_trader import MetaTrader from ...core.config import Config from ...core.errors import Error from ...core.constants import TimeFrame, CopyTicks, OrderType from ...core.models import (AccountInfo, SymbolInfo, BookInfo, TradeOrder, OrderCheckResult, OrderSendResult, TradePosition, TradeDeal) - +from ...utils import backoff_decorator +from .test_data import TestData logger = getLogger(__name__) class MetaTester(MetaTrader): + """A class for testing trading strategies in the MetaTrader 5 terminal. A subclass of MetaTrader.""" - def __init__(self, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str], - interval: int = 60, name: str = '', tz: str = 'Etc/UTC'): - """""" + def __init__(self, data: TestData): super().__init__() - self.tz = pytz.timezone(tz) - self.start = start.replace(tzinfo=self.tz) - self.end = end.replace(tzinfo=self.tz) - self.interval = interval - self.symbols = symbols - self.timeframes = timeframes - self.counter = 0 - self.name = name or f"{start:%d-%m-%y}_{end:%d-%m-%y}" - diff = int((self.end - self.start).total_seconds()) - self.span = range(start := int(self.start.timestamp()), diff + start) + self.error = None + self.data = data - def __iter__(self): - yield - - async def get_test_data(self) -> dict: + async def account_info(self) -> AccountInfo: """""" - data = {} - rates, ticks, prices = await asyncio.gather(self.get_symbols_rates(), self.get_symbols_ticks(), - self.get_symbols_prices()) - - data['rates'] = rates - data['ticks'] = ticks - data['prices'] = prices - data['symbols'] = await self.get_symbols_info() - data['account'] = self.get_account_info() - - return data - - async def get_and_save_data(self) -> None: - """""" - data = await self.get_test_data() - fh = open(f'{self.config.root}/data/{self.name}', 'wb') - data_file = pickle.dump(data, fh) - # data_file.update(data) - # data_file.sync() - # data_file.close() - fh.close() - - def load_data(self, name: str = '') -> dict: - """""" - name = name or self.name - data_file = shelve.open(f'{self.config.root}/data/{name}', writeback=True) - data = dict(data_file) - data_file.close() - return data - - async def get_symbols_info(self): - """""" - tasks = [self.get_symbol_info(symbol) for symbol in self.symbols] - res = await asyncio.gather(*tasks) - return {symbol: info for symbol, info in res} - - async def get_symbols_ticks(self): - """""" - tasks = [self.get_symbol_ticks(symbol) for symbol in self.symbols] - res = await asyncio.gather(*tasks) - return {symbol: ticks for symbol, ticks in res} - - async def get_account_info(self): - """""" - res = await super().account_info() - return res._asdict() - - async def get_symbols_prices(self): - """""" - tasks = [self.get_symbol_prices(symbol) for symbol in self.symbols] - res = await asyncio.gather(*tasks) - return {symbol: prices for symbol, prices in res} - - async def get_symbols_rates(self): - """""" - tasks = [self.get_symbol_rates(symbol, timeframe) for symbol in self.symbols for timeframe in self.timeframes] - res = await asyncio.gather(*tasks) - data = {} - for symbol, timeframe, rates in res: - data.setdefault(symbol, {}).setdefault(timeframe.name, rates) - return data - - async def get_symbol_info(self, symbol: str): - """""" - res = await super().symbol_info(symbol) - return symbol, res._asdict() - - async def get_symbol_ticks(self, symbol: str): - """""" - res = await super().copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL) - # res = pd.DataFrame(res) - # res.drop_duplicates(subset=['time'], keep='last', inplace=True) - # res.set_index('time', inplace=True, drop=False) - return symbol, res - - async def get_symbol_prices(self, symbol: str): - """""" - res = await super().copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL) - # res = pd.DataFrame(res) - # res.drop_duplicates(subset=['time'], keep='last', inplace=True) - # res.set_index('time', inplace=True, drop=False) - # res = res.reindex(self.span, method='nearest') - return symbol, res - - async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame): - """""" - res = await super().copy_rates_range(symbol, timeframe, self.start, self.end) - # res = pd.DataFrame(res) - # res.drop_duplicates(subset=['time'], keep='last', inplace=True) - # res.set_index('time', inplace=True, drop=False) - return symbol, timeframe, res - - async def account_info(self) -> AccountInfo | None: - """""" - res = await asyncio.to_thread(self._account_info) - if res is None: - err = await self.last_error() - self.error = Error(*err) - logger.warning(f'Error in obtaining account information.{self.error.description}') + res = self.data.account return res async def symbols_total(self) -> int: - return await asyncio.to_thread(self._symbols_total) + return len(self.data.symbols) - async def symbols_get(self, group: str = "") -> tuple[SymbolInfo] | None: - kwargs = {'group': group} if group else {} - res = await asyncio.to_thread(self._symbols_get, **kwargs) - if res is None: - err = await self.last_error() - self.error = Error(*err) - logger.warning(f'Error in obtaining symbols.{self.error.description}') - return res - return res + async def symbols_get(self, group: str = "") -> tuple[SymbolInfo]: + """""" + symbols = self.data.symbols.values() + return tuple(symbols) async def symbol_info(self, symbol: str) -> SymbolInfo | None: - res = await asyncio.to_thread(self._symbol_info, symbol) - if res is None: - err = await self.last_error() - self.error = Error(*err) - logger.warning(f'Error in obtaining information for {symbol}.{self.error.description}') - return res - return res + return self.data.symbols.get(symbol) async def symbol_info_tick(self, symbol: str) -> Tick | None: res = await asyncio.to_thread(self._symbol_info_tick, symbol) diff --git a/src/aiomql/lib/backtester/test_data.py b/src/aiomql/lib/backtester/test_data.py index 2c4b15c..83533f0 100644 --- a/src/aiomql/lib/backtester/test_data.py +++ b/src/aiomql/lib/backtester/test_data.py @@ -1,23 +1,14 @@ -from typing import Literal from ...core.models import AccountInfo, SymbolInfo -from ...core.constants import TimeFrame -from ...core.meta_trader import MetaTrader -# from ...account import Account +from .get_data import Data, GetData class TestData: - - def __init__(self, data): + def __init__(self, data: Data): self._data = data - - def __getitem__(self, item: tuple[Literal['ticks', 'rates'], SymbolInfo, TimeFrame]): - type_, symbol, time_frame = item - if type_ == 'ticks': - return self._data[type_][symbol.name] - return self._data[type_][symbol.name][time_frame.name] - - - -# res = pd.DataFrame(res) -# res.drop_duplicates(subset=['time'], keep='last', inplace=True) -# res.set_index('time', inplace=True, drop=False) + self.account = data['account'] + self.symbols = data['symbols'] + self.prices = data['prices'] + self.ticks = data['ticks'] + self.rates = data['rates'] + self.span = data['span'] + self.cursor = 0 diff --git a/src/aiomql/ticks.py b/src/aiomql/ticks.py index fe62229..4a3a947 100644 --- a/src/aiomql/ticks.py +++ b/src/aiomql/ticks.py @@ -32,7 +32,7 @@ class Tick: last: float volume: float time_msc: float - flags: float + flags: TickFlag volume_real: float Index: int diff --git a/src/aiomql/utils.py b/src/aiomql/utils.py index 29ebeb1..f14ad9d 100644 --- a/src/aiomql/utils.py +++ b/src/aiomql/utils.py @@ -60,4 +60,5 @@ def backoff_decorator(func=None, *, max_retries: int = 3, retries: int = 0, dela delay += 1 retries += 1 return await wrapper(*args, **kwargs) + return wrapper