diff --git a/.gitignore b/.gitignore index 6410b1a..7d2b6c4 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ notebooks/ *.egg-info/ .installed.cfg *.egg +*.pkl # PyInstaller @@ -76,3 +77,4 @@ config.json aiomql.json config/ test_data/ + diff --git a/check.json b/check.json deleted file mode 100644 index 71fe5fa..0000000 --- a/check.json +++ /dev/null @@ -1 +0,0 @@ -{"a": 5}{"af": 5} \ No newline at end of file diff --git a/data.json b/data.json deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiomql.rar b/src/aiomql.rar deleted file mode 100644 index 4d1274f..0000000 Binary files a/src/aiomql.rar and /dev/null differ diff --git a/src/aiomql/contrib/backtester/backtest_engine.py b/src/aiomql/contrib/backtester/backtest_engine.py index 6a4105d..ebd7b13 100644 --- a/src/aiomql/contrib/backtester/backtest_engine.py +++ b/src/aiomql/contrib/backtester/backtest_engine.py @@ -14,11 +14,12 @@ from MetaTrader5 import (Tick, SymbolInfo, AccountInfo, TradeOrder, TradePositio TradeRequest, OrderCheckResult, OrderSendResult, TerminalInfo) from ...core.meta_trader import MetaTrader -from ...core.constants import TimeFrame, CopyTicks, OrderType, TradeAction, AccountStopOutMode, PositionReason +from ...core.constants import (TimeFrame, CopyTicks, OrderType, TradeAction, AccountStopOutMode, PositionReason, + DealType, DealReason, DealEntry, OrderReason) from ...core.config import Config from ...utils import round_down, round_up, error_handler, error_handler_sync, async_cache -from .get_data import Data, GetData, Cursor +from .get_data import TestData, GetData, Cursor from .test_account import TestAccount from .types import PositionsManager, OrdersManager, DealsManager @@ -33,26 +34,54 @@ class BackTestEngine: range: range cursor: Cursor iter: zip_longest + rates: dict[str, dict[int, DataFrame]] + ticks: dict[str, DataFrame] + prices: dict[str, DataFrame] + orders: OrdersManager + deals: DealsManager + positions: PositionsManager + _account: TestAccount + margins: dict[int, float] - def __init__(self, data: Data = None, speed: int = 1, start: float | datetime = 0, end: float | datetime = 0): - self._data = data or Data() - self._account: TestAccount = TestAccount(**self._data.account) - self.positions: PositionsManager = PositionsManager() - self.orders: OrdersManager = OrdersManager() - self.deals: DealsManager = DealsManager() - self.margins: dict[int, float] = {} - self.config = Config(test_data=self) - self.set_up(start=start, end=end, speed=speed) - self.cursor: Cursor = self._data.cursor or Cursor(index=self.range.start, time=self.span.start) - self._data.name = self._data.name or f"{datetime.fromtimestamp(self.span[0]):%d-%m-%y}_{datetime.fromtimestamp(self.span[-1]):%d-%m-%y}" + def __init__(self, *, data: TestData = None, speed: int = 1, start: float | datetime = 0, + end: float | datetime = 0, restart: bool = False): + self._data = data or TestData() + self.config = Config(backtest_engine=self) + self.set_up(start=start, end=end, speed=speed, restart=restart) + self.prepare_data() - def set_up(self, start: float | datetime = 0, end: float | datetime = 0, speed: int = 1): + def set_up(self, *, start: float | datetime = 0, end: float | datetime = 0, speed: int = 1, restart: bool = False): span_start = (int(start.timestamp()) if isinstance(start, datetime) else int(start)) or self._data.span.start span_end = (int(end.timestamp()) if isinstance(end, datetime) else int(end)) or self._data.span.stop self.span = range(span_start, span_end, speed) self.range = range(0, span_end - span_start, speed) self.iter = zip_longest(self.range, self.span) + if restart and self._data.cursor is not None: + self.cursor = self._data.cursor + self.go_to(time=self.cursor.time) + else: + self.cursor: Cursor = self._data.cursor or Cursor(index=self.range.start, time=self.span.start) + + def prepare_data(self): + orders = {} + for ticket, order in self._data.orders.items(): + orders[ticket] = TradeOrder((order.get(k) for k in TradeOrder.__match_args__)) + self.orders = OrdersManager(data=orders) + + positions = {} + for ticket, position in self._data.positions.items(): + positions[ticket] = TradePosition((position.get(k) for k in TradePosition.__match_args__)) + self.positions = PositionsManager(data=positions) + + deals = {} + for ticket, deal in self._data.deals.items(): + deals[ticket] = TradeDeal((deal.get(k) for k in TradeDeal.__match_args__)) + + self._account: TestAccount = TestAccount(**self._data.account) + + self.margins = self._data.margins + def __next__(self) -> Cursor: try: index, time = next(self.iter) @@ -75,12 +104,12 @@ class BackTestEngine: self.iter = zip_longest(self.range, self.span) self.cursor = Cursor(index=self.range.start, time=self.span.start) - def go_to(self, *, time: datetime | int): + def go_to(self, *, time: datetime | float): time = int(time.timestamp()) if isinstance(time, datetime) else int(time) steps = time - self.cursor.time if steps > 0: - self.fast_forward(steps) + self.fast_forward(steps=steps) return range_start = time - self.span.start @@ -126,6 +155,7 @@ class BackTestEngine: if self.config.use_terminal_for_backtesting: tick = await self.mt5.copy_ticks_from(symbol, time, 1, CopyTicks.ALL) return Tick(tick[-1]) if tick else None + tick = self.prices[symbol].loc[self.cursor.time] return Tick(tick) except Exception as exe: @@ -217,15 +247,38 @@ class BackTestEngine: @cached_property def prices(self) -> dict[str, DataFrame]: - return self._data.prices + prices = {} + for symbol in self._data.prices.keys(): + res = self._data.prices[symbol] + res = pd.DataFrame(res) + res.drop_duplicates(subset=['time'], keep='last', inplace=True) + res.set_index('time', inplace=True, drop=False) + res.reindex(self.span) # fill in missing values with NaN + prices[symbol] = res + return prices @cached_property def ticks(self) -> dict[str, DataFrame]: - return self._data.ticks + ticks = {} + for symbol in self._data.ticks.keys(): + res = self._data.ticks[symbol] + res = pd.DataFrame(res) + res.drop_duplicates(subset=['time'], keep='last', inplace=True) + res.set_index('time', inplace=True, drop=False) + ticks[symbol] = res + return ticks - @property - def rates(self) -> dict[str, dict[str, DataFrame]]: - return self._data.rates + @cached_property + def rates(self) -> dict[str, dict[int, DataFrame]]: + rates = {} + for symbol in self._data.rates.keys(): + for timeframe in self._data.rates[symbol].keys(): + res = self._data.rates[symbol][timeframe] + res = pd.DataFrame(res) + res.drop_duplicates(subset=['time'], keep='last', inplace=True) + res.set_index('time', inplace=True, drop=False) + rates[symbol][timeframe] = res + return rates @cached_property def symbols(self) -> dict[str, SymbolInfo]: @@ -257,18 +310,24 @@ class BackTestEngine: if action == TradeAction.DEAL and current_position and order_type.opposite == current_position.type: res = self.close_position(current_position.ticket) if res: - trade_order.update({'comment': 'Done', 'position_id': deal_ticket, 'ticket': order_ticket, - 'position_by_id': current_position.ticket, 'time_setup': current_tick.time, 'time_expiration': current_tick.time, - 'time_setup_msc': current_tick.time_msc, 'time_done': current_tick.time, 'time_done_msc': current_tick.time_msc}) - # ToDo: Create a deal object here? - # ToDo: Update trade order with more information? + trade_order.update({'comment': '', 'position_id': current_position.ticket, 'ticket': order_ticket, + 'time_setup': current_tick.time, 'time_expiration': current_tick.time, + 'time_setup_msc': current_tick.time_msc, 'time_done': current_tick.time, + 'time_done_msc': current_tick.time_msc, 'type': order_type, 'symbol': symbol, + 'price_current': current_position.price_current, 'reason': OrderReason.EXPERT, + 'volume_initial': current_position.volume}) + deal = {'ticket': deal_ticket, 'position_id': current_position.ticket, 'order': order_ticket, + 'symbol': symbol, 'commission': 0, 'swap': 0, 'fee': 0, 'time': current_tick.time, + 'time_msc': current_tick.time_msc, 'volume': current_position.volume, + 'price': current_position.price_current, 'type': DealType(order_type), 'reason': DealReason.EXPERT, + 'entry': DealEntry.OUT, 'comment': ''} + order = TradeOrder((trade_order.get(k, 0) for k in TradeOrder.__match_args__)) self.orders[order.ticket] = order - del self.orders[order.ticket] + deal = TradeDeal((deal.get(k, 0) for k in TradeDeal.__match_args__)) + self.deals[deal.ticket] = deal osr.update({'comment': 'Request completed', 'retcode': 10009, 'order': order_ticket, 'deal': deal_ticket,}) - # ToDo: remove later - self.to_json(osr) return OrderSendResult((osr.get(k, 0) for k in OrderSendResult.__match_args__)) if action == TradeAction.SLTP and current_position: @@ -282,33 +341,37 @@ class BackTestEngine: if res: # ToDo: Create a deal object here osr.update({'comment': 'Request completed', 'retcode': 10009, 'order': order_ticket, 'deal': deal_ticket,}) - self.to_json(osr) # ToDo: remove later - return OrderSendResult((osr.get(k, 0) for k in OrderSendResult.__match_args__)) if action == TradeAction.DEAL and order_type in (OrderType.BUY, OrderType.SELL): check = await self.order_check(request=request) + if check.retcode != 0: osr = {'retcode': check.retcode, 'comment': check.comment, 'request': check.request} return OrderSendResult((osr.get(k, 0) for k in OrderSendResult.__match_args__)) - # self.to_json(osr) # ToDo: remove later - # return OrderSendResult(osr) + price = current_tick.ask if order_type == OrderType.BUY else current_tick.bid - # ToDo: Cross check this values with actual values. - position = {'comment': 'Position Opened', 'ticket': order_ticket, 'symbol': symbol, 'volume': volume, - 'price_open': price, 'price_current': price, 'type': order_type, 'profit': 0, 'reason': PositionReason.EXPERT, + position = {'ticket': order_ticket, 'symbol': symbol, 'volume': volume, + 'price_open': price, 'price_current': price, 'type': order_type, 'profit': 0, + 'reason': PositionReason.EXPERT, 'identifier': order_ticket, 'sl': sl, 'tp': tp, 'time': current_tick.time, 'time_msc': current_tick.time_msc, 'time_update': current_tick.time, 'time_update_msc': current_tick.time_msc} + deal = {'ticket': deal_ticket, 'position': order_ticket, 'symbol': symbol, 'commission': 0, 'swap': 0, + 'position_id': order_ticket, 'fee': 0, 'time': current_tick.time, 'time_msc': current_tick.time_msc, + 'volume': volume, 'price': price, 'type': DealType(order_type), 'reason': DealReason.EXPERT, + 'entry': DealEntry.IN} + # ToDo: set time_expiration based on order_type_time - trade_order.update({'ticket': order_ticket, 'symbol': symbol, 'volume': volume, 'price': price, 'price_current': price, 'sl': sl, - 'tp': tp, 'price_open': price, 'type': order_type, 'time_setup': current_tick.time, 'time_setup_msc': current_tick.time_msc, - 'volume_current': volume, 'volume_initial': volume, 'position_id': order_ticket}) + trade_order.update({'ticket': order_ticket, 'symbol': symbol, 'volume': volume, 'price': price, + 'price_current': price, 'sl': sl, 'time_setup_msc': current_tick.time_msc, + 'tp': tp, 'price_open': price, 'type': order_type, 'time_setup': current_tick.time, + 'volume_current': volume, 'volume_initial': volume, 'position_id': order_ticket}) pos = TradePosition((position.get(k, 0) for k in TradePosition.__match_args__)) order = TradeOrder((trade_order.get(k, 0) for k in TradeOrder.__match_args__)) - # ToDo: Create a deal object here - + deal = TradeDeal((deal.get(k, 0) for k in TradeDeal.__match_args__)) + self.deals[deal_ticket] = deal self.positions[order.ticket] = pos self.orders[order.ticket] = order osr.update({'order': order_ticket, 'price': price, 'volume': volume, 'bid': current_tick.bid, @@ -316,7 +379,6 @@ class BackTestEngine: margin = await self.order_calc_margin(action, symbol, volume, price) self.margins[order_ticket] = margin self.update_account(margin=margin) - self.to_json(osr) # ToDo: remove later return OrderSendResult((osr.get(k, 0) for k in OrderSendResult.__match_args__)) @error_handler diff --git a/src/aiomql/contrib/backtester/check1.py b/src/aiomql/contrib/backtester/check1.py index c1eb3b9..b12eb33 100644 --- a/src/aiomql/contrib/backtester/check1.py +++ b/src/aiomql/contrib/backtester/check1.py @@ -1,9 +1,10 @@ +class Form: + rest: str -def add(a, b): - return a + b + @property + def rest(self): + return 'rest' -def sum(a, b): - return add(a, b) -f = sum(1, 2) -print(f) \ No newline at end of file +g = Form() +print(g.rest) diff --git a/src/aiomql/contrib/backtester/d.pkl b/src/aiomql/contrib/backtester/d.pkl deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiomql/contrib/backtester/get_data.py b/src/aiomql/contrib/backtester/get_data.py index ecfc708..2a0ceea 100644 --- a/src/aiomql/contrib/backtester/get_data.py +++ b/src/aiomql/contrib/backtester/get_data.py @@ -31,7 +31,7 @@ Cursor = namedtuple('Cursor', ['index', 'time']) @dataclass -class Data: +class TestData: name: str = '' terminal: dict[str, [str | int | bool | float]] = field(default_factory=dict) version: tuple[int, int, str] = (0, 0, '') @@ -39,7 +39,7 @@ class Data: symbols: dict[str, dict] = field(default_factory=dict) prices: dict[str, ndarray] = field(default_factory=dict) ticks: dict[str, ndarray] = field(default_factory=dict) - rates: dict[str, dict[str, ndarray]] = field(default_factory=dict) + rates: dict[str, dict[int, ndarray]] = field(default_factory=dict) span: range = range(0) range: range = range(0) orders: dict[int, dict] = field(default_factory=lambda: {}) @@ -48,6 +48,7 @@ class Data: active_orders: tuple[int, ...] = field(default_factory=lambda: ()) open_positions: tuple[int, ...] = field(default_factory=lambda: ()) cursor: Cursor = None + margins: dict[int, float] = field(default_factory=lambda: {}) def __str__(self): return f""" @@ -74,7 +75,7 @@ class Data: class GetData: - data: Data | None + data: TestData def __init__(self, *, start: datetime, end: datetime, symbols: Sequence[str], timeframes: Sequence[TimeFrame], name: str = '', tz: str = 'Etc/UTC'): @@ -89,22 +90,22 @@ class GetData: diff = int((self.end - self.start).total_seconds()) self.range = range(diff) self.span = range(start := int(self.start.timestamp()), diff + start) - self.data = Data(name=name, span=self.span, range=self.range) + self.data = TestData(name=name, span=self.span, range=self.range) self.mt5 = MetaTrader() self.task_queue = TaskQueue(workers=250) @classmethod - def pickle_data(cls, *, data: Data, name: str | Path): + def pickle_data(cls, *, data: TestData, name: str | Path): """""" try: with open(name, 'wb') as fo: - data = pickle.dump(data, fo, protocol=pickle.HIGHEST_PROTOCOL) + pickle.dump(data, fo, protocol=pickle.HIGHEST_PROTOCOL) except Exception as err: logger.error(f"Error in dump_data: {err}") @classmethod - def load_data(cls, *, name: str | Path): + def load_data(cls, *, name: str | Path) -> TestData: """""" try: with open(name, 'rb') as fo: @@ -112,12 +113,11 @@ class GetData: return data except Exception as err: logger.error(f"Error: {err}") - return None - def pickle_data(self, *, name: str | Path = ''): + def save_data(self, *, name: str | Path = ''): name = name or self.name - self.__class___.pickle_data(data=self.data, name=name) - + with open(name, 'wb') as fo: + pickle.dump(self.data, fo, protocol=pickle.HIGHEST_PROTOCOL) async def get_data(self, workers: int = None): """""" @@ -192,26 +192,16 @@ class GetData: async def get_symbol_ticks(self, *, symbol: str): """""" res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL) - # res = pd.DataFrame(res) - # res.drop_duplicates(subset=['time'], keep='last', inplace=True) - # res.set_index('time', inplace=True, drop=False) self.data.ticks[symbol] = res @backoff_decorator async def get_symbol_prices(self, *, symbol: str): """""" res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL) - # res = pd.DataFrame(res) - # res.drop_duplicates(subset=['time'], keep='last', inplace=True) - # res.set_index('time', inplace=True, drop=False) - # res = res.reindex(self.span) # fill in missing values with NaN self.data.prices[symbol] = res @backoff_decorator async def get_symbol_rates(self, *, symbol: str, timeframe: TimeFrame): """""" res = await self.mt5.copy_rates_range(symbol, timeframe, self.start, self.end) - # res = pd.DataFrame(res) - # res.drop_duplicates(subset=['time'], keep='last', inplace=True) - # res.set_index('time', inplace=True, drop=False) self.data.rates.setdefault(symbol, {})[timeframe] = res