testdata

src/aiomql/account.py

@@ -29,10 +29,8 @@ class Account(AccountInfo):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        acc = self.config.account_info()
-        acc_details = {k: v for k, v in self.get_dict(include={'login', 'server', 'password'}).items() if v}
-        acc |= acc_details
-        self.config.set_attributes(**acc)
+        self.exclude = self.exclude | {'_instance', 'symbols'}
+        acc = {k: (self.dict[k] or v) for k, v in self.config.account_info().items()}
         self.set_attributes(**acc)
 
     async def refresh(self):

src/aiomql/core/backtester/check.py

@@ -1,21 +1,23 @@
-from collections import namedtuple
-class ITR:
+import socket
 
-    def __init__(self) -> None:
-        self.span = iter(range(0, 10))
-        self.start = 0
-    
-    def __next__(self):
-        self.start = next(self.span)
-        return self.start
-    
 
-Gender = namedtuple('Gender', ['man', 'woman'])
-gen = Gender(man='Manny', woman='Babe')
-gend = gen._asdict()
-genz = Gender(gend)
-print(gen, genz)
-# b = ITR()
-# print(next(b))
-# print(next(b))
-# print(next(b))
+class socketserver:
+    def __init__(self, address = '192.168.1.15', port = 9090):
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.address = address
+        self.port = port
+        self.sock.bind((self.address, self.port))
+        self.cummdata = ''
+
+
+    def recvmsg(self):
+        g=self.sock.listen(1)
+        print(g)
+        self.conn, self.addr = self.sock.accept()
+        print('connected to', self.addr)
+        data = self.conn.recv(10)
+        self.cummdata += data.decode("utf-8")
+
+
+so = socketserver()
+so.recvmsg()

src/aiomql/core/backtester/get_data.py

@@ -1,4 +1,4 @@
-from typing import TypedDict
+from dataclasses import dataclass
 import pickle
 import lzma
 from datetime import datetime
@@ -16,24 +16,36 @@ from ...core.constants import TimeFrame, CopyTicks
 from ...utils import backoff_decorator
 
 logger = getLogger(__name__)
+from MetaTrader5 import TradePosition, TradeOrder, TradeDeal
+
+tof = list(TradeOrder._fields)
+tof.append('symbol')
+tpf = list(TradePosition._fields)
+tpf.append('symbol')
+tdf = list(TradeDeal._fields)
+tdf.append('symbol')
 
 
-class Data(TypedDict):
+@dataclass
+class Data:
     account: dict
     symbols: dict[str, dict]
     prices: dict[str, DataFrame]
     ticks: dict[str, DataFrame]
     rates: dict[str, dict[str, DataFrame]]
-    interval: range
+    span: range
+    range: range
+    history_orders: DataFrame = DataFrame([], columns=tof)
+    history_deals: DataFrame = DataFrame([], columns=tdf)
+    positions: DataFrame = DataFrame([], columns=tpf)
 
 
 class GetData:
-    config: Config = Config()
 
     def __init__(self, *, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str],
                  name: str = '', tz: str = 'Etc/UTC'):
         """"""
-        super().__init__()
+        self.config = Config()
         self.tz = pytz.timezone(tz)
         self.start = start.replace(tzinfo=self.tz)
         self.end = end.replace(tzinfo=self.tz)
@@ -41,24 +53,18 @@ class GetData:
         self.timeframes = timeframes
         self.name = name or f"{start:%d-%m-%y}_{end:%d-%m-%y}"
         diff = int((self.end - self.start).total_seconds())
-        self.interval = range(start := int(self.start.timestamp()), diff + start)
+        self.range = range(diff)
+        self.span = range(start := int(self.start.timestamp()), diff + start)
         self.mt5 = MetaTrader()
 
     async def get_data(self) -> Data:
         """"""
-        data = {}
+
         rates, ticks, prices, symbols, account = await asyncio.gather(self.get_symbols_rates(), self.get_symbols_ticks(),
                                                                 self.get_symbols_prices(), self.get_symbols_info(),
                                                     self.get_account_info())
-
-        data['rates'] = rates
-        data['ticks'] = ticks
-        data['prices'] = prices
-        data['symbols'] = symbols
-        data['account'] = account
-        data['range'] = self.interval
-
-        return Data(**data)
+        return Data(account=account, symbols=symbols, prices=prices, ticks=ticks, rates=rates,
+                    span=self.span, range=self.range)
 
     async def pickle_data(self) -> None:
         """"""
@@ -145,7 +151,7 @@ class GetData:
         res = pd.DataFrame(res)
         res.drop_duplicates(subset=['time'], keep='last', inplace=True)
         res.set_index('time', inplace=True, drop=False)
-        res = res.reindex(self.interval, method='nearest')
+        res = res.reindex(self.span, method='nearest')
         return symbol, res
 
     @backoff_decorator(max_retries=5)

src/aiomql/core/backtester/test_data.py

@@ -1,19 +1,22 @@
-from datetime import datetime, tzinfo
+from collections import namedtuple
+from datetime import datetime
 from typing import Literal
+from itertools import zip_longest
+import random
 
 import pytz
 import numpy as np
-import pandas as pd
 from pandas import DataFrame
 from MetaTrader5 import (Tick, SymbolInfo, AccountInfo, TradeOrder, TradePosition, TradeDeal,
-                         ORDER_TYPE_BUY, ORDER_TYPE_SELL, TradeRequest)
-import MetaTrader5 as mt5
+                         ORDER_TYPE_BUY, ORDER_TYPE_SELL, TradeRequest, OrderCheckResult, OrderSendResult,
+                         ACCOUNT_STOPOUT_MODE_PERCENT)
 from ..meta_trader import MetaTrader
-from ..constants import TimeFrame, CopyTicks, OrderType
-from .get_data import Data, GetData
+from ..constants import TimeFrame, CopyTicks, OrderType, TradeAction
+from .get_data import Data
 from ...utils import round_down, round_up
 
 tz = pytz.timezone('Etc/UTC')
+Cursor = namedtuple('Cursor', ['index', 'time'])
 
 
 class TestData:
@@ -22,27 +25,33 @@ class TestData:
     
     def __init__(self, data: Data):
         self._data = data
-        self.account = AccountInfo(**data['account'])
-        self.symbols = {symbol: SymbolInfo(**info) for symbol, info in data['symbols'].items()}
-        self.prices = data['prices']
-        self.ticks = data['ticks']
-        self.rates = data['rates']
-        self.interval = data['interval']
-        self.cursor = 0
-        self.iter = iter(self.interval)
+        self.account = AccountInfo(**data.account)
+        self.symbols = {symbol: SymbolInfo(**info) for symbol, info in data.symbols.items()}
+        self.prices = data.prices
+        self.ticks = data.ticks
+        self.rates = data.rates
+        self.span = data.span
+        self.range = data.range
+        self.cursor = Cursor(index=self.range[0], time=self.span[0])
+        self.iter = zip_longest(self.range, self.span)
         self.orders: dict[str, dict[int, TradeOrder]] = {}
         self.open_orders: dict[int, TradeOrder] = {}
         self.positions: dict[str, dict[int, TradePosition]] = {}
         self.open_positions: dict[int, TradePosition] = {}
-        self.mt = MetaTrader()
+        self.history_orders = data.history_orders
+        self.history_deals = data.history_deals
+        self.margins: dict[int, float] = {}
+        self.mt5 = MetaTrader()
 
     def __next__(self):
-        self.cursor = next(self.iter)
+        index, time = next(self.iter)
+        self.cursor = Cursor(index=index, time=time)
         return self.cursor
     
     def reset(self):
-        self.iter = iter(self.interval)
-        return self.iter
+        self.iter = zip_longest(self.range, self.span)
+        self.cursor = Cursor(index=self.range[0], time=self.span[0])
+        return self.cursor
 
     def get_symbols_total(self) -> int:
         return len(self.symbols)
@@ -54,7 +63,7 @@ class TestData:
         return AccountInfo(**self.account._asdict())
 
     def get_symbol_info_tick(self, symbol: str) -> Tick:
-        tick = self.prices[symbol].iloc[self.cursor]
+        tick = self.prices[symbol].iloc[self.cursor.index]
         return Tick(**tick)
     
     def get_symbol_info(self, symbol: str) -> SymbolInfo:
@@ -108,25 +117,136 @@ class TestData:
 
     async def order_calc_margin(self, action: Literal[OrderType.BUY, OrderType.SELL], symbol: str, volume: float,
                                 price: float, use_terminal=False):
-        if use_terminal or self.mt.config.use_terminal:
-            return await self.mt.order_calc_margin(OrderType(action), symbol, volume, price)
+        if use_terminal and self.mt5.config.use_terminal_for_backtesting:
+            return await self.mt5.order_calc_margin(OrderType(action), symbol, volume, price)
         sym = self.symbols[symbol]
         margin = (volume * sym.trade_contract_size * price) / (self.account.leverage / (sym.margin_initial or 1))
         return margin
 
     async def order_calc_profit(self, action: Literal[OrderType.BUY, OrderType.SELL], symbol: str, volume: float,
-                                price_open: float, price_close: float, use_terminal=False):
-        if use_terminal or self.mt.config.use_terminal:
-            return await self.mt.order_calc_profit(action, symbol, volume, price_open, price_close)
+                                price_open: float, price_close: float, use_terminal=True):
+        if use_terminal and self.mt5.config.use_terminal_for_backtesting:
+            return await self.mt5.order_calc_profit(action, symbol, volume, price_open, price_close)
         sym = self.symbols[symbol]
         profit = volume * sym.trade_contract_size * (price_close - price_open)
         return profit
 
-    def order_send(self, request: dict) -> dict:
+    def check_order(self, order: TradeOrder) -> bool:
         ...
 
-    def order_check(self, request: dict) -> dict:
+    def check_position(self, position: TradePosition) -> bool:
         ...
+
+    def close_position(self, position: TradePosition):
+        profit = position.profit
+        self.open_orders.pop(position.ticket)
+        self.open_positions.pop(position.ticket)
+        margin = self.margins.pop(position.ticket)
+        self.update_account(profit, margin=margin)
+
+    def modify_stops(self, ticket: int, sl: int = None, tp: int = None):
+        ...
+
+    def update_account(self, profit: float, margin: float = 0):
+        self.account.balance += profit
+        self.account.equity += profit
+        self.account.margin -= margin
+        self.account.margin_free = self.account.equity - self.account.margin
+        self.account.margin_level = (self.account.equity / self.account.margin) * 100
+
+    async def order_send(self, request: dict, use_terminal: bool = True) -> OrderSendResult:
+        osr = {'retcode': 10009, 'comment': 'Request completed', 'request': TradeRequest(**request)}
+
+        if (position := request.get('position')) in self.open_positions:
+            pos = self.open_positions[position]
+            order_type = OrderType(request['type'])
+            pos_type = OrderType(pos.type)
+            if order_type.opposite == pos_type:  # ToDo: is there another way to check if the order is a close order?
+                # close position
+                self.close_position(pos)
+                return OrderSendResult(**osr)  # ToDo: Create a deal object here
+            action = request['action']
+            if action == TradeAction.SLTP:
+                self.modify_stops(position, request['sl'], request['tp'])
+                return OrderSendResult(**osr)
+
+        if (action := request.get('action')) == TradeAction.DEAL:
+            ocr = await self.order_check(request, use_terminal=use_terminal)
+            if ocr.retcode != 0:
+                osr.update({'comment': ocr.comment, 'retcode': ocr.retcode})
+                return OrderSendResult(**osr)
+
+            ticket = random.randint(100_000_000, 999_999_999)
+            deal_ticket = random.randint(100_000_000, 999_999_999)
+            tick = self.get_symbol_info_tick(request['symbol'])
+            order_type = request['type']
+            price = tick.ask if request['type'] == ORDER_TYPE_BUY else tick.bid
+            volume = request['volume']
+            sl, tp = request.get('sl', 0), request.get('tp', 0)
+            symbol = request['symbol']
+            pos = {'comment': 'open position', 'ticket': ticket, 'symbol': symbol, 'volume': volume,
+                   'price_open': price, 'price_current': price, 'type': order_type, 'profit': 0,
+                   'sl': sl, 'tp': tp, 'time': tick.time,
+                   'time_msc': tick.time_msc}
+            order = {'ticket': ticket, 'symbol': symbol, 'volume': volume, 'price': price, 'price_current': price,
+                     'price_open': price, 'type': order_type, 'time_setup': tick.time,
+                     'time_setup_msc': tick.time_msc, 'volume_current': volume, 'sl': sl, 'tp': tp,}
+            pos = TradePosition(**pos)
+            order = TradeOrder(**order)
+            # ToDo: Create a deal object here
+            self.open_positions[pos.ticket] = pos
+            self.open_orders[order.ticket] = order
+            self.orders.setdefault(order.symbol, {})[order.ticket] = order
+            self.positions.setdefault(pos.symbol, {})[pos.ticket] = pos
+            osr.update({'order': ticket, 'price': price, 'volume': volume, 'bid': tick.bid,
+                        'ask': tick.ask, 'deal': deal_ticket})
+            margin = await self.order_calc_margin(action, symbol, volume, price)
+            self.margins[ticket] = margin
+            return OrderSendResult(**osr)
+
+    async def order_check(self, request: dict, use_terminal=True) -> OrderCheckResult:
+        action, symbol, volume = request.get('action'), request.get('symbol'), request.get('volume')
+        price = request.get('price')
+        ocr = {'retcode': 0, 'balance': 0, 'profit': 0, 'margin': 0, 'equity': 0, 'margin_free': 0,
+               'margin_level': 0, 'comment': 'Done', request: TradeRequest(**request)}
+
+        margin = 0
+        if all([action, symbol, volume, price]):
+            margin = await self.order_calc_margin(action, symbol, volume, price)
+
+        acc = self.get_account_info()
+        equity = acc.equity
+        used_margin = acc.margin + margin
+        free_margin = acc.margin_free - margin
+        margin_level = (equity / used_margin) * 100
+
+        if use_terminal and self.mt5.config.use_terminal_for_backtesting:
+            ocr_t = await self.mt5.order_check(request)
+            # return order check result if invalid stops level are detected or bad request
+            if ocr_t.retcode in (10016, 10013, 10014):
+                return ocr_t
+        else:
+            sym = self.symbols[symbol]
+            tsl = sym.trade_stops_level
+            sl, tp = request.get('sl', 0), request.get('tp', 0)
+
+            if tp or sl:
+                min_sl = min(sl, tp)
+                dsl = abs(price - min_sl) / sym.point
+                if dsl < tsl:
+                    ocr['retcode'] = 10016
+                    ocr['comment'] = 'Invalid stops'
+                    return OrderCheckResult(**ocr)
+
+        level = margin_level if acc.margin_mode == ACCOUNT_STOPOUT_MODE_PERCENT else free_margin
+        if level < acc.margin_so_call or free_margin <= 0:
+            ocr['retcode'] = 10019
+            ocr['comment'] = 'No money'
+
+        ocr.update({'balance': acc.balance, 'profit': acc.profit, 'margin': used_margin, 'equity': equity,
+                    'margin_free': free_margin, 'margin_level': margin_level})
+
+        return OrderCheckResult(**ocr)
     
     def get_orders_total(self) -> int:
         return len(self.open_orders)

src/aiomql/core/config.py

@@ -23,54 +23,49 @@ class Config:
         server (str): Broker server
         path (str): Path to terminal file
         timeout (int): Timeout for terminal connection
-        _initialize (bool): First time initialization flag
         state (dict): A global state dictionary for storing data across the framework
-        root_dir (str): The root directory of the project
+        root (str): Root directory of the project
+
     Notes:
         By default, the config class looks for a file named aiomql.json.
         You can change this by passing the filename and/or the config_dir keyword argument(s) to the constructor
         or the load_config method.
         By passing reload=True to the load_config method, you can reload and search again for the config file.
     """
-    login: int = 0
-    trade_record_mode: Literal['csv', 'json'] = 'csv'
-    password: str = ""
-    server: str = ""
-    path: str | Path = ""
-    timeout: int = 60000
-    record_trades: bool = True
-    filename: str = "aiomql.json"
-    _initialize = True
-    state: dict = {}
+    login: int
+    trade_record_mode: Literal['csv', 'json']
+    password: str
+    server: str
+    path: str | Path
+    timeout: int
+    filename: str
+    state: dict
     root: Path
-    root_dir: Path
+    record_trades: bool
     records_dir: Path
-    config_dir: str = ''
-    task_queue: TaskQueue = TaskQueue()
-    bot: Bot = None
+    records_dir_name: str
+    test_data_dir: Path
+    test_data_dir_name: str
+    task_queue: TaskQueue
+    bot: Bot
     _instance: 'Config'
-    mode: Literal['backtest', 'live'] = 'live'
-    test_data_dir: str = 'test_data'
-    use_terminal: bool = False
+    mode: Literal['backtest', 'live']
+    use_terminal_for_backtesting: bool
+    _defaults = {"timeout": 60000, "record_trades": True, "trade_record_mode": "csv", "mode": "live",
+                'filename': "aiomql.json", "records_dir_name": "trade_records", "test_data_dir_name": "test_data",
+                "use_terminal_for_backtesting": True, 'path': '', 'login': 0, 'password': '', 'server': ''}
 
     def __new__(cls, *args, **kwargs):
         if not hasattr(cls, "_instance"):
             cls._instance = super().__new__(cls)
+            cls._instance.state = {}
+            cls._instance.task_queue = TaskQueue()
+            cls._instance.set_attributes(**cls._defaults)
+            cls._instance.load_config(**kwargs)
         return cls._instance
 
     def __init__(self, **kwargs):
-        reload = kwargs.pop('reload', False)
-        self.load_config(reload=reload, **kwargs)
-
-    def set_root(self, *, root: str | Path):
-        root = Path(root) if str else root
-        self.root = root.absolute().resolve()
-        self.root_dir = self.root
-
-    def __setattr__(self, key, value):
-        if key == 'path':
-            value = str(self.root_dir / Path(value).absolute().resolve())
-        super().__setattr__(key, value)
+        self.set_attributes(**kwargs)
 
     def set_attributes(self, **kwargs):
         """Set keyword arguments as object attributes
@@ -95,10 +90,9 @@ class Config:
             parent_dir = os.path.abspath(os.path.join(current_dir, os.path.pardir))
             last_dir, current_dir = current_dir, parent_dir
 
-    def find_config(self):
+    def find_config_file(self):
         try:
-            path = self.root_dir / self.config_dir
-            for dirname in self.walk_to_root(path):
+            for dirname in self.walk_to_root(self.root):
                 check_path = os.path.join(dirname, self.filename)
                 if os.path.isfile(check_path):
                     return check_path
@@ -106,54 +100,51 @@ class Config:
         except Exception as _:
             return
 
-    def create_records_dir(self, *, records_dir: str | Path = 'records'):
-        """Create records directory if it does not exist. By default, it is relative to the root directory of the
-         project unless an absolute path is provided.
-
-        Keyword Args:
-            records_dir (str|Path): The directory to save trade records. Default is 'trade_records'
-        """
-        try:
-            if isinstance(records_dir, str):
-                records_dir = self.root_dir / records_dir
-            elif isinstance(records_dir, Path):
-                records_dir = records_dir.absolute().resolve()
-            records_dir.mkdir(parents=True, exist_ok=True)
-            self.records_dir = records_dir
-        except Exception as err:
-            logger.warning(f"{err}: Unable to create records directory")
-
-    def load_config(self, *, file: str = None, reload: bool = True, filename: str = None,
-                    config_dir: str = '', **kwargs):
+    def load_config(self, *, file: str | Path = None, filename: str = None, root: str | Path = None, **kwargs):
         """Load configuration settings from a file.
-        Keyword Args:
-            file (str): The path to the file to load. If not provided, the file is searched for
-            reload (bool): Whether to reload the config object. Default is True
-            filename (str): The name of the file to load. If not provided, the default filename is used
-            config_dir (str): The name of the directory to search for the file. Default is the root directory
-            root_dir (str): The root directory of the project
-            kwargs: Additional keyword arguments
-        """
-        if not (self._initialize or reload):
-            return
-        data = {}
-        self.filename = filename or self.filename
-        self.config_dir = config_dir or self.config_dir
-        root_dir = kwargs.pop('root_dir', None)
-        records_dir = kwargs.pop('records_dir', 'records')
-        if self._initialize or (root_dir is not None):
-            self.set_root(root=(root_dir or '.'))
-            self.create_records_dir(records_dir=records_dir)
 
-        if (file := (file or self.find_config())) is None:
+        Keyword Args:
+            file (str | Path): The absolute path to the config file.
+            filename (str): The name of the file to load if file path is not specified. If not provided aiomql.json is used
+            root (str): The root directory of the project.
+            kwargs: Additional keyword arguments to set as object attributes.
+        """
+        if root is not None:
+            root = Path(root).resolve()
+            root.mkdir(parents=True, exist_ok=True) if not root.exists() else ...
+            self.root = root
+        else:
+            self.root = self.root if hasattr(self, 'root') else Path.cwd()
+
+        if file is not None:
+            file = Path(file).resolve()
+            if not file.exists():
+                self.filename = filename or self.filename
+                file = self.find_config_file()
+            else:
+                self.filename = file.name
+        else:
+            self.filename = filename or self.filename
+            file = self.find_config_file()
+
+        if file is None:
             logger.warning("No Config File Found")
+            file_config = {}
         else:
             fh = open(file, mode="r")
-            data = json.load(fh)
+            file_config = json.load(fh)
             fh.close()
-        data |= kwargs
+
+        data = file_config | kwargs
         self.set_attributes(**data)
-        self._initialize = False
+
+        if self.record_trades and not hasattr(self, "records_dir"):
+            self.records_dir = self.root / self.records_dir_name
+            self.records_dir.mkdir(parents=True, exist_ok=True)
+
+        if self.mode == "backtest" and not hasattr(self, "test_data_dir"):
+            self.test_data_dir = self.root / self.test_data_dir_name
+            self.test_data_dir.mkdir(parents=True, exist_ok=True)
 
     def account_info(self) -> dict[str, int | str]:
         """Returns Account login details as found in the config object if available
@@ -161,4 +152,4 @@ class Config:
         Returns:
             dict: A dictionary of login details
         """
-        return {"login": self.login, "password": self.password, "server": self.server}
+        return {'login': self.login, 'password': self.password, 'server': self.server}