gavindiaz/aiomql
1
0
Fork 0
mirror of https://github.com/Ichinga-Samuel/aiomql.git synced 2026-08-07 17:27:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

testdata

Browse Source
This commit is contained in:
Ichinga Samuel
2024-08-26 05:54:45 +01:00
parent 3c3b9bfa9a 64850d79c1
commit 8f67eb3e8e
6 changed files with 107 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aiomql/core/models.py
+25 -3
View File
@@ -4,7 +4,7 @@ from .constants import BookType, TradeAction, OrderType, OrderTime, OrderFilling
DealReason, SymbolChartMode, SymbolTradeMode, SymbolCalcMode, SymbolOptionMode, SymbolOrderGTCMode, \
SymbolOptionRight, \
SymbolTradeExecution, SymbolSwapMode, DayOfWeek, AccountTradeMode, AccountStopOutMode, AccountMarginMode, \
OrderReason
OrderReason, TickFlag
from .base import Base
@@ -334,9 +334,9 @@ class SymbolInfo(Base):
path: str
def __init__(self, **kwargs):
if 'name' not in kwargs:
if name := kwargs.pop('name', None):
raise AttributeError('Symbol Object Must be initialized with a name')
self.name = kwargs.pop('name')
self.name = name
super().__init__(**kwargs)
def __repr__(self):
@@ -351,6 +351,28 @@ class SymbolInfo(Base):
def __hash__(self):
return hash(self.name)
class TickInfo(Base):
"""Price Tick of a Financial Instrument.
Attributes:
time (int): Time of the last prices update for the symbol
bid (float): Current Bid price
ask (float): Current Ask price
last (float): Price of the last deal (Last)
volume (float): Volume for the current Last price
time_msc (int): Time of the last prices update for the symbol in milliseconds
flags (TickFlag): Tick flags
volume_real (float): Volume for the current Last price
Index (int): Custom attribute representing the position of the tick in a sequence.
"""
time: float
bid: float
ask: float
last: float
volume: float
time_msc: float
flags: TickFlag
volume_real: float
class BookInfo(Base):
"""Book Information Class.
src/aiomql/lib/backtester/get_data.py
+55 -29
View File
@@ -1,5 +1,6 @@
from typing import TypedDict
import pickle
import random
import lzma
from datetime import datetime
from logging import getLogger
import asyncio
@@ -7,22 +8,31 @@ import asyncio
import pytz
from MetaTrader5 import Tick, SymbolInfo
import pandas as pd
from pandas import DataFrame
from ...core.meta_trader import MetaTrader
from ...core.config import Config
from ...core.errors import Error
from ...core.constants import TimeFrame, CopyTicks, OrderType
from ...core.models import (AccountInfo, SymbolInfo, BookInfo, TradeOrder, OrderCheckResult, OrderSendResult,
TradePosition, TradeDeal)
TradePosition, TradeDeal, TickInfo)
from ...utils import backoff_decorator
logger = getLogger(__name__)
class Data(TypedDict):
account: AccountInfo
symbols: dict[str, SymbolInfo]
prices: DataFrame
ticks: DataFrame
rates: DataFrame
span: range
class GetData:
config = Config()
config: Config = Config()
def __init__(self, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str],
def __init__(self, *, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str],
interval: int = 60, name: str = '', tz: str = 'Etc/UTC'):
""""""
super().__init__()
@@ -38,55 +48,71 @@ class GetData:
self.span = range(start := int(self.start.timestamp()), diff + start)
self.mt5 = MetaTrader()
@classmethod
def load_data(cls, name: str = '') -> dict:
""""""
name = name
file = open(f'{cls.config.root}/data/{name}', 'rb')
data = pickle.load(file)
file.close()
return data
async def get_test_data(self) -> dict:
async def get_data(self) -> Data:
""""""
data = {}
rates, ticks, prices, symbols, account = await asyncio.gather(self.get_symbols_rates(), self.get_symbols_ticks(),
self.get_symbols_prices(), self.get_symbols_info(),
self.get_account_info())
self.get_account_info())
data['rates'] = rates
data['ticks'] = ticks
data['prices'] = prices
data['symbols'] = symbols
data['account'] = account
data['range'] = self.span
return data
return Data(**data)
async def get_and_save_data(self) -> None:
async def pickle_data(self) -> None:
""""""
data = await self.get_test_data()
data = await self.get_data()
fh = open(f'{self.config.root}/data/{self.name}', 'wb')
pickle.dump(data, fh)
fh.close()
async def compress_data(self):
""""""
data = await self.get_data()
bdata = pickle.dumps(data)
name = self.name + 'xz'
with lzma.open(name, 'w') as fh:
fh.write(bdata)
async def get_symbols_info(self):
@classmethod
def load_data(cls, name: str, compressed=False) -> dict:
""""""
fo = open(f'{cls.config.root}/data/{name}', 'rb')
data = fo.read()
if compressed:
data = lzma.decompress(data)
else:
data = pickle.loads(data)
fo.close()
return data
async def get_symbols_info(self) -> dict[str, SymbolInfo]:
""""""
tasks = [self.get_symbol_info(symbol) for symbol in self.symbols]
res = await asyncio.gather(*tasks)
return {symbol: info for symbol, info in res}
return {symbol: SymbolInfo(**info) for symbol, info in res}
async def get_symbols_ticks(self):
async def get_symbols_ticks(self) -> dict[str, DataFrame]:
""""""
tasks = [self.get_symbol_ticks(symbol) for symbol in self.symbols]
res = await asyncio.gather(*tasks)
return {symbol: ticks for symbol, ticks in res}
return {symbol: tick for symbol, tick in res}
async def get_symbols_prices(self):
async def get_symbols_prices(self) -> dict[str, DataFrame]:
""""""
tasks = [self.get_symbol_prices(symbol) for symbol in self.symbols]
res = await asyncio.gather(*tasks)
return {symbol: prices for symbol, prices in res}
async def get_symbols_rates(self):
async def get_symbols_rates(self) -> dict[str: dict[TimeFrame: DataFrame]]:
""""""
tasks = [self.get_symbol_rates(symbol, timeframe) for symbol in self.symbols for timeframe in self.timeframes]
res = await asyncio.gather(*tasks)
@@ -96,19 +122,19 @@ class GetData:
return data
@backoff_decorator(max_retries=5)
async def get_account_info(self) -> AccountInfo | None:
async def get_account_info(self) -> dict:
""""""
res = await self.mt5.account_info()
return res._asdict()
@backoff_decorator(max_retries=5)
async def get_symbol_info(self, symbol: str):
async def get_symbol_info(self, symbol: str) -> tuple[str, dict]:
""""""
res = await self.mt5.symbol_info(symbol)
return symbol, res._asdict()
@backoff_decorator(max_retries=5)
async def get_symbol_ticks(self, symbol: str):
async def get_symbol_ticks(self, symbol: str) -> tuple[str, DataFrame]:
""""""
res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
res = pd.DataFrame(res)
@@ -117,7 +143,7 @@ class GetData:
return symbol, res
@backoff_decorator(max_retries=5)
async def get_symbol_prices(self, symbol: str):
async def get_symbol_prices(self, symbol: str) -> tuple[str, DataFrame]:
""""""
res = await self.mt5.copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
res = pd.DataFrame(res)
@@ -127,7 +153,7 @@ class GetData:
return symbol, res
@backoff_decorator(max_retries=5)
async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame):
async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame) -> tuple[str, TimeFrame, DataFrame]:
""""""
res = await self.mt5.copy_rates_range(symbol, timeframe, self.start, self.end)
res = pd.DataFrame(res)
src/aiomql/lib/backtester/meta_tester.py
+16 -134
View File
@@ -2,165 +2,47 @@ import pickle
from datetime import datetime
from logging import getLogger
import asyncio
import re
import pytz
from MetaTrader5 import Tick, SymbolInfo
import pandas as pd
from ... import TestData
from ...core.meta_trader import MetaTrader
from ...core.config import Config
from ...core.errors import Error
from ...core.constants import TimeFrame, CopyTicks, OrderType
from ...core.models import (AccountInfo, SymbolInfo, BookInfo, TradeOrder, OrderCheckResult, OrderSendResult,
TradePosition, TradeDeal)
from ...utils import backoff_decorator
from .test_data import TestData
logger = getLogger(__name__)
class MetaTester(MetaTrader):
"""A class for testing trading strategies in the MetaTrader 5 terminal. A subclass of MetaTrader."""
def __init__(self, start: datetime, end: datetime, timeframes: set[TimeFrame], symbols: set[str],
interval: int = 60, name: str = '', tz: str = 'Etc/UTC'):
""""""
def __init__(self, data: TestData):
super().__init__()
self.tz = pytz.timezone(tz)
self.start = start.replace(tzinfo=self.tz)
self.end = end.replace(tzinfo=self.tz)
self.interval = interval
self.symbols = symbols
self.timeframes = timeframes
self.counter = 0
self.name = name or f"{start:%d-%m-%y}_{end:%d-%m-%y}"
diff = int((self.end - self.start).total_seconds())
self.span = range(start := int(self.start.timestamp()), diff + start)
self.error = None
self.data = data
def __iter__(self):
yield
async def get_test_data(self) -> dict:
async def account_info(self) -> AccountInfo:
""""""
data = {}
rates, ticks, prices = await asyncio.gather(self.get_symbols_rates(), self.get_symbols_ticks(),
self.get_symbols_prices())
data['rates'] = rates
data['ticks'] = ticks
data['prices'] = prices
data['symbols'] = await self.get_symbols_info()
data['account'] = self.get_account_info()
return data
async def get_and_save_data(self) -> None:
""""""
data = await self.get_test_data()
fh = open(f'{self.config.root}/data/{self.name}', 'wb')
data_file = pickle.dump(data, fh)
# data_file.update(data)
# data_file.sync()
# data_file.close()
fh.close()
def load_data(self, name: str = '') -> dict:
""""""
name = name or self.name
data_file = shelve.open(f'{self.config.root}/data/{name}', writeback=True)
data = dict(data_file)
data_file.close()
return data
async def get_symbols_info(self):
""""""
tasks = [self.get_symbol_info(symbol) for symbol in self.symbols]
res = await asyncio.gather(*tasks)
return {symbol: info for symbol, info in res}
async def get_symbols_ticks(self):
""""""
tasks = [self.get_symbol_ticks(symbol) for symbol in self.symbols]
res = await asyncio.gather(*tasks)
return {symbol: ticks for symbol, ticks in res}
async def get_account_info(self):
""""""
res = await super().account_info()
return res._asdict()
async def get_symbols_prices(self):
""""""
tasks = [self.get_symbol_prices(symbol) for symbol in self.symbols]
res = await asyncio.gather(*tasks)
return {symbol: prices for symbol, prices in res}
async def get_symbols_rates(self):
""""""
tasks = [self.get_symbol_rates(symbol, timeframe) for symbol in self.symbols for timeframe in self.timeframes]
res = await asyncio.gather(*tasks)
data = {}
for symbol, timeframe, rates in res:
data.setdefault(symbol, {}).setdefault(timeframe.name, rates)
return data
async def get_symbol_info(self, symbol: str):
""""""
res = await super().symbol_info(symbol)
return symbol, res._asdict()
async def get_symbol_ticks(self, symbol: str):
""""""
res = await super().copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
# res = pd.DataFrame(res)
# res.drop_duplicates(subset=['time'], keep='last', inplace=True)
# res.set_index('time', inplace=True, drop=False)
return symbol, res
async def get_symbol_prices(self, symbol: str):
""""""
res = await super().copy_ticks_range(symbol, self.start, self.end, CopyTicks.ALL)
# res = pd.DataFrame(res)
# res.drop_duplicates(subset=['time'], keep='last', inplace=True)
# res.set_index('time', inplace=True, drop=False)
# res = res.reindex(self.span, method='nearest')
return symbol, res
async def get_symbol_rates(self, symbol: str, timeframe: TimeFrame):
""""""
res = await super().copy_rates_range(symbol, timeframe, self.start, self.end)
# res = pd.DataFrame(res)
# res.drop_duplicates(subset=['time'], keep='last', inplace=True)
# res.set_index('time', inplace=True, drop=False)
return symbol, timeframe, res
async def account_info(self) -> AccountInfo | None:
""""""
res = await asyncio.to_thread(self._account_info)
if res is None:
err = await self.last_error()
self.error = Error(*err)
logger.warning(f'Error in obtaining account information.{self.error.description}')
res = self.data.account
return res
async def symbols_total(self) -> int:
return await asyncio.to_thread(self._symbols_total)
return len(self.data.symbols)
async def symbols_get(self, group: str = "") -> tuple[SymbolInfo] | None:
kwargs = {'group': group} if group else {}
res = await asyncio.to_thread(self._symbols_get, **kwargs)
if res is None:
err = await self.last_error()
self.error = Error(*err)
logger.warning(f'Error in obtaining symbols.{self.error.description}')
return res
return res
async def symbols_get(self, group: str = "") -> tuple[SymbolInfo]:
""""""
symbols = self.data.symbols.values()
return tuple(symbols)
async def symbol_info(self, symbol: str) -> SymbolInfo | None:
res = await asyncio.to_thread(self._symbol_info, symbol)
if res is None:
err = await self.last_error()
self.error = Error(*err)
logger.warning(f'Error in obtaining information for {symbol}.{self.error.description}')
return res
return res
return self.data.symbols.get(symbol)
async def symbol_info_tick(self, symbol: str) -> Tick | None:
res = await asyncio.to_thread(self._symbol_info_tick, symbol)
src/aiomql/lib/backtester/test_data.py
+9 -18
View File
@@ -1,23 +1,14 @@
from typing import Literal
from ...core.models import AccountInfo, SymbolInfo
from ...core.constants import TimeFrame
from ...core.meta_trader import MetaTrader
# from ...account import Account
from .get_data import Data, GetData
class TestData:
def __init__(self, data):
def __init__(self, data: Data):
self._data = data
def __getitem__(self, item: tuple[Literal['ticks', 'rates'], SymbolInfo, TimeFrame]):
type_, symbol, time_frame = item
if type_ == 'ticks':
return self._data[type_][symbol.name]
return self._data[type_][symbol.name][time_frame.name]
# res = pd.DataFrame(res)
# res.drop_duplicates(subset=['time'], keep='last', inplace=True)
# res.set_index('time', inplace=True, drop=False)
self.account = data['account']
self.symbols = data['symbols']
self.prices = data['prices']
self.ticks = data['ticks']
self.rates = data['rates']
self.span = data['span']
self.cursor = 0
src/aiomql/ticks.py
+1 -1
View File
@@ -32,7 +32,7 @@ class Tick:
last: float
volume: float
time_msc: float
flags: float
flags: TickFlag
volume_real: float
Index: int
src/aiomql/utils.py
+1
View File
@@ -60,4 +60,5 @@ def backoff_decorator(func=None, *, max_retries: int = 3, retries: int = 0, dela
delay += 1
retries += 1
return await wrapper(*args, **kwargs)
return wrapper