diff --git a/config.yaml b/config.yaml index 880a736..2a006f1 100644 --- a/config.yaml +++ b/config.yaml @@ -3,20 +3,21 @@ calculate: from: days: 10 timeframe: 15 - min_prices: 500 + min_prices: 600 max_set_size_diff_pct: 90 overlap_pct: 90 max_p_value: 0.05 monitor: from: - minutes: 60 + minutes: 10 interval: 10 - min_prices: 500 - max_set_size_diff_pct: 90 - overlap_pct: 90 + min_prices: 300 + max_set_size_diff_pct: 50 + overlap_pct: 50 max_p_value: 0.05 monitoring_threshold: 0.9 divergence_threshold: 0.8 + tick_cache_time: 10 logging: version: 1 disable_existing_loggers: false diff --git a/mt5_correlation/correlation.py b/mt5_correlation/correlation.py index b2d6a23..860e170 100644 --- a/mt5_correlation/correlation.py +++ b/mt5_correlation/correlation.py @@ -1,12 +1,14 @@ import math import logging import pandas as pd -from datetime import datetime +from datetime import datetime, timedelta import time import sched import threading import pytz from scipy.stats.stats import pearsonr +import yaml +import pickle from mt5_correlation.mt5 import MT5 @@ -16,6 +18,9 @@ class Correlation: A class to maintain the state of the calculated correlation coefficients. """ + # Connection to metatrader + __mt5 = None + # Minimum base coefficient for monitoring. Symbol pairs with a lower correlation # coefficient than ths won't be monitored. monitoring_threshold = 0.9 @@ -24,13 +29,25 @@ class Correlation: __monitoring = False __monitoring_params = {} + # The price data used to calculate the correlations + __price_data = None + + # Coefficient data and history. Will be created as dataframes in Init + coefficient_data = None + coefficient_history = None + + # Cache for ticks. Dict: {Symbol: [retrieved datetime, ticks dataframe]} + __ticks = {} + def __init__(self): + # Logger self.__log = logging.getLogger(__name__) - # Create dataframe - self.__columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe', - 'Last Check', 'Last Coefficient'] - self.coefficient_data = pd.DataFrame(columns=self.__columns) + # Connection to metatrader + self.__mt5 = MT5() + + # Create dataframe for coefficient data + self.__reset_coefficient_data() # Create timer for continuous monitoring self.__scheduler = sched.scheduler(time.time, time.sleep) @@ -45,22 +62,38 @@ class Correlation: else: return None - def load(self, filename): + def load(self, filename, price_data_filename=None): """ - Loads a csv file containing calculated coefficients - :param filename: + Loads a csv file containing calculated coefficients, and optionally the price data used to calculate those + coefficients + :param filename: The filename for the coefficient data to load. + :param price_data_filename: The filename for the price data to load. :return: """ + # Load coefficients file self.coefficient_data = pd.read_csv(filename) - def save(self, filename): + # If specified, load price data yaml file + if price_data_filename is not None: + self.__price_data = {} # Clear + with open(price_data_filename, 'rb') as file: + self.__price_data = pickle.load(file) + + def save(self, filename, price_data_filename=None): """ Saves the calculated coefficients as a csv file - :param filename: + :param filename: The filename for the coefficient data to save to. + :param price_data_filename: The filename for the price data to save to. :return: """ + # Save the coefficient data self.coefficient_data.to_csv(filename, index=False) + # Save the price data if required + if price_data_filename is not None: + with open(price_data_filename, 'wb') as file: + pickle.dump(self.__price_data, file, protocol=pickle.HIGHEST_PROTOCOL) + def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, max_p_value=0.05): """ @@ -79,25 +112,24 @@ class Correlation: :return: """ + coefficient = None + # If we are monitoring, stop. We will need to restart later was_monitoring = self.__monitoring if self.__monitoring: self.stop_monitor() # Clear the existing correlations - self.coefficient_data = pd.DataFrame(columns=self.__columns) + self.__reset_coefficient_data() - # Create mt5 class. This contains required methods for interacting with MT5. - mt5 = MT5() - - # Gte all visible symbols - symbols = mt5.get_symbols() + # Get all visible symbols + symbols = self.__mt5.get_symbols() # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict. - price_data = {} + self.__price_data = {} for symbol in symbols: - price_data[symbol] = mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to, - timeframe=timeframe) + self.__price_data[symbol] = self.__mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to, + timeframe=timeframe) # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to @@ -114,16 +146,18 @@ class Correlation: index += 1 # Get price data for both symbols - symbol1_price_data = price_data[symbol1] - symbol2_price_data = price_data[symbol2] + symbol1_price_data = self.__price_data[symbol1] + symbol2_price_data = self.__price_data[symbol2] - # Get coefficient and store if valid - coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data, - symbol2_prices=symbol2_price_data, - min_prices=min_prices, - max_set_size_diff_pct=max_set_size_diff_pct, - overlap_pct=overlap_pct, max_p_value=max_p_value) + # Get coefficient + if symbol1_price_data is not None and symbol2_price_data is not None: + coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data, + symbol2_prices=symbol2_price_data, + min_prices=min_prices, + max_set_size_diff_pct=max_set_size_diff_pct, + overlap_pct=overlap_pct, max_p_value=max_p_value) + # Store if valid if coefficient is not None: self.coefficient_data = \ @@ -143,28 +177,40 @@ class Correlation: # If we were monitoring, we stopped, so start again. if was_monitoring: self.start_monitor(interval=self.__monitoring_params['interval'], - date_from=self.__monitoring_params['date_from'], - date_to=self.__monitoring_params['date_to'], + from_mins=self.__monitoring_params['from_mins'], min_prices=self.__monitoring_params['min_prices'], max_set_size_diff_pct=self.__monitoring_params['max_set_size_diff_pct'], overlap_pct=self.__monitoring_params['overlap_pct'], max_p_value=self.__monitoring_params['max_p_value']) - def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05): + def get_price_data(self, symbol): + """ + Returns the price data used to calculate the base coefficients for the specified symbol + :param symbol: Symbol to get price data for. + :return: price data + """ + price_data = None + if symbol in self.__price_data: + price_data = self.__price_data[symbol] + + return price_data + + def start_monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05, cache_time=10): """ Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient threshold. :param interval: How often to check in seconds - :param date_from: From date for tick data from which to calculate correlation coefficients - :param date_to: To date for tick data from which to calculate correlation coefficients + :param from_mins: The number of minutes of tick data to use for calculations :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold is not met then returned coefficient will be None :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are within this pct of each other :param overlap_pct: :param max_p_value: The maximum p value for the correlation to be meaningful + :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse + the tick data. Number of seconds to cache tick data for before it becomes stale. :return: correlation coefficient, or None if coefficient could not be calculated. """ @@ -180,9 +226,9 @@ class Correlation: # Create thread to run monitoring This will call private __monitor method that will run the calculation and # keep scheduling itself while self.monitoring is True. Store the params. We will need to use these if we have # to stop and restart the monitor. Note, this happens during calculate - self.__monitoring_params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, + self.__monitoring_params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct, - 'overlap_pct': overlap_pct, 'max_p_value': max_p_value} + 'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time} thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params) thread.start() @@ -197,23 +243,27 @@ class Correlation: else: self.__log.debug(f"Request to stop monitor when it is not running. No action taken.") - @staticmethod - def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90, - overlap_pct=90, max_p_value=0.05): + def calculate_coefficient(self, symbol1_prices, symbol2_prices, min_prices: int = 100, + max_set_size_diff_pct: int = 90, overlap_pct: int = 90, + max_p_value: float = 0.05): """ Calculates the correlation coefficient between two sets of price data. Uses close price. - :param symbol1_prices: prices or ticks for symbol 1 - :param symbol2_prices: prices or ticks for symbol 2 + :param symbol1_prices: Pandas dataframe containing prices or ticks for symbol 1 + :param symbol2_prices: Pandas dataframe containing prices or ticks for symbol 2 :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold is not met then returned coefficient will be None :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are within this pct of each other :param overlap_pct: :param max_p_value: The maximum p value for the correlation to be meaningful + :return: correlation coefficient, or None if coefficient could not be calculated. + :rtype: float or None """ + assert symbol1_prices is not None and symbol2_prices is not None + # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set? @@ -237,29 +287,47 @@ class Correlation: # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null # hypothesis is false). coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close']) - coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0] + coefficient = None if coefficient_with_p_value[1] > max_p_value else coefficient_with_p_value[0] # If NaN, change to None if coefficient is not None and math.isnan(coefficient): coefficient = None + self.__log.debug(f"Calculate coefficient returning {coefficient}. " + f"Symbol 1 Prices: {len(symbol1_prices)} Symbol 2 Prices: {len(symbol2_prices)} " + f"Overlap Prices: {len(intersect_dates)} Similar size: {similar_size} " + f"Enough overlap: {enough_overlap} Enough prices: {enough_prices} Suitable: {suitable}.") + return coefficient - def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05): + def get_coefficient_history(self, symbol1, symbol2): + """ + Returns the coefficient history for the specified symbol pair calculated during this instance. + Coefficient history does not persist between instances. + :param symbol1: + :param symbol2: + :return: dataframe containing history of coefficient data. + """ + history = self.coefficient_history[(self.coefficient_history['Symbol 1'] == symbol1) & + (self.coefficient_history['Symbol 2'] == symbol2)] + return history + + def __monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05, cache_time=10): """ The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and stop_monitoring. :param interval: How often to check in seconds - :param date_from: From date for tick data from which to calculate correlation coefficients - :param date_to: To date for tick data from which to calculate correlation coefficients + :param from_mins: The number of minutes of tick data to use for calculations :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold is not met then returned coefficient will be None :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are within this pct of each other :param overlap_pct: :param max_p_value: The maximum p value for the correlation to be meaningful + :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse + the tick data. Number of seconds to cache tick data for before it becomes stale. :return: correlation coefficient, or None if coefficient could not be calculated. """ @@ -268,66 +336,166 @@ class Correlation: # Only run if monitor is not stopped if self.__monitoring: # Update all coefficients - self.__update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices, + self.__update_all_coefficients(from_mins=from_mins, min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct, - max_p_value=max_p_value) + max_p_value=max_p_value, cache_time=cache_time) # Schedule the timer to run again - params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices, + params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct, - 'max_p_value': max_p_value} + 'max_p_value': max_p_value, "cache_time": cache_time} self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params) self.__scheduler.run() - def __update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, - overlap_pct=90, max_p_value=0.05): + def __update_coefficient(self, symbol1, symbol2, from_mins, min_prices=100, max_set_size_diff_pct=90, + overlap_pct=90, max_p_value=0.05, cache_time=10): """ Updates the coefficient for the specified symbol pair :param symbol1: Name of symbol to calculate coefficient for. :param symbol2: Name of symbol to calculate coefficient for. - :param date_from: From date for tick data from which to calculate correlation coefficients - :param date_to: To date for tick data from which to calculate correlation coefficients + :param from_mins: The number of minutes of tick data to use for calculations :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold is not met then returned coefficient will be None :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are within this pct of each other :param overlap_pct: :param max_p_value: The maximum p value for the correlation to be meaningful + :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse + the tick data. Number of seconds to cache tick data for before it becomes stale. :return: correlation coefficient, or None if coefficient could not be calculated. """ + coefficient = None + + # Get dates + # From and to dates for calculations. + timezone = pytz.timezone("Etc/UTC") + date_to = datetime.now(tz=timezone) + date_from = date_to - timedelta(minutes=from_mins) + # Get the tick data - mt5 = MT5() - symbol1ticks = mt5.get_ticks(symbol=symbol1, from_date=date_from, to_date=date_to) - symbol2ticks = mt5.get_ticks(symbol=symbol2, from_date=date_from, to_date=date_to) + symbol1ticks = self.__get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time) + symbol2ticks = self.__get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time) # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one # tick per second and ensuring that times can match. We will need to set the index to time for the resample # then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price if symbol1ticks is not None and symbol2ticks is not None and \ len(symbol1ticks.index) > 0 and len(symbol2ticks.index) > 0: - symbol1ticks.set_index('time', inplace=True) - symbol2ticks.set_index('time', inplace=True) - symbol1prices = symbol1ticks['ask'].resample('1S').ohlc() - symbol2prices = symbol2ticks['ask'].resample('1S').ohlc() - symbol1prices.reset_index(inplace=True) - symbol2prices.reset_index(inplace=True) - symbol1prices = symbol1prices[symbol1prices['close'].notna()] - symbol2prices = symbol2prices[symbol2prices['close'].notna()] + symbol1ticks = symbol1ticks.set_index('time') + symbol2ticks = symbol2ticks.set_index('time') + try: + symbol1prices = symbol1ticks['ask'].resample('1S').ohlc() + symbol2prices = symbol2ticks['ask'].resample('1S').ohlc() + except RecursionError: + self.__log.warning(f"Coefficient could not be calculated for {symbol1}:{symbol2} as prices could not " + f"be resampled.") + else: + symbol1prices.reset_index(inplace=True) + symbol2prices.reset_index(inplace=True) + symbol1prices = symbol1prices[symbol1prices['close'].notna()] + symbol2prices = symbol2prices[symbol2prices['close'].notna()] - # Calculate the coefficient - coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices, - min_prices=min_prices, - max_set_size_diff_pct=max_set_size_diff_pct, - overlap_pct=overlap_pct, max_p_value=max_p_value) + # Calculate the coefficient + coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices, + min_prices=min_prices, + max_set_size_diff_pct=max_set_size_diff_pct, + overlap_pct=overlap_pct, max_p_value=max_p_value) + + self.__log.debug(f"Symbol pair {symbol1}:{symbol2} has a coefficient of {coefficient}.") else: coefficient = None - # Find the correct row in the coefficient data and update with calculation date and calculated coefficient + # Update the coefficient data + if coefficient is not None: + self.__update_coefficient_data(symbol1=symbol1, symbol2=symbol2, coefficient=coefficient, + date_from=date_from, date_to=date_to) + + return coefficient + + def __update_all_coefficients(self, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05, cache_time=10): + """ + Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet + the threshold can be accessed through the filtered_coefficient_data property. + + :param from_mins: The number of minutes of tick data to use for calculations + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse + the tick data. Number of seconds to cache tick data for before it becomes stale. + + :return: correlation coefficient, or None if coefficient could not be calculated. + """ + # Update latest coefficient for every pair + for index, row in self.filtered_coefficient_data.iterrows(): + symbol1 = row['Symbol 1'] + symbol2 = row['Symbol 2'] + self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, from_mins=from_mins, + min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct, + overlap_pct=overlap_pct, max_p_value=max_p_value, cache_time=cache_time) + + def __get_ticks(self, symbol, date_from, date_to, cache_time): + """ + Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe. + + :param symbol: Name of symbol to get ticks for. + :param date_from: + :param date_to: + :param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed, + get data from source and refresh cache. + + :return: + """ + + timezone = pytz.timezone("Etc/UTC") + utc_now = datetime.now(tz=timezone) + + # Check if in cache and not stale + if symbol in self.__ticks and utc_now < self.__ticks[symbol][0] + timedelta(seconds=cache_time): + # Cached ticks are not stale. Get them + ticks = self.__ticks[symbol][1] + self.__log.debug(f"Ticks for {symbol} retrieved from cache.") + else: + # Data does not exist in cache or cached data is stale. Retrieve from source and cache. + ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to) + self.__ticks[symbol] = [utc_now, ticks] + self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.") + return ticks + + def __reset_coefficient_data(self): + """ + Clears coefficient data and history. + :return: + """ + # Create dataframe for coefficient data + coefficient_data_columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', + 'Timeframe', 'Last Check', 'Last Coefficient'] + self.coefficient_data = pd.DataFrame(columns=coefficient_data_columns) + + # Create dataframe for coefficient history + coefficient_history_columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To'] + self.coefficient_history = pd.DataFrame(columns=coefficient_history_columns) + + def __update_coefficient_data(self, symbol1, symbol2, coefficient, date_from, date_to): + """ + Updates the coefficient data with the latest coefficient and adds to coefficient history. + :param symbol1: + :param symbol2: + :param coefficient: + :param date_from: + :param date_to: + :return: + """ + timezone = pytz.timezone("Etc/UTC") now = datetime.now(tz=timezone) - # Update data if we have a coefficient + # Update data if we have a coefficient and add to history if coefficient is not None: self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) & (self.coefficient_data['Symbol 2'] == symbol2), @@ -337,28 +505,6 @@ class Correlation: (self.coefficient_data['Symbol 2'] == symbol2), 'Last Coefficient'] = coefficient - return coefficient - - def __update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05): - """ - Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet - the threshold can be accessed through the filtered_coefficient_data property. - - :param date_from: From date for tick data from which to calculate correlation coefficients - :param date_to: To date for tick data from which to calculate correlation coefficients - :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold - is not met then returned coefficient will be None - :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are - within this pct of each other - :param overlap_pct: - :param max_p_value: The maximum p value for the correlation to be meaningful - :return: correlation coefficient, or None if coefficient could not be calculated. - """ - # Update latest coefficient for every pair - for index, row in self.filtered_coefficient_data.iterrows(): - symbol1 = row['Symbol 1'] - symbol2 = row['Symbol 2'] - self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to, - min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct, - overlap_pct=overlap_pct, max_p_value=max_p_value) + row = pd.DataFrame(columns=self.coefficient_history.columns, + data=[[symbol1, symbol2, coefficient, date_from, date_to]]) + self.coefficient_history = self.coefficient_history.append(row) diff --git a/mt5_correlation/gui.py b/mt5_correlation/gui.py index 3d4b86f..d3d8671 100644 --- a/mt5_correlation/gui.py +++ b/mt5_correlation/gui.py @@ -1,5 +1,10 @@ import wx import wx.grid +import matplotlib.pyplot as plt +from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas +import matplotlib.dates +import matplotlib + from mt5_correlation.correlation import Correlation from mt5_correlation.config import Config, SettingsDialog from datetime import datetime, timedelta @@ -7,15 +12,21 @@ import pytz import pandas as pd import logging import logging.config +import os + +matplotlib.use('WXAgg') class MonitorFrame(wx.Frame): cor = None rows = 0 # Need to track as we need to notify grid if row count changes. - opened_filename = None # So we can save to same file as we opened + __opened_filename = None # So we can save to same file as we opened config = None # The applications config + __selected_correlation = [] # List of Symbol 1 & Symbol 2 + + # Columns for coefficient table COLUMN_INDEX = 0 COLUMN_SYMBOL1 = 1 COLUMN_SYMBOL2 = 2 @@ -31,7 +42,7 @@ class MonitorFrame(wx.Frame): wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title="Divergence Monitor") # Create logger and get config - self.log = logging.getLogger(__name__) + self.__log = logging.getLogger(__name__) self.config = Config() # Create correlation instance to maintain state of calculated coefficients. Set min coefficient from config @@ -76,8 +87,8 @@ class MonitorFrame(wx.Frame): panel = wx.Panel(self, wx.ID_ANY) toggle_sizer = wx.BoxSizer(wx.HORIZONTAL) # Label and toggle correlations_sizer = wx.BoxSizer(wx.VERTICAL) # Toggle sizer and correlations grid - main_sizer = wx.BoxSizer(wx.HORIZONTAL) # Correlations sizer and graphs panel - panel.SetSizer(main_sizer) + self.__main_sizer = wx.BoxSizer(wx.HORIZONTAL) # Correlations sizer and graphs panel + panel.SetSizer(self.__main_sizer) # Create the label and toggle, populate the toggle sizer and add the toggle sizer to the correlations sizer monitor_toggle_label = wx.StaticText(panel, id=wx.ID_ANY, label="Monitoring") @@ -111,12 +122,13 @@ class MonitorFrame(wx.Frame): self.grid_correlations.SetMaxSize((520, -1)) correlations_sizer.Add(self.grid_correlations, 1, wx.ALL | wx.EXPAND, 1) - # Create the charts - charts = wx.StaticText(panel, wx.ID_ANY, "Charts Go Here", style=wx.ALIGN_CENTER_HORIZONTAL) + # Create the charts and hide as we have no data to display yet + self.__graph = GraphPanel(panel) + self.__graph.Hide() # Add the correlations sizer and the charts to the main sizer. - main_sizer.Add(correlations_sizer, 1, wx.ALL | wx.EXPAND, 1) - main_sizer.Add(charts, 1, wx.ALL | wx.EXPAND, 1) + self.__main_sizer.Add(correlations_sizer, 0, wx.ALL | wx.EXPAND, 1) + self.__main_sizer.Add(self.__graph, 1, wx.ALL | wx.EXPAND, 1) # Size the window. self.SetSize((800, 500)) @@ -129,7 +141,7 @@ class MonitorFrame(wx.Frame): self.monitor_toggle.Bind(wx.EVT_TOGGLEBUTTON, self.monitor) # Bind timer - self.Bind(wx.EVT_TIMER, self.refresh_grid, self.timer) + self.Bind(wx.EVT_TIMER, self.__timer_event, self.timer) # Bind menu items self.Bind(wx.EVT_MENU, self.open_file, menu_item_open) @@ -139,6 +151,9 @@ class MonitorFrame(wx.Frame): self.Bind(wx.EVT_MENU, self.open_settings, menu_item_settings) self.Bind(wx.EVT_MENU, self.quit, menu_item_exit) + # Bind row select + self.Bind(wx.grid.EVT_GRID_SELECT_CELL, self.select_cell, self.grid_correlations) + # Bind window close event self.Bind(wx.EVT_CLOSE, self.on_close, self) @@ -149,18 +164,28 @@ class MonitorFrame(wx.Frame): if fileDialog.ShowModal() == wx.ID_CANCEL: return # the user changed their mind - # Load the file chosen by the user - self.opened_filename = fileDialog.GetPath() - self.cor.load(self.opened_filename) + # Load the file chosen by the user. Also load the corresponding data file if there is one. + self.__opened_filename = fileDialog.GetPath() + data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data" + if os.path.isfile(data_filename): + self.cor.load(self.__opened_filename, price_data_filename=data_filename) + else: + self.cor.load(self.__opened_filename) # Refresh data in grid - self.refresh_grid(event) + self.refresh_grid() - self.SetStatusText(f"File {self.opened_filename} loaded.") + self.SetStatusText(f"File {self.__opened_filename} loaded.") def save_file(self, event): - self.cor.save(self.opened_filename) - self.SetStatusText(f"File saved as {self.opened_filename}") + self.SetStatusText(f"Saving file as {self.__opened_filename}") + + if self.__opened_filename is None: + self.save_file_as(event) + else: + self.cor.save(self.__opened_filename) + + self.SetStatusText(f"File saved as {self.__opened_filename}") def save_file_as(self, event): with wx.FileDialog(self, "Save Coefficients file", wildcard="CSV (*.csv)|*.csv", @@ -168,11 +193,14 @@ class MonitorFrame(wx.Frame): if fileDialog.ShowModal() == wx.ID_CANCEL: return # the user changed their mind - # Save the file, changing opened filename so next save writes to new file - self.opened_filename = fileDialog.GetPath() - self.cor.save(self.opened_filename) + # Save the file and price data file, changing opened filename so next save writes to new file + self.SetStatusText(f"Saving file as {self.__opened_filename}") - self.SetStatusText(f"File saved as {self.opened_filename}") + self.__opened_filename = fileDialog.GetPath() + data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data" + self.cor.save(self.__opened_filename, price_data_filename=data_filename) + + self.SetStatusText(f"File saved as {self.__opened_filename}") def calculate_coefficients(self, event): # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today) @@ -191,30 +219,30 @@ class MonitorFrame(wx.Frame): self.SetStatusText("") # Show calculated data - self.refresh_grid(event) + self.refresh_grid() def quit(self, event): self.Close() - def refresh_grid(self, event): + def refresh_grid(self): """ Refreshes grid. Notifies if rows have been added or deleted. :return: """ - self.log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}") + self.__log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}") # Update data self.table.data = self.cor.coefficient_data.copy() # Format - self.table.data['Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format) - self.table.data['Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True) - self.table.data['Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S') - self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format) + self.table.data.loc[:, 'Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format) + self.table.data.loc[:, 'Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True) + self.table.data.loc[:, 'Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S') + self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format) - # Remove nans. The ones from the float column wil be str nan as they have been formatted + # Remove nans. The ones from the float column will be str nan as they have been formatted self.table.data = self.table.data.fillna('') - self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '') + self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '') # Start refresh self.grid_correlations.BeginBatch() @@ -244,24 +272,21 @@ class MonitorFrame(wx.Frame): def monitor(self, event): # Check state of toggle button. If on, then start monitoring, else stop if self.monitor_toggle.GetValue(): - self.log.debug("Starting monitoring.") + self.__log.info("Starting monitoring.") self.monitor_toggle.SetBackgroundColour(wx.GREEN) self.monitor_toggle.SetLabelText("On") self.SetStatusText("Monitoring for changes to coefficients.") - # From and to dates for calculations. - timezone = pytz.timezone("Etc/UTC") - utc_to = datetime.now(tz=timezone) - utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes')) - self.timer.Start(self.config.get('monitor.interval')*1000) - self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to, + self.cor.start_monitor(interval=self.config.get('monitor.interval'), + from_mins=self.config.get('monitor.from.minutes'), min_prices=self.config.get('monitor.min_prices'), max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'), overlap_pct=self.config.get('monitor.overlap_pct'), - max_p_value=self.config.get('monitor.max_p_value')) + max_p_value=self.config.get('monitor.max_p_value'), + cache_time=self.config.get('monitor.tick_cache_time')) else: - self.log.debug("Stopping monitoring.") + self.__log.info("Stopping monitoring.") self.monitor_toggle.SetBackgroundColour(wx.RED) self.monitor_toggle.SetLabelText("Off") self.SetStatusText("Monitoring stopped.") @@ -288,7 +313,7 @@ class MonitorFrame(wx.Frame): # If 'monitor.interval has changed then restart gui timer. # If 'monitor.monitoring_threshold' has changed, then refresh correlation data. # If any 'logging.' settings have changed, then reload logger config. - if setting.startswith('monitor. ') and setting != 'monitor.divergence_threshold': + if setting.startswith('monitor.') and setting != 'monitor.divergence_threshold': restart_monitor_timer = True if setting == 'monitor.interval': restart_gui_timer = True @@ -299,29 +324,30 @@ class MonitorFrame(wx.Frame): # Now perform the actions if restart_monitor_timer: - self.log.debug("Settings updated. Reloading monitoring timer.") + self.__log.info("Settings updated. Reloading monitoring timer.") self.cor.stop_monitor() # From and to dates for calculations. timezone = pytz.timezone("Etc/UTC") utc_to = datetime.now(tz=timezone) utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes')) - self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to, + self.cor.start_monitor(interval=self.config.get('monitor.interval'), + from_mins=self.config.get('monitor.from.minutes'), min_prices=self.config.get('monitor.min_prices'), max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'), overlap_pct=self.config.get('monitor.overlap_pct'), max_p_value=self.config.get('monitor.max_p_value')) if restart_gui_timer: - self.log.debug("Settings updated. Restarting gui timer.") + self.__log.info("Settings updated. Restarting gui timer.") self.timer.Stop() self.timer.Start(self.config.get('monitor.interval') * 1000) if reload_correlations: - self.log.debug("Settings updated. Updating monitoring threshold and reloading grid.") + self.__log.info("Settings updated. Updating monitoring threshold and reloading grid.") self.cor.monitoring_threshold = self.config.get("monitor.monitoring_threshold") - self.refresh_grid(event) + self.refresh_grid() if reload_logger: - self.log.debug("Settings updated. Reloading logger.") + self.__log.info("Settings updated. Reloading logger.") log_config = Config().get('logging') logging.config.dictConfig(log_config) @@ -331,13 +357,64 @@ class MonitorFrame(wx.Frame): :param event: :return: """ - if self.opened_filename is not None: - self.cor.save(self.opened_filename) + if self.__opened_filename is not None: + self.cor.save(self.__opened_filename) self.cor.stop_monitor() + # Kill graph as it seems to be stopping script from ending + self.__graph = None + + # End event.Skip() + def select_cell(self, event): + """ + A cell was selected. Show the graph for the correlation. + :param event: + :return: + """ + # Get row and symbols. + row = event.GetRow() + symbol1 = self.grid_correlations.GetCellValue(row, self.COLUMN_SYMBOL1) + symbol2 = self.grid_correlations.GetCellValue(row, self.COLUMN_SYMBOL2) + self.__selected_correlation = [symbol1, symbol2] + + self.show_graph(symbol1, symbol2) + + def show_graph(self, symbol1, symbol2): + """ + Displays the graph for the specified symbols correlation history + :param symbol1: + :param symbol2: + :return: + """ + # Get the data price data for the base coefficient calculation and the coefficient history data + symbol_1_price_data = self.cor.get_price_data(symbol1) + symbol_2_price_data = self.cor.get_price_data(symbol2) + history_data = self.cor.get_coefficient_history(symbol1, symbol2) + times = history_data['UTC Date To'] + coefficients = history_data['Coefficient'] + + # Display if we have any data + self.__log.debug(f"Refreshing history graph {symbol1}:{symbol2}.") + self.__graph.draw(times=times, coefficients=coefficients, prices=[symbol_1_price_data, symbol_2_price_data], + symbols=[symbol1, symbol2]) + + # Un-hide and layout if hidden + if not self.__graph.IsShown(): + self.__graph.Show() + self.__main_sizer.Layout() + + def __timer_event(self, event): + """ + Called on timer event. Refreshes grid and updatates selected graph. + :return: + """ + self.refresh_grid() + if len(self.__selected_correlation) == 2: + self.show_graph(symbol1=self.__selected_correlation[0], symbol2=self.__selected_correlation[1]) + class DataTable(wx.grid.GridTableBase): """ @@ -393,3 +470,72 @@ class DataTable(wx.grid.GridTableBase): attr.SetBackgroundColour(wx.WHITE) return attr + + +class GraphPanel(wx.Panel): + def __init__(self, parent): + # Super + wx.Panel.__init__(self, parent) + + # 3 axis, price data 1, price data 2 and coefficient data. All will have axis labels and top and right boarders + # removed + self.__fig, self.__axes = plt.subplots(nrows=3, ncols=1) + + # Create the canvas + self.__canvas = FigureCanvas(self, -1, self.__fig) + + # Date format for x axes + self.__tick_fmt_date = matplotlib.dates.DateFormatter('%d-%b') + self.__tick_fmt_time = matplotlib.dates.DateFormatter('%H:%M:%S') + + # Sizer etc. + self.__sizer = wx.BoxSizer(wx.VERTICAL) + self.__sizer.Add(self.__canvas, 1, wx.LEFT | wx.TOP | wx.GROW) + self.SetSizer(self.__sizer) + self.Fit() + + def __del__(self): + # Close all plots + plt.close('all') + self.__axes = None + self.__fig = None + + def draw(self, times, coefficients, prices=None, symbols=None): + """ + Plot the correlations. + :param times: Series of time values for x axis + :param coefficients: Series of coefficients values for y axis + :param prices: Price data used to calculate base coefficient. List [Symbol1 Price Data, Symbol 2 Price Data] + :param symbols: Symbols. List [Symbol1, Symbol2] + :return: + """ + # Clear. We will need to redraw + for ax in self.__axes: + ax.clear() + + if symbols is not None and len(symbols) == 2: + # Price history chart for both symbols + for i in range(0, 2): + if symbols[i] is not None and prices is not None and len(prices) == 2 and prices[i] is not None: + self.__axes[i].set_title(f"Base Coefficient Price Data for {symbols[i]}") + self.__axes[i].set_xlabel('Time') + self.__axes[i].set_ylabel('Price') + self.__axes[i].plot(prices[i]['time'], prices[i]['close']) + self.__axes[i].xaxis.set_major_formatter(self.__tick_fmt_date) + self.__axes[i].xaxis.set_minor_formatter(self.__tick_fmt_time) + plt.setp(self.__axes[i].xaxis.get_majorticklabels(), rotation=45) + + # Coefficient history chart + self.__axes[2].set_title(f"Coefficient History for {symbols[0]}:{symbols[1]}") + self.__axes[2].set_xlabel('Time') + self.__axes[2].set_ylabel('Coefficient') + self.__axes[2].plot(times, coefficients) + self.__axes[2].set_ylim([-1, 1]) + self.__axes[2].xaxis.set_major_formatter(self.__tick_fmt_time) + plt.setp(self.__axes[2].xaxis.get_majorticklabels(), rotation=45) + + # Layout with padding between charts + self.__fig.tight_layout(pad=0.5) + + # Redraw canvas + self.__canvas.draw() diff --git a/mt5_correlation/mt5.py b/mt5_correlation/mt5.py index 11bf8a3..ef9f319 100644 --- a/mt5_correlation/mt5.py +++ b/mt5_correlation/mt5.py @@ -35,18 +35,18 @@ class MT5: # Connect to MetaTrader5. Opens if not already open. # Logger - self.log = logging.getLogger(__name__) + self.__log = logging.getLogger(__name__) # Open MT5 and log error if it could not open if not MetaTrader5.initialize(): - self.log.error("initialize() failed") + self.__log.error("initialize() failed") MetaTrader5.shutdown() # Print connection status - self.log.debug(MetaTrader5.terminal_info()) + self.__log.debug(MetaTrader5.terminal_info()) # Print data on MetaTrader 5 version - self.log.debug(MetaTrader5.version()) + self.__log.debug(MetaTrader5.version()) def __del__(self): # shut down connection to the MetaTrader 5 terminal @@ -67,7 +67,7 @@ class MT5: # Log symbol counts total_symbols = MetaTrader5.symbols_total() num_selected_symbols = len(selected_symbols) - self.log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.") + self.__log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.") return selected_symbols @@ -102,13 +102,16 @@ class MT5: :return: Price data for symbol as dataframe """ + prices_dataframe = None + # Get prices from MT5 prices = MetaTrader5.copy_rates_range(symbol, timeframe, from_date, to_date) - self.log.debug(f"{len(prices)} prices retrieved for {symbol}.") + if prices is not None: + self.__log.debug(f"{len(prices)} prices retrieved for {symbol}.") - # Create dataframe from data and convert time in seconds to datetime format - prices_dataframe = pd.DataFrame(prices) - prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s') + # Create dataframe from data and convert time in seconds to datetime format + prices_dataframe = pd.DataFrame(prices) + prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s') return prices_dataframe @@ -121,19 +124,23 @@ class MT5: :return: Tick data for symbol as dataframe """ + ticks_dataframe = None + # Get ticks from MT5 ticks = MetaTrader5.copy_ticks_range(symbol, from_date, to_date, MetaTrader5.COPY_TICKS_ALL) # If ticks is None, there was an error if ticks is None: error = MetaTrader5.last_error() - self.log.error(f"Error retrieving ticks for {symbol}: {error}") - return None + self.__log.error(f"Error retrieving ticks for {symbol}: {error}") else: - self.log.debug(f"{len(ticks)} ticks retrieved for {symbol}.") + self.__log.debug(f"{len(ticks)} ticks retrieved for {symbol}.") # Create dataframe from data and convert time in seconds to datetime format - ticks_dataframe = pd.DataFrame(ticks) - ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s') + try: + ticks_dataframe = pd.DataFrame(ticks) + ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s') + except RecursionError: + self.__log.warning("Error converting ticks to dataframe.") return ticks_dataframe