From 862a55d649baf99e346dc80fd2342913ed3492eb Mon Sep 17 00:00:00 2001 From: Jamie Cash Date: Wed, 17 Feb 2021 18:24:48 +0000 Subject: [PATCH] Split notebook page (tab) out as seperate class from Settings Dialog. --- config.yaml | 8 +- mt5_correlation.py | 13 +- mt5_correlation/config.py | 257 +++++++++++++++++++++++++- mt5_correlation/correlation.py | 324 ++++++++++++++++++--------------- mt5_correlation/gui.py | 281 +++++----------------------- 5 files changed, 497 insertions(+), 386 deletions(-) diff --git a/config.yaml b/config.yaml index 084b3ab..05f17aa 100644 --- a/config.yaml +++ b/config.yaml @@ -9,11 +9,11 @@ calculate: max_p_value: 0.05 monitor: from: - minutes: 15 + minutes: 60 interval: 10 - min_prices: 400 - max_set_size_diff_pct: 50 - overlap_pct: 50 + min_prices: 1000 + max_set_size_diff_pct: 90 + overlap_pct: 90 max_p_value: 0.05 monitoring_threshold: 0.9 divergence_threshold: 0.8 diff --git a/mt5_correlation.py b/mt5_correlation.py index 2744c37..3f1e6f9 100644 --- a/mt5_correlation.py +++ b/mt5_correlation.py @@ -2,11 +2,20 @@ Application to monitor previously correlated symbol pairs for correlation divergence. """ import definitions -import yaml import logging.config from mt5_correlation.gui import MonitorFrame from mt5_correlation.config import Config import wx +import wx.lib.mixins.inspection as wit + + +class CorrelationMonitorApp(wx.App, wit.InspectionMixin): + # Override app to use inspection. + # TODO Remove wit.InspectionMixin from overrides when live. + def OnInit(self): + self.Init() # initialize the inspection tool + return True + if __name__ == "__main__": # Load the config @@ -17,7 +26,7 @@ if __name__ == "__main__": logging.config.dictConfig(log_config) # Start the app - app = wx.App(False) + app = CorrelationMonitorApp() frame = MonitorFrame() frame.Show() app.MainLoop() diff --git a/mt5_correlation/config.py b/mt5_correlation/config.py index 7815c0a..40646ef 100644 --- a/mt5_correlation/config.py +++ b/mt5_correlation/config.py @@ -1,5 +1,6 @@ import yaml -import definitions +import wx +import logging class Config(object): @@ -86,3 +87,257 @@ class Config(object): obj = obj[k] obj[key_list[-1]] = value + + +class SettingsDialog(wx.Dialog): + + # Store any settings that have changed + changed_settings = {} + + def __init__(self, *args, **kwargs): + # Super Constructor + wx.Dialog.__init__(self, *args, **kwargs) + self.SetTitle("Settings") + + # Create logger and get config + self.__log = logging.getLogger(__name__) + self.__settings = Config() + + # Dict of changes. Will commit only on ok + self.__changes = {} + + # We want 2 vertical sections, the tabbed notebook and the buttons. The buttons sizer will have 2 horizontal + # sections, one for each button. + main_sizer = wx.BoxSizer(wx.VERTICAL) # Notebook panel + button_sizer = wx.BoxSizer(wx.HORIZONTAL) # Button sizer + + # Notebook + self.__notebook = wx.Notebook(self, wx.ID_ANY) # The notebook + + # A tab for each root node in config. We will store the tabs components in lists which can be accessed by the + # index returned from notebook.GetSelectedItem() + root_nodes = self.__settings.get_root_nodes() + self.__tabs = [] + for node in root_nodes: + # Create new tab + self.__tabs.append(SettingsTab(self, self.__notebook, node)) + + # Add tab to notebook + self.__notebook.AddPage(self.__tabs[-1], node) + + # Buttons + button_ok = wx.Button(self, label="Update") + button_cancel = wx.Button(self, label="Cancel") + button_sizer.Add(button_ok, 0, wx.ALL, 1) + button_sizer.Add(button_cancel, 0, wx.ALL, 1) + + # Add notebook and button sizer to main sizer and set main sizer for window + main_sizer.Add(self.__notebook, 1, wx.ALL | wx.EXPAND, 5) + main_sizer.Add(button_sizer) + self.SetSizer(main_sizer) + + # Bind buttons & notebook page select. + button_ok.Bind(wx.EVT_BUTTON, self.__on_ok) + button_cancel.Bind(wx.EVT_BUTTON, self.__on_cancel) + self.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.__on_page_select) + + # Call on_page_select to select the first page + self.__on_page_select(event=None) + + def __on_page_select(self, event): + # Call the tabs select method to populate + index = self.__notebook.GetSelection() + self.__tabs[index].select() + + def __on_cancel(self, event): + # Clear changed settings and close + self.changed_settings = {} + self.EndModal(wx.ID_CANCEL) + self.Destroy() + + def __on_ok(self, event): + # Update settings and save + delkeys = [] + for setting in self.changed_settings: + # Get the current and new setting + orig_value = self.__settings.get(setting) + new_value = self.changed_settings[setting] + + # If they are the same, discard from changes. We will use a list of items to delete (delkeys) as we cant + # delete whilst iterating. If they are different, update settings. + if orig_value == new_value: + delkeys.append(setting) + else: + # We need to retain data type. New values will all be string as they were retrieved from textctl. + # Get the data type of the original and cast new to it. + new_value = type(orig_value)(new_value) + self.__settings.set(setting, new_value) + + # Now delete the items that were the same from changed_settings. changed_settings may be used by settings + # dialog caller. + for key in delkeys: + del(self.changed_settings[key]) + + # Save the settings and close dialog + self.__settings.save() + self.EndModal(wx.ID_OK) + self.Destroy() + + +class SettingsTab(wx.Panel): + """ + A notebook tab containing the settings tree and values for a settings root node. + """ + + # Parent frame. Set during constructor + __parent_frame = None + + # Each tab has: a tree view; a values panel; and a list of value text boxes bound to a change + # event. + __tree = None + __tab_sizer = None + __value_sizer = None + __value_boxes = [] + + def __init__(self, parent_frame, notebook, root_node): + """ + Creates a tab for the settings notebook. + + :param parent_frame: The frame containing the notebook. + :param notebook. The notebook that this tab should be part of. + :param root_node. The root node for the settings + """ + # Super Constructor + wx.Panel.__init__(self, parent=notebook) + + # Store the parent frame and get the settings for this tab. + self.__parent_frame = parent_frame + settings = Config().get(root_node) + + # Create logger + self.__log = logging.getLogger(__name__) + + # Build the tab and set it's sizer. + self.__tab_sizer = wx.BoxSizer(wx.HORIZONTAL) + self.SetSizer(self.__tab_sizer) + + # Create tree control + self.__tree = wx.TreeCtrl(self, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize) + + # Add root to tree and use item data to store settings path + root = self.__tree.AddRoot(root_node) + self.__tree.SetItemData(root, root_node) + + # Add items to root + self.__tree = self.__build_tree(self.__tree, root, settings) + + # expand tree and add it to the sizer + self.__tree.Expand(root) + self.__tab_sizer.Add(self.__tree, 1, wx.ALL | wx.EXPAND, 1) + + # Add the value sizer for settings values. This is only used for spacing, as it will be overwritten when + # tree items are selected. + self.__value_sizer = wx.FlexGridSizer(rows=1, cols=2, vgap=2, hgap=2) + self.__tab_sizer.Add(self.__value_sizer, 1, wx.ALL | wx.EXPAND, 1) + label = wx.StaticText(self, wx.ID_ANY, " ".ljust(50), style=wx.ALIGN_LEFT) + self.__value_sizer.Add(label) + + # Bind tree selection changed + self.__tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.__on_tree_select) + + def select(self): + """ + To be called when this tab is selected. Populate value sizer for the selected item, If no item is selected, + populate for root. + :return: + """ + selected_item = self.__tree.GetSelection() + if selected_item.ID is None: + root_node = self.__tree.GetRootItem() + setting_path = self.__tree.GetItemData(root_node) + else: + setting_path = self.__tree.GetItemData(selected_item) + + self.__populate_settings_values(setting_path) + + def __build_tree(self, tree, node, settings): + """ + Recursive function to build the tree from the node, using the settings + :param tree: The tree to build + :param node: The tree view node + :param settings: The settings dict for the node. + :return: The built tree + """ + for setting in settings: + # Get value. If dict, add the node and recursively call this function again. + value = settings[setting] + if type(value) is dict: + # Add the node and set its settings path + node_id = tree.AppendItem(node, setting) + current_settings_path = tree.GetItemData(node) + tree.SetItemData(node_id, f"{current_settings_path}.{setting}") + + # Recurse + tree = self.__build_tree(tree, node_id, value) + + return tree + + def __populate_settings_values(self, setting_path): + """ + Populates the settings in the value sizer for a settings path. + + :param setting_path: + + :return: + """ + + # Get the settings for the path + settings = Config().get(setting_path) + + # Clear the value sizer and set its rows + self.__value_sizer.Clear(True) + self.__value_sizer.SetRows(len(settings)) + + # Display every value that is a leaf (not dict) + for setting in settings: + value = settings[setting] + if type(value) is not dict: + # Add a label and value text box + label = wx.StaticText(self, wx.ID_ANY, setting, style=wx.ALIGN_LEFT) + self.__value_boxes.append(wx.TextCtrl(self, wx.ID_ANY, f"{value}", style=wx.ALIGN_LEFT)) + self.__value_sizer.AddMany([(label, 0, wx.EXPAND), (self.__value_boxes[-1], 0, wx.EXPAND)]) + + # Bind to text change. We need to generate a handler as this will have a parameter. + self.__value_boxes[-1].Bind(wx.EVT_TEXT, + self.__get_on_change_evt_handler(setting_path=f'{setting_path}.{setting}')) + + self.__value_sizer.Layout() + + def __get_on_change_evt_handler(self, setting_path): + """ + Returns a new event handler with a parameter of the settings path + :param setting_path: + :return: + """ + def on_value_changed(event): + self.__parent_frame.changed_settings[setting_path] = event.String + self.__log.debug(f"Value changed for {setting_path}.") + + return on_value_changed + + def __on_tree_select(self, event): + """ + Called when an item in the tree is selected. Populates the settings values + :param event: + :return: + """ + # Get Selected item and check that it is a tree item + tree_item = event.GetItem() + if not tree_item.IsOk(): + return + + # Get the setting path from item data + setting_path = self.__tree.GetItemData(tree_item) + + # Populate value_sizer + self.__populate_settings_values(setting_path) diff --git a/mt5_correlation/correlation.py b/mt5_correlation/correlation.py index a63680b..b2d6a23 100644 --- a/mt5_correlation/correlation.py +++ b/mt5_correlation/correlation.py @@ -17,22 +17,33 @@ class Correlation: """ # Minimum base coefficient for monitoring. Symbol pairs with a lower correlation - # coefficient than ths wont be monitored. + # coefficient than ths won't be monitored. monitoring_threshold = 0.9 # Toggle on whether we are monitoring or not. Set through start_monitor and stop_monitor - _monitoring = False + __monitoring = False + __monitoring_params = {} def __init__(self): - self.log = logging.getLogger(__name__) + self.__log = logging.getLogger(__name__) # Create dataframe - columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe', - 'Last Check', 'Last Coefficient'] - self.coefficient_data = pd.DataFrame(columns=columns) + self.__columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe', + 'Last Check', 'Last Coefficient'] + self.coefficient_data = pd.DataFrame(columns=self.__columns) # Create timer for continuous monitoring - self.scheduler = sched.scheduler(time.time, time.sleep) + self.__scheduler = sched.scheduler(time.time, time.sleep) + + @property + def filtered_coefficient_data(self): + """ + :return: Coefficient data filtered so that all base coefficients >= monitoring_threshold + """ + if self.coefficient_data is not None: + return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold] + else: + return None def load(self, filename): """ @@ -68,6 +79,14 @@ class Correlation: :return: """ + # If we are monitoring, stop. We will need to restart later + was_monitoring = self.__monitoring + if self.__monitoring: + self.stop_monitor() + + # Clear the existing correlations + self.coefficient_data = pd.DataFrame(columns=self.__columns) + # Create mt5 class. This contains required methods for interacting with MT5. mt5 = MT5() @@ -112,17 +131,156 @@ class Correlation: 'Base Coefficient': coefficient, 'UTC Date From': date_from, 'UTC Date To': date_to, 'Timeframe': timeframe}, ignore_index=True) - self.log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a " - f"coefficient of {coefficient}.") + self.__log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a " + f"coefficient of {coefficient}.") else: - self.log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:" - f"{symbol2} could no be calculated.") + self.__log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:" + f"{symbol2} could no be calculated.") # Sort, highest correlated first self.coefficient_data = self.coefficient_data.sort_values('Base Coefficient', ascending=False) - def update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, - overlap_pct=90, max_p_value=0.05): + # If we were monitoring, we stopped, so start again. + if was_monitoring: + self.start_monitor(interval=self.__monitoring_params['interval'], + date_from=self.__monitoring_params['date_from'], + date_to=self.__monitoring_params['date_to'], + min_prices=self.__monitoring_params['min_prices'], + max_set_size_diff_pct=self.__monitoring_params['max_set_size_diff_pct'], + overlap_pct=self.__monitoring_params['overlap_pct'], + max_p_value=self.__monitoring_params['max_p_value']) + + def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): + """ + Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient + threshold. + + :param interval: How often to check in seconds + :param date_from: From date for tick data from which to calculate correlation coefficients + :param date_to: To date for tick data from which to calculate correlation coefficients + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + + :return: correlation coefficient, or None if coefficient could not be calculated. + """ + + if self.__monitoring: + self.__log.debug(f"Request to start monitor when monitor is already running. Monitor will be stopped and" + f"restarted with new parameters.") + self.stop_monitor() + + self.__log.debug(f"Starting monitor.") + self.__monitoring = True + + # Create thread to run monitoring This will call private __monitor method that will run the calculation and + # keep scheduling itself while self.monitoring is True. Store the params. We will need to use these if we have + # to stop and restart the monitor. Note, this happens during calculate + self.__monitoring_params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, + 'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct, + 'overlap_pct': overlap_pct, 'max_p_value': max_p_value} + thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params) + thread.start() + + def stop_monitor(self): + """ + Stops monitoring symbol pairs for correlation. + :return: + """ + if self.__monitoring: + self.__log.debug(f"Stopping monitor.") + self.__monitoring = False + else: + self.__log.debug(f"Request to stop monitor when it is not running. No action taken.") + + @staticmethod + def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90, + overlap_pct=90, max_p_value=0.05): + """ + Calculates the correlation coefficient between two sets of price data. Uses close price. + + :param symbol1_prices: prices or ticks for symbol 1 + :param symbol2_prices: prices or ticks for symbol 2 + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + :return: correlation coefficient, or None if coefficient could not be calculated. + """ + + # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for + # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of + # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set? + coefficient = None + + intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time'])) + len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)])) + len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)])) + similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set + enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100) + enough_prices = len_smallest_set >= min_prices + suitable = similar_size and enough_overlap and enough_prices + + if suitable: + # Calculate coefficient on close prices + + # First filter prices to only include those that intersect + symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)] + symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)] + + # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null + # hypothesis is false). + coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close']) + coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0] + + # If NaN, change to None + if coefficient is not None and math.isnan(coefficient): + coefficient = None + + return coefficient + + def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): + """ + The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and + stop_monitoring. + + :param interval: How often to check in seconds + :param date_from: From date for tick data from which to calculate correlation coefficients + :param date_to: To date for tick data from which to calculate correlation coefficients + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + + :return: correlation coefficient, or None if coefficient could not be calculated. + """ + self.__log.debug(f"In monitor event. Monitoring: {self.__monitoring}.") + + # Only run if monitor is not stopped + if self.__monitoring: + # Update all coefficients + self.__update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices, + max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct, + max_p_value=max_p_value) + + # Schedule the timer to run again + params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices, + 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct, + 'max_p_value': max_p_value} + self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params) + self.__scheduler.run() + + def __update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, + overlap_pct=90, max_p_value=0.05): """ Updates the coefficient for the specified symbol pair :param symbol1: Name of symbol to calculate coefficient for. @@ -181,8 +339,8 @@ class Correlation: return coefficient - def update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05): + def __update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): """ Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet the threshold can be accessed through the filtered_coefficient_data property. @@ -201,136 +359,6 @@ class Correlation: for index, row in self.filtered_coefficient_data.iterrows(): symbol1 = row['Symbol 1'] symbol2 = row['Symbol 2'] - self.update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to, - min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct, - overlap_pct=overlap_pct, max_p_value=max_p_value) - - def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05): - """ - Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient - threshold. - - :param interval: How often to check in seconds - :param date_from: From date for tick data from which to calculate correlation coefficients - :param date_to: To date for tick data from which to calculate correlation coefficients - :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold - is not met then returned coefficient will be None - :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are - within this pct of each other - :param overlap_pct: - :param max_p_value: The maximum p value for the correlation to be meaningful - :return: correlation coefficient, or None if coefficient could not be calculated. - - :return: - """ - self.log.debug(f"Starting monitor.") - self._monitoring = True - - # Create thread to run monitoring This will call private __monitor method that will run the calculation and - # keep scheduling itself while self.monitoring is True - params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices, - 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct, - 'max_p_value': max_p_value} - thread = threading.Thread(target=self.__monitor, kwargs=params) - thread.start() - - def stop_monitor(self): - """ - Stops monitoring symbol pairs for correlation. - :return: - """ - self.log.debug(f"Stopping monitor.") - self._monitoring = False - - def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05): - """ - The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and - stop_monitoring. - - :param interval: How often to check in seconds - :param date_from: From date for tick data from which to calculate correlation coefficients - :param date_to: To date for tick data from which to calculate correlation coefficients - :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold - is not met then returned coefficient will be None - :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are - within this pct of each other - :param overlap_pct: - :param max_p_value: The maximum p value for the correlation to be meaningful - :return: correlation coefficient, or None if coefficient could not be calculated. - :return: - """ - self.log.debug(f"In monitor event. Monitoring: {self._monitoring}.") - - # Only run if monitor is not stopped - if self._monitoring: - # Update all coefficients - self.update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices, - max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct, - max_p_value=max_p_value) - - # Schedule the timer to run again - params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices, - 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct, - 'max_p_value': max_p_value} - self.scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params) - self.scheduler.run() - - @property - def filtered_coefficient_data(self): - """ - :return: Coefficient data filtered so that all base coefficients >= monitoring_threshold - """ - if self.coefficient_data is not None: - return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold] - else: - return None - - @staticmethod - def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90, - overlap_pct=90, max_p_value=0.05): - """ - Calculates the correlation coefficient between two sets of price data. Uses close price. - - :param symbol1_prices: prices or ticks for symbol 1 - :param symbol2_prices: prices or ticks for symbol 2 - :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold - is not met then returned coefficient will be None - :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are - within this pct of each other - :param overlap_pct: - :param max_p_value: The maximum p value for the correlation to be meaningful - :return: correlation coefficient, or None if coefficient could not be calculated. - """ - - # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for - # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of - # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set? - coefficient = None - - intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time'])) - len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)])) - len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)])) - similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set - enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100) - enough_prices = len_smallest_set >= min_prices - suitable = similar_size and enough_overlap and enough_prices - - if suitable: - # Calculate coefficient on close prices - - # First filter prices to only include those that intersect - symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)] - symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)] - - # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null - # hypothesis is false). - coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close']) - coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0] - - # If NaN, change to None - if coefficient is not None and math.isnan(coefficient): - coefficient = None - - return coefficient + self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to, + min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct, + overlap_pct=overlap_pct, max_p_value=max_p_value) diff --git a/mt5_correlation/gui.py b/mt5_correlation/gui.py index a912c47..f7873da 100644 --- a/mt5_correlation/gui.py +++ b/mt5_correlation/gui.py @@ -1,7 +1,7 @@ import wx import wx.grid from mt5_correlation.correlation import Correlation -from mt5_correlation.config import Config +from mt5_correlation.config import Config, SettingsDialog from datetime import datetime, timedelta import pytz import pandas as pd @@ -180,9 +180,6 @@ class MonitorFrame(wx.Frame): utc_to = datetime.now(tz=timezone) utc_from = utc_to - timedelta(days=self.config.get('calculate.from.days')) - # Set timeframe - timeframe = self.config.get('calculate.timeframe') - # Calculate self.SetStatusText("Calculating coefficients.") self.cor.calculate(date_from=utc_from, date_to=utc_to, @@ -252,12 +249,12 @@ class MonitorFrame(wx.Frame): self.monitor_toggle.SetLabelText("On") self.SetStatusText("Monitoring for changes to coefficients.") - # Calculate correlations fro last 10 mins + # From and to dates for calculations. timezone = pytz.timezone("Etc/UTC") utc_to = datetime.now(tz=timezone) utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes')) - self.timer.Start(10000) + self.timer.Start(self.config.get('monitor.interval')*1000) self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to, min_prices=self.config.get('monitor.min_prices'), max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'), @@ -282,10 +279,53 @@ class MonitorFrame(wx.Frame): # Reload relevant parts of app # TODO: Reload relevant parts of app once settings have changed. Stop and restart monitoring, # reload logger, filter data, refresh data window. - self.log.debug("Settings updated. Reloading logger.") - log_config = Config().get('logging') - logging.config.dictConfig(log_config) - settings_dialog.Destroy() + restart_monitor_timer = False + restart_gui_timer = False + reload_correlations = False + reload_logger = False + + for setting in settings_dialog.changed_settings: + # If any 'monitor.' settings except 'monitor.divergence_threshold have changed then restart + # monitoring timer with new settings. + # If 'monitor.interval has changed then restart gui timer. + # If 'monitor.monitoring_threshold' has changed, then refresh correlation data. + # If any 'logging.' settings have changed, then reload logger config. + if setting.startswith('monitor. ') and setting != 'monitor.divergence_threshold': + restart_monitor_timer = True + if setting == 'monitor.interval': + restart_gui_timer = True + if setting == 'monitor.monitoring_threshold': + reload_correlations = True + if setting.startswith('logging.'): + reload_logger = True + + # Now perform the actions + if restart_monitor_timer: + self.log.debug("Settings updated. Reloading monitoring timer.") + self.cor.stop_monitor() + # From and to dates for calculations. + timezone = pytz.timezone("Etc/UTC") + utc_to = datetime.now(tz=timezone) + utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes')) + self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to, + min_prices=self.config.get('monitor.min_prices'), + max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'), + overlap_pct=self.config.get('monitor.overlap_pct'), + max_p_value=self.config.get('monitor.max_p_value')) + if restart_gui_timer: + self.log.debug("Settings updated. Restarting gui timer.") + self.timer.Stop() + self.timer.Start(self.config.get('monitor.interval') * 1000) + + if reload_correlations: + self.log.debug("Settings updated. Updating monitoring threshold and reloading grid.") + self.cor.monitoring_threshold = self.config.get("monitor.monitoring_threshold") + self.refresh_grid(event) + + if reload_logger: + self.log.debug("Settings updated. Reloading logger.") + log_config = Config().get('logging') + logging.config.dictConfig(log_config) def on_close(self, event): """ @@ -355,224 +395,3 @@ class DataTable(wx.grid.GridTableBase): attr.SetBackgroundColour(wx.WHITE) return attr - - -class SettingsDialog(wx.Dialog): - - # Store any settings that have changed - changed_settings = {} - - def __init__(self, *args, **kwargs): - # Super Constructor - wx.Dialog.__init__(self, *args, **kwargs) - self.SetTitle("Settings") - - # Create logger and get config - self.log = logging.getLogger(__name__) - - # Get settings - self.__settings = Config() - - # Dict of changes. Will commit only on ok - self.__changes = {} - - # We want 2 vertical sections, the tabbed notebook and the buttons. The buttons sizer will have 2 horizontal - # sections, one for each button. - # ------------------------- - # |Tabbed notebook | - # | | - # | | - # | | - # | | - # |-----------------------| - # |ok | cancel | - # ------------------------- - main_sizer = wx.BoxSizer(wx.VERTICAL) # Notebook panel - button_sizer = wx.BoxSizer(wx.HORIZONTAL) # Button sizer - - # Notebook - self.__notebook = wx.Notebook(self, wx.ID_ANY) # The notebook - - # A tab for each root node in config. We will store the tabs components in lists which can be accessed by the - # index returned from notebook.GetSelectedItem() - root_nodes = self.__settings.get_root_nodes() - self.__tabs = [] - self.__trees = [] - self.__tab_sizers = [] - self.__value_sizers = [] - self.__value_boxes = [] # We need to store these as they will all be bound to a change event - # self.roots = [] - for node in root_nodes: - # Create new tab - self.__tabs.append(wx.Panel(self.__notebook, wx.ID_ANY)) - self.__tab_sizers.append(wx.BoxSizer(wx.HORIZONTAL)) - self.__tabs[-1].SetSizer(self.__tab_sizers[-1]) - - # Get settings items for tab / node - node_settings = self.__settings.get(node) - - # Create tree control - self.__trees.append(wx.TreeCtrl(self.__tabs[-1], wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize)) - - # Add root to tree and use item data to store settings path - root = self.__trees[-1].AddRoot(node) - self.__trees[-1].SetItemData(root, node) - - # Add items to root - self.__trees[-1] = self.__build_tree(self.__trees[-1], root, node_settings) - - # expand tree and add it to the sizer - self.__trees[-1].Expand(root) - self.__tab_sizers[-1].Add(self.__trees[-1], 1, wx.ALL | wx.EXPAND, 1) - - # Add the value sizer for settings values. This is only used for spacing, as it will be overwritten when - # tree items are selected. - self.__value_sizers.append(wx.FlexGridSizer(rows=1, cols=2, vgap=2, hgap=2)) - self.__tab_sizers[-1].Add(self.__value_sizers[-1], 1, wx.ALL | wx.EXPAND, 1) - label = wx.StaticText(self.__tabs[-1], wx.ID_ANY, " ".ljust(50), style=wx.ALIGN_LEFT) - self.__value_sizers[-1].Add(label) - - # Add tab to notebook - self.__notebook.AddPage(self.__tabs[-1], node) - - # Buttons - button_ok = wx.Button(self, label="Update") - button_cancel = wx.Button(self, label="Cancel") - button_sizer.Add(button_ok, 0, wx.ALL, 1) - button_sizer.Add(button_cancel, 0, wx.ALL, 1) - - # Add notebook and button sizer to main sizer and set main sizer for window - main_sizer.Add(self.__notebook, 1, wx.ALL | wx.EXPAND, 5) - main_sizer.Add(button_sizer) - self.SetSizer(main_sizer) - - # Bind buttons, notebook page select and tree control select item - button_ok.Bind(wx.EVT_BUTTON, self.__on_ok) - button_cancel.Bind(wx.EVT_BUTTON, self.__on_cancel) - self.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.__on_page_select) - self.Bind(wx.EVT_TREE_SEL_CHANGED, self.__on_tree_select) - - # Call on_page_select to select the first page - self.__on_page_select(event=None) - - def __build_tree(self, tree, node, settings): - """ - Recursive function to build the tree from the node, using the settings - :param tree: The tree to build - :param node: The tree view node - :param settings: The settings dict for the node. - :return: The built tree - """ - for setting in settings: - # Get value. If dict, add the node and recursively call this function again. - value = settings[setting] - if type(value) is dict: - # Add the node and set its settings path - node_id = tree.AppendItem(node, setting) - current_settings_path = tree.GetItemData(node) - tree.SetItemData(node_id, f"{current_settings_path}.{setting}") - - # Recurse - tree = self.__build_tree(tree, node_id, value) - - return tree - - def __populate_settings_values(self, setting_path, index): - """ - Populates the settings in the value sizer for a settings path. - - :param setting_path: - :param index: The tab index containing the value_sizer to populate - - :return: - """ - # Get the setting values - settings = self.__settings.get(setting_path) - - # Get the value sizer, clear it and set its rows - value_sizer = self.__value_sizers[index] - value_sizer.Clear(True) - value_sizer.SetRows(len(settings)) - - # Display every value that is a leaf (not dict) - for setting in settings: - value = settings[setting] - if type(value) is not dict: - # Add a label and value text box - label = wx.StaticText(self.__tabs[index], wx.ID_ANY, setting, style=wx.ALIGN_LEFT) - self.__value_boxes.append(wx.TextCtrl(self.__tabs[index], wx.ID_ANY, f"{value}", style=wx.ALIGN_LEFT)) - value_sizer.AddMany([(label, 0, wx.EXPAND), (self.__value_boxes[-1], 0, wx.EXPAND)]) - - # Bind to text change. We need to generate a handler as this will have a parameter. - self.__value_boxes[-1].Bind(wx.EVT_TEXT, - self.__get_on_change_evt_handler(setting_path=f'{setting_path}.{setting}')) - - value_sizer.Layout() - - def __on_page_select(self, event): - # Populate value sizer for the selected item, If no item is selected, populate for root. - index = self.__notebook.GetSelection() - selected_item = self.__trees[index].GetSelection() - if selected_item.ID is None: - root_node = self.__trees[index].GetRootItem() - setting_path = self.__trees[index].GetItemData(root_node) - else: - setting_path = self.__trees[index].GetItemData(selected_item) - - self.__populate_settings_values(setting_path, index) - - def __on_tree_select(self, event): - # Get Selected item and check that it is a tree item - tree_item = event.GetItem() - if not tree_item.IsOk(): - return - - # Get the index of the current tab - index = self.__notebook.GetSelection() - - # Get the setting path from item data - setting_path = self.__trees[index].GetItemData(tree_item) - - # Populate value_sizer - self.__populate_settings_values(setting_path, index) - - def __on_cancel(self, e): - # Clear changed settings and close - self.changed_settings = {} - self.EndModal(wx.ID_CANCEL) - self.Destroy() - - def __on_ok(self, e): - # Update settings and save - delkeys = [] - for setting in self.changed_settings: - # Get the current and new setting - orig_value = self.__settings.get(setting) - new_value = self.changed_settings[setting] - - # If they are the same, discard from changes. We will use a list of items to delete (delkeys) as we cant - # delete whilst iterating. If they are different, update settings. - if orig_value == new_value: - delkeys.append(setting) - else: - # We need to retain data type. New values will all be string as they were retrieved from textctl. - # Get the data type of the original and cast new to it. - new_value = type(orig_value)(new_value) - self.__settings.set(setting, new_value) - - # Now delete the items that were the same from changed_settings. changed_settings may be used by settings - # dialog caller. - for key in delkeys: - del(self.changed_settings[key]) - - # Save the settings and close dialog - self.__settings.save() - self.EndModal(wx.ID_OK) - self.Destroy() - - def __get_on_change_evt_handler(self, setting_path): - def on_value_changed(event): - self.changed_settings[setting_path] = event.String - self.log.debug(f"Value changed for {setting_path}.") - - return on_value_changed