Split notebook page (tab) out as seperate class from Settings Dialog.

config.yaml

@@ -9,11 +9,11 @@ calculate:
   max_p_value: 0.05
 monitor:
   from:
-    minutes: 15
+    minutes: 60
   interval: 10
-  min_prices: 400
-  max_set_size_diff_pct: 50
-  overlap_pct: 50
+  min_prices: 1000
+  max_set_size_diff_pct: 90
+  overlap_pct: 90
   max_p_value: 0.05
   monitoring_threshold: 0.9
   divergence_threshold: 0.8

mt5_correlation.py

@@ -2,11 +2,20 @@
 Application to monitor previously correlated symbol pairs for correlation divergence.
 """
 import definitions
-import yaml
 import logging.config
 from mt5_correlation.gui import MonitorFrame
 from mt5_correlation.config import Config
 import wx
+import wx.lib.mixins.inspection as wit
+
+
+class CorrelationMonitorApp(wx.App, wit.InspectionMixin):
+    # Override app to use inspection.
+    # TODO Remove wit.InspectionMixin from overrides when live.
+    def OnInit(self):
+        self.Init()  # initialize the inspection tool
+        return True
+
 
 if __name__ == "__main__":
     # Load the config
@@ -17,7 +26,7 @@ if __name__ == "__main__":
     logging.config.dictConfig(log_config)
 
     # Start the app
-    app = wx.App(False)
+    app = CorrelationMonitorApp()
     frame = MonitorFrame()
     frame.Show()
     app.MainLoop()

mt5_correlation/config.py

@@ -1,5 +1,6 @@
 import yaml
-import definitions
+import wx
+import logging
 
 
 class Config(object):
@@ -86,3 +87,257 @@ class Config(object):
             obj = obj[k]
 
         obj[key_list[-1]] = value
+
+
+class SettingsDialog(wx.Dialog):
+
+    # Store any settings that have changed
+    changed_settings = {}
+
+    def __init__(self, *args, **kwargs):
+        # Super Constructor
+        wx.Dialog.__init__(self, *args, **kwargs)
+        self.SetTitle("Settings")
+
+        # Create logger and get config
+        self.__log = logging.getLogger(__name__)
+        self.__settings = Config()
+
+        # Dict of changes. Will commit only on ok
+        self.__changes = {}
+
+        # We want 2 vertical sections, the tabbed notebook and the buttons. The buttons sizer will have 2 horizontal
+        # sections, one for each button.
+        main_sizer = wx.BoxSizer(wx.VERTICAL)  # Notebook panel
+        button_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Button sizer
+
+        # Notebook
+        self.__notebook = wx.Notebook(self, wx.ID_ANY)  # The notebook
+
+        # A tab for each root node in config. We will store the tabs components in lists which can be accessed by the
+        # index returned from notebook.GetSelectedItem()
+        root_nodes = self.__settings.get_root_nodes()
+        self.__tabs = []
+        for node in root_nodes:
+            # Create new tab
+            self.__tabs.append(SettingsTab(self, self.__notebook, node))
+
+            # Add tab to notebook
+            self.__notebook.AddPage(self.__tabs[-1], node)
+
+        # Buttons
+        button_ok = wx.Button(self, label="Update")
+        button_cancel = wx.Button(self, label="Cancel")
+        button_sizer.Add(button_ok, 0, wx.ALL, 1)
+        button_sizer.Add(button_cancel, 0, wx.ALL, 1)
+
+        # Add notebook and button sizer to main sizer and set main sizer for window
+        main_sizer.Add(self.__notebook, 1, wx.ALL | wx.EXPAND, 5)
+        main_sizer.Add(button_sizer)
+        self.SetSizer(main_sizer)
+
+        # Bind buttons &  notebook page select.
+        button_ok.Bind(wx.EVT_BUTTON, self.__on_ok)
+        button_cancel.Bind(wx.EVT_BUTTON, self.__on_cancel)
+        self.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.__on_page_select)
+
+        # Call on_page_select to select the first page
+        self.__on_page_select(event=None)
+
+    def __on_page_select(self, event):
+        # Call the tabs select method to populate
+        index = self.__notebook.GetSelection()
+        self.__tabs[index].select()
+
+    def __on_cancel(self, event):
+        # Clear changed settings and close
+        self.changed_settings = {}
+        self.EndModal(wx.ID_CANCEL)
+        self.Destroy()
+
+    def __on_ok(self, event):
+        # Update settings and save
+        delkeys = []
+        for setting in self.changed_settings:
+            # Get the current and new setting
+            orig_value = self.__settings.get(setting)
+            new_value = self.changed_settings[setting]
+
+            # If they are the same, discard from changes. We will use a list of items to delete (delkeys) as we cant
+            # delete whilst iterating. If they are different, update settings.
+            if orig_value == new_value:
+                delkeys.append(setting)
+            else:
+                # We need to retain data type. New values will all be string as they were retrieved from textctl.
+                # Get the data type of the original and cast new to it.
+                new_value = type(orig_value)(new_value)
+                self.__settings.set(setting, new_value)
+
+        # Now delete the items that were the same from changed_settings. changed_settings may be used by settings
+        # dialog caller.
+        for key in delkeys:
+            del(self.changed_settings[key])
+
+        # Save the settings and close dialog
+        self.__settings.save()
+        self.EndModal(wx.ID_OK)
+        self.Destroy()
+
+
+class SettingsTab(wx.Panel):
+    """
+    A notebook tab containing the settings tree and values for a settings root node.
+    """
+
+    # Parent frame. Set during constructor
+    __parent_frame = None
+
+    # Each tab has: a tree view; a values panel; and a list of value text boxes bound to a change
+    # event.
+    __tree = None
+    __tab_sizer = None
+    __value_sizer = None
+    __value_boxes = []
+
+    def __init__(self, parent_frame, notebook, root_node):
+        """
+        Creates a tab for the settings notebook.
+
+        :param parent_frame: The frame containing the notebook.
+        :param notebook. The notebook that this tab should be part of.
+        :param root_node. The root node for the settings
+        """
+        # Super Constructor
+        wx.Panel.__init__(self, parent=notebook)
+
+        # Store the parent frame and get the settings for this tab.
+        self.__parent_frame = parent_frame
+        settings = Config().get(root_node)
+
+        # Create logger
+        self.__log = logging.getLogger(__name__)
+
+        # Build the tab and set it's sizer.
+        self.__tab_sizer = wx.BoxSizer(wx.HORIZONTAL)
+        self.SetSizer(self.__tab_sizer)
+
+        # Create tree control
+        self.__tree = wx.TreeCtrl(self, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize)
+
+        # Add root to tree and use item data to store settings path
+        root = self.__tree.AddRoot(root_node)
+        self.__tree.SetItemData(root, root_node)
+
+        # Add items to root
+        self.__tree = self.__build_tree(self.__tree, root, settings)
+
+        # expand tree and add it to the sizer
+        self.__tree.Expand(root)
+        self.__tab_sizer.Add(self.__tree, 1, wx.ALL | wx.EXPAND, 1)
+
+        # Add the value sizer for settings values. This is only used for spacing, as it will be overwritten when
+        # tree items are selected.
+        self.__value_sizer =  wx.FlexGridSizer(rows=1, cols=2, vgap=2, hgap=2)
+        self.__tab_sizer.Add(self.__value_sizer, 1, wx.ALL | wx.EXPAND, 1)
+        label = wx.StaticText(self, wx.ID_ANY, " ".ljust(50), style=wx.ALIGN_LEFT)
+        self.__value_sizer.Add(label)
+
+        # Bind tree selection changed
+        self.__tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.__on_tree_select)
+
+    def select(self):
+        """
+        To be called when this tab is selected. Populate value sizer for the selected item, If no item is selected,
+        populate for root.
+        :return:
+        """
+        selected_item = self.__tree.GetSelection()
+        if selected_item.ID is None:
+            root_node = self.__tree.GetRootItem()
+            setting_path = self.__tree.GetItemData(root_node)
+        else:
+            setting_path = self.__tree.GetItemData(selected_item)
+
+        self.__populate_settings_values(setting_path)
+
+    def __build_tree(self, tree, node, settings):
+        """
+        Recursive function to build the tree from the node, using the settings
+        :param tree: The tree to build
+        :param node: The tree view node
+        :param settings: The settings dict for the node.
+        :return: The built tree
+        """
+        for setting in settings:
+            # Get value. If dict, add the node and recursively call this function again.
+            value = settings[setting]
+            if type(value) is dict:
+                # Add the node and set its settings path
+                node_id = tree.AppendItem(node, setting)
+                current_settings_path = tree.GetItemData(node)
+                tree.SetItemData(node_id, f"{current_settings_path}.{setting}")
+
+                # Recurse
+                tree = self.__build_tree(tree, node_id, value)
+
+        return tree
+
+    def __populate_settings_values(self, setting_path):
+        """
+        Populates the settings in the value sizer for a settings path.
+
+        :param setting_path:
+
+        :return:
+        """
+
+        # Get the settings for the path
+        settings = Config().get(setting_path)
+
+        # Clear the value sizer and set its rows
+        self.__value_sizer.Clear(True)
+        self.__value_sizer.SetRows(len(settings))
+
+        # Display every value that is a leaf (not dict)
+        for setting in settings:
+            value = settings[setting]
+            if type(value) is not dict:
+                # Add a label and value text box
+                label = wx.StaticText(self, wx.ID_ANY, setting, style=wx.ALIGN_LEFT)
+                self.__value_boxes.append(wx.TextCtrl(self, wx.ID_ANY, f"{value}", style=wx.ALIGN_LEFT))
+                self.__value_sizer.AddMany([(label, 0, wx.EXPAND), (self.__value_boxes[-1], 0, wx.EXPAND)])
+
+                # Bind to text change. We need to generate a handler as this will have a parameter.
+                self.__value_boxes[-1].Bind(wx.EVT_TEXT,
+                                            self.__get_on_change_evt_handler(setting_path=f'{setting_path}.{setting}'))
+
+        self.__value_sizer.Layout()
+
+    def __get_on_change_evt_handler(self, setting_path):
+        """
+        Returns a new event handler with a parameter of the settings path
+        :param setting_path:
+        :return:
+        """
+        def on_value_changed(event):
+            self.__parent_frame.changed_settings[setting_path] = event.String
+            self.__log.debug(f"Value changed for {setting_path}.")
+
+        return on_value_changed
+
+    def __on_tree_select(self, event):
+        """
+        Called when an item in the tree is selected. Populates the settings values
+        :param event:
+        :return:
+        """
+        # Get Selected item and check that it is a tree item
+        tree_item = event.GetItem()
+        if not tree_item.IsOk():
+            return
+
+        # Get the setting path from item data
+        setting_path = self.__tree.GetItemData(tree_item)
+
+        # Populate value_sizer
+        self.__populate_settings_values(setting_path)

mt5_correlation/correlation.py

@@ -17,22 +17,33 @@ class Correlation:
     """
 
     # Minimum base coefficient for monitoring. Symbol pairs with a lower correlation
-    # coefficient than ths wont be monitored.
+    # coefficient than ths won't be monitored.
     monitoring_threshold = 0.9
 
     # Toggle on whether we are monitoring or not. Set through start_monitor and stop_monitor
-    _monitoring = False
+    __monitoring = False
+    __monitoring_params = {}
 
     def __init__(self):
-        self.log = logging.getLogger(__name__)
+        self.__log = logging.getLogger(__name__)
 
         # Create dataframe
-        columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe',
-                   'Last Check', 'Last Coefficient']
-        self.coefficient_data = pd.DataFrame(columns=columns)
+        self.__columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe',
+                          'Last Check', 'Last Coefficient']
+        self.coefficient_data = pd.DataFrame(columns=self.__columns)
 
         # Create timer for continuous monitoring
-        self.scheduler = sched.scheduler(time.time, time.sleep)
+        self.__scheduler = sched.scheduler(time.time, time.sleep)
+
+    @property
+    def filtered_coefficient_data(self):
+        """
+        :return: Coefficient data filtered so that all base coefficients >= monitoring_threshold
+        """
+        if self.coefficient_data is not None:
+            return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold]
+        else:
+            return None
 
     def load(self, filename):
         """
@@ -68,6 +79,14 @@ class Correlation:
         :return:
         """
 
+        # If we are monitoring, stop. We will need to restart later
+        was_monitoring = self.__monitoring
+        if self.__monitoring:
+            self.stop_monitor()
+
+        # Clear the existing correlations
+        self.coefficient_data = pd.DataFrame(columns=self.__columns)
+
         # Create mt5 class. This contains required methods for interacting with MT5.
         mt5 = MT5()
 
@@ -112,17 +131,156 @@ class Correlation:
                                                       'Base Coefficient': coefficient, 'UTC Date From': date_from,
                                                       'UTC Date To': date_to, 'Timeframe': timeframe},
                                                      ignore_index=True)
-                    self.log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a "
-                                   f"coefficient of {coefficient}.")
+                    self.__log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a "
+                                     f"coefficient of {coefficient}.")
                 else:
-                    self.log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:"
-                                   f"{symbol2} could no be calculated.")
+                    self.__log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:"
+                                     f"{symbol2} could no be calculated.")
 
         # Sort, highest correlated first
         self.coefficient_data = self.coefficient_data.sort_values('Base Coefficient', ascending=False)
 
-    def update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90,
-                           overlap_pct=90, max_p_value=0.05):
+        # If we were monitoring, we stopped, so start again.
+        if was_monitoring:
+            self.start_monitor(interval=self.__monitoring_params['interval'],
+                               date_from=self.__monitoring_params['date_from'],
+                               date_to=self.__monitoring_params['date_to'],
+                               min_prices=self.__monitoring_params['min_prices'],
+                               max_set_size_diff_pct=self.__monitoring_params['max_set_size_diff_pct'],
+                               overlap_pct=self.__monitoring_params['overlap_pct'],
+                               max_p_value=self.__monitoring_params['max_p_value'])
+
+    def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                      max_p_value=0.05):
+        """
+        Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
+        threshold.
+
+        :param interval: How often to check in seconds
+        :param date_from: From date for tick data from which to calculate correlation coefficients
+        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+
+        if self.__monitoring:
+            self.__log.debug(f"Request to start monitor when monitor is already running. Monitor will be stopped and"
+                             f"restarted with new parameters.")
+            self.stop_monitor()
+
+        self.__log.debug(f"Starting monitor.")
+        self.__monitoring = True
+
+        # Create thread to run monitoring This will call private __monitor method that will run the calculation and
+        # keep scheduling itself while self.monitoring is True. Store the params. We will need to use these if we have
+        # to stop and restart the monitor. Note, this happens during calculate
+        self.__monitoring_params = {'interval': interval, 'date_from': date_from, 'date_to': date_to,
+                                    'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct,
+                                    'overlap_pct': overlap_pct, 'max_p_value': max_p_value}
+        thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params)
+        thread.start()
+
+    def stop_monitor(self):
+        """
+        Stops monitoring symbol pairs for correlation.
+        :return:
+        """
+        if self.__monitoring:
+            self.__log.debug(f"Stopping monitor.")
+            self.__monitoring = False
+        else:
+            self.__log.debug(f"Request to stop monitor when it is not running. No action taken.")
+
+    @staticmethod
+    def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90,
+                              overlap_pct=90, max_p_value=0.05):
+        """
+        Calculates the correlation coefficient between two sets of price data. Uses close price.
+
+        :param symbol1_prices: prices or ticks for symbol 1
+        :param symbol2_prices: prices or ticks for symbol 2
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+
+        # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
+        # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of
+        # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set?
+        coefficient = None
+
+        intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time']))
+        len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)]))
+        len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
+        similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set
+        enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100)
+        enough_prices = len_smallest_set >= min_prices
+        suitable = similar_size and enough_overlap and enough_prices
+
+        if suitable:
+            # Calculate coefficient on close prices
+
+            # First filter prices to only include those that intersect
+            symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)]
+            symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)]
+
+            # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
+            # hypothesis is false).
+            coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close'])
+            coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0]
+
+            # If NaN, change to None
+            if coefficient is not None and math.isnan(coefficient):
+                coefficient = None
+
+        return coefficient
+
+    def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                  max_p_value=0.05):
+        """
+        The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
+        stop_monitoring.
+
+        :param interval: How often to check in seconds
+        :param date_from: From date for tick data from which to calculate correlation coefficients
+        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+        self.__log.debug(f"In monitor event. Monitoring: {self.__monitoring}.")
+
+        # Only run if monitor is not stopped
+        if self.__monitoring:
+            # Update all coefficients
+            self.__update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices,
+                                           max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
+                                           max_p_value=max_p_value)
+
+            # Schedule the timer to run again
+            params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
+                      'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
+                      'max_p_value': max_p_value}
+            self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
+            self.__scheduler.run()
+
+    def __update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90,
+                             overlap_pct=90, max_p_value=0.05):
         """
         Updates the coefficient for the specified symbol pair
         :param symbol1: Name of symbol to calculate coefficient for.
@@ -181,8 +339,8 @@ class Correlation:
 
         return coefficient
 
-    def update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                                max_p_value=0.05):
+    def __update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                                  max_p_value=0.05):
         """
         Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
         the threshold can be accessed through the filtered_coefficient_data property.
@@ -201,136 +359,6 @@ class Correlation:
         for index, row in self.filtered_coefficient_data.iterrows():
             symbol1 = row['Symbol 1']
             symbol2 = row['Symbol 2']
-            self.update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to,
-                                    min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
-                                    overlap_pct=overlap_pct, max_p_value=max_p_value)
-
-    def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                      max_p_value=0.05):
-        """
-        Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
-        threshold.
-
-        :param interval: How often to check in seconds
-        :param date_from: From date for tick data from which to calculate correlation coefficients
-        :param date_to: To date for tick data from which to calculate correlation coefficients
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
-        :return: correlation coefficient, or None if coefficient could not be calculated.
-
-        :return:
-        """
-        self.log.debug(f"Starting monitor.")
-        self._monitoring = True
-
-        # Create thread to run monitoring This will call private __monitor method that will run the calculation and
-        # keep scheduling itself while self.monitoring is True
-        params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
-                  'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
-                  'max_p_value': max_p_value}
-        thread = threading.Thread(target=self.__monitor, kwargs=params)
-        thread.start()
-
-    def stop_monitor(self):
-        """
-        Stops monitoring symbol pairs for correlation.
-        :return:
-        """
-        self.log.debug(f"Stopping monitor.")
-        self._monitoring = False
-
-    def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                  max_p_value=0.05):
-        """
-        The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
-        stop_monitoring.
-
-        :param interval: How often to check in seconds
-        :param date_from: From date for tick data from which to calculate correlation coefficients
-        :param date_to: To date for tick data from which to calculate correlation coefficients
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
-        :return: correlation coefficient, or None if coefficient could not be calculated.
-        :return:
-        """
-        self.log.debug(f"In monitor event. Monitoring: {self._monitoring}.")
-
-        # Only run if monitor is not stopped
-        if self._monitoring:
-            # Update all coefficients
-            self.update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices,
-                                         max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
-                                         max_p_value=max_p_value)
-
-            # Schedule the timer to run again
-            params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
-                      'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
-                      'max_p_value': max_p_value}
-            self.scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
-            self.scheduler.run()
-
-    @property
-    def filtered_coefficient_data(self):
-        """
-        :return: Coefficient data filtered so that all base coefficients >= monitoring_threshold
-        """
-        if self.coefficient_data is not None:
-            return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold]
-        else:
-            return None
-
-    @staticmethod
-    def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90,
-                              overlap_pct=90, max_p_value=0.05):
-        """
-        Calculates the correlation coefficient between two sets of price data. Uses close price.
-
-        :param symbol1_prices: prices or ticks for symbol 1
-        :param symbol2_prices: prices or ticks for symbol 2
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
-        :return: correlation coefficient, or None if coefficient could not be calculated.
-        """
-
-        # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
-        # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of
-        # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set?
-        coefficient = None
-
-        intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time']))
-        len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)]))
-        len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
-        similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set
-        enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100)
-        enough_prices = len_smallest_set >= min_prices
-        suitable = similar_size and enough_overlap and enough_prices
-
-        if suitable:
-            # Calculate coefficient on close prices
-
-            # First filter prices to only include those that intersect
-            symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)]
-            symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)]
-
-            # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
-            # hypothesis is false).
-            coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close'])
-            coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0]
-
-            # If NaN, change to None
-            if coefficient is not None and math.isnan(coefficient):
-                coefficient = None
-
-        return coefficient
+            self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to,
+                                      min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
+                                      overlap_pct=overlap_pct, max_p_value=max_p_value)

mt5_correlation/gui.py

@@ -1,7 +1,7 @@
 import wx
 import wx.grid
 from mt5_correlation.correlation import Correlation
-from mt5_correlation.config import Config
+from mt5_correlation.config import Config, SettingsDialog
 from datetime import datetime, timedelta
 import pytz
 import pandas as pd
@@ -180,9 +180,6 @@ class MonitorFrame(wx.Frame):
         utc_to = datetime.now(tz=timezone)
         utc_from = utc_to - timedelta(days=self.config.get('calculate.from.days'))
 
-        # Set timeframe
-        timeframe = self.config.get('calculate.timeframe')
-
         # Calculate
         self.SetStatusText("Calculating coefficients.")
         self.cor.calculate(date_from=utc_from, date_to=utc_to,
@@ -252,12 +249,12 @@ class MonitorFrame(wx.Frame):
             self.monitor_toggle.SetLabelText("On")
             self.SetStatusText("Monitoring for changes to coefficients.")
 
-            # Calculate correlations fro last 10 mins
+            # From and to dates for calculations.
             timezone = pytz.timezone("Etc/UTC")
             utc_to = datetime.now(tz=timezone)
             utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes'))
 
-            self.timer.Start(10000)
+            self.timer.Start(self.config.get('monitor.interval')*1000)
             self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to,
                                    min_prices=self.config.get('monitor.min_prices'),
                                    max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'),
@@ -282,10 +279,53 @@ class MonitorFrame(wx.Frame):
             # Reload relevant parts of app
             # TODO: Reload relevant parts of app once settings have changed. Stop and restart monitoring,
             #       reload logger, filter data, refresh data window.
-            self.log.debug("Settings updated. Reloading logger.")
-            log_config = Config().get('logging')
-            logging.config.dictConfig(log_config)
-        settings_dialog.Destroy()
+            restart_monitor_timer = False
+            restart_gui_timer = False
+            reload_correlations = False
+            reload_logger = False
+
+            for setting in settings_dialog.changed_settings:
+                # If any 'monitor.' settings except 'monitor.divergence_threshold have changed then restart
+                # monitoring timer with new settings.
+                # If 'monitor.interval has changed then restart gui timer.
+                # If 'monitor.monitoring_threshold' has changed, then refresh correlation data.
+                # If any 'logging.' settings have changed, then reload logger config.
+                if setting.startswith('monitor. ') and setting != 'monitor.divergence_threshold':
+                    restart_monitor_timer = True
+                if setting == 'monitor.interval':
+                    restart_gui_timer = True
+                if setting == 'monitor.monitoring_threshold':
+                    reload_correlations = True
+                if setting.startswith('logging.'):
+                    reload_logger = True
+
+            # Now perform the actions
+            if restart_monitor_timer:
+                self.log.debug("Settings updated. Reloading monitoring timer.")
+                self.cor.stop_monitor()
+                # From and to dates for calculations.
+                timezone = pytz.timezone("Etc/UTC")
+                utc_to = datetime.now(tz=timezone)
+                utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes'))
+                self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to,
+                                       min_prices=self.config.get('monitor.min_prices'),
+                                       max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'),
+                                       overlap_pct=self.config.get('monitor.overlap_pct'),
+                                       max_p_value=self.config.get('monitor.max_p_value'))
+            if restart_gui_timer:
+                self.log.debug("Settings updated. Restarting gui timer.")
+                self.timer.Stop()
+                self.timer.Start(self.config.get('monitor.interval') * 1000)
+
+            if reload_correlations:
+                self.log.debug("Settings updated. Updating monitoring threshold and reloading grid.")
+                self.cor.monitoring_threshold = self.config.get("monitor.monitoring_threshold")
+                self.refresh_grid(event)
+
+            if reload_logger:
+                self.log.debug("Settings updated. Reloading logger.")
+                log_config = Config().get('logging')
+                logging.config.dictConfig(log_config)
 
     def on_close(self, event):
         """
@@ -355,224 +395,3 @@ class DataTable(wx.grid.GridTableBase):
                     attr.SetBackgroundColour(wx.WHITE)
 
         return attr
-
-
-class SettingsDialog(wx.Dialog):
-
-    # Store any settings that have changed
-    changed_settings = {}
-
-    def __init__(self, *args, **kwargs):
-        # Super Constructor
-        wx.Dialog.__init__(self, *args, **kwargs)
-        self.SetTitle("Settings")
-
-        # Create logger and get config
-        self.log = logging.getLogger(__name__)
-
-        # Get settings
-        self.__settings = Config()
-
-        # Dict of changes. Will commit only on ok
-        self.__changes = {}
-
-        # We want 2 vertical sections, the tabbed notebook and the buttons. The buttons sizer will have 2 horizontal
-        # sections, one for each button.
-        # -------------------------
-        # |Tabbed notebook        |
-        # |                       |
-        # |                       |
-        # |                       |
-        # |                       |
-        # |-----------------------|
-        # |ok | cancel            |
-        # -------------------------
-        main_sizer = wx.BoxSizer(wx.VERTICAL)  # Notebook panel
-        button_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Button sizer
-
-        # Notebook
-        self.__notebook = wx.Notebook(self, wx.ID_ANY)  # The notebook
-
-        # A tab for each root node in config. We will store the tabs components in lists which can be accessed by the
-        # index returned from notebook.GetSelectedItem()
-        root_nodes = self.__settings.get_root_nodes()
-        self.__tabs = []
-        self.__trees = []
-        self.__tab_sizers = []
-        self.__value_sizers = []
-        self.__value_boxes = []  # We need to store these as they will all be bound to a change event
-        # self.roots = []
-        for node in root_nodes:
-            # Create new tab
-            self.__tabs.append(wx.Panel(self.__notebook, wx.ID_ANY))
-            self.__tab_sizers.append(wx.BoxSizer(wx.HORIZONTAL))
-            self.__tabs[-1].SetSizer(self.__tab_sizers[-1])
-
-            # Get settings items for tab / node
-            node_settings = self.__settings.get(node)
-
-            # Create tree control
-            self.__trees.append(wx.TreeCtrl(self.__tabs[-1], wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize))
-
-            # Add root to tree and use item data to store settings path
-            root = self.__trees[-1].AddRoot(node)
-            self.__trees[-1].SetItemData(root, node)
-
-            # Add items to root
-            self.__trees[-1] = self.__build_tree(self.__trees[-1], root, node_settings)
-
-            # expand tree and add it to the sizer
-            self.__trees[-1].Expand(root)
-            self.__tab_sizers[-1].Add(self.__trees[-1], 1, wx.ALL | wx.EXPAND, 1)
-
-            # Add the value sizer for settings values. This is only used for spacing, as it will be overwritten when
-            # tree items are selected.
-            self.__value_sizers.append(wx.FlexGridSizer(rows=1, cols=2, vgap=2, hgap=2))
-            self.__tab_sizers[-1].Add(self.__value_sizers[-1], 1, wx.ALL | wx.EXPAND, 1)
-            label = wx.StaticText(self.__tabs[-1], wx.ID_ANY, " ".ljust(50), style=wx.ALIGN_LEFT)
-            self.__value_sizers[-1].Add(label)
-
-            # Add tab to notebook
-            self.__notebook.AddPage(self.__tabs[-1], node)
-
-        # Buttons
-        button_ok = wx.Button(self, label="Update")
-        button_cancel = wx.Button(self, label="Cancel")
-        button_sizer.Add(button_ok, 0, wx.ALL, 1)
-        button_sizer.Add(button_cancel, 0, wx.ALL, 1)
-
-        # Add notebook and button sizer to main sizer and set main sizer for window
-        main_sizer.Add(self.__notebook, 1, wx.ALL | wx.EXPAND, 5)
-        main_sizer.Add(button_sizer)
-        self.SetSizer(main_sizer)
-
-        # Bind buttons, notebook page select and tree control select item
-        button_ok.Bind(wx.EVT_BUTTON, self.__on_ok)
-        button_cancel.Bind(wx.EVT_BUTTON, self.__on_cancel)
-        self.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.__on_page_select)
-        self.Bind(wx.EVT_TREE_SEL_CHANGED, self.__on_tree_select)
-
-        # Call on_page_select to select the first page
-        self.__on_page_select(event=None)
-
-    def __build_tree(self, tree, node, settings):
-        """
-        Recursive function to build the tree from the node, using the settings
-        :param tree: The tree to build
-        :param node: The tree view node
-        :param settings: The settings dict for the node.
-        :return: The built tree
-        """
-        for setting in settings:
-            # Get value. If dict, add the node and recursively call this function again.
-            value = settings[setting]
-            if type(value) is dict:
-                # Add the node and set its settings path
-                node_id = tree.AppendItem(node, setting)
-                current_settings_path = tree.GetItemData(node)
-                tree.SetItemData(node_id, f"{current_settings_path}.{setting}")
-
-                # Recurse
-                tree = self.__build_tree(tree, node_id, value)
-
-        return tree
-
-    def __populate_settings_values(self, setting_path, index):
-        """
-        Populates the settings in the value sizer for a settings path.
-
-        :param setting_path:
-        :param index: The tab index containing the value_sizer to populate
-
-        :return:
-        """
-        # Get the setting values
-        settings = self.__settings.get(setting_path)
-
-        # Get the value sizer, clear it and set its rows
-        value_sizer = self.__value_sizers[index]
-        value_sizer.Clear(True)
-        value_sizer.SetRows(len(settings))
-
-        # Display every value that is a leaf (not dict)
-        for setting in settings:
-            value = settings[setting]
-            if type(value) is not dict:
-                # Add a label and value text box
-                label = wx.StaticText(self.__tabs[index], wx.ID_ANY, setting, style=wx.ALIGN_LEFT)
-                self.__value_boxes.append(wx.TextCtrl(self.__tabs[index], wx.ID_ANY, f"{value}", style=wx.ALIGN_LEFT))
-                value_sizer.AddMany([(label, 0, wx.EXPAND), (self.__value_boxes[-1], 0, wx.EXPAND)])
-
-                # Bind to text change. We need to generate a handler as this will have a parameter.
-                self.__value_boxes[-1].Bind(wx.EVT_TEXT,
-                                            self.__get_on_change_evt_handler(setting_path=f'{setting_path}.{setting}'))
-
-        value_sizer.Layout()
-
-    def __on_page_select(self, event):
-        # Populate value sizer for the selected item, If no item is selected, populate for root.
-        index = self.__notebook.GetSelection()
-        selected_item = self.__trees[index].GetSelection()
-        if selected_item.ID is None:
-            root_node = self.__trees[index].GetRootItem()
-            setting_path = self.__trees[index].GetItemData(root_node)
-        else:
-            setting_path = self.__trees[index].GetItemData(selected_item)
-
-        self.__populate_settings_values(setting_path, index)
-
-    def __on_tree_select(self, event):
-        # Get Selected item and check that it is a tree item
-        tree_item = event.GetItem()
-        if not tree_item.IsOk():
-            return
-
-        # Get the index of the current tab
-        index = self.__notebook.GetSelection()
-
-        # Get the setting path from item data
-        setting_path = self.__trees[index].GetItemData(tree_item)
-
-        # Populate value_sizer
-        self.__populate_settings_values(setting_path, index)
-
-    def __on_cancel(self, e):
-        # Clear changed settings and close
-        self.changed_settings = {}
-        self.EndModal(wx.ID_CANCEL)
-        self.Destroy()
-
-    def __on_ok(self, e):
-        # Update settings and save
-        delkeys = []
-        for setting in self.changed_settings:
-            # Get the current and new setting
-            orig_value = self.__settings.get(setting)
-            new_value = self.changed_settings[setting]
-
-            # If they are the same, discard from changes. We will use a list of items to delete (delkeys) as we cant
-            # delete whilst iterating. If they are different, update settings.
-            if orig_value == new_value:
-                delkeys.append(setting)
-            else:
-                # We need to retain data type. New values will all be string as they were retrieved from textctl.
-                # Get the data type of the original and cast new to it.
-                new_value = type(orig_value)(new_value)
-                self.__settings.set(setting, new_value)
-
-        # Now delete the items that were the same from changed_settings. changed_settings may be used by settings
-        # dialog caller.
-        for key in delkeys:
-            del(self.changed_settings[key])
-
-        # Save the settings and close dialog
-        self.__settings.save()
-        self.EndModal(wx.ID_OK)
-        self.Destroy()
-
-    def __get_on_change_evt_handler(self, setting_path):
-        def on_value_changed(event):
-            self.changed_settings[setting_path] = event.String
-            self.log.debug(f"Value changed for {setting_path}.")
-
-        return on_value_changed