Documentation and tidy up before major release

.gitignore

@@ -1,4 +1,4 @@
 /venv/
-/log/
-/out/
 /.idea/workspace.xml
+/*.log
+/*.log.?

.idea/mt5-correlation.iml

@@ -10,4 +10,7 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
+  <component name="PyDocumentationSettings">
+    <option name="renderExternalDocumentation" value="true" />
+  </component>
 </module>

.idea/other.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PySciProjectComponent">
+    <option name="PY_SCI_VIEW" value="true" />
+    <option name="PY_SCI_VIEW_SUGGESTED" value="true" />
+  </component>
+</project>

README.md

@@ -1,77 +1,76 @@
 # mt5-correlation
-Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch.
+Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch and monitors previously correlated symbol pairs for divergence
 
 # Setup
 1) Set up your MetaTrader 5 environment ensuring that all symbols that you would like to assess for correlation are shown in your Market Watch window;
 2) Set up your python environment; and
 3) Install the required libraries.
+
 ```
-pip install -r mt5-correlation\requirements.txt
+pip install -r mt5-correlation/requirements.txt
 ```
 
 # Usage
 If you set up a virtual environment in the Setup step, ensure this is activated. Then run the script.
+
 ```
-python -m mt5_correlations\mt5_correlations.py
+python -m mt5_correlations/mt5_correlations.py
 ```
 
-A .csv file containing the correlation coefficient for all combinations of sybmols from the MetaTrader market watch will be produced in the current directory.
+This will open a GUI.
 
-|Symbol 1    |Symbol 2    |Coefficient|UTC Date From      |UTC Date To        |Timeframe|
-|------------|------------|-----------|-------------------|-------------------|---------|
-|OIL-MAR21   |OILMn-MAR21 |1.0        |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
-|EURUSD      |EURHKD      |0.99980    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
-|OILMn-MAR21 |BRENT-APR21 |0.99894    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
-|OIL-MAR21   |BRENT-APR21 |0.99894    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
-|GSOIL-FEB21 |BRENT-APR21 |0.99605    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
-|GSOIL-FEB21 |OILMn-MAR21 |0.99543    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
-|GSOIL-FEB21 |OIL-MAR21   |0.99543    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
-|EU50Cash    |FRA40Cash   |0.99072    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+## Calculating Baseline Coefficients
+On the first time that you run, you will want to calculate the initial set of correlations.
+1) Open settings and review the settings under the 'Calculate' tab. These settings are:
+    - from.days: The number of days of data to be used to calculate the coefficient.
+    - timeframe: The timeframe for price candles to use for the calculation. Possible values are:
+        * 1: 1 Minute candles
+        * 2: 2 Minute candles
+        * 3: 3 Minute candles
+        * 4: 4 Minute candles
+        * 5: 5 Minute candles
+        * 6: 6 Minute candles
+        * 10: 10 Minute candles
+        * 15: 15 Minute candles
+        * 20: 20 Minute candles
+        * 30: 30 Minute candles
+        * 16385: 1 Hour candles
+        * 16386: 2 Hour candles
+        * 16387: 3 Hour candles
+        * 16388: 4 Hour candles
+        * 16390: 6 Hour candles
+        * 16392: 8 Hour candles
+        * 16396: 12 Hour candles
+        * 16408: 1 Day candles
+        * 32769: 1 Week candles
+        * 49153: 1 Month candles
+        
+    - min_prices: The minimum number of candles required to calculate a coefficient from. If any of the symbols do not have at least this number of candles then the coefficient won't be calculated.
+    - max_set_size_diff_pct: For a meaningful coefficient calculation, the two sets of data should be of a similar size. This setting specifies the % difference allowed for a correlation to be calculated. The smallest set of candle data must be at least this values % of the largest set.
+    - overlap_pct: The dates and times in the two sets of data must match. The coefficient will only be calculated against the dates that overlap. Any non overlapping dates will be discarded. This setting specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient will not be calculated if this threshold is not met.
+    - max_p_value: The maximum P value for the coefficient to be considered valid. A full explanation on the correlation coefficient P value is available here: https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.pearsonr.html.
+    
+2) Set the threshold for the correlations to monitor. This can be set under the Settings 'Monitoring' tab and is named monitoring.threshold. Only settings with a coefficient over this threshold will be displayed and monitored.
 
-# Customising
-Edit mt5_correlations.py to customise.
+3) Calculate the coefficients by selecting File/Calculate. All symbol pairs that have a correlation coefficient greater than the monitoring threshold will be displayed. A graph showing the price candle data used to calculate the coefficient for the pair will be displayed when you select the row. 
 
-The coefficients are calculated only if:
-* The smallest set of price data is no less than 90% of the size of the largest set;
-* The overlapping prices between both sets of price data contains no less than 90% of the prices in the smallest set;
-* The pearsonr p-value for the calculated coefficient is less than 0.05.
+4) You may want to save. Select File/Save and choose a file name. This file will contain all the calculated coefficients, and the price data used to calculate them. This file can be loaded to avoid having to recalculate the baseline coefficients every time you use the application.
 
-These settings can all be changed in the call to Correlation.calculate_coefficient by passing values for max_set_size_diff_pct; overlap_pct; or max_p_value.
-```
-coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data,
-                                                        symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90,
-                                                        overlap_pct=90, max_p_value=0.05)
-```
-
-The price data compared is 15 minute price data for the last 7 days. This can be changed by changing the values for the following variables:
-```
-utc_to = datetime.now(tz=timezone)
-utc_from = utc_to - timedelta(days=7)
-timeframe = mt5.TIMEFRAME_M15
-```
-
-The possible values for timeframe are:
-
-|Timeframe|Description|
-|--------------|-----------|
-|TIMEFRAME_M1  |1 minute   |
-|TIMEFRAME_M2  |2 minutes  |
-|TIMEFRAME_M3  |3 minutes  |
-|TIMEFRAME_M4  |4 minutes  |
-|TIMEFRAME_M5  |5 minutes  |
-|TIMEFRAME_M6  |6 minutes  |
-|TIMEFRAME_M10 |10 minutes |
-|TIMEFRAME_M12 |12 minutes |
-|TIMEFRAME_M12 |15 minutes |
-|TIMEFRAME_M20 |20 minutes |
-|TIMEFRAME_M30 |30 minutes |
-|TIMEFRAME_H1  |1 hour     |
-|TIMEFRAME_H2  |2 hours    |
-|TIMEFRAME_H3  |3 hours    |
-|TIMEFRAME_H4  |4 hours    |
-|TIMEFRAME_H6  |6 hours    |
-|TIMEFRAME_H8  |8 hours    |
-|TIMEFRAME_H12 |12 hours   |
-|TIMEFRAME_D1  |1 day      |
-|TIMEFRAME_W1  |1 week     |
-|TIMEFRAME_MN1 |1 month    |
+## Monitoring for Divergence
+Once you have calculated or loaded the baseline coefficients, they can be monitored for divergence.
+1) Open settings and review the settings under the 'Monitor' tab. These settings are:
+    - from.minutes: The number of minutes of data to be used to calculate the coefficient.
+    - interval: The number of seconds between monitoring events. The application will monitor for divergence every {interval} seconds.
+    - min_prices: Tick data will be converted to 1 second candles prior to calculation. This will enable data to be matched between symbols. This setting specifies the minimum number of price candles required to calculate a coefficient from. If any of the symbols do not have at least number of candles then the coefficient won't be calculated.
+    - max_set_size_diff_pct: For a meaningful coefficient calculation, the two sets of data should be of a similar size. This setting specifies the % difference allowed for a correlation to be calculated. The smallest set of price candles must be at least this values % of the largest set.
+    - overlap_pct: The dates and times in the two sets of data must match. The ticks will be converted to 1 second price candles before calculation. The coefficient will only be calculated against the times from the candles that overlap. Any non overlapping times will be discarded. This setting specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient will not be calculated if this threshold is not met.
+    - max_p_value: The maximum P value for the coefficient to be considered valid. A full explanation on the correlation coefficient P value is available here: https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.pearsonr.html.
+    - monitoring_threshold: The application will only display correlated pairs that have a correlation coefficient greater than or equal to this value.
+    - divergence_threshold: The application will consider a pair to have diverged if the correlation coefficient falls below this threshold. These will be highlighted in yellow.
+    - tick_cache_time: Every calculation requires tick data for both symbols. Tick data will be cached for this number of seconds before being retrieved from MetaTrader. Some caching is recommended as a single monitoring run will request the same data for symbols that form multiple correlated pairs.
+    - autosave: Whether to auto save after every monitoring event. If a file was opened or has been saved, then the data will be saved to this file, otherwise the data will be saved to a file named autosave.cpd.
+    
+2) Switch the monitoring toggle to on. The application will continuously monitor for divergence. The data frame will be updated with the last time that the correlation was checked, and the last coefficient. The chart frame will contain 5 charts which will be updated after every monitoring event:
+    - 2 charts, one for each symbol in the correlated pair, showing the price data used to calculate the baseline coefficient.
+    - 2 charts, one for each symbol in the correlated pair, showing the tick data used to calculate the latest coefficient.
+    - 1 chart showing every correlation coefficient calculated for the symbol pair.

config.yaml

@@ -0,0 +1,72 @@
+---
+calculate:
+  from:
+    days: 10
+  timeframe: 15
+  min_prices: 400
+  max_set_size_diff_pct: 90
+  overlap_pct: 90
+  max_p_value: 0.05
+monitor:
+  from:
+    minutes: 30
+  interval: 10
+  min_prices: 400
+  max_set_size_diff_pct: 90
+  overlap_pct: 80
+  max_p_value: 0.05
+  monitoring_threshold: 0.9
+  divergence_threshold: 0.8
+  tick_cache_time: 10
+  autosave: true
+logging:
+  version: 1
+  disable_existing_loggers: false
+  formatters:
+    brief:
+      format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+      datefmt: '%H:%M:%S'
+    precice:
+      format: '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s'
+      datefmt: '%Y-%m-%d %H:%M:%S'
+  handlers:
+    console:
+      level: INFO
+      class: logging.StreamHandler
+      formatter: brief
+      stream: ext://sys.stdout
+    file:
+      level: DEBUG
+      class: logging.handlers.RotatingFileHandler
+      formatter: precice
+      filename: debug.log
+      mode: a
+      maxBytes: 2560000
+      backupCount: 1
+  root:
+    level: DEBUG
+    handlers:
+    - console
+    - file
+  loggers:
+    mt5-correlation:
+      level: DEBUG
+      handlers:
+      - console
+      - file
+      propagate: 0
+developer:
+  inspection: false
+window:
+  x: 42
+  y: 51
+  width: 1512
+  height: 975
+  style: 541072960
+settings_window:
+  x: 354
+  y: 299
+  width: 664
+  height: 317
+  style: 524352
+...

logging_conf.yaml

@@ -1,29 +0,0 @@
----
-version: 1
-disable_existing_loggers: False
-
-formatters:
-    brief:
-        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-        datefmt: '%H:%M:%S'
-
-    precice:
-        format: "%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s"
-        datefmt: '%Y-%m-%d %H:%M:%S'
-
-handlers:
-    console:
-        level: WARNING
-        class: logging.StreamHandler
-        formatter: brief
-        stream: ext://sys.stdout
-
-root:
-    level: DEBUG
-    handlers: [console]
-
-loggers:
-    mt5-correlation:
-        level: DEBUG
-        handlers: [console]
-        propagate: 0

mt5_correlation.py

@@ -1,79 +1,37 @@
-from datetime import datetime, timedelta
-import logging.config
-import pandas as pd
-import pytz
-import yaml
-from mt5_correlation.mt5 import MT5
-from mt5_correlation.correlation import Correlation
+"""
+Application to monitor previously correlated symbol pairs for correlation divergence.
+"""
 import definitions
+import logging.config
+from mt5_correlation.gui import MonitorFrame
+from mt5_correlation.config import Config
+import wx
+import wx.lib.mixins.inspection as wit
 
-# Configure logger
-with open(fr'{definitions.ROOT_DIR}\logging_conf.yaml', 'rt') as file:
-    config = yaml.safe_load(file.read())
-    logging.config.dictConfig(config)
-    log = logging.getLogger()
 
-# Create mt5 class. This contains required methods for interacting with MT5.
-mt5 = MT5()
+class InspectionApp(wx.App, wit.InspectionMixin):
+    # Override app to use inspection.
+    def OnInit(self):
+        self.Init()  # initialize the inspection tool
+        return True
 
-# Gte all visible symbols
-symbols = mt5.get_symbols()
 
-# set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
-timezone = pytz.timezone("Etc/UTC")
-utc_to = datetime.now(tz=timezone)
-utc_from = utc_to - timedelta(days=7)
+if __name__ == "__main__":
+    # Load the config
+    Config().load(fr"{definitions.ROOT_DIR}\config.yaml")
 
-# Set timeframe
-timeframe = mt5.TIMEFRAME_M15
+    # Get logging config and configure the logger
+    log_config = Config().get('logging')
+    logging.config.dictConfig(log_config)
 
-# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
-price_data = {}
-for symbol in symbols:
-    price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to,
-                                             timeframe=timeframe)
+    # Do we have inspection turned on. Create correct version of app
+    inspection = Config().get('developer.inspection')
+    if inspection:
+        app = InspectionApp()
+    else:
+        app = wx.App(False)
 
-# Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
-# eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
-# avoid duplicating. We will store all coefficients in a dataframe for export as CSV.
-columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe']
-coefficients = pd.DataFrame(columns=columns)
-
-index = 0
-# There will be (x^2 - x) / 2 pairs where x is number of symbols
-num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
-
-for i in range(0, len(symbols)):
-    symbol1 = symbols[i]
-
-    for j in range(i + 1, len(symbols)):
-        symbol2 = symbols[j]
-        index += 1
-
-        # Get price data for both symbols
-        symbol1_price_data = price_data[symbol1.name]
-        symbol2_price_data = price_data[symbol2.name]
-
-        # Get coefficient and store if valid
-        coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data,
-                                                        symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90,
-                                                        overlap_pct=90, max_p_value=0.05)
-
-        if coefficient is not None:
-            coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
-                                                'Coefficient': coefficient, 'UTC Date From': utc_from,
-                                                'UTC Date To': utc_to, 'Timeframe': timeframe}, ignore_index=True)
-
-            log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of "
-                     f"{coefficient}.")
-        else:
-            log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not"
-                     f" be calculated.")
-
-# Sort, highest correlated first
-coefficients = coefficients.sort_values('Coefficient', ascending=False)
-
-# Save as CSV
-filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S}.csv"
-log.info(f"Saving coefficients as '{filename}'.")
-coefficients.to_csv(filename, index=False)
+    # Start the app
+    frame = MonitorFrame()
+    frame.Show()
+    app.MainLoop()

mt5_correlation/config.py

@@ -0,0 +1,474 @@
+import yaml
+import wx
+import logging
+
+
+class Config(object):
+    """
+    Provides access to application configuration parameters stored in config.yaml.
+    """
+
+    config_filepath = None
+    __config = None
+    __instance = None
+
+    def __new__(cls):
+        """
+        Singleton. Get instance of this class. Create if not already created.
+        :return:
+        """
+        if cls.__instance is None:
+            cls.__instance = super(Config, cls).__new__(cls)
+        return cls.__instance
+
+    def load(self, path):
+        """
+        Loads the applications config file
+        :param path: Path to config file
+        :return:
+        """
+        with open(path, 'r') as yamlfile:
+            self.__config = yaml.safe_load(yamlfile)
+
+        # Store path so that we can save later
+        self.config_filepath = path
+
+    def save(self):
+        """
+        Saves config file
+        :return:
+        """
+
+        with open(self.config_filepath, 'w') as file:
+            file.write("---\n")
+            yaml.dump(self.__config, file, sort_keys=False)
+            file.write("...")
+
+    def get(self, path):
+        """
+        Gets a config property value.
+        :param path: path to property. Path separated by .
+        :return: property value
+        """
+
+        elements = path.split('.')
+        last = None
+
+        for element in elements:
+            if last is None:
+                last = self.__config[element]
+            else:
+                last = last[element]
+
+        return last
+
+    def get_root_nodes(self):
+        """
+        Returns all root notes as a list
+        :return: dict of root notes of YAML config file
+        """
+        nodes = []
+        for key in self.__config:
+            nodes.append(key)
+
+        return nodes
+
+    def set(self, path, value):
+        """
+        Sets a config property value
+        :param path: path to property. Path separated by .
+        :param value: Value to set property to
+        :return:
+        """
+        obj = self.__config
+        key_list = path.split(".")
+
+        for k in key_list[:-1]:
+            obj = obj[k]
+
+        obj[key_list[-1]] = value
+
+
+class SettingsDialog(wx.Dialog):
+    """
+    A dialog box for changing settings. A tab for each root node, with a tree view on left for every branch and a text
+    box for every value.
+    """
+
+    # Settings
+    __settings = None  # Will set in init.
+
+    # Store any settings that have changed
+    changed_settings = {}
+
+    def __init__(self, parent, exclude=None):
+        """
+        Open the settings dialog
+        :param parent: The parent frame for this dialog
+        :param exclude: List of settings root nodes to exclude from this dialog
+        """
+        # Super Constructor
+        wx.Dialog.__init__(self, parent=parent, id=wx.ID_ANY, title="Settings",
+                           pos=wx.Point(x=Config().get('settings_window.x'),
+                                        y=Config().get('settings_window.y')),
+                           size=wx.Size(width=Config().get('settings_window.width'),
+                                        height=Config().get('settings_window.height')),
+                           style=Config().get('settings_window.style'))
+
+        self.SetTitle("Settings")
+
+        # Create logger and get config
+        self.__log = logging.getLogger(__name__)
+        self.__settings = Config()
+
+        # Dict of changes. Will commit only on ok
+        self.__changes = {}
+
+        # Dialog should be resizable
+        self.SetWindowStyle(wx.RESIZE_BORDER)
+
+        # Settings to exclude. Just settings_window if None. Add settings_window if not specified.
+        exclude = ['settings_window'] if exclude is None else exclude
+        if 'settings_window' not in exclude:
+            exclude.append('settings_window')
+
+        # We want 2 vertical sections, the tabbed notebook and the buttons. The buttons sizer will have 2 horizontal
+        # sections, one for each button.
+        main_sizer = wx.BoxSizer(wx.VERTICAL)  # Notebook panel
+        button_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Button sizer
+
+        # Notebook
+        self.__notebook = wx.Notebook(self, wx.ID_ANY)  # The notebook
+
+        # A tab for each root node in config. We will store the tabs components in lists which can be accessed by the
+        # index returned from notebook.GetSelectedItem()
+        root_nodes = self.__settings.get_root_nodes()
+        self.__tabs = []
+        for node in root_nodes:
+            # Exclude?
+            if node not in exclude:
+                # Create new tab
+                self.__tabs.append(SettingsTab(self, self.__notebook, node))
+
+                # Add tab to notebook
+                self.__notebook.AddPage(self.__tabs[-1], node)
+
+        # Buttons
+        button_ok = wx.Button(self, label="Update")
+        button_cancel = wx.Button(self, label="Cancel")
+        button_sizer.Add(button_ok, 0, wx.ALL, 1)
+        button_sizer.Add(button_cancel, 0, wx.ALL, 1)
+
+        # Add notebook and button sizer to main sizer and set main sizer for window
+        main_sizer.Add(self.__notebook, 1, wx.ALL | wx.EXPAND, 5)
+        main_sizer.Add(button_sizer)
+        self.SetSizer(main_sizer)
+
+        # Bind buttons &  notebook page select.
+        button_ok.Bind(wx.EVT_BUTTON, self.__on_ok)
+        button_cancel.Bind(wx.EVT_BUTTON, self.__on_cancel)
+        self.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.__on_page_select)
+
+        # Bind window close event
+        self.Bind(wx.EVT_CLOSE, self.__on_close, self)
+
+        # Call on_page_select to select the first page
+        self.__on_page_select(event=None)
+
+    def __on_page_select(self, event):
+        # Call the tabs select method to populate
+        index = self.__notebook.GetSelection()
+        self.__tabs[index].select()
+
+    def __on_cancel(self, event):
+        # Clear changed settings and close
+        self.changed_settings = {}
+        self.EndModal(wx.ID_CANCEL)
+        self.Close()
+
+    def __on_ok(self, event):
+        # Update settings and save
+        delkeys = []
+        for setting in self.changed_settings:
+            # Get the current and new setting
+            orig_value = self.__settings.get(setting)
+            new_value = self.changed_settings[setting]
+
+            # We need to retain data type. New values will all be string as they were retrieved from textctl.
+            # Get the data type of the original and cast new to it. Note boolean needs to be handled differently as it
+            # doesn't cast directly.
+            if isinstance(orig_value, bool):
+                new_value = new_value.lower() in ['true', '1', 'yes', 't']
+            else:
+                new_value = type(orig_value)(new_value)
+
+            # If they are the same, discard from changes. We will use a list of items to delete (delkeys) as we cant
+            # delete whilst iterating. If they are different, update settings.
+            if orig_value == new_value:
+                delkeys.append(setting)
+            else:
+                self.__settings.set(setting, new_value)
+
+        # Now delete the items that were the same from changed_settings. changed_settings may be used by settings
+        # dialog caller.
+        for key in delkeys:
+            del(self.changed_settings[key])
+
+        # Save the settings and close dialog
+        self.__settings.save()
+        self.EndModal(wx.ID_OK)
+        self.Destroy()
+
+    def __on_close(self, event):
+        # Save pos and size
+        x, y = self.GetPosition()
+        width, height = self.GetSize()
+        self.__settings.set('settings_window.x', x)
+        self.__settings.set('settings_window.y', y)
+        self.__settings.set('settings_window.width', width)
+        self.__settings.set('settings_window.height', height)
+
+        # Style
+        style = self.GetWindowStyle()
+        self.__settings.set('settings_window.style', style)
+
+
+class SettingsTab(wx.Panel):
+    """
+    A notebook tab containing the settings tree and values for a settings root node.
+    """
+
+    # Root node for this tab
+    __root_node_name = None
+
+    # Parent frame. Set during constructor
+    __parent_frame = None
+
+    # Each tab has: a tree view; a values panel; and a list of value text boxes bound to a change
+    # event.
+    __tree = None
+    __tab_sizer = None
+
+    # Currently displayed value panel. Will be switched when tree menu items are selected.
+    __current_value_panel = None
+
+    def __init__(self, parent_frame, notebook, root_node):
+        """
+        Creates a tab for the settings notebook.
+
+        :param parent_frame: The frame containing the notebook.
+        :param notebook. The notebook that this tab should be part of.
+        :param root_node. The root node name for the settings
+        """
+        # Super Constructor
+        wx.Panel.__init__(self, parent=notebook)
+
+        # Store the parent frame and get the settings for this tab.
+        self.__parent_frame = parent_frame
+
+        # Store the root node for this tab
+        self.__root_node_name = root_node
+
+        # Create logger
+        self.__log = logging.getLogger(__name__)
+
+        # Build the tab and set it's sizer.
+        self.__tab_sizer = wx.BoxSizer(wx.HORIZONTAL)
+        self.SetSizer(self.__tab_sizer)
+
+        # Create tree control and add it to sizer
+        self.__tree = SettingsTree(self, self.__root_node_name)
+        self.__tab_sizer.Add(self.__tree, 1, wx.ALL | wx.EXPAND, 1)
+
+        # Bind tree selection changed
+        self.__tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.__on_tree_select)
+
+    def select(self):
+        """
+        To be called when this tab is selected. Populate value sizer for the selected item, If no item is selected,
+        populate for root.
+        :return:
+        """
+        selected_item = self.__tree.GetSelection()
+        if selected_item.ID is None:
+            root_node = self.__tree.GetRootItem()
+            setting_path = self.__tree.GetItemData(root_node)
+        else:
+            setting_path = self.__tree.GetItemData(selected_item)
+
+        # Set the panel
+        self.__switch_value_panel(setting_path)
+
+    def __on_tree_select(self, event):
+        """
+        Called when an item in the tree is selected. Displays the correct settings panel
+        :param event:
+        :return:
+        """
+        # Get Selected item and check that it is a tree item
+        tree_item = event.GetItem()
+        if not tree_item.IsOk():
+            return
+
+        # Get the setting path from item data
+        setting_path = self.__tree.GetItemData(tree_item)
+
+        # Switch the panel
+        self.__switch_value_panel(setting_path)
+
+    def __switch_value_panel(self, setting_path):
+        """
+        Switched the value panel to the correct one for the settings path
+        :param setting_path:
+        :return:
+        """
+        # Get current panel and delete.
+        if self.__current_value_panel is not None:
+            self.__current_value_panel.Destroy()
+
+        # Create the new value panel and add to sizer.
+        self.__current_value_panel = SettingsValuePanel(self.__parent_frame, self, setting_path)
+        self.__tab_sizer.Add(self.__current_value_panel, 1, wx.ALL | wx.EXPAND, 1)
+
+        # Redraw
+        self.__tab_sizer.Layout()
+
+
+class SettingsTree(wx.TreeCtrl):
+    """
+    A Tree control containing the settings nodes settings node
+    """
+
+    __root_node_name = None
+
+    def __init__(self, settings_tab, settings_node):
+        """
+        Creates a tree control for specified settings node.
+
+        :param settings_tab. The settings_tab on which this tree control should be displayed.
+        :param settings_node. The node name for the settings who's values will be presented
+        """
+        # Super Constructor
+        wx.TreeCtrl.__init__(self, parent=settings_tab)
+
+        # Set root node
+        self.__root_node_name = settings_node
+
+        # Build the tree
+        self.__build_tree(None, self.__root_node_name)
+
+        # Expand it
+        # self.ExpandAll()
+
+        # Set max size. Width should be best size width, height should be auto (-1)
+        best_width = self.GetBestSize()[0] * 2  # Hack, best size not working
+        self.SetMaxSize((best_width, -1))
+
+    def __build_tree(self, node, node_name):
+        """
+        Recursive function to build the tree and value panels from the node using the settings
+        :param node: The tree view node to build from. If none, builds from root.
+        :param node_name. The name of the node to build from.
+        """
+
+        # Build root node if node is None
+        if node is None:
+            node = self.AddRoot(self.__root_node_name)
+            self.SetItemData(node, self.__root_node_name)
+
+        # Get settings
+        node_path = self.GetItemData(node)
+        settings = Config().get(node_path)
+
+        # Iterate settings, adding branches. Recurse to add sub branches.
+        for setting in settings:
+            # Get settings path
+            settings_path = f"{self.GetItemData(node)}.{setting}"
+
+            # Get value. If dict, add the node and recursively call this function again.
+            value = settings[setting]
+            if type(value) is dict:
+                # Add the node and set its settings path
+                node_id = self.AppendItem(node, setting)
+                self.SetItemData(node_id, settings_path)
+
+                # Recurse
+                self.__build_tree(node_id, value)
+
+
+class SettingsValuePanel(wx.ScrolledWindow):
+    """
+    A panel containing text boxes for editing values for a settings node
+    """
+
+    __value_sizer = None
+    __value_boxes = []
+
+    def __init__(self, parent_frame, settings_tab, node):
+        """
+        Creates a panel containing values for a settings node.
+
+        :param parent_frame: The frame containing the notebook.
+        :param settings_tab. The settings_tab on which this panel should be displayed.
+        :param node. The node name for the settings who's values will be presented
+        """
+        # Super Constructor
+        wx.ScrolledWindow.__init__(self, parent=settings_tab)
+
+        # Create logger
+        self.__log = logging.getLogger(__name__)
+
+        # Store the parent frame and get the settings for this node.
+        self.__parent_frame = parent_frame
+        self.__settings = Config().get(node)
+
+        leaf_settings = {}
+        for setting in self.__settings:
+            if type(self.__settings[setting]) is not dict:
+                leaf_settings[setting] = self.__settings[setting]
+
+        # Add the value sizer for settings values.
+        self.__value_sizer = wx.FlexGridSizer(rows=len(leaf_settings), cols=2, vgap=2, hgap=2)
+
+        # Add the sizer to the panel
+        self.SetSizer(self.__value_sizer)
+
+        # Display every value
+        for setting in leaf_settings:
+            # Setting path
+            setting_path = f"{node}.{setting}"
+
+            # Value. Make sure that we display changed value if already changed
+            if setting_path in self.__parent_frame.changed_settings:
+                value = self.__parent_frame.changed_settings[setting_path]
+            else:
+                value = leaf_settings[setting]
+
+            # Add a label and value text box
+            label = wx.StaticText(self, wx.ID_ANY, setting, style=wx.ALIGN_LEFT)
+            self.__value_boxes.append(wx.TextCtrl(self, wx.ID_ANY, f"{value}", style=wx.ALIGN_LEFT))
+            self.__value_sizer.AddMany([(label, 0, wx.EXPAND), (self.__value_boxes[-1], 0, wx.EXPAND)])
+
+            # Bind to text change. We need to generate a handler as this will have a parameter.
+            self.__value_boxes[-1].Bind(wx.EVT_TEXT, self.__get_on_change_evt_handler(setting_path=setting_path))
+
+        # Layout the value sizer
+        self.__value_sizer.Layout()
+
+        # Setup scrollbars
+        self.SetScrollbars(1, 1, 1000, 1000)
+
+    def __get_on_change_evt_handler(self, setting_path):
+        """
+        Returns a new event handler with a parameter of the settings path
+        :param setting_path:
+        :return:
+        """
+
+        def on_value_changed(event):
+            old_val = self.__settings.get(setting_path)
+            self.__parent_frame.changed_settings[setting_path] = event.String
+            self.__log.debug(f"Value changed from {old_val} to {self.__parent_frame.changed_settings[setting_path]} "
+                             f"for {setting_path}.")
+
+        return on_value_changed

mt5_correlation/correlation.py

@@ -1,26 +1,275 @@
 import math
+import logging
+import pandas as pd
+from datetime import datetime, timedelta
+import time
+import sched
+import threading
+import pytz
 from scipy.stats.stats import pearsonr
+import pickle
+import inspect
+import sys
+
+from mt5_correlation.mt5 import MT5
 
 
 class Correlation:
     """
-    A class to calculate the correlation coefficient between two sets of price data
+    A class to maintain the state of the calculated correlation coefficients.
     """
 
-    @staticmethod
-    def calculate_coefficient(symbol1_prices, symbol2_prices, max_set_size_diff_pct=90, overlap_pct=90,
-                              max_p_value=0.05):
-        """
-        Calculates the correlation coefficient between two sets of price data. Uses close price.
+    # Connection to metatrader
+    __mt5 = None
 
-        :param symbol1_prices:
-        :param symbol2_prices:
+    # Minimum base coefficient for monitoring. Symbol pairs with a lower correlation
+    # coefficient than ths won't be monitored.
+    monitoring_threshold = 0.9
+
+    # Toggle on whether we are monitoring or not. Set through start_monitor and stop_monitor
+    __monitoring = False
+    __monitoring_params = {}
+
+    # First run of scheduler
+    __first_run = True
+
+    # The price data used to calculate the correlations
+    __price_data = None
+
+    # Coefficient data and history. Will be created as dataframes in Init
+    coefficient_data = None
+    coefficient_history = None
+
+    # Stores tick data used to calculate coefficient during Monitor.
+    # Dict: {Symbol: [retrieved datetime, ticks dataframe]}
+    __monitor_tick_data = {}
+
+    def __init__(self):
+        # Logger
+        self.__log = logging.getLogger(__name__)
+
+        # Connection to metatrader
+        self.__mt5 = MT5()
+
+        # Create dataframe for coefficient data
+        self.__reset_coefficient_data()
+
+        # Create timer for continuous monitoring
+        self.__scheduler = sched.scheduler(time.time, time.sleep)
+
+    @property
+    def filtered_coefficient_data(self):
+        """
+        :return: Coefficient data filtered so that all base coefficients >= monitoring_threshold
+        """
+        if self.coefficient_data is not None:
+            return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold]
+        else:
+            return None
+
+    def load(self, filename):
+        """
+        Loads calculated coefficients, price data used to calculate them and tick data used during monitoring.
+        coefficients
+        :param filename: The filename for the coefficient data to load.
+        :return:
+        """
+        # Load data
+        with open(filename, 'rb') as file:
+            loaded_dict = pickle.load(file)
+
+        # Get data from loaded dict and save
+        self.coefficient_data = loaded_dict["coefficient_data"]
+        self.__price_data = loaded_dict["price_data"]
+        self.__monitor_tick_data = loaded_dict["monitor_tick_data"]
+        self.coefficient_history = loaded_dict["coefficient_history"]
+
+    def save(self, filename):
+        """
+        Saves the calculated coefficients, the price data used to calculate and the tick data for monitoring to a file.
+        :param filename: The filename to save the data to.
+        :return:
+        """
+        # Add data to dict then use pickle to save
+        save_dict = {"coefficient_data": self.coefficient_data, "price_data": self.__price_data,
+                     "monitor_tick_data": self.__monitor_tick_data, "coefficient_history": self.coefficient_history}
+        with open(filename, 'wb') as file:
+            pickle.dump(save_dict, file, protocol=pickle.HIGHEST_PROTOCOL)
+
+    def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                  max_p_value=0.05):
+        """
+        Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch. Updates coefficient data.
+
+        :param date_from: From date for price data from which to calculate correlation coefficients
+        :param date_to: To date for price data from which to calculate correlation coefficients
+        :param timeframe: Timeframe for price data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
             within this pct of each other
         :param overlap_pct:
         :param max_p_value: The maximum p value for the correlation to be meaningful
+
+        :return:
+        """
+
+        coefficient = None
+
+        # If we are monitoring, stop. We will need to restart later
+        was_monitoring = self.__monitoring
+        if self.__monitoring:
+            self.stop_monitor()
+
+        # Clear the existing correlations
+        self.__reset_coefficient_data()
+
+        # Get all visible symbols
+        symbols = self.__mt5.get_symbols()
+
+        # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
+        self.__price_data = {}
+        for symbol in symbols:
+            self.__price_data[symbol] = self.__mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to,
+                                                              timeframe=timeframe)
+
+        # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
+        # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
+        # avoid duplicating. We will store all coefficients in a dataframe for export as CSV.
+        index = 0
+        # There will be (x^2 - x) / 2 pairs where x is number of symbols
+        num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
+
+        for i in range(0, len(symbols)):
+            symbol1 = symbols[i]
+
+            for j in range(i + 1, len(symbols)):
+                symbol2 = symbols[j]
+                index += 1
+
+                # Get price data for both symbols
+                symbol1_price_data = self.__price_data[symbol1]
+                symbol2_price_data = self.__price_data[symbol2]
+
+                # Get coefficient
+                if symbol1_price_data is not None and symbol2_price_data is not None:
+                    coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data,
+                                                             symbol2_prices=symbol2_price_data,
+                                                             min_prices=min_prices,
+                                                             max_set_size_diff_pct=max_set_size_diff_pct,
+                                                             overlap_pct=overlap_pct, max_p_value=max_p_value)
+
+                # Store if valid
+                if coefficient is not None:
+
+                    self.coefficient_data = \
+                        self.coefficient_data.append({'Symbol 1': symbol1, 'Symbol 2': symbol2,
+                                                      'Base Coefficient': coefficient, 'UTC Date From': date_from,
+                                                      'UTC Date To': date_to, 'Timeframe': timeframe},
+                                                     ignore_index=True)
+                    self.__log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a "
+                                     f"coefficient of {coefficient}.")
+                else:
+                    self.__log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:"
+                                     f"{symbol2} could no be calculated.")
+
+        # Sort, highest correlated first
+        self.coefficient_data = self.coefficient_data.sort_values('Base Coefficient', ascending=False)
+
+        # If we were monitoring, we stopped, so start again.
+        if was_monitoring:
+            self.start_monitor(interval=self.__monitoring_params['interval'],
+                               from_mins=self.__monitoring_params['from_mins'],
+                               min_prices=self.__monitoring_params['min_prices'],
+                               max_set_size_diff_pct=self.__monitoring_params['max_set_size_diff_pct'],
+                               overlap_pct=self.__monitoring_params['overlap_pct'],
+                               max_p_value=self.__monitoring_params['max_p_value'])
+
+    def get_price_data(self, symbol):
+        """
+        Returns the price data used to calculate the base coefficients for the specified symbol
+        :param symbol: Symbol to get price data for.
+        :return: price data
+        """
+        price_data = None
+        if symbol in self.__price_data:
+            price_data = self.__price_data[symbol]
+
+        return price_data
+
+    def start_monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                      max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'):
+        """
+        Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
+        threshold.
+
+        :param interval: How often to check in seconds
+        :param from_mins: The number of minutes of tick data to use for calculations
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
+        :param autosave: Whether to autosave after every monitor run. If there is no filename specified then will
+            create one named autosave.cpd
+        :param filename: Filename for autosave. Default is autosave.cpd.
+
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
+
+        if self.__monitoring:
+            self.__log.debug(f"Request to start monitor when monitor is already running. Monitor will be stopped and"
+                             f"restarted with new parameters.")
+            self.stop_monitor()
+
+        self.__log.debug(f"Starting monitor.")
+        self.__monitoring = True
+
+        # Create thread to run monitoring This will call private __monitor method that will run the calculation and
+        # keep scheduling itself while self.monitoring is True. Store the params. We will need to use these if we have
+        # to stop and restart the monitor. Note, this happens during calculate
+        self.__monitoring_params = {'interval': interval, 'from_mins': from_mins,
+                                    'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct,
+                                    'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time,
+                                    'autosave': autosave, 'filename': filename}
+        thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params)
+        thread.start()
+
+    def stop_monitor(self):
+        """
+        Stops monitoring symbol pairs for correlation.
+        :return:
+        """
+        if self.__monitoring:
+            self.__log.debug(f"Stopping monitor.")
+            self.__monitoring = False
+        else:
+            self.__log.debug(f"Request to stop monitor when it is not running. No action taken.")
+
+    def calculate_coefficient(self, symbol1_prices, symbol2_prices, min_prices: int = 100,
+                              max_set_size_diff_pct: int = 90, overlap_pct: int = 90,
+                              max_p_value: float = 0.05):
+        """
+        Calculates the correlation coefficient between two sets of price data. Uses close price.
+
+        :param symbol1_prices: Pandas dataframe containing prices or ticks for symbol 1
+        :param symbol2_prices: Pandas dataframe containing prices or ticks for symbol 2
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        :rtype: float or None
+        """
+
+        assert symbol1_prices is not None and symbol2_prices is not None
+
         # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
         # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of
         # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set?
@@ -31,7 +280,8 @@ class Correlation:
         len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
         similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set
         enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100)
-        suitable = similar_size and enough_overlap
+        enough_prices = len_smallest_set >= min_prices
+        suitable = similar_size and enough_overlap and enough_prices
 
         if suitable:
             # Calculate coefficient on close prices
@@ -43,13 +293,248 @@ class Correlation:
             # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
             # hypothesis is false).
             coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close'])
-            coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0]
+            coefficient = None if coefficient_with_p_value[1] > max_p_value else coefficient_with_p_value[0]
 
             # If NaN, change to None
             if coefficient is not None and math.isnan(coefficient):
                 coefficient = None
 
+        self.__log.debug(f"Calculate coefficient returning {coefficient}. "
+                         f"Symbol 1 Prices: {len(symbol1_prices)}  Symbol 2 Prices: {len(symbol2_prices)} "
+                         f"Overlap Prices: {len(intersect_dates)} Similar size: {similar_size} "
+                         f"Enough overlap: {enough_overlap} Enough prices: {enough_prices} Suitable: {suitable}.")
+
         return coefficient
 
+    def get_coefficient_history(self, symbol1, symbol2):
+        """
+        Returns the coefficient history for the specified symbol pair calculated during this instance.
+        Coefficient history does not persist between instances.
+        :param symbol1:
+        :param symbol2:
+        :return: dataframe containing history of coefficient data.
+        """
+        history = self.coefficient_history[(self.coefficient_history['Symbol 1'] == symbol1) &
+                                           (self.coefficient_history['Symbol 2'] == symbol2)]
+        return history
 
+    def get_ticks(self, symbol, date_from=None, date_to=None, cache_time=0, cache_only=False):
+        """
+        Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe.
 
+        :param symbol: Name of symbol to get ticks for.
+        :param date_from: Date to get ticks from. Can only be None if getting from cache (cache_only=True)
+        :param date_to:Date to get ticks to. Can only be None if getting from cache (cache_only=True)
+        :param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed,
+            get data from source and refresh cache.
+        :param cache_only: Only retrieve from cache. cache_time is ignored. Returns None if symbol is not available in
+            cache.
+
+        :return:
+        """
+
+        timezone = pytz.timezone("Etc/UTC")
+        utc_now = datetime.now(tz=timezone)
+
+        ticks = None
+
+        # Cache only
+        if cache_only:
+            if symbol in self.__monitor_tick_data:
+                ticks = self.__monitor_tick_data[symbol][1]
+        # Check if we already have it and it is not stale
+        elif symbol in self.__monitor_tick_data and utc_now < \
+                self.__monitor_tick_data[symbol][0] + timedelta(seconds=cache_time):
+            # Cached ticks are not stale. Get them
+            ticks = self.__monitor_tick_data[symbol][1]
+            self.__log.debug(f"Ticks for {symbol} retrieved from cache.")
+        else:
+            # Data does not exist in cache or cached data is stale. Retrieve from source and cache.
+            ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to)
+            self.__monitor_tick_data[symbol] = [utc_now, ticks]
+            self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.")
+        return ticks
+
+    def __monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                  max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'):
+        """
+        The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
+        stop_monitoring.
+
+        :param interval: How often to check in seconds
+        :param from_mins: The number of minutes of tick data to use for calculations
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
+        :param autosave: Whether to autosave after every monitor run. If there is no filename specified then will
+            create one named autosave.cpd
+        :param filename: Filename for autosave. Default is autosave.cpd.
+
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+        self.__log.debug(f"In monitor event. Monitoring: {self.__monitoring}.")
+
+        # Only run if monitor is not stopped
+        if self.__monitoring:
+            # Update all coefficients
+            self.__update_all_coefficients(from_mins=from_mins, min_prices=min_prices,
+                                           max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
+                                           max_p_value=max_p_value, cache_time=cache_time)
+
+            # Autosave
+            if autosave:
+                self.save(filename=filename)
+
+            # Schedule the timer to run again
+            params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices,
+                      'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
+                      'max_p_value': max_p_value, "cache_time": cache_time, 'autosave': autosave,
+                      'filename': filename}
+            self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
+
+            # Log the stack. Debug stack overflow
+            self.__log.debug(f"Current stack size: {len(inspect.stack())} Recursion limit: {sys.getrecursionlimit()}")
+
+            # Run
+            if self.__first_run:
+                self.__first_run = False
+                self.__scheduler.run()
+
+    def __update_coefficient(self, symbol1, symbol2, from_mins, min_prices=100, max_set_size_diff_pct=90,
+                             overlap_pct=90, max_p_value=0.05, cache_time=10):
+        """
+        Updates the coefficient for the specified symbol pair
+        :param symbol1: Name of symbol to calculate coefficient for.
+        :param symbol2: Name of symbol to calculate coefficient for.
+        :param from_mins: The number of minutes of tick data to use for calculations
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+
+        coefficient = None
+
+        # Get dates
+        # From and to dates for calculations.
+        timezone = pytz.timezone("Etc/UTC")
+        date_to = datetime.now(tz=timezone)
+        date_from = date_to - timedelta(minutes=from_mins)
+
+        # Get the tick data
+        symbol1ticks = self.get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time)
+        symbol2ticks = self.get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time)
+
+        # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one
+        # tick per second and ensuring that times can match. We will need to set the index to time for the resample
+        # then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price
+        if symbol1ticks is not None and symbol2ticks is not None and \
+                len(symbol1ticks.index) > 0 and len(symbol2ticks.index) > 0:
+            symbol1ticks = symbol1ticks.set_index('time')
+            symbol2ticks = symbol2ticks.set_index('time')
+            try:
+                symbol1prices = symbol1ticks['ask'].resample('1S').ohlc()
+                symbol2prices = symbol2ticks['ask'].resample('1S').ohlc()
+            except RecursionError:
+                self.__log.warning(f"Coefficient could not be calculated for {symbol1}:{symbol2} as prices could not "
+                                   f"be resampled.")
+            else:
+                symbol1prices.reset_index(inplace=True)
+                symbol2prices.reset_index(inplace=True)
+                symbol1prices = symbol1prices[symbol1prices['close'].notna()]
+                symbol2prices = symbol2prices[symbol2prices['close'].notna()]
+
+                # Calculate the coefficient
+                coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices,
+                                                         min_prices=min_prices,
+                                                         max_set_size_diff_pct=max_set_size_diff_pct,
+                                                         overlap_pct=overlap_pct, max_p_value=max_p_value)
+
+                self.__log.debug(f"Symbol pair {symbol1}:{symbol2} has a coefficient of {coefficient}.")
+        else:
+            coefficient = None
+
+        # Update the coefficient data
+        if coefficient is not None:
+            self.__update_coefficient_data(symbol1=symbol1, symbol2=symbol2, coefficient=coefficient,
+                                           date_from=date_from, date_to=date_to)
+
+        return coefficient
+
+    def __update_all_coefficients(self, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                                  max_p_value=0.05, cache_time=10):
+        """
+        Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
+        the threshold can be accessed through the filtered_coefficient_data property.
+
+        :param from_mins: The number of minutes of tick data to use for calculations
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
+
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+        # Update  latest coefficient for every pair
+        for index, row in self.filtered_coefficient_data.iterrows():
+            symbol1 = row['Symbol 1']
+            symbol2 = row['Symbol 2']
+            self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, from_mins=from_mins,
+                                      min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
+                                      overlap_pct=overlap_pct, max_p_value=max_p_value, cache_time=cache_time)
+
+    def __reset_coefficient_data(self):
+        """
+        Clears coefficient data and history.
+        :return:
+        """
+        # Create dataframe for coefficient data
+        coefficient_data_columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To',
+                                    'Timeframe', 'Last Check', 'Last Coefficient']
+        self.coefficient_data = pd.DataFrame(columns=coefficient_data_columns)
+
+        # Create dataframe for coefficient history
+        coefficient_history_columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To']
+        self.coefficient_history = pd.DataFrame(columns=coefficient_history_columns)
+
+    def __update_coefficient_data(self, symbol1, symbol2, coefficient, date_from, date_to):
+        """
+        Updates the coefficient data with the latest coefficient and adds to coefficient history.
+        :param symbol1:
+        :param symbol2:
+        :param coefficient:
+        :param date_from:
+        :param date_to:
+        :return:
+        """
+
+        timezone = pytz.timezone("Etc/UTC")
+        now = datetime.now(tz=timezone)
+
+        # Update data if we have a coefficient and add to history
+        if coefficient is not None:
+            self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
+                                      (self.coefficient_data['Symbol 2'] == symbol2),
+                                      'Last Check'] = now
+
+            self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
+                                      (self.coefficient_data['Symbol 2'] == symbol2),
+                                      'Last Coefficient'] = coefficient
+
+            row = pd.DataFrame(columns=self.coefficient_history.columns,
+                               data=[[symbol1, symbol2, coefficient, date_from, date_to]])
+            self.coefficient_history = self.coefficient_history.append(row)

mt5_correlation/gui.py

@@ -0,0 +1,604 @@
+import wx
+import wx.grid
+import matplotlib.pyplot as plt
+from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas
+import matplotlib.dates
+import matplotlib
+
+from mt5_correlation.correlation import Correlation
+from mt5_correlation.config import Config, SettingsDialog
+from datetime import datetime, timedelta
+import pytz
+import pandas as pd
+import logging
+import logging.config
+
+matplotlib.use('WXAgg')
+
+
+class MonitorFrame(wx.Frame):
+
+    __cor = None
+    __rows = 0  # Need to track as we need to notify grid if row count changes.
+    __opened_filename = None  # So we can save to same file as we opened
+    __config = None  # The applications config
+
+    __selected_correlation = []  # List of Symbol 1 & Symbol 2
+
+    # Columns for coefficient table
+    COLUMN_INDEX = 0
+    COLUMN_SYMBOL1 = 1
+    COLUMN_SYMBOL2 = 2
+    COLUMN_BASE_COEFFICIENT = 3
+    COLUMN_DATE_FROM = 4
+    COLUMN_DATE_TO = 5
+    COLUMN_TIMEFRAME = 6
+    COLUMN_LAST_CHECK = 7
+    COLUMN_LAST_COEFFICIENT = 8
+
+    def __init__(self):
+        # Super
+        wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title="Divergence Monitor",
+                          pos=wx.Point(x=Config().get('window.x'),
+                                       y=Config().get('window.y')),
+                          size=wx.Size(width=Config().get('window.width'),
+                                       height=Config().get('window.height')),
+                          style=Config().get('window.style'))
+
+        # Create logger and get config
+        self.__log = logging.getLogger(__name__)
+        self.__config = Config()
+
+        # Create correlation instance to maintain state of calculated coefficients. Set min coefficient from config
+        self.__cor = Correlation()
+        self.__cor.monitoring_threshold = self.__config.get("monitor.monitoring_threshold")
+
+        # Status bar
+        self.statusbar = self.CreateStatusBar(1)
+
+        # Menu Bar and file menu
+        self.menubar = wx.MenuBar()
+        file_menu = wx.Menu()
+
+        # File menu items
+        menu_item_open = file_menu.Append(wx.ID_ANY, "Open", "Open correlations file.")
+        menu_item_save = file_menu.Append(wx.ID_ANY, "Save", "Save correlations file.")
+        menu_item_saveas = file_menu.Append(wx.ID_ANY, "Save As", "Save correlations file.")
+        file_menu.AppendSeparator()
+        menu_item_calculate = file_menu.Append(wx.ID_ANY, "Calculate", "Calculate base coefficients.")
+        file_menu.AppendSeparator()
+        menu_item_settings = file_menu.Append(wx.ID_ANY, "Settings", "Change application settings.")
+        file_menu.AppendSeparator()
+        menu_item_exit = file_menu.Append(wx.ID_ANY, "Exit", "Close the application")
+
+        # Add file menu and set menu bar
+        self.menubar.Append(file_menu, "File")
+        self.SetMenuBar(self.menubar)
+
+        # Main window. We want 2 horizontal sections, the grid showing correlations and a graph. In the correlations
+        # section, we want 2 vertical sections, the monitor toggle and the correlations grid. For the toggle we want 2
+        # sections, a label and a toggle.
+        # ---------------------------------------------------------------
+        # |label  |  toggle       |                                      |
+        # |-----------------------|                                      |
+        # |                       |                                      |
+        # |Correlations Grid      |         Graphs                       |
+        # |                       |                                      |
+        # |                       |                                      |
+        # |                       |                                      |
+        # |                       |                                      |
+        # ----------------------------------------------------------------
+        panel = wx.Panel(self, wx.ID_ANY)
+        toggle_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Label and toggle
+        correlations_sizer = wx.BoxSizer(wx.VERTICAL)  # Toggle sizer and correlations grid
+        self.__main_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Correlations sizer and graphs panel
+        panel.SetSizer(self.__main_sizer)
+
+        # Create the label and toggle, populate the toggle sizer and add the toggle sizer to the correlations sizer
+        monitor_toggle_label = wx.StaticText(panel, id=wx.ID_ANY, label="Monitoring")
+        toggle_sizer.Add(monitor_toggle_label, 0, wx.ALL, 1)
+        self.monitor_toggle = wx.ToggleButton(panel, wx.ID_ANY, label="Off")
+        self.monitor_toggle.SetBackgroundColour(wx.RED)
+        toggle_sizer.Add(self.monitor_toggle, 0, wx.ALL, 1)
+        correlations_sizer.Add(toggle_sizer, 0, wx.ALL, 1)
+
+        # Create the correlations grid. This is a data table using pandas dataframe for underlying data. Add the
+        # correlations_grid to the correlations sizer.
+        self.table = DataTable(self.__cor.filtered_coefficient_data)
+        self.grid_correlations = wx.grid.Grid(panel, wx.ID_ANY)
+        self.grid_correlations.SetTable(self.table, takeOwnership=True)
+        self.grid_correlations.EnableEditing(False)
+        self.grid_correlations.EnableDragRowSize(False)
+        self.grid_correlations.EnableDragColSize(False)
+        self.grid_correlations.EnableDragGridSize(False)
+        self.grid_correlations.SetSelectionMode(wx.grid.Grid.SelectRows)
+        self.grid_correlations.SetRowLabelSize(0)
+        self.grid_correlations.SetColSize(self.COLUMN_INDEX, 0)  # Index. Hide
+        self.grid_correlations.SetColSize(self.COLUMN_SYMBOL1, 100)  # Symbol 1
+        self.grid_correlations.SetColSize(self.COLUMN_SYMBOL2, 100)  # Symbol 2
+        self.grid_correlations.SetColSize(self.COLUMN_BASE_COEFFICIENT, 100)  # Base Coefficient
+        self.grid_correlations.SetColSize(self.COLUMN_DATE_FROM, 0)  # UTC Date From. Hide
+        self.grid_correlations.SetColSize(self.COLUMN_DATE_TO, 0)  # UTC Date To. Hide
+        self.grid_correlations.SetColSize(self.COLUMN_TIMEFRAME, 0)  # Timeframe. Hide.
+        self.grid_correlations.SetColSize(self.COLUMN_LAST_CHECK, 100)  # Last Check
+        self.grid_correlations.SetColSize(self.COLUMN_LAST_COEFFICIENT, 100)  # Last Coefficient
+        self.grid_correlations.SetMinSize((520, 500))
+        self.grid_correlations.SetMaxSize((520, -1))
+        correlations_sizer.Add(self.grid_correlations, 1, wx.ALL | wx.EXPAND, 1)
+
+        # Create the charts and hide as we have no data to display yet
+        self.__graph = GraphPanel(panel)
+        self.__graph.Hide()
+
+        # Add the correlations sizer and the charts to the main sizer.
+        self.__main_sizer.Add(correlations_sizer, 0, wx.ALL | wx.EXPAND, 1)
+        self.__main_sizer.Add(self.__graph, 1, wx.ALL | wx.EXPAND, 1)
+
+        # Layout the window.
+        self.Layout()
+
+        # Set up timer to refresh grid
+        self.timer = wx.Timer(self)
+
+        # Bind monitor button
+        self.monitor_toggle.Bind(wx.EVT_TOGGLEBUTTON, self.monitor)
+
+        # Bind timer
+        self.Bind(wx.EVT_TIMER, self.__timer_event, self.timer)
+
+        # Bind menu items
+        self.Bind(wx.EVT_MENU, self.open_file, menu_item_open)
+        self.Bind(wx.EVT_MENU, self.save_file, menu_item_save)
+        self.Bind(wx.EVT_MENU, self.save_file_as, menu_item_saveas)
+        self.Bind(wx.EVT_MENU, self.calculate_coefficients, menu_item_calculate)
+        self.Bind(wx.EVT_MENU, self.open_settings, menu_item_settings)
+        self.Bind(wx.EVT_MENU, self.quit, menu_item_exit)
+
+        # Bind row select
+        self.Bind(wx.grid.EVT_GRID_SELECT_CELL, self.select_cell, self.grid_correlations)
+
+        # Bind window close event
+        self.Bind(wx.EVT_CLOSE, self.on_close, self)
+
+    def open_file(self, event):
+        with wx.FileDialog(self, "Open Coefficients file", wildcard="cpd (*.cpd)|*.cpd",
+                           style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog:
+
+            if fileDialog.ShowModal() == wx.ID_CANCEL:
+                return  # the user changed their mind
+
+            # Load the file chosen by the user.
+            self.__opened_filename = fileDialog.GetPath()
+
+            self.SetStatusText(f"Loading file {self.__opened_filename}.")
+            self.__cor.load(self.__opened_filename)
+
+            # Refresh data in grid
+            self.refresh_grid()
+
+            self.SetStatusText(f"File {self.__opened_filename} loaded.")
+
+    def save_file(self, event):
+        self.SetStatusText(f"Saving file as {self.__opened_filename}")
+
+        if self.__opened_filename is None:
+            self.save_file_as(event)
+        else:
+            self.__cor.save(self.__opened_filename)
+
+        self.SetStatusText(f"File saved as {self.__opened_filename}")
+
+    def save_file_as(self, event):
+        with wx.FileDialog(self, "Save Coefficients file", wildcard="cpd (*.cpd)|*.cpd",
+                           style=wx.FD_SAVE) as fileDialog:
+            if fileDialog.ShowModal() == wx.ID_CANCEL:
+                return  # the user changed their mind
+
+            # Save the file and price data file, changing opened filename so next save writes to new file
+            self.SetStatusText(f"Saving file as {self.__opened_filename}")
+
+            self.__opened_filename = fileDialog.GetPath()
+            self.__cor.save(self.__opened_filename)
+
+            self.SetStatusText(f"File saved as {self.__opened_filename}")
+
+    def calculate_coefficients(self, event):
+        # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
+        timezone = pytz.timezone("Etc/UTC")
+        utc_to = datetime.now(tz=timezone)
+        utc_from = utc_to - timedelta(days=self.__config.get('calculate.from.days'))
+
+        # Calculate
+        self.SetStatusText("Calculating coefficients.")
+        self.__cor.calculate(date_from=utc_from, date_to=utc_to,
+                             timeframe=self.__config.get('calculate.timeframe'),
+                             min_prices=self.__config.get('calculate.min_prices'),
+                             max_set_size_diff_pct=self.__config.get('calculate.max_set_size_diff_pct'),
+                             overlap_pct=self.__config.get('calculate.overlap_pct'),
+                             max_p_value=self.__config.get('calculate.max_p_value'))
+        self.SetStatusText("")
+
+        # Show calculated data
+        self.refresh_grid()
+
+    def quit(self, event):
+        # Close
+        self.Close()
+
+    def refresh_grid(self):
+        """
+        Refreshes grid. Notifies if rows have been added or deleted.
+        :return:
+        """
+        self.__log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}")
+
+        # Update data
+        self.table.data = self.__cor.coefficient_data.copy()
+
+        # Format
+        self.table.data.loc[:, 'Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format)
+        self.table.data.loc[:, 'Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True)
+        self.table.data.loc[:, 'Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S')
+        self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format)
+
+        # Remove nans. The ones from the float column will be str nan as they have been formatted
+        self.table.data = self.table.data.fillna('')
+        self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '')
+
+        # Start refresh
+        self.grid_correlations.BeginBatch()
+
+        # Check if num rows in dataframe has changed, and send appropriate APPEND or DELETE messages
+        cur_rows = len(self.__cor.filtered_coefficient_data.index)
+        if cur_rows < self.__rows:
+            # Data has been deleted. Send message
+            msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_DELETED,
+                                           self.__rows - cur_rows, self.__rows - cur_rows)
+            self.grid_correlations.ProcessTableMessage(msg)
+        elif cur_rows > self.__rows:
+            # Data has been added. Send message
+            msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_APPENDED,
+                                           cur_rows - self.__rows)  # how many
+            self.grid_correlations.ProcessTableMessage(msg)
+
+        self.grid_correlations.EndBatch()
+
+        # Send updated message
+        msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_REQUEST_VIEW_GET_VALUES)
+        self.grid_correlations.ProcessTableMessage(msg)
+
+        # Update row count
+        self.__rows = cur_rows
+
+    def monitor(self, event):
+        # Check state of toggle button. If on, then start monitoring, else stop
+        if self.monitor_toggle.GetValue():
+            self.__log.info("Starting monitoring.")
+            self.monitor_toggle.SetBackgroundColour(wx.GREEN)
+            self.monitor_toggle.SetLabelText("On")
+            self.SetStatusText("Monitoring for changes to coefficients.")
+
+            self.timer.Start(self.__config.get('monitor.interval')*1000)
+
+            # Autosave filename
+            filename = self.__opened_filename if self.__opened_filename is not None else 'autosave.cpd'
+
+            self.__cor.start_monitor(interval=self.__config.get('monitor.interval'),
+                                     from_mins=self.__config.get('monitor.from.minutes'),
+                                     min_prices=self.__config.get('monitor.min_prices'),
+                                     max_set_size_diff_pct=self.__config.get('monitor.max_set_size_diff_pct'),
+                                     overlap_pct=self.__config.get('monitor.overlap_pct'),
+                                     max_p_value=self.__config.get('monitor.max_p_value'),
+                                     cache_time=self.__config.get('monitor.tick_cache_time'),
+                                     autosave=self.__config.get('monitor.autosave'),
+                                     filename=filename)
+        else:
+            self.__log.info("Stopping monitoring.")
+            self.monitor_toggle.SetBackgroundColour(wx.RED)
+            self.monitor_toggle.SetLabelText("Off")
+            self.SetStatusText("Monitoring stopped.")
+            self.timer.Stop()
+            self.__cor.stop_monitor()
+
+    def open_settings(self, event):
+        """
+        Opens the settings dialog
+        :return:
+        """
+        settings_dialog = SettingsDialog(parent=self, exclude=['window'])
+        res = settings_dialog.ShowModal()
+        if res == wx.ID_OK:
+            # Reload relevant parts of app
+            restart_monitor_timer = False
+            restart_gui_timer = False
+            reload_correlations = False
+            reload_logger = False
+            reload_graph = False
+
+            for setting in settings_dialog.changed_settings:
+                # If any 'monitor.' settings except 'monitor.divergence_threshold have changed then restart
+                # monitoring timer with new settings.
+                # If 'monitor.interval has changed then restart gui timer.
+                # If 'monitor.monitoring_threshold' has changed, then refresh correlation data.
+                # If any 'logging.' settings have changed, then reload logger config.
+                if setting.startswith('monitor.') and setting != 'monitor.divergence_threshold':
+                    restart_monitor_timer = True
+                if setting == 'monitor.interval':
+                    restart_gui_timer = True
+                if setting == 'monitor.monitoring_threshold':
+                    reload_correlations = True
+                if setting.startswith('logging.'):
+                    reload_logger = True
+                if setting.startswith('monitor.from.'):
+                    reload_graph = True
+
+            # Now perform the actions
+            if restart_monitor_timer:
+                self.__log.info("Settings updated. Reloading monitoring timer.")
+                self.__cor.stop_monitor()
+                self.__cor.start_monitor(interval=self.__config.get('monitor.interval'),
+                                         from_mins=self.__config.get('monitor.from.minutes'),
+                                         min_prices=self.__config.get('monitor.min_prices'),
+                                         max_set_size_diff_pct=self.__config.get('monitor.max_set_size_diff_pct'),
+                                         overlap_pct=self.__config.get('monitor.overlap_pct'),
+                                         max_p_value=self.__config.get('monitor.max_p_value'))
+            if restart_gui_timer:
+                self.__log.info("Settings updated. Restarting gui timer.")
+                self.timer.Stop()
+                self.timer.Start(self.__config.get('monitor.interval') * 1000)
+
+            if reload_correlations:
+                self.__log.info("Settings updated. Updating monitoring threshold and reloading grid.")
+                self.__cor.monitoring_threshold = self.__config.get("monitor.monitoring_threshold")
+                self.refresh_grid()
+
+            if reload_logger:
+                self.__log.info("Settings updated. Reloading logger.")
+                log_config = Config().get('logging')
+                logging.config.dictConfig(log_config)
+
+            if reload_graph:
+                self.__log.info("Settings updated. Reloading graph.")
+                if len(self.__selected_correlation) == 2:
+                    self.show_graph(symbol1=self.__selected_correlation[0], symbol2=self.__selected_correlation[1])
+
+    def on_close(self, event):
+        """
+        Window closing. Save coefficients and stop monitoring.
+        :param event:
+        :return:
+        """
+        # Save pos and size
+        x, y = self.GetPosition()
+        width, height = self.GetSize()
+        self.__config.set('window.x', x)
+        self.__config.set('window.y', y)
+        self.__config.set('window.width', width)
+        self.__config.set('window.height', height)
+
+        # Style
+        style = self.GetWindowStyle()
+        self.__config.set('window.style', style)
+
+        self.__config.save()
+
+        # Stop monitoring
+        self.__cor.stop_monitor()
+
+        # Kill graph as it seems to be stopping script from ending
+        self.__graph = None
+
+        # End
+        event.Skip()
+
+    def select_cell(self, event):
+        """
+        A cell was selected. Show the graph for the correlation.
+        :param event:
+        :return:
+        """
+        # Get row and symbols.
+        row = event.GetRow()
+        symbol1 = self.grid_correlations.GetCellValue(row, self.COLUMN_SYMBOL1)
+        symbol2 = self.grid_correlations.GetCellValue(row, self.COLUMN_SYMBOL2)
+        self.__selected_correlation = [symbol1, symbol2]
+
+        self.show_graph(symbol1, symbol2)
+
+    def show_graph(self, symbol1, symbol2):
+        """
+        Displays the graph for the specified symbols correlation history
+        :param symbol1:
+        :param symbol2:
+        :return:
+        """
+        # Get the price data for the base coefficient calculation, tick data to calculate last coefficient and  and the
+        # coefficient history data
+        symbol_1_price_data = self.__cor.get_price_data(symbol1)
+        symbol_2_price_data = self.__cor.get_price_data(symbol2)
+        symbol_1_ticks = self.__cor.get_ticks(symbol1, cache_only=True)
+        symbol_2_ticks = self.__cor.get_ticks(symbol2, cache_only=True)
+        history_data = self.__cor.get_coefficient_history(symbol1, symbol2)
+        times = history_data['UTC Date To']
+        coefficients = history_data['Coefficient']
+
+        # Display if we have any data
+        self.__log.debug(f"Refreshing history graph {symbol1}:{symbol2}.")
+        self.__graph.draw(times=times, coefficients=coefficients, prices=[symbol_1_price_data, symbol_2_price_data],
+                          ticks=[symbol_1_ticks, symbol_2_ticks], symbols=[symbol1, symbol2])
+
+        # Un-hide and layout if hidden
+        if not self.__graph.IsShown():
+            self.__graph.Show()
+            self.__main_sizer.Layout()
+
+    def __timer_event(self, event):
+        """
+        Called on timer event. Refreshes grid and updatates selected graph.
+        :return:
+        """
+        self.refresh_grid()
+        if len(self.__selected_correlation) == 2:
+            self.show_graph(symbol1=self.__selected_correlation[0], symbol2=self.__selected_correlation[1])
+
+
+class DataTable(wx.grid.GridTableBase):
+    """
+    A data table that holds data in a pandas dataframe
+    """
+    def __init__(self, data=None):
+        wx.grid.GridTableBase.__init__(self)
+        self.headerRows = 1
+        if data is None:
+            data = pd.DataFrame()
+        self.data = data
+
+        # Get divergence threshold from app config
+        self.divergence_threshold = Config().get('monitor.divergence_threshold')
+
+    def GetNumberRows(self):
+        return len(self.data)
+
+    def GetNumberCols(self):
+        return len(self.data.columns) + 1
+
+    def GetValue(self, row, col):
+        if col == 0:
+            return self.data.index[row]
+        return self.data.iloc[row, col - 1]
+
+    def SetValue(self, row, col, value):
+        self.data.iloc[row, col - 1] = value
+
+    def GetColLabelValue(self, col):
+        if col == 0:
+            if self.data.index.name is None:
+                return 'Index'
+            else:
+                return self.data.index.name
+        return str(self.data.columns[col - 1])
+
+    def GetTypeName(self, row, col):
+        return wx.grid.GRID_VALUE_STRING
+
+    def GetAttr(self, row, col, prop):
+        attr = wx.grid.GridCellAttr()
+
+        # If column is last coefficient, get value and check against threshold. Highlight if diverged.
+        threshold = Config().get('monitor.divergence_threshold')
+        if col == MonitorFrame.COLUMN_LAST_COEFFICIENT:
+            value = self.GetValue(row, col)
+            if value != "":
+                value = float(value)
+                if value <= threshold:
+                    attr.SetBackgroundColour(wx.YELLOW)
+                else:
+                    attr.SetBackgroundColour(wx.WHITE)
+
+        return attr
+
+
+class GraphPanel(wx.Panel):
+    def __init__(self, parent):
+        """
+        A panel to show the graphs
+        :param parent: The parent panel
+        """
+        # Super
+        wx.Panel.__init__(self, parent)
+
+        # 3  axis, 2 price data for calculate, 2 price data for last coefficient and coefficient history.
+        # All will have axis labels and top and right boarders
+        # removed
+        self.__fig, self.__axes = plt.subplots(nrows=5, ncols=1)
+
+        # Create the canvas
+        self.__canvas = FigureCanvas(self, -1, self.__fig)
+
+        # Date format for x axes
+        self.__tick_fmt_date = matplotlib.dates.DateFormatter('%d-%b')
+        self.__tick_fmt_time = matplotlib.dates.DateFormatter('%H:%M:%S')
+
+        # Sizer etc.
+        self.__sizer = wx.BoxSizer(wx.VERTICAL)
+        self.__sizer.Add(self.__canvas, 1, wx.LEFT | wx.TOP | wx.GROW)
+        self.SetSizer(self.__sizer)
+        self.Fit()
+
+    def __del__(self):
+        # Close all plots
+        plt.close('all')
+        self.__axes = None
+        self.__fig = None
+
+    def draw(self, times, coefficients, prices=None, symbols=None, ticks=None):
+        """
+        Plot the correlations.
+        :param times: Series of time values for x axis for coefficient history chart
+        :param coefficients: Series of coefficients values for y axis of coefficient history chart
+        :param prices: Price data used to calculate base coefficient. List [Symbol1 Price Data, Symbol 2 Price Data]
+        :param symbols: Symbols. List [Symbol1, Symbol2]
+        :param ticks: Ticks used to calculate last coefficient. List [Symbol1, Symbol2]
+        :return:
+        """
+        # Clear. We will need to redraw
+        for ax in self.__axes:
+            ax.clear()
+
+        if symbols is not None and len(symbols) == 2:
+            # Axis ranges
+            price_chart_date_range = [min(min(prices[0]['time']), min(prices[1]['time'])),
+                                      max(max(prices[0]['time']), max(prices[1]['time']))]
+            tick_chart_date_range = [min(min(ticks[0]['time']), min(ticks[1]['time'])),
+                                     max(max(ticks[0]['time']), max(ticks[1]['time']))]
+
+            # Chart config
+            titles = [f"Base Coefficient Price Data for {symbols[0]}", f"Base Coefficient Price Data for {symbols[1]}",
+                      f"Coefficient Tick Data for {symbols[0]}", f"Coefficient Tick Data for {symbols[1]}",
+                      f"Coefficient History for {symbols[0]}:{symbols[1]}"]
+            xlims = [price_chart_date_range, price_chart_date_range, tick_chart_date_range, tick_chart_date_range, None]
+            ylims = [None, None, None, None, [-1, 1]]
+            xlabels = [None, None, None, None, None]
+            ylabels = ['Price', 'Price', 'Price', 'Price', 'Coefficient']
+            tick_labels = [[], prices[1]['time'], [], ticks[1]['time'], times]
+            mtick_fmts = [None, self.__tick_fmt_date, None, self.__tick_fmt_time, self.__tick_fmt_time]
+            mtick_rot = [0, 45, 0, 45, 45]
+            xdata = [prices[0]['time'], prices[1]['time'], ticks[0]['time'], ticks[1]['time'], times]
+            ydata = [prices[0]['close'], prices[1]['close'], ticks[0]['ask'], ticks[1]['ask'], coefficients]
+            types = ['plot', 'plot', 'plot', 'plot', 'scatter']
+
+            # Draw 5 charts
+            for index in range(0, len(self.__axes)):
+                # Titles and axis labels
+                self.__axes[index].set_title(titles[index])
+                self.__axes[index].set_xlabel(xlabels[index])
+                self.__axes[index].set_ylabel(ylabels[index])
+
+                # Limits
+                if xlims[index] is not None:
+                    self.__axes[index].set_xlim(xlims[index])
+
+                if ylims[index] is not None:
+                    self.__axes[index].set_ylim(ylims[index])
+
+                # Tick labels and formats
+                self.__axes[index].xaxis.set_ticklabels(tick_labels[index])
+                if mtick_fmts[index] is not None:
+                    self.__axes[index].xaxis.set_major_formatter(mtick_fmts[index])
+                plt.setp(self.__axes[index].xaxis.get_majorticklabels(), rotation=mtick_rot[index])
+
+                # Remove typ and right boarders
+                self.__axes[index].spines["top"].set_visible(False)
+                self.__axes[index].spines["right"].set_visible(False)
+
+                # Plot
+                if types[index] == 'plot':
+                    self.__axes[index].plot(xdata[index], ydata[index])
+                elif types[index] == 'scatter':
+                    self.__axes[index].scatter(xdata[index], ydata[index], s=1)
+
+            # Layout with padding between charts
+            self.__fig.tight_layout(pad=0.5)
+
+        # Redraw canvas
+        self.__canvas.draw()

mt5_correlation/mt5.py

@@ -1,80 +1,80 @@
 import pandas as pd
-import MetaTrader5 as mt5
+import MetaTrader5
 import logging
 
+# Timeframes
+TIMEFRAME_M1 = MetaTrader5.TIMEFRAME_M1
+TIMEFRAME_M2 = MetaTrader5.TIMEFRAME_M2
+TIMEFRAME_M3 = MetaTrader5.TIMEFRAME_M3
+TIMEFRAME_M4 = MetaTrader5.TIMEFRAME_M4
+TIMEFRAME_M5 = MetaTrader5.TIMEFRAME_M5
+TIMEFRAME_M6 = MetaTrader5.TIMEFRAME_M6
+TIMEFRAME_M10 = MetaTrader5.TIMEFRAME_M10
+TIMEFRAME_M12 = MetaTrader5.TIMEFRAME_M12
+TIMEFRAME_M15 = MetaTrader5.TIMEFRAME_M15
+TIMEFRAME_M20 = MetaTrader5.TIMEFRAME_M20
+TIMEFRAME_M30 = MetaTrader5.TIMEFRAME_M30
+TIMEFRAME_H1 = MetaTrader5.TIMEFRAME_H1
+TIMEFRAME_H2 = MetaTrader5.TIMEFRAME_H2
+TIMEFRAME_H3 = MetaTrader5.TIMEFRAME_H3
+TIMEFRAME_H4 = MetaTrader5.TIMEFRAME_H4
+TIMEFRAME_H6 = MetaTrader5.TIMEFRAME_H6
+TIMEFRAME_H8 = MetaTrader5.TIMEFRAME_H8
+TIMEFRAME_H12 = MetaTrader5.TIMEFRAME_H12
+TIMEFRAME_D1 = MetaTrader5.TIMEFRAME_D1
+TIMEFRAME_W1 = MetaTrader5.TIMEFRAME_W1
+TIMEFRAME_MN1 = MetaTrader5.TIMEFRAME_MN1
+
 
 class MT5:
     """
     A class to connect to and interface with MetaTrader 5
     """
 
-    # Timeframes
-    TIMEFRAME_M1 = mt5.TIMEFRAME_M1
-    TIMEFRAME_M2 = mt5.TIMEFRAME_M2
-    TIMEFRAME_M3 = mt5.TIMEFRAME_M3
-    TIMEFRAME_M4 = mt5.TIMEFRAME_M4
-    TIMEFRAME_M5 = mt5.TIMEFRAME_M5
-    TIMEFRAME_M6 = mt5.TIMEFRAME_M6
-    TIMEFRAME_M10 = mt5.TIMEFRAME_M10
-    TIMEFRAME_M12 = mt5.TIMEFRAME_M10
-    TIMEFRAME_M15 = mt5.TIMEFRAME_M15
-    TIMEFRAME_M20 = mt5.TIMEFRAME_M20
-    TIMEFRAME_M30 = mt5.TIMEFRAME_M30
-    TIMEFRAME_H1 = mt5.TIMEFRAME_H1
-    TIMEFRAME_H2 = mt5.TIMEFRAME_H2
-    TIMEFRAME_H3 = mt5.TIMEFRAME_H3
-    TIMEFRAME_H4 = mt5.TIMEFRAME_H4
-    TIMEFRAME_H6 = mt5.TIMEFRAME_H6
-    TIMEFRAME_H8 = mt5.TIMEFRAME_H8
-    TIMEFRAME_H12 = mt5.TIMEFRAME_H12
-    TIMEFRAME_D1 = mt5.TIMEFRAME_D1
-    TIMEFRAME_W1 = mt5.TIMEFRAME_W1
-    TIMEFRAME_MN1 = mt5.TIMEFRAME_MN1
-
     def __init__(self):
         # Connect to MetaTrader5. Opens if not already open.
 
         # Logger
-        self.log = logging.getLogger(__name__)
+        self.__log = logging.getLogger(__name__)
 
         # Open MT5 and log error if it could not open
-        if not mt5.initialize():
-            self.log.error("initialize() failed")
-            mt5.shutdown()
+        if not MetaTrader5.initialize():
+            self.__log.error("initialize() failed")
+            MetaTrader5.shutdown()
 
         # Print connection status
-        self.log.debug(mt5.terminal_info())
+        self.__log.debug(MetaTrader5.terminal_info())
 
         # Print data on MetaTrader 5 version
-        self.log.debug(mt5.version())
+        self.__log.debug(MetaTrader5.version())
 
     def __del__(self):
         # shut down connection to the MetaTrader 5 terminal
-        mt5.shutdown()
+        MetaTrader5.shutdown()
 
     def get_symbols(self):
         """
         Gets list of symbols open in MT5 market watch.
-        :return: list of symbols
+        :return: list of symbol names
         """
         # Iterate symbols and get those in market watch.
-        symbols = mt5.symbols_get()
+        symbols = MetaTrader5.symbols_get()
         selected_symbols = []
         for symbol in symbols:
             if symbol.visible:
-                selected_symbols.append(symbol)
+                selected_symbols.append(symbol.name)
 
         # Log symbol counts
-        total_symbols = mt5.symbols_total()
+        total_symbols = MetaTrader5.symbols_total()
         num_selected_symbols = len(selected_symbols)
-        self.log.info(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
+        self.__log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
 
         return selected_symbols
 
     def get_prices(self, symbol, from_date, to_date, timeframe):
         """
         Gets OHLC price data for the specified symbol.
-        :param symbol: The MT5 symbol to get the price data for
+        :param symbol: The name of the symbol to get the price data for.
         :param from_date: Date from when to retrieve data
         :param to_date: Date where to receive data to
         :param timeframe: The timeframe for the candes. Possible values are:
@@ -101,15 +101,46 @@ class MT5:
             TIMEFRAME_MN1: 1 month
         :return: Price data for symbol as dataframe
         """
-        # Get prices from MT5
-        prices = mt5.copy_rates_range(symbol.name, timeframe, from_date, to_date)
-        self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.")
 
-        # Create dataframe from data and convert time in seconds to datetime format
-        prices_dataframe = pd.DataFrame(prices)
-        prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
+        prices_dataframe = None
+
+        # Get prices from MT5
+        prices = MetaTrader5.copy_rates_range(symbol, timeframe, from_date, to_date)
+        if prices is not None:
+            self.__log.debug(f"{len(prices)} prices retrieved for {symbol}.")
+
+            # Create dataframe from data and convert time in seconds to datetime format
+            prices_dataframe = pd.DataFrame(prices)
+            prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
 
         return prices_dataframe
 
+    def get_ticks(self, symbol, from_date, to_date):
+        """
+        Gets OHLC price data for the specified symbol.
+        :param symbol: The name of the symbol to get the price data for.
+        :param from_date: Date from when to retrieve data
+        :param to_date: Date where to receive data to
+        :return: Tick data for symbol as dataframe
+        """
 
+        ticks_dataframe = None
 
+        # Get ticks from MT5
+        ticks = MetaTrader5.copy_ticks_range(symbol, from_date, to_date, MetaTrader5.COPY_TICKS_ALL)
+
+        # If ticks is None, there was an error
+        if ticks is None:
+            error = MetaTrader5.last_error()
+            self.__log.error(f"Error retrieving ticks for {symbol}: {error}")
+        else:
+            self.__log.debug(f"{len(ticks)} ticks retrieved for {symbol}.")
+
+            # Create dataframe from data and convert time in seconds to datetime format
+            try:
+                ticks_dataframe = pd.DataFrame(ticks)
+                ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s')
+            except RecursionError:
+                self.__log.warning("Error converting ticks to dataframe.")
+
+        return ticks_dataframe

requirements.txt

@@ -4,4 +4,5 @@ MetaTrader5==5.0.34
 pytz==2021.1
 scipy==1.6.0
 logging==0.4.9.6
-pyyaml==5.4.1
+pyyaml==5.4.1
+wxpython==4.1.1

test/test_config.py

@@ -0,0 +1,49 @@
+import unittest
+from mt5_correlation.config import Config
+
+
+class TestConfig(unittest.TestCase):
+    def test_load_and_get(self):
+        config = Config()
+        config.load("testconfig.yaml")
+        val121 = config.get('test1.test1_2.val1_2_1')
+        self.assertEqual(val121, 'val1_2_1', "Get returned incorrect value.")
+
+    def test_set_and_save(self):
+        config = Config()
+        config.load("testconfig.yaml")
+
+        path = 'test1.test1_2.val1_2_1'
+
+        # Save new value, storing orig so we can restore later
+        orig_value = config.get(path)
+        config.set(path, "newval")
+        config.save()
+
+        # Reopen config and get value to see if it is previously saved value
+        config = Config()
+        config.load("testconfig.yaml")
+        saved_value = config.get(path)
+        self.assertEqual(saved_value, 'newval', "New value was not saved and returned.")
+
+        # Restore and save file
+        config.set(path, orig_value)
+        config.save()
+
+    def test_get_root_nodes(self):
+        config = Config()
+        config.load("testconfig.yaml")
+
+        # Get root nodes
+        root_nodes = config.get_root_nodes()
+
+        # There should be 2
+        self.assertTrue(len(root_nodes) == 2, "There should be 2 root nodes.")
+
+        # The first should be test1 and the second should be test2
+        self.assertEqual(root_nodes[0], 'test1', "First root node should be 'test1'.")
+        self.assertEqual(root_nodes[1], 'test2', "Second root node should be 'test2'.")
+
+
+if __name__ == '__main__':
+    unittest.main()

test/testconfig.yaml

@@ -0,0 +1,20 @@
+---
+test1:
+  test1_1:
+    val1_1_1: val1_1_1
+    val1_1_2: val1_1_2
+    val1_1_3: val1_1_3
+  test1_2:
+    val1_2_1: val1_2_1
+    val1_2_2: val1_2_2
+    val1_2_3: val1_2_3
+test2:
+  test2_1:
+    val2_1_1: val1_1_1
+    val2_1_2: val1_1_2
+    val2_1_3: val1_1_3
+  test2_2:
+    val2_2_1: val1_2_1
+    val2_2_2: val1_2_2
+    val2_2_3: val1_2_3
+...