Added UI and basic monitoring

.gitignore

@@ -1,4 +1,7 @@
 /venv/
-/log/
-/out/
 /.idea/workspace.xml
+/gui/*.bak
+/gui/*.py
+/gui/#~wxg.autosave~app_design.wxg#
+/*.log
+/*.log.?

.idea/mt5-correlation.iml

@@ -10,4 +10,7 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
+  <component name="PyDocumentationSettings">
+    <option name="renderExternalDocumentation" value="true" />
+  </component>
 </module>

README.md

@@ -5,14 +5,16 @@ Calculates correlation coefficient between all symbols in MetaTrader5 Market Wat
 1) Set up your MetaTrader 5 environment ensuring that all symbols that you would like to assess for correlation are shown in your Market Watch window;
 2) Set up your python environment; and
 3) Install the required libraries.
+
 ```
-pip install -r mt5-correlation\requirements.txt
+pip install -r mt5-correlation/requirements.txt
 ```
 
 # Usage
 If you set up a virtual environment in the Setup step, ensure this is activated. Then run the script.
+
 ```
-python -m mt5_correlations\mt5_correlations.py
+python -m mt5_correlations/get_correlations.py
 ```
 
 A .csv file containing the correlation coefficient for all combinations of sybmols from the MetaTrader market watch will be produced in the current directory.
@@ -29,7 +31,7 @@ A .csv file containing the correlation coefficient for all combinations of sybmo
 |EU50Cash    |FRA40Cash   |0.99072    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
 
 # Customising
-Edit mt5_correlations.py to customise.
+Edit get_correlations.py to customise.
 
 The coefficients are calculated only if:
 * The smallest set of price data is no less than 90% of the size of the largest set;

config.yaml

@@ -0,0 +1,19 @@
+---
+calculate:
+  from:
+    days: 7
+  timeframe: 15
+  min_prices: 400
+  max_set_size_diff_pct: 90
+  overlap_pct: 90
+  max_p_value: 0.05
+monitor:
+  from:
+    minutes: 10
+  interval: 10
+  min_prices: 400
+  max_set_size_diff_pct: 50
+  overlap_pct: 50
+  max_p_value: 0.05
+  divergence_threshold: 0.8
+...

logging_conf.yaml

@@ -13,17 +13,25 @@ formatters:
 
 handlers:
     console:
-        level: WARNING
+        level: INFO
         class: logging.StreamHandler
         formatter: brief
         stream: ext://sys.stdout
+    file:
+        level: DEBUG
+        class: logging.handlers.RotatingFileHandler
+        formatter: precice
+        filename: debug.log
+        mode: a
+        maxBytes: 2560000
+        backupCount: 1
 
 root:
     level: DEBUG
-    handlers: [console]
+    handlers: [console, file]
 
 loggers:
     mt5-correlation:
         level: DEBUG
-        handlers: [console]
+        handlers: [console, file]
         propagate: 0

mt5_correlation.py

@@ -1,79 +1,25 @@
-from datetime import datetime, timedelta
-import logging.config
-import pandas as pd
-import pytz
-import yaml
-from mt5_correlation.mt5 import MT5
-from mt5_correlation.correlation import Correlation
+"""
+Application to monitor previously correlated symbol pairs for correlation divergence.
+"""
 import definitions
+import yaml
+import logging.config
+from mt5_correlation.gui import MonitorFrame
+from mt5_correlation.config import Config
+import wx
 
-# Configure logger
-with open(fr'{definitions.ROOT_DIR}\logging_conf.yaml', 'rt') as file:
-    config = yaml.safe_load(file.read())
-    logging.config.dictConfig(config)
-    log = logging.getLogger()
+if __name__ == "__main__":
+    # Configure the logger
+    with open(fr'{definitions.ROOT_DIR}\logging_conf.yaml', 'rt') as file:
+        config = yaml.safe_load(file.read())
+        logging.config.dictConfig(config)
 
-# Create mt5 class. This contains required methods for interacting with MT5.
-mt5 = MT5()
+    # Load the config
+    config = Config.instance()
+    config.load(fr"{definitions.ROOT_DIR}\config.yaml")
 
-# Gte all visible symbols
-symbols = mt5.get_symbols()
-
-# set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
-timezone = pytz.timezone("Etc/UTC")
-utc_to = datetime.now(tz=timezone)
-utc_from = utc_to - timedelta(days=7)
-
-# Set timeframe
-timeframe = mt5.TIMEFRAME_M15
-
-# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
-price_data = {}
-for symbol in symbols:
-    price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to,
-                                             timeframe=timeframe)
-
-# Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
-# eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
-# avoid duplicating. We will store all coefficients in a dataframe for export as CSV.
-columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe']
-coefficients = pd.DataFrame(columns=columns)
-
-index = 0
-# There will be (x^2 - x) / 2 pairs where x is number of symbols
-num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
-
-for i in range(0, len(symbols)):
-    symbol1 = symbols[i]
-
-    for j in range(i + 1, len(symbols)):
-        symbol2 = symbols[j]
-        index += 1
-
-        # Get price data for both symbols
-        symbol1_price_data = price_data[symbol1.name]
-        symbol2_price_data = price_data[symbol2.name]
-
-        # Get coefficient and store if valid
-        coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data,
-                                                        symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90,
-                                                        overlap_pct=90, max_p_value=0.05)
-
-        if coefficient is not None:
-            coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
-                                                'Coefficient': coefficient, 'UTC Date From': utc_from,
-                                                'UTC Date To': utc_to, 'Timeframe': timeframe}, ignore_index=True)
-
-            log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of "
-                     f"{coefficient}.")
-        else:
-            log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not"
-                     f" be calculated.")
-
-# Sort, highest correlated first
-coefficients = coefficients.sort_values('Coefficient', ascending=False)
-
-# Save as CSV
-filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S}.csv"
-log.info(f"Saving coefficients as '{filename}'.")
-coefficients.to_csv(filename, index=False)
+    # Start the app
+    app = wx.App(False)
+    frame = MonitorFrame(None, wx.ID_ANY, "")
+    frame.Show()
+    app.MainLoop()

mt5_correlation/config.py

@@ -0,0 +1,84 @@
+import yaml
+import definitions
+
+
+class Config(object):
+    """
+    Provides access to application configuration parameters stored in config.yaml.
+    """
+
+    _config = None
+    _path = None
+    _instance = None
+
+    def __init__(self):
+        """
+        Singleton. Raise runtime error
+        """
+        raise RuntimeError('Call instance() instead')
+
+    @classmethod
+    def instance(cls):
+        """
+        Singleton. Get instance of this class. Create if not already created.
+        :return:
+        """
+        if cls._instance is None:
+            cls._instance = cls.__new__(cls)
+        return cls._instance
+
+    def load(self, path):
+        """
+        Loads the applications config file
+        :param path: Path to config file
+        :return:
+        """
+        with open(path, 'r') as yamlfile:
+            self._config = yaml.safe_load(yamlfile)
+
+        # Store path so that we can save later
+        self._path = path
+
+    def save(self):
+        """
+        Saves config file
+        :return:
+        """
+
+        with open(self._path, 'w') as file:
+            file.write("---\n")
+            yaml.dump(self._config, file, sort_keys=False)
+            file.write("...")
+
+    def get(self, path):
+        """
+        Gets a config property value.
+        :param path: path to property. Path separated by .
+        :return: property value
+        """
+
+        elements = path.split('.')
+        last = None
+
+        for element in elements:
+            if last is None:
+                last = self._config[element]
+            else:
+                last = last[element]
+
+        return last
+
+    def set(self, path, value):
+        """
+        Sets a config property value
+        :param path: path to property. Path separated by .
+        :param value: Value to set property to
+        :return:
+        """
+        obj = self._config
+        key_list = path.split(".")
+
+        for k in key_list[:-1]:
+            obj = obj[k]
+
+        obj[key_list[-1]] = value

mt5_correlation/correlation.py

@@ -1,26 +1,305 @@
 import math
+import logging
+import pandas as pd
+from datetime import datetime
+import time
+import sched
+import threading
+import pytz
 from scipy.stats.stats import pearsonr
 
+from mt5_correlation.mt5 import MT5
+
 
 class Correlation:
     """
-    A class to calculate the correlation coefficient between two sets of price data
+    A class to maintain the state of the calculated correlation coefficients.
     """
 
-    @staticmethod
-    def calculate_coefficient(symbol1_prices, symbol2_prices, max_set_size_diff_pct=90, overlap_pct=90,
-                              max_p_value=0.05):
-        """
-        Calculates the correlation coefficient between two sets of price data. Uses close price.
+    min_coefficient = 0.9
+    monitoring = False  # Monitoring cor correlations
 
-        :param symbol1_prices:
-        :param symbol2_prices:
+    def __init__(self):
+        self.log = logging.getLogger(__name__)
+
+        # Create dataframe
+        columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe',
+                   'Last Check', 'Last Coefficient']
+        self.coefficient_data = pd.DataFrame(columns=columns)
+
+        # Create timer for continuous monitoring
+        self.scheduler = sched.scheduler(time.time, time.sleep)
+
+    def load(self, filename):
+        """
+        Loads a csv file containing calculated coefficients
+        :param filename:
+        :return:
+        """
+        self.coefficient_data = pd.read_csv(filename)
+
+    def save(self, filename):
+        """
+        Saves the calculated coefficients as a csv file
+        :param filename:
+        :return:
+        """
+        self.coefficient_data.to_csv(filename, index=False)
+
+    def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                  max_p_value=0.05):
+        """
+        Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch. Updates coefficient data.
+
+        :param date_from: From date for price data from which to calculate correlation coefficients
+        :param date_to: To date for price data from which to calculate correlation coefficients
+        :param timeframe: Timeframe for price data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+
+        :return:
+        """
+
+        # Create mt5 class. This contains required methods for interacting with MT5.
+        mt5 = MT5()
+
+        # Gte all visible symbols
+        symbols = mt5.get_symbols()
+
+        # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
+        price_data = {}
+        for symbol in symbols:
+            price_data[symbol] = mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to,
+                                                timeframe=timeframe)
+
+        # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
+        # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
+        # avoid duplicating. We will store all coefficients in a dataframe for export as CSV.
+        index = 0
+        # There will be (x^2 - x) / 2 pairs where x is number of symbols
+        num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
+
+        for i in range(0, len(symbols)):
+            symbol1 = symbols[i]
+
+            for j in range(i + 1, len(symbols)):
+                symbol2 = symbols[j]
+                index += 1
+
+                # Get price data for both symbols
+                symbol1_price_data = price_data[symbol1]
+                symbol2_price_data = price_data[symbol2]
+
+                # Get coefficient and store if valid
+                coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data,
+                                                         symbol2_prices=symbol2_price_data,
+                                                         min_prices=min_prices,
+                                                         max_set_size_diff_pct=max_set_size_diff_pct,
+                                                         overlap_pct=overlap_pct, max_p_value=max_p_value)
+
+                if coefficient is not None:
+
+                    self.coefficient_data = \
+                        self.coefficient_data.append({'Symbol 1': symbol1, 'Symbol 2': symbol2,
+                                                      'Base Coefficient': coefficient, 'UTC Date From': date_from,
+                                                      'UTC Date To': date_to, 'Timeframe': timeframe},
+                                                     ignore_index=True)
+                    self.log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a "
+                                   f"coefficient of {coefficient}.")
+                else:
+                    self.log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:"
+                                   f"{symbol2} could no be calculated.")
+
+        # Sort, highest correlated first
+        self.coefficient_data = self.coefficient_data.sort_values('Base Coefficient', ascending=False)
+
+    def update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90,
+                           overlap_pct=90, max_p_value=0.05):
+        """
+        Updates the coefficient for the specified symbol pair
+        :param symbol1: Name of symbol to calculate coefficient for.
+        :param symbol2: Name of symbol to calculate coefficient for.
+        :param date_from: From date for tick data from which to calculate correlation coefficients
+        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
             within this pct of each other
         :param overlap_pct:
         :param max_p_value: The maximum p value for the correlation to be meaningful
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
+
+        # Get the tick data
+        mt5 = MT5()
+        symbol1ticks = mt5.get_ticks(symbol=symbol1, from_date=date_from, to_date=date_to)
+        symbol2ticks = mt5.get_ticks(symbol=symbol2, from_date=date_from, to_date=date_to)
+
+        # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one
+        # tick per second and ensuring that times can match. We will need to set the index to time for the resample
+        # then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price
+        if symbol1ticks is not None and symbol2ticks is not None and \
+                len(symbol1ticks.index) > 0 and len(symbol2ticks.index) > 0:
+            symbol1ticks.set_index('time', inplace=True)
+            symbol2ticks.set_index('time', inplace=True)
+            symbol1prices = symbol1ticks['ask'].resample('1S').ohlc()
+            symbol2prices = symbol2ticks['ask'].resample('1S').ohlc()
+            symbol1prices.reset_index(inplace=True)
+            symbol2prices.reset_index(inplace=True)
+            symbol1prices = symbol1prices[symbol1prices['close'].notna()]
+            symbol2prices = symbol2prices[symbol2prices['close'].notna()]
+
+            # Calculate the coefficient
+            coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices,
+                                                     min_prices=min_prices,
+                                                     max_set_size_diff_pct=max_set_size_diff_pct,
+                                                     overlap_pct=overlap_pct, max_p_value=max_p_value)
+        else:
+            coefficient = None
+
+        # Find the correct row in the coefficient data and update with calculation date and calculated coefficient
+        timezone = pytz.timezone("Etc/UTC")
+        now = datetime.now(tz=timezone)
+
+        # Update data if we have a coefficient
+        if coefficient is not None:
+            self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
+                                      (self.coefficient_data['Symbol 2'] == symbol2),
+                                      'Last Check'] = now
+
+            self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
+                                      (self.coefficient_data['Symbol 2'] == symbol2),
+                                      'Last Coefficient'] = coefficient
+
+        return coefficient
+
+    def update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                                max_p_value=0.05):
+        """
+        Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
+        the threshold can be accessed through the filtered_coefficient_data property.
+
+        :param date_from: From date for tick data from which to calculate correlation coefficients
+        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+        # Update  latest coefficient for every pair
+        for index, row in self.filtered_coefficient_data.iterrows():
+            symbol1 = row['Symbol 1']
+            symbol2 = row['Symbol 2']
+            self.update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to,
+                                    min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
+                                    overlap_pct=overlap_pct, max_p_value=max_p_value)
+
+    def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                      max_p_value=0.05):
+        """
+        Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
+        threshold.
+
+        :param interval: How often to check in seconds
+        :param date_from: From date for tick data from which to calculate correlation coefficients
+        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+
+        :return:
+        """
+        self.log.debug(f"Starting monitor.")
+        self.monitoring = True
+
+        # Create thread to run monitoring This will call private __monitor method that will run the calculation and
+        # keep scheduling itself while self.monitoring is True
+        params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
+                  'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
+                  'max_p_value': max_p_value}
+        thread = threading.Thread(target=self.__monitor, kwargs=params)
+        thread.start()
+
+    def stop_monitor(self):
+        """
+        Stops monitoring symbol pairs for correlation.
+        :return:
+        """
+        self.log.debug(f"Stopping monitor.")
+        self.monitoring = False
+
+    def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                  max_p_value=0.05):
+        """
+        The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
+        stop_monitoring.
+
+        :param interval: How often to check in seconds
+        :param date_from: From date for tick data from which to calculate correlation coefficients
+        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        :return:
+        """
+        self.log.debug(f"In monitor event. Monitoring: {self.monitoring}.")
+
+        # Only run if monitor is not stopped
+        if self.monitoring:
+            # Update all coefficients
+            self.update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices,
+                                         max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
+                                         max_p_value=max_p_value)
+
+            # Schedule the timer to run again
+            params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
+                      'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
+                      'max_p_value': max_p_value}
+            self.scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
+            self.scheduler.run()
+
+    @property
+    def filtered_coefficient_data(self):
+        """
+        :return: Coefficient data filtered so that all base coefficients >= min coefficient
+        """
+        if self.coefficient_data is not None:
+            return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.min_coefficient]
+        else:
+            return None
+
+    @staticmethod
+    def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90,
+                              overlap_pct=90, max_p_value=0.05):
+        """
+        Calculates the correlation coefficient between two sets of price data. Uses close price.
+
+        :param symbol1_prices: prices or ticks for symbol 1
+        :param symbol2_prices: prices or ticks for symbol 2
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+
         # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
         # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of
         # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set?
@@ -31,7 +310,8 @@ class Correlation:
         len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
         similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set
         enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100)
-        suitable = similar_size and enough_overlap
+        enough_prices = len_smallest_set >= min_prices
+        suitable = similar_size and enough_overlap and enough_prices
 
         if suitable:
             # Calculate coefficient on close prices
@@ -50,6 +330,3 @@ class Correlation:
                 coefficient = None
 
         return coefficient
-
-
-

mt5_correlation/gui.py

@@ -0,0 +1,343 @@
+import wx
+import wx.grid
+import wx.lib.masked as masked
+from mt5_correlation.correlation import Correlation
+from mt5_correlation.config import Config
+from datetime import datetime, timedelta
+import pytz
+import pandas as pd
+import logging
+import definitions
+
+
+class MonitorFrame(wx.Frame):
+
+    cor = None
+    rows = 0  # Need to track as we need to notify grid if row count changes.
+    opened_filename = None  # So we can save to same file as we opened
+    config = None  # The applications config
+
+    COLUMN_INDEX = 0
+    COLUMN_SYMBOL1 = 1
+    COLUMN_SYMBOL2 = 2
+    COLUMN_BASE_COEFFICIENT = 3
+    COLUMN_DATE_FROM = 4
+    COLUMN_DATE_TO = 5
+    COLUMN_TIMEFRAME = 6
+    COLUMN_LAST_CHECK = 7
+    COLUMN_LAST_COEFFICIENT = 8
+
+    def __init__(self, *args, **kwds):
+        self.log = logging.getLogger(__name__)
+        self.config = Config.instance()
+
+        # Create correlation instance to maintain state of calculated coefficients
+        self.cor = Correlation()
+
+        kwds["style"] = kwds.get("style", 0) | wx.DEFAULT_FRAME_STYLE
+        wx.Frame.__init__(self, *args, **kwds)
+        self.SetSize((1235, 800))
+        self.SetTitle("Monitor for Divergence")
+
+        # Status bar
+        self.statusbar = self.CreateStatusBar(1)
+
+        # Menu Bar
+        self.menubar = wx.MenuBar()
+        file_menu = wx.Menu()
+
+        # Open and save
+        menu_item_open = file_menu.Append(wx.ID_ANY, "Open", "Open correlations file.")
+        menu_item_save = file_menu.Append(wx.ID_ANY, "Save", "Save correlations file.")
+        menu_item_saveas = file_menu.Append(wx.ID_ANY, "Save As", "Save correlations file.")
+
+        # Calculate
+        file_menu.AppendSeparator()
+        menu_item_calculate = file_menu.Append(wx.ID_ANY, "Calculate", "Calculate base coefficients.")
+
+        # Close
+        file_menu.AppendSeparator()
+        menu_item_exit = file_menu.Append(wx.ID_ANY, "Exit", "Close the application")
+
+        # Add file menu and set menu bar
+        self.menubar.Append(file_menu, "File")
+        self.SetMenuBar(self.menubar)
+
+        # Main window
+        self.window = wx.SplitterWindow(self, wx.ID_ANY)
+        self.window.SetMinimumPaneSize(20)
+
+        self.correlations_pane = wx.Panel(self.window, wx.ID_ANY)
+
+        sizer_grid = wx.BoxSizer(wx.VERTICAL)
+
+        # Filter label, input box and button
+        sizer_coefficient = wx.BoxSizer(wx.HORIZONTAL)
+        sizer_grid.Add(sizer_coefficient, 0, 0, 0)
+
+        label_min_coefficient = wx.StaticText(self.correlations_pane, wx.ID_ANY, "Min Coefficient (Range -1 - 1)")
+        sizer_coefficient.Add(label_min_coefficient, 0, wx.ALL, 0)
+
+        self.edit_ctrl_min_coefficient = masked.NumCtrl(self.correlations_pane, value=self.cor.min_coefficient,
+                                                        allowNegative=True, min=-1, max=1, integerWidth=1,
+                                                        fractionWidth=5)
+        sizer_coefficient.Add(self.edit_ctrl_min_coefficient, 0, wx.ALL, 0)
+
+        self.filter_button = wx.Button(self.correlations_pane, wx.ID_ANY, label="Filter")
+        sizer_coefficient.Add(self.filter_button, 0, wx.ALL, 0)
+
+        self.monitor_toggle = wx.ToggleButton(self.correlations_pane, wx.ID_ANY, label="Monitoring")
+        sizer_coefficient.Add(self.monitor_toggle, 0, wx.ALL, 0)
+
+        # Data table using pandas dataframe for underlying data
+        self.table = DataTable(self.cor.filtered_coefficient_data)
+        self.grid_correlations = wx.grid.Grid(self.correlations_pane, wx.ID_ANY, size=(1, 1))
+        self.grid_correlations.SetTable(self.table, takeOwnership=True)
+        self.grid_correlations.EnableEditing(0)
+        self.grid_correlations.EnableDragRowSize(0)
+        self.grid_correlations.EnableDragGridSize(0)
+        self.grid_correlations.SetSelectionMode(wx.grid.Grid.SelectRows)
+        self.grid_correlations.SetColSize(self.COLUMN_INDEX, 0)  # Index. Hide
+        self.grid_correlations.SetColSize(self.COLUMN_SYMBOL1, 100)  # Symbol 1
+        self.grid_correlations.SetColSize(self.COLUMN_SYMBOL2, 100)  # Symbol 2
+        self.grid_correlations.SetColSize(self.COLUMN_BASE_COEFFICIENT, 100)  # Base Coefficient
+        self.grid_correlations.SetColSize(self.COLUMN_DATE_FROM, 0)  # UTC Date From. Hide
+        self.grid_correlations.SetColSize(self.COLUMN_DATE_TO, 0)  # UTC Date To. Hide
+        self.grid_correlations.SetColSize(self.COLUMN_TIMEFRAME, 0)  # Timeframe. Hide.
+        self.grid_correlations.SetColSize(self.COLUMN_LAST_CHECK, 100)  # Last Check
+        self.grid_correlations.SetColSize(self.COLUMN_LAST_COEFFICIENT, 100)  # Last Coefficient
+        sizer_grid.Add(self.grid_correlations, 1, wx.ALL | wx.EXPAND, 0)
+
+        self.charts_pane = wx.Panel(self.window, wx.ID_ANY)
+
+        # Charts
+        sizer_chart = wx.BoxSizer(wx.VERTICAL)
+
+        label_2 = wx.StaticText(self.charts_pane, wx.ID_ANY, "Charts Go Here", style=wx.ALIGN_CENTER_HORIZONTAL)
+        sizer_chart.Add(label_2, 0, wx.ALIGN_CENTER_HORIZONTAL, 0)
+
+        # Add the sizers to the panes
+        self.charts_pane.SetSizer(sizer_chart)
+        self.correlations_pane.SetSizer(sizer_grid)
+
+        # Set window split to 2 panes and layout
+        self.window.SplitVertically(self.correlations_pane, self.charts_pane)
+        self.Layout()
+
+        # Set up timer to refresh grid
+        self.timer = wx.Timer(self)
+
+        # Bind my buttons, timer, menu
+        self.filter_button.Bind(wx.EVT_BUTTON, self.change_min_coefficient)
+        self.monitor_toggle.Bind(wx.EVT_TOGGLEBUTTON, self.monitor)
+        self.Bind(wx.EVT_TIMER, self.refresh_grid, self.timer)
+        self.Bind(wx.EVT_MENU, self.open_file, menu_item_open)
+        self.Bind(wx.EVT_MENU, self.save_file, menu_item_save)
+        self.Bind(wx.EVT_MENU, self.save_file_as, menu_item_saveas)
+        self.Bind(wx.EVT_MENU, self.calculate_coefficients, menu_item_calculate)
+        self.Bind(wx.EVT_MENU, self.quit, menu_item_exit)
+
+        # Bind window close event
+        self.Bind(wx.EVT_CLOSE, self.on_close, self)
+
+    def open_file(self, event):
+        with wx.FileDialog(self, "Open Coefficients file", wildcard="CSV (*.csv)|*.csv",
+                           style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog:
+
+            if fileDialog.ShowModal() == wx.ID_CANCEL:
+                return  # the user changed their mind
+
+            # Load the file chosen by the user
+            self.opened_filename = fileDialog.GetPath()
+            self.cor.load(self.opened_filename)
+
+            # Refresh data in grid
+            self.refresh_grid(event)
+
+            self.SetStatusText(f"File {self.opened_filename} loaded.")
+
+    def save_file(self, event):
+        self.cor.save(self.opened_filename)
+        self.SetStatusText(f"File saved as {self.opened_filename}")
+
+    def save_file_as(self, event):
+        with wx.FileDialog(self, "Save Coefficients file", wildcard="CSV (*.csv)|*.csv",
+                           style=wx.FD_SAVE) as fileDialog:
+            if fileDialog.ShowModal() == wx.ID_CANCEL:
+                return  # the user changed their mind
+
+            # Save the file, changing opened filename so next save writes to new file
+            self.opened_filename = fileDialog.GetPath()
+            self.cor.save(self.opened_filename)
+
+            self.SetStatusText(f"File saved as {self.opened_filename}")
+
+    def calculate_coefficients(self, event):
+        # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
+        timezone = pytz.timezone("Etc/UTC")
+        utc_to = datetime.now(tz=timezone)
+        utc_from = utc_to - timedelta(days=self.config.get('calculate.from.days'))
+
+        # Set timeframe
+        timeframe = self.config.get('calculate.timeframe')
+
+        # Calculate
+        self.SetStatusText("Calculating coefficients.")
+        self.cor.calculate(date_from=utc_from, date_to=utc_to,
+                           timeframe=self.config.get('calculate.timeframe'),
+                           min_prices=self.config.get('calculate.min_prices'),
+                           max_set_size_diff_pct=self.config.get('calculate.max_set_size_diff_pct'),
+                           overlap_pct=self.config.get('calculate.overlap_pct'),
+                           max_p_value=self.config.get('calculate.max_p_value'))
+        self.SetStatusText("")
+
+        # Show calculated data
+        self.refresh_grid(event)
+
+    def quit(self, event):
+        self.Close()
+
+    def change_min_coefficient(self, event):
+        self.cor.min_coefficient = self.edit_ctrl_min_coefficient.GetValue()
+        self.refresh_grid(event)
+
+    def refresh_grid(self, event):
+        """
+        Refreshes grid. Notifies if rows have been added or deleted.
+        :return:
+        """
+        self.log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}")
+
+        # Update data
+        self.table.data = self.cor.coefficient_data.copy()
+
+        # Format
+        self.table.data['Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format)
+        self.table.data['Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True)
+        self.table.data['Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S')
+        self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format)
+
+        # Remove nans. The ones from the float column wil be str nan as they have been formatted
+        self.table.data = self.table.data.fillna('')
+        self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '')
+
+        # Start refresh
+        self.grid_correlations.BeginBatch()
+
+        # Check if num rows in dataframe has changed, and send appropriate APPEND or DELETE messages
+        cur_rows = len(self.cor.filtered_coefficient_data.index)
+        if cur_rows < self.rows:
+            # Data has been deleted. Send message
+            msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_DELETED,
+                                           self.rows - cur_rows, self.rows - cur_rows)
+            self.grid_correlations.ProcessTableMessage(msg)
+        elif cur_rows > self.rows:
+            # Data has been added. Send message
+            msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_APPENDED,
+                                           cur_rows - self.rows)  # how many
+            self.grid_correlations.ProcessTableMessage(msg)
+
+        self.grid_correlations.EndBatch()
+
+        # Send updated message
+        msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_REQUEST_VIEW_GET_VALUES)
+        self.grid_correlations.ProcessTableMessage(msg)
+
+        # Update row count
+        self.rows = cur_rows
+
+    def monitor(self, event):
+        # Check state of toggle button. If on, then start monitoring, else stop
+        if self.monitor_toggle.GetValue():
+            self.log.debug("Starting monitoring.")
+            self.SetStatusText("Monitoring for changes to coefficients.")
+
+            # Calculate correlations fro last 10 mins
+            timezone = pytz.timezone("Etc/UTC")
+            utc_to = datetime.now(tz=timezone)
+            utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes'))
+
+            self.timer.Start(10000)
+            self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to,
+                                   min_prices=self.config.get('monitor.min_prices'),
+                                   max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'),
+                                   overlap_pct=self.config.get('monitor.overlap_pct'),
+                                   max_p_value=self.config.get('monitor.max_p_value'))
+        else:
+            self.log.debug("Stopping monitoring.")
+            self.SetStatusText("Monitoring stopped.")
+            self.timer.Stop()
+            self.cor.stop_monitor()
+
+    def on_close(self, event):
+        """
+        Window closing. Save coefficients and stop monitoring.
+        :param event:
+        :return:
+        """
+        if self.opened_filename is not None:
+            self.cor.save(self.opened_filename)
+
+        self.cor.stop_monitor()
+
+        event.Skip()
+
+
+class DataTable(wx.grid.GridTableBase):
+    """
+    A data table that holds data in a pandas dataframe
+    """
+    def __init__(self, data=None):
+        wx.grid.GridTableBase.__init__(self)
+        self.headerRows = 1
+        if data is None:
+            data = pd.DataFrame()
+        self.data = data
+
+        # Get divergence threshold from app config
+        # Get application config
+        self.config = Config.instance()
+
+        # Get divergence threshold. This will be used by DataTable to highlight cells
+        self.divergence_threshold = self.config.get('monitor.divergence_threshold')
+
+    def GetNumberRows(self):
+        return len(self.data)
+
+    def GetNumberCols(self):
+        return len(self.data.columns) + 1
+
+    def GetValue(self, row, col):
+        if col == 0:
+            return self.data.index[row]
+        return self.data.iloc[row, col - 1]
+
+    def SetValue(self, row, col, value):
+        self.data.iloc[row, col - 1] = value
+
+    def GetColLabelValue(self, col):
+        if col == 0:
+            if self.data.index.name is None:
+                return 'Index'
+            else:
+                return self.data.index.name
+        return str(self.data.columns[col - 1])
+
+    def GetTypeName(self, row, col):
+        return wx.grid.GRID_VALUE_STRING
+
+    def GetAttr(self, row, col, prop):
+        attr = wx.grid.GridCellAttr()
+
+        # If column is last coefficient, get value and check against threshold. Highlight if diverged.
+        threshold = self.config.get('monitor.divergence_threshold')
+        if col == MonitorFrame.COLUMN_LAST_COEFFICIENT:
+            value = self.GetValue(row, col)
+            if value != "":
+                value = float(value)
+                if value <= threshold:
+                    attr.SetBackgroundColour(wx.YELLOW)
+                else:
+                    attr.SetBackgroundColour(wx.WHITE)
+
+        return attr

mt5_correlation/mt5.py

@@ -1,5 +1,5 @@
 import pandas as pd
-import MetaTrader5 as mt5
+import MetaTrader5
 import logging
 
 
@@ -9,27 +9,27 @@ class MT5:
     """
 
     # Timeframes
-    TIMEFRAME_M1 = mt5.TIMEFRAME_M1
-    TIMEFRAME_M2 = mt5.TIMEFRAME_M2
-    TIMEFRAME_M3 = mt5.TIMEFRAME_M3
-    TIMEFRAME_M4 = mt5.TIMEFRAME_M4
-    TIMEFRAME_M5 = mt5.TIMEFRAME_M5
-    TIMEFRAME_M6 = mt5.TIMEFRAME_M6
-    TIMEFRAME_M10 = mt5.TIMEFRAME_M10
-    TIMEFRAME_M12 = mt5.TIMEFRAME_M10
-    TIMEFRAME_M15 = mt5.TIMEFRAME_M15
-    TIMEFRAME_M20 = mt5.TIMEFRAME_M20
-    TIMEFRAME_M30 = mt5.TIMEFRAME_M30
-    TIMEFRAME_H1 = mt5.TIMEFRAME_H1
-    TIMEFRAME_H2 = mt5.TIMEFRAME_H2
-    TIMEFRAME_H3 = mt5.TIMEFRAME_H3
-    TIMEFRAME_H4 = mt5.TIMEFRAME_H4
-    TIMEFRAME_H6 = mt5.TIMEFRAME_H6
-    TIMEFRAME_H8 = mt5.TIMEFRAME_H8
-    TIMEFRAME_H12 = mt5.TIMEFRAME_H12
-    TIMEFRAME_D1 = mt5.TIMEFRAME_D1
-    TIMEFRAME_W1 = mt5.TIMEFRAME_W1
-    TIMEFRAME_MN1 = mt5.TIMEFRAME_MN1
+    TIMEFRAME_M1 = MetaTrader5.TIMEFRAME_M1
+    TIMEFRAME_M2 = MetaTrader5.TIMEFRAME_M2
+    TIMEFRAME_M3 = MetaTrader5.TIMEFRAME_M3
+    TIMEFRAME_M4 = MetaTrader5.TIMEFRAME_M4
+    TIMEFRAME_M5 = MetaTrader5.TIMEFRAME_M5
+    TIMEFRAME_M6 = MetaTrader5.TIMEFRAME_M6
+    TIMEFRAME_M10 = MetaTrader5.TIMEFRAME_M10
+    TIMEFRAME_M12 = MetaTrader5.TIMEFRAME_M10
+    TIMEFRAME_M15 = MetaTrader5.TIMEFRAME_M15
+    TIMEFRAME_M20 = MetaTrader5.TIMEFRAME_M20
+    TIMEFRAME_M30 = MetaTrader5.TIMEFRAME_M30
+    TIMEFRAME_H1 = MetaTrader5.TIMEFRAME_H1
+    TIMEFRAME_H2 = MetaTrader5.TIMEFRAME_H2
+    TIMEFRAME_H3 = MetaTrader5.TIMEFRAME_H3
+    TIMEFRAME_H4 = MetaTrader5.TIMEFRAME_H4
+    TIMEFRAME_H6 = MetaTrader5.TIMEFRAME_H6
+    TIMEFRAME_H8 = MetaTrader5.TIMEFRAME_H8
+    TIMEFRAME_H12 = MetaTrader5.TIMEFRAME_H12
+    TIMEFRAME_D1 = MetaTrader5.TIMEFRAME_D1
+    TIMEFRAME_W1 = MetaTrader5.TIMEFRAME_W1
+    TIMEFRAME_MN1 = MetaTrader5.TIMEFRAME_MN1
 
     def __init__(self):
         # Connect to MetaTrader5. Opens if not already open.
@@ -38,43 +38,43 @@ class MT5:
         self.log = logging.getLogger(__name__)
 
         # Open MT5 and log error if it could not open
-        if not mt5.initialize():
+        if not MetaTrader5.initialize():
             self.log.error("initialize() failed")
-            mt5.shutdown()
+            MetaTrader5.shutdown()
 
         # Print connection status
-        self.log.debug(mt5.terminal_info())
+        self.log.debug(MetaTrader5.terminal_info())
 
         # Print data on MetaTrader 5 version
-        self.log.debug(mt5.version())
+        self.log.debug(MetaTrader5.version())
 
     def __del__(self):
         # shut down connection to the MetaTrader 5 terminal
-        mt5.shutdown()
+        MetaTrader5.shutdown()
 
     def get_symbols(self):
         """
         Gets list of symbols open in MT5 market watch.
-        :return: list of symbols
+        :return: list of symbol names
         """
         # Iterate symbols and get those in market watch.
-        symbols = mt5.symbols_get()
+        symbols = MetaTrader5.symbols_get()
         selected_symbols = []
         for symbol in symbols:
             if symbol.visible:
-                selected_symbols.append(symbol)
+                selected_symbols.append(symbol.name)
 
         # Log symbol counts
-        total_symbols = mt5.symbols_total()
+        total_symbols = MetaTrader5.symbols_total()
         num_selected_symbols = len(selected_symbols)
-        self.log.info(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
+        self.log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
 
         return selected_symbols
 
     def get_prices(self, symbol, from_date, to_date, timeframe):
         """
         Gets OHLC price data for the specified symbol.
-        :param symbol: The MT5 symbol to get the price data for
+        :param symbol: The name of the symbol to get the price data for.
         :param from_date: Date from when to retrieve data
         :param to_date: Date where to receive data to
         :param timeframe: The timeframe for the candes. Possible values are:
@@ -101,9 +101,10 @@ class MT5:
             TIMEFRAME_MN1: 1 month
         :return: Price data for symbol as dataframe
         """
+
         # Get prices from MT5
-        prices = mt5.copy_rates_range(symbol.name, timeframe, from_date, to_date)
-        self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.")
+        prices = MetaTrader5.copy_rates_range(symbol, timeframe, from_date, to_date)
+        self.log.debug(f"{len(prices)} prices retrieved for {symbol}.")
 
         # Create dataframe from data and convert time in seconds to datetime format
         prices_dataframe = pd.DataFrame(prices)
@@ -111,5 +112,28 @@ class MT5:
 
         return prices_dataframe
 
+    def get_ticks(self, symbol, from_date, to_date):
+        """
+        Gets OHLC price data for the specified symbol.
+        :param symbol: The name of the symbol to get the price data for.
+        :param from_date: Date from when to retrieve data
+        :param to_date: Date where to receive data to
+        :return: Tick data for symbol as dataframe
+        """
 
+        # Get ticks from MT5
+        ticks = MetaTrader5.copy_ticks_range(symbol, from_date, to_date, MetaTrader5.COPY_TICKS_ALL)
 
+        # If ticks is None, there was an error
+        if ticks is None:
+            error = MetaTrader5.last_error()
+            self.log.error(f"Error retrieving ticks for {symbol}: {error}")
+            return None
+        else:
+            self.log.debug(f"{len(ticks)} ticks retrieved for {symbol}.")
+
+            # Create dataframe from data and convert time in seconds to datetime format
+            ticks_dataframe = pd.DataFrame(ticks)
+            ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s')
+
+        return ticks_dataframe

requirements.txt

@@ -4,4 +4,5 @@ MetaTrader5==5.0.34
 pytz==2021.1
 scipy==1.6.0
 logging==0.4.9.6
-pyyaml==5.4.1
+pyyaml==5.4.1
+wxpython==4.1.1

test/test_config.py

@@ -0,0 +1,35 @@
+import unittest
+from mt5_correlation.config import Config
+
+
+class TestConfig(unittest.TestCase):
+    def test_load_and_get(self):
+        config = Config.instance()
+        config.load("testconfig.yaml")
+        val121 = config.get('test1.test1_2.val1_2_1')
+        self.assertEqual(val121, 'val1_2_1', "Get returned incorrect value.")
+
+    def test_set_and_save(self):
+        config = Config.instance()
+        config.load("testconfig.yaml")
+
+        path = 'test1.test1_2.val1_2_1'
+
+        # Save new value, storing orig so we can restore later
+        orig_value = config.get(path)
+        config.set(path, "newval")
+        config.save()
+
+        # Reopen config and get value to see if it is previously saved value
+        config = Config.instance()
+        config.load("testconfig.yaml")
+        saved_value = config.get(path)
+        self.assertEqual(saved_value, 'newval', "New value was not saved and returned.")
+
+        # Restore and save file
+        config.set(path, orig_value)
+        config.save()
+
+
+if __name__ == '__main__':
+    unittest.main()

test/testconfig.yaml

@@ -0,0 +1,20 @@
+---
+test1:
+  test1_1:
+    val1_1_1: val1_1_1
+    val1_1_2: val1_1_2
+    val1_1_3: val1_1_3
+  test1_2:
+    val1_2_1: val1_2_1
+    val1_2_2: val1_2_2
+    val1_2_3: val1_2_3
+test2:
+  test2_1:
+    val2_1_1: val1_1_1
+    val2_1_2: val1_1_2
+    val2_1_3: val1_1_3
+  test2_2:
+    val2_2_1: val1_2_1
+    val2_2_2: val1_2_2
+    val2_2_3: val1_2_3
+...