Added charts

config.yaml

@@ -3,20 +3,21 @@ calculate:
   from:
     days: 10
   timeframe: 15
-  min_prices: 500
+  min_prices: 600
   max_set_size_diff_pct: 90
   overlap_pct: 90
   max_p_value: 0.05
 monitor:
   from:
-    minutes: 60
+    minutes: 10
   interval: 10
-  min_prices: 500
-  max_set_size_diff_pct: 90
-  overlap_pct: 90
+  min_prices: 300
+  max_set_size_diff_pct: 50
+  overlap_pct: 50
   max_p_value: 0.05
   monitoring_threshold: 0.9
   divergence_threshold: 0.8
+  tick_cache_time: 10
 logging:
   version: 1
   disable_existing_loggers: false

mt5_correlation/correlation.py

@@ -1,12 +1,14 @@
 import math
 import logging
 import pandas as pd
-from datetime import datetime
+from datetime import datetime, timedelta
 import time
 import sched
 import threading
 import pytz
 from scipy.stats.stats import pearsonr
+import yaml
+import pickle
 
 from mt5_correlation.mt5 import MT5
 
@@ -16,6 +18,9 @@ class Correlation:
     A class to maintain the state of the calculated correlation coefficients.
     """
 
+    # Connection to metatrader
+    __mt5 = None
+
     # Minimum base coefficient for monitoring. Symbol pairs with a lower correlation
     # coefficient than ths won't be monitored.
     monitoring_threshold = 0.9
@@ -24,13 +29,25 @@ class Correlation:
     __monitoring = False
     __monitoring_params = {}
 
+    # The price data used to calculate the correlations
+    __price_data = None
+
+    # Coefficient data and history. Will be created as dataframes in Init
+    coefficient_data = None
+    coefficient_history = None
+
+    # Cache for ticks. Dict: {Symbol: [retrieved datetime, ticks dataframe]}
+    __ticks = {}
+
     def __init__(self):
+        # Logger
         self.__log = logging.getLogger(__name__)
 
-        # Create dataframe
-        self.__columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe',
-                          'Last Check', 'Last Coefficient']
-        self.coefficient_data = pd.DataFrame(columns=self.__columns)
+        # Connection to metatrader
+        self.__mt5 = MT5()
+
+        # Create dataframe for coefficient data
+        self.__reset_coefficient_data()
 
         # Create timer for continuous monitoring
         self.__scheduler = sched.scheduler(time.time, time.sleep)
@@ -45,22 +62,38 @@ class Correlation:
         else:
             return None
 
-    def load(self, filename):
+    def load(self, filename, price_data_filename=None):
         """
-        Loads a csv file containing calculated coefficients
-        :param filename:
+        Loads a csv file containing calculated coefficients, and optionally the price data used to calculate those
+        coefficients
+        :param filename: The filename for the coefficient data to load.
+        :param price_data_filename: The filename for the price data to load.
         :return:
         """
+        # Load coefficients file
         self.coefficient_data = pd.read_csv(filename)
 
-    def save(self, filename):
+        # If specified, load price data yaml file
+        if price_data_filename is not None:
+            self.__price_data = {}  # Clear
+            with open(price_data_filename, 'rb') as file:
+                self.__price_data = pickle.load(file)
+
+    def save(self, filename, price_data_filename=None):
         """
         Saves the calculated coefficients as a csv file
-        :param filename:
+        :param filename: The filename for the coefficient data to save to.
+        :param price_data_filename: The filename for the price data to save to.
         :return:
         """
+        # Save the coefficient data
         self.coefficient_data.to_csv(filename, index=False)
 
+        # Save the price data if required
+        if price_data_filename is not None:
+            with open(price_data_filename, 'wb') as file:
+                pickle.dump(self.__price_data, file, protocol=pickle.HIGHEST_PROTOCOL)
+
     def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
                   max_p_value=0.05):
         """
@@ -79,25 +112,24 @@ class Correlation:
         :return:
         """
 
+        coefficient = None
+
         # If we are monitoring, stop. We will need to restart later
         was_monitoring = self.__monitoring
         if self.__monitoring:
             self.stop_monitor()
 
         # Clear the existing correlations
-        self.coefficient_data = pd.DataFrame(columns=self.__columns)
+        self.__reset_coefficient_data()
 
-        # Create mt5 class. This contains required methods for interacting with MT5.
-        mt5 = MT5()
-
-        # Gte all visible symbols
-        symbols = mt5.get_symbols()
+        # Get all visible symbols
+        symbols = self.__mt5.get_symbols()
 
         # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
-        price_data = {}
+        self.__price_data = {}
         for symbol in symbols:
-            price_data[symbol] = mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to,
-                                                timeframe=timeframe)
+            self.__price_data[symbol] = self.__mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to,
+                                                              timeframe=timeframe)
 
         # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
         # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
@@ -114,16 +146,18 @@ class Correlation:
                 index += 1
 
                 # Get price data for both symbols
-                symbol1_price_data = price_data[symbol1]
-                symbol2_price_data = price_data[symbol2]
+                symbol1_price_data = self.__price_data[symbol1]
+                symbol2_price_data = self.__price_data[symbol2]
 
-                # Get coefficient and store if valid
-                coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data,
-                                                         symbol2_prices=symbol2_price_data,
-                                                         min_prices=min_prices,
-                                                         max_set_size_diff_pct=max_set_size_diff_pct,
-                                                         overlap_pct=overlap_pct, max_p_value=max_p_value)
+                # Get coefficient
+                if symbol1_price_data is not None and symbol2_price_data is not None:
+                    coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data,
+                                                             symbol2_prices=symbol2_price_data,
+                                                             min_prices=min_prices,
+                                                             max_set_size_diff_pct=max_set_size_diff_pct,
+                                                             overlap_pct=overlap_pct, max_p_value=max_p_value)
 
+                # Store if valid
                 if coefficient is not None:
 
                     self.coefficient_data = \
@@ -143,28 +177,40 @@ class Correlation:
         # If we were monitoring, we stopped, so start again.
         if was_monitoring:
             self.start_monitor(interval=self.__monitoring_params['interval'],
-                               date_from=self.__monitoring_params['date_from'],
-                               date_to=self.__monitoring_params['date_to'],
+                               from_mins=self.__monitoring_params['from_mins'],
                                min_prices=self.__monitoring_params['min_prices'],
                                max_set_size_diff_pct=self.__monitoring_params['max_set_size_diff_pct'],
                                overlap_pct=self.__monitoring_params['overlap_pct'],
                                max_p_value=self.__monitoring_params['max_p_value'])
 
-    def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                      max_p_value=0.05):
+    def get_price_data(self, symbol):
+        """
+        Returns the price data used to calculate the base coefficients for the specified symbol
+        :param symbol: Symbol to get price data for.
+        :return: price data
+        """
+        price_data = None
+        if symbol in self.__price_data:
+            price_data = self.__price_data[symbol]
+
+        return price_data
+
+    def start_monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                      max_p_value=0.05, cache_time=10):
         """
         Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
         threshold.
 
         :param interval: How often to check in seconds
-        :param date_from: From date for tick data from which to calculate correlation coefficients
-        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param from_mins: The number of minutes of tick data to use for calculations
         :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
             is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
             within this pct of each other
         :param overlap_pct:
         :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
 
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
@@ -180,9 +226,9 @@ class Correlation:
         # Create thread to run monitoring This will call private __monitor method that will run the calculation and
         # keep scheduling itself while self.monitoring is True. Store the params. We will need to use these if we have
         # to stop and restart the monitor. Note, this happens during calculate
-        self.__monitoring_params = {'interval': interval, 'date_from': date_from, 'date_to': date_to,
+        self.__monitoring_params = {'interval': interval, 'from_mins': from_mins,
                                     'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct,
-                                    'overlap_pct': overlap_pct, 'max_p_value': max_p_value}
+                                    'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time}
         thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params)
         thread.start()
 
@@ -197,23 +243,27 @@ class Correlation:
         else:
             self.__log.debug(f"Request to stop monitor when it is not running. No action taken.")
 
-    @staticmethod
-    def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90,
-                              overlap_pct=90, max_p_value=0.05):
+    def calculate_coefficient(self, symbol1_prices, symbol2_prices, min_prices: int = 100,
+                              max_set_size_diff_pct: int = 90, overlap_pct: int = 90,
+                              max_p_value: float = 0.05):
         """
         Calculates the correlation coefficient between two sets of price data. Uses close price.
 
-        :param symbol1_prices: prices or ticks for symbol 1
-        :param symbol2_prices: prices or ticks for symbol 2
+        :param symbol1_prices: Pandas dataframe containing prices or ticks for symbol 1
+        :param symbol2_prices: Pandas dataframe containing prices or ticks for symbol 2
         :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
             is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
             within this pct of each other
         :param overlap_pct:
         :param max_p_value: The maximum p value for the correlation to be meaningful
+
         :return: correlation coefficient, or None if coefficient could not be calculated.
+        :rtype: float or None
         """
 
+        assert symbol1_prices is not None and symbol2_prices is not None
+
         # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
         # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of
         # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set?
@@ -237,29 +287,47 @@ class Correlation:
             # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
             # hypothesis is false).
             coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close'])
-            coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0]
+            coefficient = None if coefficient_with_p_value[1] > max_p_value else coefficient_with_p_value[0]
 
             # If NaN, change to None
             if coefficient is not None and math.isnan(coefficient):
                 coefficient = None
 
+        self.__log.debug(f"Calculate coefficient returning {coefficient}. "
+                         f"Symbol 1 Prices: {len(symbol1_prices)}  Symbol 2 Prices: {len(symbol2_prices)} "
+                         f"Overlap Prices: {len(intersect_dates)} Similar size: {similar_size} "
+                         f"Enough overlap: {enough_overlap} Enough prices: {enough_prices} Suitable: {suitable}.")
+
         return coefficient
 
-    def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                  max_p_value=0.05):
+    def get_coefficient_history(self, symbol1, symbol2):
+        """
+        Returns the coefficient history for the specified symbol pair calculated during this instance.
+        Coefficient history does not persist between instances.
+        :param symbol1:
+        :param symbol2:
+        :return: dataframe containing history of coefficient data.
+        """
+        history = self.coefficient_history[(self.coefficient_history['Symbol 1'] == symbol1) &
+                                           (self.coefficient_history['Symbol 2'] == symbol2)]
+        return history
+
+    def __monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                  max_p_value=0.05, cache_time=10):
         """
         The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
         stop_monitoring.
 
         :param interval: How often to check in seconds
-        :param date_from: From date for tick data from which to calculate correlation coefficients
-        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param from_mins: The number of minutes of tick data to use for calculations
         :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
             is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
             within this pct of each other
         :param overlap_pct:
         :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
 
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
@@ -268,66 +336,166 @@ class Correlation:
         # Only run if monitor is not stopped
         if self.__monitoring:
             # Update all coefficients
-            self.__update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices,
+            self.__update_all_coefficients(from_mins=from_mins, min_prices=min_prices,
                                            max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
-                                           max_p_value=max_p_value)
+                                           max_p_value=max_p_value, cache_time=cache_time)
 
             # Schedule the timer to run again
-            params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
+            params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices,
                       'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
-                      'max_p_value': max_p_value}
+                      'max_p_value': max_p_value, "cache_time": cache_time}
             self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
             self.__scheduler.run()
 
-    def __update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90,
-                             overlap_pct=90, max_p_value=0.05):
+    def __update_coefficient(self, symbol1, symbol2, from_mins, min_prices=100, max_set_size_diff_pct=90,
+                             overlap_pct=90, max_p_value=0.05, cache_time=10):
         """
         Updates the coefficient for the specified symbol pair
         :param symbol1: Name of symbol to calculate coefficient for.
         :param symbol2: Name of symbol to calculate coefficient for.
-        :param date_from: From date for tick data from which to calculate correlation coefficients
-        :param date_to: To date for tick data from which to calculate correlation coefficients
+        :param from_mins: The number of minutes of tick data to use for calculations
         :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
             is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
             within this pct of each other
         :param overlap_pct:
         :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
 
+        coefficient = None
+
+        # Get dates
+        # From and to dates for calculations.
+        timezone = pytz.timezone("Etc/UTC")
+        date_to = datetime.now(tz=timezone)
+        date_from = date_to - timedelta(minutes=from_mins)
+
         # Get the tick data
-        mt5 = MT5()
-        symbol1ticks = mt5.get_ticks(symbol=symbol1, from_date=date_from, to_date=date_to)
-        symbol2ticks = mt5.get_ticks(symbol=symbol2, from_date=date_from, to_date=date_to)
+        symbol1ticks = self.__get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time)
+        symbol2ticks = self.__get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time)
 
         # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one
         # tick per second and ensuring that times can match. We will need to set the index to time for the resample
         # then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price
         if symbol1ticks is not None and symbol2ticks is not None and \
                 len(symbol1ticks.index) > 0 and len(symbol2ticks.index) > 0:
-            symbol1ticks.set_index('time', inplace=True)
-            symbol2ticks.set_index('time', inplace=True)
-            symbol1prices = symbol1ticks['ask'].resample('1S').ohlc()
-            symbol2prices = symbol2ticks['ask'].resample('1S').ohlc()
-            symbol1prices.reset_index(inplace=True)
-            symbol2prices.reset_index(inplace=True)
-            symbol1prices = symbol1prices[symbol1prices['close'].notna()]
-            symbol2prices = symbol2prices[symbol2prices['close'].notna()]
+            symbol1ticks = symbol1ticks.set_index('time')
+            symbol2ticks = symbol2ticks.set_index('time')
+            try:
+                symbol1prices = symbol1ticks['ask'].resample('1S').ohlc()
+                symbol2prices = symbol2ticks['ask'].resample('1S').ohlc()
+            except RecursionError:
+                self.__log.warning(f"Coefficient could not be calculated for {symbol1}:{symbol2} as prices could not "
+                                   f"be resampled.")
+            else:
+                symbol1prices.reset_index(inplace=True)
+                symbol2prices.reset_index(inplace=True)
+                symbol1prices = symbol1prices[symbol1prices['close'].notna()]
+                symbol2prices = symbol2prices[symbol2prices['close'].notna()]
 
-            # Calculate the coefficient
-            coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices,
-                                                     min_prices=min_prices,
-                                                     max_set_size_diff_pct=max_set_size_diff_pct,
-                                                     overlap_pct=overlap_pct, max_p_value=max_p_value)
+                # Calculate the coefficient
+                coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices,
+                                                         min_prices=min_prices,
+                                                         max_set_size_diff_pct=max_set_size_diff_pct,
+                                                         overlap_pct=overlap_pct, max_p_value=max_p_value)
+
+                self.__log.debug(f"Symbol pair {symbol1}:{symbol2} has a coefficient of {coefficient}.")
         else:
             coefficient = None
 
-        # Find the correct row in the coefficient data and update with calculation date and calculated coefficient
+        # Update the coefficient data
+        if coefficient is not None:
+            self.__update_coefficient_data(symbol1=symbol1, symbol2=symbol2, coefficient=coefficient,
+                                           date_from=date_from, date_to=date_to)
+
+        return coefficient
+
+    def __update_all_coefficients(self, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+                                  max_p_value=0.05, cache_time=10):
+        """
+        Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
+        the threshold can be accessed through the filtered_coefficient_data property.
+
+        :param from_mins: The number of minutes of tick data to use for calculations
+        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+            is not met then returned coefficient will be None
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
+            the tick data. Number of seconds to cache tick data for before it becomes stale.
+
+        :return: correlation coefficient, or None if coefficient could not be calculated.
+        """
+        # Update  latest coefficient for every pair
+        for index, row in self.filtered_coefficient_data.iterrows():
+            symbol1 = row['Symbol 1']
+            symbol2 = row['Symbol 2']
+            self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, from_mins=from_mins,
+                                      min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
+                                      overlap_pct=overlap_pct, max_p_value=max_p_value, cache_time=cache_time)
+
+    def __get_ticks(self, symbol, date_from, date_to, cache_time):
+        """
+        Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe.
+
+        :param symbol: Name of symbol to get ticks for.
+        :param date_from:
+        :param date_to:
+        :param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed,
+            get data from source and refresh cache.
+
+        :return:
+        """
+
+        timezone = pytz.timezone("Etc/UTC")
+        utc_now = datetime.now(tz=timezone)
+
+        # Check if in cache and not stale
+        if symbol in self.__ticks and utc_now < self.__ticks[symbol][0] + timedelta(seconds=cache_time):
+            # Cached ticks are not stale. Get them
+            ticks = self.__ticks[symbol][1]
+            self.__log.debug(f"Ticks for {symbol} retrieved from cache.")
+        else:
+            # Data does not exist in cache or cached data is stale. Retrieve from source and cache.
+            ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to)
+            self.__ticks[symbol] = [utc_now, ticks]
+            self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.")
+        return ticks
+
+    def __reset_coefficient_data(self):
+        """
+        Clears coefficient data and history.
+        :return:
+        """
+        # Create dataframe for coefficient data
+        coefficient_data_columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To',
+                                    'Timeframe', 'Last Check', 'Last Coefficient']
+        self.coefficient_data = pd.DataFrame(columns=coefficient_data_columns)
+
+        # Create dataframe for coefficient history
+        coefficient_history_columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To']
+        self.coefficient_history = pd.DataFrame(columns=coefficient_history_columns)
+
+    def __update_coefficient_data(self, symbol1, symbol2, coefficient, date_from, date_to):
+        """
+        Updates the coefficient data with the latest coefficient and adds to coefficient history.
+        :param symbol1:
+        :param symbol2:
+        :param coefficient:
+        :param date_from:
+        :param date_to:
+        :return:
+        """
+
         timezone = pytz.timezone("Etc/UTC")
         now = datetime.now(tz=timezone)
 
-        # Update data if we have a coefficient
+        # Update data if we have a coefficient and add to history
         if coefficient is not None:
             self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
                                       (self.coefficient_data['Symbol 2'] == symbol2),
@@ -337,28 +505,6 @@ class Correlation:
                                       (self.coefficient_data['Symbol 2'] == symbol2),
                                       'Last Coefficient'] = coefficient
 
-        return coefficient
-
-    def __update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                                  max_p_value=0.05):
-        """
-        Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
-        the threshold can be accessed through the filtered_coefficient_data property.
-
-        :param date_from: From date for tick data from which to calculate correlation coefficients
-        :param date_to: To date for tick data from which to calculate correlation coefficients
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
-        :return: correlation coefficient, or None if coefficient could not be calculated.
-        """
-        # Update  latest coefficient for every pair
-        for index, row in self.filtered_coefficient_data.iterrows():
-            symbol1 = row['Symbol 1']
-            symbol2 = row['Symbol 2']
-            self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to,
-                                      min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
-                                      overlap_pct=overlap_pct, max_p_value=max_p_value)
+            row = pd.DataFrame(columns=self.coefficient_history.columns,
+                               data=[[symbol1, symbol2, coefficient, date_from, date_to]])
+            self.coefficient_history = self.coefficient_history.append(row)

mt5_correlation/gui.py

@@ -1,5 +1,10 @@
 import wx
 import wx.grid
+import matplotlib.pyplot as plt
+from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas
+import matplotlib.dates
+import matplotlib
+
 from mt5_correlation.correlation import Correlation
 from mt5_correlation.config import Config, SettingsDialog
 from datetime import datetime, timedelta
@@ -7,15 +12,21 @@ import pytz
 import pandas as pd
 import logging
 import logging.config
+import os
+
+matplotlib.use('WXAgg')
 
 
 class MonitorFrame(wx.Frame):
 
     cor = None
     rows = 0  # Need to track as we need to notify grid if row count changes.
-    opened_filename = None  # So we can save to same file as we opened
+    __opened_filename = None  # So we can save to same file as we opened
     config = None  # The applications config
 
+    __selected_correlation = []  # List of Symbol 1 & Symbol 2
+
+    # Columns for coefficient table
     COLUMN_INDEX = 0
     COLUMN_SYMBOL1 = 1
     COLUMN_SYMBOL2 = 2
@@ -31,7 +42,7 @@ class MonitorFrame(wx.Frame):
         wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title="Divergence Monitor")
 
         # Create logger and get config
-        self.log = logging.getLogger(__name__)
+        self.__log = logging.getLogger(__name__)
         self.config = Config()
 
         # Create correlation instance to maintain state of calculated coefficients. Set min coefficient from config
@@ -76,8 +87,8 @@ class MonitorFrame(wx.Frame):
         panel = wx.Panel(self, wx.ID_ANY)
         toggle_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Label and toggle
         correlations_sizer = wx.BoxSizer(wx.VERTICAL)  # Toggle sizer and correlations grid
-        main_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Correlations sizer and graphs panel
-        panel.SetSizer(main_sizer)
+        self.__main_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Correlations sizer and graphs panel
+        panel.SetSizer(self.__main_sizer)
 
         # Create the label and toggle, populate the toggle sizer and add the toggle sizer to the correlations sizer
         monitor_toggle_label = wx.StaticText(panel, id=wx.ID_ANY, label="Monitoring")
@@ -111,12 +122,13 @@ class MonitorFrame(wx.Frame):
         self.grid_correlations.SetMaxSize((520, -1))
         correlations_sizer.Add(self.grid_correlations, 1, wx.ALL | wx.EXPAND, 1)
 
-        # Create the charts
-        charts = wx.StaticText(panel, wx.ID_ANY, "Charts Go Here", style=wx.ALIGN_CENTER_HORIZONTAL)
+        # Create the charts and hide as we have no data to display yet
+        self.__graph = GraphPanel(panel)
+        self.__graph.Hide()
 
         # Add the correlations sizer and the charts to the main sizer.
-        main_sizer.Add(correlations_sizer, 1, wx.ALL | wx.EXPAND, 1)
-        main_sizer.Add(charts, 1, wx.ALL | wx.EXPAND, 1)
+        self.__main_sizer.Add(correlations_sizer, 0, wx.ALL | wx.EXPAND, 1)
+        self.__main_sizer.Add(self.__graph, 1, wx.ALL | wx.EXPAND, 1)
 
         # Size the window.
         self.SetSize((800, 500))
@@ -129,7 +141,7 @@ class MonitorFrame(wx.Frame):
         self.monitor_toggle.Bind(wx.EVT_TOGGLEBUTTON, self.monitor)
 
         # Bind timer
-        self.Bind(wx.EVT_TIMER, self.refresh_grid, self.timer)
+        self.Bind(wx.EVT_TIMER, self.__timer_event, self.timer)
 
         # Bind menu items
         self.Bind(wx.EVT_MENU, self.open_file, menu_item_open)
@@ -139,6 +151,9 @@ class MonitorFrame(wx.Frame):
         self.Bind(wx.EVT_MENU, self.open_settings, menu_item_settings)
         self.Bind(wx.EVT_MENU, self.quit, menu_item_exit)
 
+        # Bind row select
+        self.Bind(wx.grid.EVT_GRID_SELECT_CELL, self.select_cell, self.grid_correlations)
+
         # Bind window close event
         self.Bind(wx.EVT_CLOSE, self.on_close, self)
 
@@ -149,18 +164,28 @@ class MonitorFrame(wx.Frame):
             if fileDialog.ShowModal() == wx.ID_CANCEL:
                 return  # the user changed their mind
 
-            # Load the file chosen by the user
-            self.opened_filename = fileDialog.GetPath()
-            self.cor.load(self.opened_filename)
+            # Load the file chosen by the user. Also load the corresponding data file if there is one.
+            self.__opened_filename = fileDialog.GetPath()
+            data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data"
+            if os.path.isfile(data_filename):
+                self.cor.load(self.__opened_filename, price_data_filename=data_filename)
+            else:
+                self.cor.load(self.__opened_filename)
 
             # Refresh data in grid
-            self.refresh_grid(event)
+            self.refresh_grid()
 
-            self.SetStatusText(f"File {self.opened_filename} loaded.")
+            self.SetStatusText(f"File {self.__opened_filename} loaded.")
 
     def save_file(self, event):
-        self.cor.save(self.opened_filename)
-        self.SetStatusText(f"File saved as {self.opened_filename}")
+        self.SetStatusText(f"Saving file as {self.__opened_filename}")
+
+        if self.__opened_filename is None:
+            self.save_file_as(event)
+        else:
+            self.cor.save(self.__opened_filename)
+
+        self.SetStatusText(f"File saved as {self.__opened_filename}")
 
     def save_file_as(self, event):
         with wx.FileDialog(self, "Save Coefficients file", wildcard="CSV (*.csv)|*.csv",
@@ -168,11 +193,14 @@ class MonitorFrame(wx.Frame):
             if fileDialog.ShowModal() == wx.ID_CANCEL:
                 return  # the user changed their mind
 
-            # Save the file, changing opened filename so next save writes to new file
-            self.opened_filename = fileDialog.GetPath()
-            self.cor.save(self.opened_filename)
+            # Save the file and price data file, changing opened filename so next save writes to new file
+            self.SetStatusText(f"Saving file as {self.__opened_filename}")
 
-            self.SetStatusText(f"File saved as {self.opened_filename}")
+            self.__opened_filename = fileDialog.GetPath()
+            data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data"
+            self.cor.save(self.__opened_filename, price_data_filename=data_filename)
+
+            self.SetStatusText(f"File saved as {self.__opened_filename}")
 
     def calculate_coefficients(self, event):
         # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
@@ -191,30 +219,30 @@ class MonitorFrame(wx.Frame):
         self.SetStatusText("")
 
         # Show calculated data
-        self.refresh_grid(event)
+        self.refresh_grid()
 
     def quit(self, event):
         self.Close()
 
-    def refresh_grid(self, event):
+    def refresh_grid(self):
         """
         Refreshes grid. Notifies if rows have been added or deleted.
         :return:
         """
-        self.log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}")
+        self.__log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}")
 
         # Update data
         self.table.data = self.cor.coefficient_data.copy()
 
         # Format
-        self.table.data['Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format)
-        self.table.data['Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True)
-        self.table.data['Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S')
-        self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format)
+        self.table.data.loc[:, 'Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format)
+        self.table.data.loc[:, 'Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True)
+        self.table.data.loc[:, 'Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S')
+        self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format)
 
-        # Remove nans. The ones from the float column wil be str nan as they have been formatted
+        # Remove nans. The ones from the float column will be str nan as they have been formatted
         self.table.data = self.table.data.fillna('')
-        self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '')
+        self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '')
 
         # Start refresh
         self.grid_correlations.BeginBatch()
@@ -244,24 +272,21 @@ class MonitorFrame(wx.Frame):
     def monitor(self, event):
         # Check state of toggle button. If on, then start monitoring, else stop
         if self.monitor_toggle.GetValue():
-            self.log.debug("Starting monitoring.")
+            self.__log.info("Starting monitoring.")
             self.monitor_toggle.SetBackgroundColour(wx.GREEN)
             self.monitor_toggle.SetLabelText("On")
             self.SetStatusText("Monitoring for changes to coefficients.")
 
-            # From and to dates for calculations.
-            timezone = pytz.timezone("Etc/UTC")
-            utc_to = datetime.now(tz=timezone)
-            utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes'))
-
             self.timer.Start(self.config.get('monitor.interval')*1000)
-            self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to,
+            self.cor.start_monitor(interval=self.config.get('monitor.interval'),
+                                   from_mins=self.config.get('monitor.from.minutes'),
                                    min_prices=self.config.get('monitor.min_prices'),
                                    max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'),
                                    overlap_pct=self.config.get('monitor.overlap_pct'),
-                                   max_p_value=self.config.get('monitor.max_p_value'))
+                                   max_p_value=self.config.get('monitor.max_p_value'),
+                                   cache_time=self.config.get('monitor.tick_cache_time'))
         else:
-            self.log.debug("Stopping monitoring.")
+            self.__log.info("Stopping monitoring.")
             self.monitor_toggle.SetBackgroundColour(wx.RED)
             self.monitor_toggle.SetLabelText("Off")
             self.SetStatusText("Monitoring stopped.")
@@ -288,7 +313,7 @@ class MonitorFrame(wx.Frame):
                 # If 'monitor.interval has changed then restart gui timer.
                 # If 'monitor.monitoring_threshold' has changed, then refresh correlation data.
                 # If any 'logging.' settings have changed, then reload logger config.
-                if setting.startswith('monitor. ') and setting != 'monitor.divergence_threshold':
+                if setting.startswith('monitor.') and setting != 'monitor.divergence_threshold':
                     restart_monitor_timer = True
                 if setting == 'monitor.interval':
                     restart_gui_timer = True
@@ -299,29 +324,30 @@ class MonitorFrame(wx.Frame):
 
             # Now perform the actions
             if restart_monitor_timer:
-                self.log.debug("Settings updated. Reloading monitoring timer.")
+                self.__log.info("Settings updated. Reloading monitoring timer.")
                 self.cor.stop_monitor()
                 # From and to dates for calculations.
                 timezone = pytz.timezone("Etc/UTC")
                 utc_to = datetime.now(tz=timezone)
                 utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes'))
-                self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to,
+                self.cor.start_monitor(interval=self.config.get('monitor.interval'),
+                                       from_mins=self.config.get('monitor.from.minutes'),
                                        min_prices=self.config.get('monitor.min_prices'),
                                        max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'),
                                        overlap_pct=self.config.get('monitor.overlap_pct'),
                                        max_p_value=self.config.get('monitor.max_p_value'))
             if restart_gui_timer:
-                self.log.debug("Settings updated. Restarting gui timer.")
+                self.__log.info("Settings updated. Restarting gui timer.")
                 self.timer.Stop()
                 self.timer.Start(self.config.get('monitor.interval') * 1000)
 
             if reload_correlations:
-                self.log.debug("Settings updated. Updating monitoring threshold and reloading grid.")
+                self.__log.info("Settings updated. Updating monitoring threshold and reloading grid.")
                 self.cor.monitoring_threshold = self.config.get("monitor.monitoring_threshold")
-                self.refresh_grid(event)
+                self.refresh_grid()
 
             if reload_logger:
-                self.log.debug("Settings updated. Reloading logger.")
+                self.__log.info("Settings updated. Reloading logger.")
                 log_config = Config().get('logging')
                 logging.config.dictConfig(log_config)
 
@@ -331,13 +357,64 @@ class MonitorFrame(wx.Frame):
         :param event:
         :return:
         """
-        if self.opened_filename is not None:
-            self.cor.save(self.opened_filename)
+        if self.__opened_filename is not None:
+            self.cor.save(self.__opened_filename)
 
         self.cor.stop_monitor()
 
+        # Kill graph as it seems to be stopping script from ending
+        self.__graph = None
+
+        # End
         event.Skip()
 
+    def select_cell(self, event):
+        """
+        A cell was selected. Show the graph for the correlation.
+        :param event:
+        :return:
+        """
+        # Get row and symbols.
+        row = event.GetRow()
+        symbol1 = self.grid_correlations.GetCellValue(row, self.COLUMN_SYMBOL1)
+        symbol2 = self.grid_correlations.GetCellValue(row, self.COLUMN_SYMBOL2)
+        self.__selected_correlation = [symbol1, symbol2]
+
+        self.show_graph(symbol1, symbol2)
+
+    def show_graph(self, symbol1, symbol2):
+        """
+        Displays the graph for the specified symbols correlation history
+        :param symbol1:
+        :param symbol2:
+        :return:
+        """
+        # Get the data price data for the base coefficient calculation and the coefficient history data
+        symbol_1_price_data = self.cor.get_price_data(symbol1)
+        symbol_2_price_data = self.cor.get_price_data(symbol2)
+        history_data = self.cor.get_coefficient_history(symbol1, symbol2)
+        times = history_data['UTC Date To']
+        coefficients = history_data['Coefficient']
+
+        # Display if we have any data
+        self.__log.debug(f"Refreshing history graph {symbol1}:{symbol2}.")
+        self.__graph.draw(times=times, coefficients=coefficients, prices=[symbol_1_price_data, symbol_2_price_data],
+                          symbols=[symbol1, symbol2])
+
+        # Un-hide and layout if hidden
+        if not self.__graph.IsShown():
+            self.__graph.Show()
+            self.__main_sizer.Layout()
+
+    def __timer_event(self, event):
+        """
+        Called on timer event. Refreshes grid and updatates selected graph.
+        :return:
+        """
+        self.refresh_grid()
+        if len(self.__selected_correlation) == 2:
+            self.show_graph(symbol1=self.__selected_correlation[0], symbol2=self.__selected_correlation[1])
+
 
 class DataTable(wx.grid.GridTableBase):
     """
@@ -393,3 +470,72 @@ class DataTable(wx.grid.GridTableBase):
                     attr.SetBackgroundColour(wx.WHITE)
 
         return attr
+
+
+class GraphPanel(wx.Panel):
+    def __init__(self, parent):
+        # Super
+        wx.Panel.__init__(self, parent)
+
+        # 3  axis, price data 1, price data 2 and coefficient data. All will have axis labels and top and right boarders
+        # removed
+        self.__fig, self.__axes = plt.subplots(nrows=3, ncols=1)
+
+        # Create the canvas
+        self.__canvas = FigureCanvas(self, -1, self.__fig)
+
+        # Date format for x axes
+        self.__tick_fmt_date = matplotlib.dates.DateFormatter('%d-%b')
+        self.__tick_fmt_time = matplotlib.dates.DateFormatter('%H:%M:%S')
+
+        # Sizer etc.
+        self.__sizer = wx.BoxSizer(wx.VERTICAL)
+        self.__sizer.Add(self.__canvas, 1, wx.LEFT | wx.TOP | wx.GROW)
+        self.SetSizer(self.__sizer)
+        self.Fit()
+
+    def __del__(self):
+        # Close all plots
+        plt.close('all')
+        self.__axes = None
+        self.__fig = None
+
+    def draw(self, times, coefficients, prices=None, symbols=None):
+        """
+        Plot the correlations.
+        :param times: Series of time values for x axis
+        :param coefficients: Series of coefficients values for y axis
+        :param prices: Price data used to calculate base coefficient. List [Symbol1 Price Data, Symbol 2 Price Data]
+        :param symbols: Symbols. List [Symbol1, Symbol2]
+        :return:
+        """
+        # Clear. We will need to redraw
+        for ax in self.__axes:
+            ax.clear()
+
+        if symbols is not None and len(symbols) == 2:
+            # Price history chart for both symbols
+            for i in range(0, 2):
+                if symbols[i] is not None and prices is not None and len(prices) == 2 and prices[i] is not None:
+                    self.__axes[i].set_title(f"Base Coefficient Price Data for {symbols[i]}")
+                    self.__axes[i].set_xlabel('Time')
+                    self.__axes[i].set_ylabel('Price')
+                    self.__axes[i].plot(prices[i]['time'], prices[i]['close'])
+                    self.__axes[i].xaxis.set_major_formatter(self.__tick_fmt_date)
+                    self.__axes[i].xaxis.set_minor_formatter(self.__tick_fmt_time)
+                    plt.setp(self.__axes[i].xaxis.get_majorticklabels(), rotation=45)
+
+            # Coefficient history chart
+            self.__axes[2].set_title(f"Coefficient History for {symbols[0]}:{symbols[1]}")
+            self.__axes[2].set_xlabel('Time')
+            self.__axes[2].set_ylabel('Coefficient')
+            self.__axes[2].plot(times, coefficients)
+            self.__axes[2].set_ylim([-1, 1])
+            self.__axes[2].xaxis.set_major_formatter(self.__tick_fmt_time)
+            plt.setp(self.__axes[2].xaxis.get_majorticklabels(), rotation=45)
+
+            # Layout with padding between charts
+            self.__fig.tight_layout(pad=0.5)
+
+        # Redraw canvas
+        self.__canvas.draw()

mt5_correlation/mt5.py

@@ -35,18 +35,18 @@ class MT5:
         # Connect to MetaTrader5. Opens if not already open.
 
         # Logger
-        self.log = logging.getLogger(__name__)
+        self.__log = logging.getLogger(__name__)
 
         # Open MT5 and log error if it could not open
         if not MetaTrader5.initialize():
-            self.log.error("initialize() failed")
+            self.__log.error("initialize() failed")
             MetaTrader5.shutdown()
 
         # Print connection status
-        self.log.debug(MetaTrader5.terminal_info())
+        self.__log.debug(MetaTrader5.terminal_info())
 
         # Print data on MetaTrader 5 version
-        self.log.debug(MetaTrader5.version())
+        self.__log.debug(MetaTrader5.version())
 
     def __del__(self):
         # shut down connection to the MetaTrader 5 terminal
@@ -67,7 +67,7 @@ class MT5:
         # Log symbol counts
         total_symbols = MetaTrader5.symbols_total()
         num_selected_symbols = len(selected_symbols)
-        self.log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
+        self.__log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
 
         return selected_symbols
 
@@ -102,13 +102,16 @@ class MT5:
         :return: Price data for symbol as dataframe
         """
 
+        prices_dataframe = None
+
         # Get prices from MT5
         prices = MetaTrader5.copy_rates_range(symbol, timeframe, from_date, to_date)
-        self.log.debug(f"{len(prices)} prices retrieved for {symbol}.")
+        if prices is not None:
+            self.__log.debug(f"{len(prices)} prices retrieved for {symbol}.")
 
-        # Create dataframe from data and convert time in seconds to datetime format
-        prices_dataframe = pd.DataFrame(prices)
-        prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
+            # Create dataframe from data and convert time in seconds to datetime format
+            prices_dataframe = pd.DataFrame(prices)
+            prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
 
         return prices_dataframe
 
@@ -121,19 +124,23 @@ class MT5:
         :return: Tick data for symbol as dataframe
         """
 
+        ticks_dataframe = None
+
         # Get ticks from MT5
         ticks = MetaTrader5.copy_ticks_range(symbol, from_date, to_date, MetaTrader5.COPY_TICKS_ALL)
 
         # If ticks is None, there was an error
         if ticks is None:
             error = MetaTrader5.last_error()
-            self.log.error(f"Error retrieving ticks for {symbol}: {error}")
-            return None
+            self.__log.error(f"Error retrieving ticks for {symbol}: {error}")
         else:
-            self.log.debug(f"{len(ticks)} ticks retrieved for {symbol}.")
+            self.__log.debug(f"{len(ticks)} ticks retrieved for {symbol}.")
 
             # Create dataframe from data and convert time in seconds to datetime format
-            ticks_dataframe = pd.DataFrame(ticks)
-            ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s')
+            try:
+                ticks_dataframe = pd.DataFrame(ticks)
+                ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s')
+            except RecursionError:
+                self.__log.warning("Error converting ticks to dataframe.")
 
         return ticks_dataframe