Merged charts to create more space.

config.yaml

@@ -8,15 +8,29 @@ calculate:
   overlap_pct: 90
   max_p_value: 0.05
 monitor:
-  from:
-    minutes: 30
   interval: 10
-  min_prices: 400
-  max_set_size_diff_pct: 90
-  overlap_pct: 80
-  max_p_value: 0.05
+  calculations:
+    long:
+      from: 60
+      min_prices: 2000
+      max_set_size_diff_pct: 50
+      overlap_pct: 50
+      max_p_value: 0.05
+    medium:
+      from: 30
+      min_prices: 1000
+      max_set_size_diff_pct: 50
+      overlap_pct: 50
+      max_p_value: 0.05
+    short:
+      from: 10
+      min_prices: 200
+      max_set_size_diff_pct: 50
+      overlap_pct: 50
+      max_p_value: 0.05
   monitoring_threshold: 0.9
   divergence_threshold: 0.8
+  monitor_inverse: true
   tick_cache_time: 10
   autosave: true
 logging:
@@ -58,10 +72,10 @@ logging:
 developer:
   inspection: false
 window:
-  x: 42
-  y: 51
-  width: 1512
-  height: 975
+  x: 24
+  y: 38
+  width: 1510
+  height: 956
   style: 541072960
 settings_window:
   x: 354

mt5_correlation/correlation.py

@@ -14,21 +14,85 @@ import sys
 from mt5_correlation.mt5 import MT5
 
 
+class CorrelationStatus:
+    """
+    The status of the monitoring event for a symbol pair.
+    """
+
+    val = None
+    text = None
+    long_text = None
+
+    def __init__(self, status_val, status_text, status_long_text=None):
+        """
+        Creates a status.
+        :param status_val:
+        :param status_text:
+        :param status_long_text
+        :return:
+        """
+        self.val = status_val
+        self.text = status_text
+        self.long_text = status_long_text
+
+    def __eq__(self, other):
+        """
+        Compare the status val. We can compare against other CorrelationStatus instances or against int.
+        :param other:
+        :return:
+        """
+        if isinstance(other, self.__class__):
+            return self.val == other.val
+        elif isinstance(other, int):
+            return self.val == other
+        else:
+            return False
+
+    def __str__(self):
+        """
+        str is the text for the status.
+        :return:
+        """
+        return self.text
+
+
+# All status's for symbol pair from monitoring. Status set from assessing coefficient for all timeframes from last run.
+STATUS_NOT_CALCULATED = CorrelationStatus(-1, 'NOT CALC', 'Coefficient could not be calculated')
+STATUS_ABOVE_DIVERGENCE_THRESHOLD = CorrelationStatus(1, 'ABOVE', 'All coefficients equal to or above the divergence '
+                                                                  'threshold')
+STATUS_BELOW_DIVERGENCE_THRESHOLD = CorrelationStatus(2, 'BELOW', 'All coefficients below the divergence threshold')
+STATUS_INCONSISTENT = CorrelationStatus(3, 'INCONSISTENT', 'Coefficients not consistently above or below divergence '
+                                                           'threshold')
+
+
 class Correlation:
     """
     A class to maintain the state of the calculated correlation coefficients.
     """
 
-    # Connection to metatrader
+    # Connection to MetaTrader5
     __mt5 = None
 
     # Minimum base coefficient for monitoring. Symbol pairs with a lower correlation
     # coefficient than ths won't be monitored.
     monitoring_threshold = 0.9
 
+    # Threshold for divergence. Correlation coefficients that were previously above the monitoring_threshold and fall
+    # below this threshold will be considered as having diverged
+    divergence_threshold = 0.8
+
+    # Flag to determine we monitor and report on inverse correlations
+    monitor_inverse = False
+
     # Toggle on whether we are monitoring or not. Set through start_monitor and stop_monitor
     __monitoring = False
-    __monitoring_params = {}
+
+    # Monitoring calculation params, interval, cache_time, autosave and filename. Passed to start_monitor
+    __monitoring_params = []
+    __interval = None
+    __cache_time = None
+    __autosave = None
+    __filename = None
 
     # First run of scheduler
     __first_run = True
@@ -36,7 +100,7 @@ class Correlation:
     # The price data used to calculate the correlations
     __price_data = None
 
-    # Coefficient data and history. Will be created as dataframes in Init
+    # Coefficient data and history. Will be created in init call to __reset_coefficient_data
     coefficient_data = None
     coefficient_history = None
 
@@ -44,11 +108,19 @@ class Correlation:
     # Dict: {Symbol: [retrieved datetime, ticks dataframe]}
     __monitor_tick_data = {}
 
-    def __init__(self):
+    def __init__(self, monitoring_threshold=0.9, divergence_threshold=0.8, monitor_inverse=False):
+        """
+        Initialises the Correlation class.
+        :param monitoring_threshold: Only correlations that are greater than or equal to this threshold will be
+            monitored.
+        :param divergence_threshold: Correlations that are being monitored and fall below this threshold are considered
+            to have diverged.
+        :param monitor_inverse: Whether we will monitor and report on negative / inverse correlations.
+        """
         # Logger
         self.__log = logging.getLogger(__name__)
 
-        # Connection to metatrader
+        # Connection to MetaTrader5
         self.__mt5 = MT5()
 
         # Create dataframe for coefficient data
@@ -57,15 +129,28 @@ class Correlation:
         # Create timer for continuous monitoring
         self.__scheduler = sched.scheduler(time.time, time.sleep)
 
+        # Set thresholds and flags
+        self.monitoring_threshold = monitoring_threshold
+        self.divergence_threshold = divergence_threshold
+        self.monitor_inverse = monitor_inverse
+
     @property
     def filtered_coefficient_data(self):
         """
         :return: Coefficient data filtered so that all base coefficients >= monitoring_threshold
         """
+        filtered_data = None
         if self.coefficient_data is not None:
-            return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold]
-        else:
-            return None
+            if self.monitor_inverse:
+                filtered_data = self.coefficient_data \
+                    .loc[(self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold) |
+                         (self.coefficient_data['Base Coefficient'] <= self.monitoring_threshold * -1)]
+
+            else:
+                filtered_data = self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >=
+                                                          self.monitoring_threshold]
+
+        return filtered_data
 
     def load(self, filename):
         """
@@ -108,7 +193,10 @@ class Correlation:
             is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
             within this pct of each other
-        :param overlap_pct:
+        :param overlap_pct:The dates and times in the two sets of data must match. The coefficient will only be
+            calculated against the dates that overlap. Any non overlapping dates will be discarded. This setting
+            specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient
+            will not be calculated if this threshold is not met.
         :param max_p_value: The maximum p value for the correlation to be meaningful
 
         :return:
@@ -165,8 +253,9 @@ class Correlation:
                     self.coefficient_data = \
                         self.coefficient_data.append({'Symbol 1': symbol1, 'Symbol 2': symbol2,
                                                       'Base Coefficient': coefficient, 'UTC Date From': date_from,
-                                                      'UTC Date To': date_to, 'Timeframe': timeframe},
+                                                      'UTC Date To': date_to, 'Timeframe': timeframe, 'Status': ''},
                                                      ignore_index=True)
+
                     self.__log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a "
                                      f"coefficient of {coefficient}.")
                 else:
@@ -178,12 +267,8 @@ class Correlation:
 
         # If we were monitoring, we stopped, so start again.
         if was_monitoring:
-            self.start_monitor(interval=self.__monitoring_params['interval'],
-                               from_mins=self.__monitoring_params['from_mins'],
-                               min_prices=self.__monitoring_params['min_prices'],
-                               max_set_size_diff_pct=self.__monitoring_params['max_set_size_diff_pct'],
-                               overlap_pct=self.__monitoring_params['overlap_pct'],
-                               max_p_value=self.__monitoring_params['max_p_value'])
+            self.start_monitor(interval=self.__interval, calculation_params=self.__monitoring_params,
+                               cache_time=self.__cache_time, autosave=self.__autosave, filename=self.__filename)
 
     def get_price_data(self, symbol):
         """
@@ -197,20 +282,26 @@ class Correlation:
 
         return price_data
 
-    def start_monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                      max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'):
+    def start_monitor(self, interval, calculation_params, cache_time=10, autosave=False, filename='autosave.cpd'):
         """
         Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
         threshold.
 
         :param interval: How often to check in seconds
-        :param from_mins: The number of minutes of tick data to use for calculations
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
+        :param calculation_params: A single dict or list of dicts containing the parameters for the coefficient
+            calculations. On every iteration, a coefficient will be calculated for every set of params in list. Params
+            contain the following values:
+                from: The number of minutes of tick data to use for calculation. This can be a single value or
+                    a list. If a list, then calculations will be performed for every from date in list.
+                min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+                    is not met then returned coefficient will be None
+                max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+                    within this pct of each other
+                overlap_pct: The dates and times in the two sets of data must match. The coefficient will only be
+                    calculated against the dates that overlap. Any non overlapping dates will be discarded. This
+                    setting specifies the minimum size of the overlapping data when compared to the smallest set as a %.
+                    A coefficient will not be calculated if this threshold is not met.
+                max_p_value: The maximum p value for the correlation to be meaningful
         :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
             the tick data. Number of seconds to cache tick data for before it becomes stale.
         :param autosave: Whether to autosave after every monitor run. If there is no filename specified then will
@@ -228,14 +319,20 @@ class Correlation:
         self.__log.debug(f"Starting monitor.")
         self.__monitoring = True
 
+        # Store the calculation params. If it isn't a list, convert to list of one to make code simpler later on.
+        self.__monitoring_params = calculation_params if isinstance(calculation_params, list) \
+            else [calculation_params, ]
+
+        # Store the other params. We will need these later if monitor is stopped and needs to be restarted. This
+        # happens in calculate.
+        self.__interval = interval
+        self.__cache_time = cache_time
+        self.__autosave = autosave
+        self.__filename = filename
+
         # Create thread to run monitoring This will call private __monitor method that will run the calculation and
-        # keep scheduling itself while self.monitoring is True. Store the params. We will need to use these if we have
-        # to stop and restart the monitor. Note, this happens during calculate
-        self.__monitoring_params = {'interval': interval, 'from_mins': from_mins,
-                                    'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct,
-                                    'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time,
-                                    'autosave': autosave, 'filename': filename}
-        thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params)
+        # keep scheduling itself while self.monitoring is True.
+        thread = threading.Thread(target=self.__monitor)
         thread.start()
 
     def stop_monitor(self):
@@ -255,8 +352,8 @@ class Correlation:
         """
         Calculates the correlation coefficient between two sets of price data. Uses close price.
 
-        :param symbol1_prices: Pandas dataframe containing prices or ticks for symbol 1
-        :param symbol2_prices: Pandas dataframe containing prices or ticks for symbol 2
+        :param symbol1_prices: Pandas dataframe containing prices for symbol 1
+        :param symbol2_prices: Pandas dataframe containing prices for symbol 2
         :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
             is not met then returned coefficient will be None
         :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
@@ -306,27 +403,56 @@ class Correlation:
 
         return coefficient
 
-    def get_coefficient_history(self, symbol1, symbol2):
+    def get_coefficient_history(self, filters=None):
         """
-        Returns the coefficient history for the specified symbol pair calculated during this instance.
-        Coefficient history does not persist between instances.
-        :param symbol1:
-        :param symbol2:
+        Returns the coefficient history that matches the supplied filter.
+        :param filters: Dict of all filters to apply. Possible values in dict are:
+            Symbol 1
+            Symbol 2
+            Coefficient
+            Timeframe
+            Date From
+            Date To
+
+        If filter is not supplied, then all history is returned.
+
         :return: dataframe containing history of coefficient data.
         """
-        history = self.coefficient_history[(self.coefficient_history['Symbol 1'] == symbol1) &
-                                           (self.coefficient_history['Symbol 2'] == symbol2)]
+        history = self.coefficient_history
+
+        # Apply filters
+        if filters is not None:
+            for key in filters:
+                if key in history.columns:
+                    history = history[history[key] == filters[key]]
+                else:
+                    self.__log.warning(f"Invalid column name provided for filter. Filter column: {key} "
+                                       f"Valid columns: {history.columns}")
+
         return history
 
-    def get_ticks(self, symbol, date_from=None, date_to=None, cache_time=0, cache_only=False):
+    def clear_coefficient_history(self):
+        """
+        Clears the coefficient history for all symbol pairs
+        :return:
+        """
+        # Create dataframes for coefficient history.
+        coefficient_history_columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'Timeframe', 'Date To']
+        self.coefficient_history = pd.DataFrame(columns=coefficient_history_columns)
+
+        # Clear tick data
+        self.__monitor_tick_data = {}
+
+        # Clear status from coefficient data
+        self.coefficient_data['Status'] = ''
+
+    def get_ticks(self, symbol, date_from=None, date_to=None, cache_only=False):
         """
         Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe.
 
         :param symbol: Name of symbol to get ticks for.
         :param date_from: Date to get ticks from. Can only be None if getting from cache (cache_only=True)
         :param date_to:Date to get ticks to. Can only be None if getting from cache (cache_only=True)
-        :param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed,
-            get data from source and refresh cache.
         :param cache_only: Only retrieve from cache. cache_time is ignored. Returns None if symbol is not available in
             cache.
 
@@ -342,9 +468,9 @@ class Correlation:
         if cache_only:
             if symbol in self.__monitor_tick_data:
                 ticks = self.__monitor_tick_data[symbol][1]
-        # Check if we already have it and it is not stale
-        elif symbol in self.__monitor_tick_data and utc_now < \
-                self.__monitor_tick_data[symbol][0] + timedelta(seconds=cache_time):
+        # Check if we have a cache time defined, if we already have the tick data and it is not stale
+        elif self.__cache_time is not None and symbol in self.__monitor_tick_data and utc_now < \
+                self.__monitor_tick_data[symbol][0] + timedelta(seconds=self.__cache_time):
             # Cached ticks are not stale. Get them
             ticks = self.__monitor_tick_data[symbol][1]
             self.__log.debug(f"Ticks for {symbol} retrieved from cache.")
@@ -355,26 +481,70 @@ class Correlation:
             self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.")
         return ticks
 
-    def __monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                  max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'):
+    def get_last_status(self, symbol1, symbol2):
+        """
+        Get the last status for the specified symbol pair.
+        :param symbol1
+        :param symbol2
+
+        :return: CorrelationStatus instance for symbol pair
+        """
+        status_col = self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
+                                               (self.coefficient_data['Symbol 2'] == symbol2), 'Status']
+        status = status_col.values[0]
+        return status
+
+    def get_last_calculation(self, symbol1=None, symbol2=None):
+        """
+        Get the last calculation time the specified symbol pair. If no symbols are specified, then gets the last
+        calculation time across all pairs
+        :param symbol1
+        :param symbol2
+
+        :return: last calculation time
+        """
+        last_calc = None
+        if self.coefficient_data is not None and len(self.coefficient_data.index) > 0:
+            data = self.coefficient_data.copy()
+
+            # Filter by symbols if specified
+            data = data.loc[data['Symbol 1'] == symbol1] if symbol1 is not None else data
+            data = data.loc[data['Symbol 2'] == symbol2] if symbol2 is not None else data
+
+            # Filter to remove blank dates
+            data = data.dropna(subset=['Last Calculation'])
+
+            # Get the column
+            col = data['Last Calculation']
+
+            # Get max date from column
+            if col is not None and len(col) > 0:
+                last_calc = max(col.values)
+
+        return last_calc
+
+    def get_base_coefficient(self, symbol1, symbol2):
+        """
+        Returns the base coefficient for the specified symbol pair
+        :param symbol1:
+        :param symbol2:
+        :return:
+        """
+        base_coefficient = None
+        if self.coefficient_data is not None:
+            row = self.coefficient_data[(self.coefficient_data['Symbol 1'] == symbol1) &
+                                        (self.coefficient_data['Symbol 2'] == symbol2)]
+
+            if row is not None and len(row) == 1:
+                base_coefficient = row.iloc[0]['Base Coefficient']
+
+        return base_coefficient
+
+    def __monitor(self):
         """
         The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
         stop_monitoring.
 
-        :param interval: How often to check in seconds
-        :param from_mins: The number of minutes of tick data to use for calculations
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
-        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
-            the tick data. Number of seconds to cache tick data for before it becomes stale.
-        :param autosave: Whether to autosave after every monitor run. If there is no filename specified then will
-            create one named autosave.cpd
-        :param filename: Filename for autosave. Default is autosave.cpd.
-
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
         self.__log.debug(f"In monitor event. Monitoring: {self.__monitoring}.")
@@ -382,20 +552,14 @@ class Correlation:
         # Only run if monitor is not stopped
         if self.__monitoring:
             # Update all coefficients
-            self.__update_all_coefficients(from_mins=from_mins, min_prices=min_prices,
-                                           max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
-                                           max_p_value=max_p_value, cache_time=cache_time)
+            self.__update_all_coefficients()
 
             # Autosave
-            if autosave:
-                self.save(filename=filename)
+            if self.__autosave:
+                self.save(filename=self.__filename)
 
             # Schedule the timer to run again
-            params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices,
-                      'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
-                      'max_p_value': max_p_value, "cache_time": cache_time, 'autosave': autosave,
-                      'filename': filename}
-            self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
+            self.__scheduler.enter(delay=self.__interval, priority=1, action=self.__monitor)
 
             # Log the stack. Debug stack overflow
             self.__log.debug(f"Current stack size: {len(inspect.stack())} Recursion limit: {sys.getrecursionlimit()}")
@@ -405,120 +569,117 @@ class Correlation:
                 self.__first_run = False
                 self.__scheduler.run()
 
-    def __update_coefficient(self, symbol1, symbol2, from_mins, min_prices=100, max_set_size_diff_pct=90,
-                             overlap_pct=90, max_p_value=0.05, cache_time=10):
+    def __update_coefficients(self, symbol1, symbol2):
         """
-        Updates the coefficient for the specified symbol pair
+        Updates the coefficients for the specified symbol pair
         :param symbol1: Name of symbol to calculate coefficient for.
         :param symbol2: Name of symbol to calculate coefficient for.
-        :param from_mins: The number of minutes of tick data to use for calculations
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
-        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
-            the tick data. Number of seconds to cache tick data for before it becomes stale.
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
 
-        coefficient = None
+        # Get the largest value of from in monitoring_params. This will be used to retrieve the data. We will only
+        # retrieve once and use for every set of params by getting subset of the data.
+        max_from = None
+        for params in self.__monitoring_params:
+            if max_from is None:
+                max_from = params['from']
+            else:
+                max_from = max(max_from, params['from'])
 
-        # Get dates
-        # From and to dates for calculations.
+        # Date range for data
         timezone = pytz.timezone("Etc/UTC")
         date_to = datetime.now(tz=timezone)
-        date_from = date_to - timedelta(minutes=from_mins)
+        date_from = date_to - timedelta(minutes=max_from)
 
-        # Get the tick data
-        symbol1ticks = self.get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time)
-        symbol2ticks = self.get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time)
+        # Get the tick data for the longest timeframe calculation.
+        symbol1ticks = self.get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to)
+        symbol2ticks = self.get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to)
+
+        # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than
+        # one tick per second and ensuring that times can match. We will need to set the index to time for the
+        # resample then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price
+        s1_prices = None
+        s2_prices = None
+        if symbol1ticks is not None and symbol2ticks is not None and len(symbol1ticks.index) > 0 and \
+                len(symbol2ticks.index) > 0:
 
-        # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one
-        # tick per second and ensuring that times can match. We will need to set the index to time for the resample
-        # then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price
-        if symbol1ticks is not None and symbol2ticks is not None and \
-                len(symbol1ticks.index) > 0 and len(symbol2ticks.index) > 0:
-            symbol1ticks = symbol1ticks.set_index('time')
-            symbol2ticks = symbol2ticks.set_index('time')
             try:
-                symbol1prices = symbol1ticks['ask'].resample('1S').ohlc()
-                symbol2prices = symbol2ticks['ask'].resample('1S').ohlc()
+                symbol1ticks = symbol1ticks.set_index('time')
+                symbol2ticks = symbol2ticks.set_index('time')
+                s1_prices = symbol1ticks['ask'].resample('1S').ohlc()
+                s2_prices = symbol2ticks['ask'].resample('1S').ohlc()
             except RecursionError:
-                self.__log.warning(f"Coefficient could not be calculated for {symbol1}:{symbol2} as prices could not "
+                self.__log.warning(f"Coefficient could not be calculated for {symbol1}:{symbol2}. prices could not "
                                    f"be resampled.")
             else:
-                symbol1prices.reset_index(inplace=True)
-                symbol2prices.reset_index(inplace=True)
-                symbol1prices = symbol1prices[symbol1prices['close'].notna()]
-                symbol2prices = symbol2prices[symbol2prices['close'].notna()]
+                s1_prices.reset_index(inplace=True)
+                s2_prices.reset_index(inplace=True)
+                s1_prices = s1_prices[s1_prices['close'].notna()]
+                s2_prices = s2_prices[s2_prices['close'].notna()]
 
-                # Calculate the coefficient
-                coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices,
-                                                         min_prices=min_prices,
-                                                         max_set_size_diff_pct=max_set_size_diff_pct,
-                                                         overlap_pct=overlap_pct, max_p_value=max_p_value)
+            # Calculate for all sets of monitoring_params
+            if s1_prices is not None and s2_prices is not None:
+                coefficients = {}
+                for params in self.__monitoring_params:
+                    # Get the from date as a datetime64
+                    date_from_subset = pd.Timestamp(date_to - timedelta(minutes=params['from'])).to_datetime64()
 
-                self.__log.debug(f"Symbol pair {symbol1}:{symbol2} has a coefficient of {coefficient}.")
-        else:
-            coefficient = None
+                    # Get subset of the price data
+                    s1_prices_subset = s1_prices[(s1_prices['time'] >= date_from_subset)]
+                    s2_prices_subset = s2_prices[(s2_prices['time'] >= date_from_subset)]
 
-        # Update the coefficient data
-        if coefficient is not None:
-            self.__update_coefficient_data(symbol1=symbol1, symbol2=symbol2, coefficient=coefficient,
-                                           date_from=date_from, date_to=date_to)
+                    # Calculate the coefficient
+                    coefficient = \
+                        self.calculate_coefficient(symbol1_prices=s1_prices_subset, symbol2_prices=s2_prices_subset,
+                                                   min_prices=params['min_prices'],
+                                                   max_set_size_diff_pct=params['max_set_size_diff_pct'],
+                                                   overlap_pct=params['overlap_pct'], max_p_value=params['max_p_value'])
 
-        return coefficient
+                    self.__log.debug(f"Symbol pair {symbol1}:{symbol2} has a coefficient of {coefficient} for last "
+                                     f"{params['from']} minutes.")
 
-    def __update_all_coefficients(self, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
-                                  max_p_value=0.05, cache_time=10):
+                    # Add the coefficient to a dict {timeframe: coefficient}. We will update together for all for
+                    # symbol pair and time
+                    coefficients[params['from']] = coefficient
+
+                # Update coefficient data for all coefficients for all timeframes for this run and symbol pair.
+                self.__update_coefficient_data(symbol1=symbol1, symbol2=symbol2, coefficients=coefficients,
+                                               date_to=date_to)
+
+    def __update_all_coefficients(self):
         """
         Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
         the threshold can be accessed through the filtered_coefficient_data property.
-
-        :param from_mins: The number of minutes of tick data to use for calculations
-        :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
-            is not met then returned coefficient will be None
-        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
-            within this pct of each other
-        :param overlap_pct:
-        :param max_p_value: The maximum p value for the correlation to be meaningful
-        :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
-            the tick data. Number of seconds to cache tick data for before it becomes stale.
-
-        :return: correlation coefficient, or None if coefficient could not be calculated.
         """
         # Update  latest coefficient for every pair
         for index, row in self.filtered_coefficient_data.iterrows():
             symbol1 = row['Symbol 1']
             symbol2 = row['Symbol 2']
-            self.__update_coefficient(symbol1=symbol1, symbol2=symbol2, from_mins=from_mins,
-                                      min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
-                                      overlap_pct=overlap_pct, max_p_value=max_p_value, cache_time=cache_time)
+            self.__update_coefficients(symbol1=symbol1, symbol2=symbol2)
 
     def __reset_coefficient_data(self):
         """
         Clears coefficient data and history.
         :return:
         """
-        # Create dataframe for coefficient data
+        # Create dataframes for coefficient data.
         coefficient_data_columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To',
-                                    'Timeframe', 'Last Check', 'Last Coefficient']
+                                    'Timeframe', 'Last Calculation', 'Status']
         self.coefficient_data = pd.DataFrame(columns=coefficient_data_columns)
 
-        # Create dataframe for coefficient history
-        coefficient_history_columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To']
-        self.coefficient_history = pd.DataFrame(columns=coefficient_history_columns)
+        # Clear coefficient history
+        self.clear_coefficient_history()
 
-    def __update_coefficient_data(self, symbol1, symbol2, coefficient, date_from, date_to):
+        # Clear price data
+        self.__price_data = None
+
+    def __update_coefficient_data(self, symbol1, symbol2, coefficients, date_to):
         """
         Updates the coefficient data with the latest coefficient and adds to coefficient history.
         :param symbol1:
         :param symbol2:
-        :param coefficient:
-        :param date_from:
-        :param date_to:
+        :param coefficients: Dict of all coefficients calculated for this run and symbol pair. {timeframe: coefficient}
+        :param date_to: The date from for which the coefficient was calculated
         :return:
         """
 
@@ -526,15 +687,53 @@ class Correlation:
         now = datetime.now(tz=timezone)
 
         # Update data if we have a coefficient and add to history
-        if coefficient is not None:
+        if coefficients is not None:
+            # Update the coefficient data table with the Last Calculation time.
             self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
                                       (self.coefficient_data['Symbol 2'] == symbol2),
-                                      'Last Check'] = now
+                                      'Last Calculation'] = now
 
+            # Are we an inverse correlation
+            inverse = self.get_base_coefficient(symbol1, symbol2) <= self.monitoring_threshold * -1
+
+            # Calculate status and update
+            status = self.__calculate_status(coefficients=coefficients, inverse=inverse)
             self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
                                       (self.coefficient_data['Symbol 2'] == symbol2),
-                                      'Last Coefficient'] = coefficient
+                                      'Status'] = status
 
-            row = pd.DataFrame(columns=self.coefficient_history.columns,
-                               data=[[symbol1, symbol2, coefficient, date_from, date_to]])
-            self.coefficient_history = self.coefficient_history.append(row)
+            # Update history data
+            for key in coefficients:
+                row = pd.DataFrame(columns=self.coefficient_history.columns,
+                                   data=[[symbol1, symbol2, coefficients[key], key, date_to]])
+                self.coefficient_history = self.coefficient_history.append(row)
+
+    def __calculate_status(self, coefficients, inverse):
+        """
+        Calculates the status from the supplied set of coefficients
+        :param coefficients: Dict of timeframes and coefficients {timeframe: coefficient} to calculate status from
+        :param: Whether we are calculating status based on normal or inverse correlation
+        :return: status
+        """
+        status = STATUS_NOT_CALCULATED
+        values = coefficients.values()
+
+        if None not in values:
+            if self.monitor_inverse and inverse:
+                # Calculation for inverse calculations
+                if all(i <= self.divergence_threshold * -1 for i in values):
+                    status = STATUS_ABOVE_DIVERGENCE_THRESHOLD
+                elif all(i > self.divergence_threshold * -1 for i in values):
+                    status = STATUS_BELOW_DIVERGENCE_THRESHOLD
+                else:
+                    status = STATUS_INCONSISTENT
+            else:
+                # Calculation for standard correlations
+                if all(i >= self.divergence_threshold for i in values):
+                    status = STATUS_ABOVE_DIVERGENCE_THRESHOLD
+                elif all(i < self.divergence_threshold for i in values):
+                    status = STATUS_BELOW_DIVERGENCE_THRESHOLD
+                else:
+                    status = STATUS_INCONSISTENT
+
+        return status

mt5_correlation/gui.py

@@ -4,8 +4,9 @@ import matplotlib.pyplot as plt
 from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas
 import matplotlib.dates
 import matplotlib
+import matplotlib.ticker as mticker
 
-from mt5_correlation.correlation import Correlation
+from mt5_correlation import correlation as cor
 from mt5_correlation.config import Config, SettingsDialog
 from datetime import datetime, timedelta
 import pytz
@@ -33,8 +34,8 @@ class MonitorFrame(wx.Frame):
     COLUMN_DATE_FROM = 4
     COLUMN_DATE_TO = 5
     COLUMN_TIMEFRAME = 6
-    COLUMN_LAST_CHECK = 7
-    COLUMN_LAST_COEFFICIENT = 8
+    COLUMN_LAST_CALCULATION = 7
+    COLUMN_STATUS = 8
 
     def __init__(self):
         # Super
@@ -50,58 +51,48 @@ class MonitorFrame(wx.Frame):
         self.__config = Config()
 
         # Create correlation instance to maintain state of calculated coefficients. Set min coefficient from config
-        self.__cor = Correlation()
-        self.__cor.monitoring_threshold = self.__config.get("monitor.monitoring_threshold")
+        self.__cor = cor.Correlation(monitoring_threshold=self.__config.get("monitor.monitoring_threshold"),
+                                     divergence_threshold=self.__config.get("monitor.divergence_threshold"),
+                                     monitor_inverse=self.__config.get("monitor.monitor_inverse"))
 
-        # Status bar
-        self.statusbar = self.CreateStatusBar(1)
+        # Status bar. 2 fields, one for monitoring status and one for general status. On open, monitoring status is not
+        # monitoring. SetBackgroundColour will change colour of both. Couldn't find a way to set on single field only.
+        self.__statusbar = self.CreateStatusBar(2)
+        self.__statusbar.SetStatusWidths([100, -1])
+        self.SetStatusText("Not Monitoring", 0)
 
-        # Menu Bar and file menu
+        # Menu Bar
         self.menubar = wx.MenuBar()
-        file_menu = wx.Menu()
 
-        # File menu items
+        # File menu and items
+        file_menu = wx.Menu()
         menu_item_open = file_menu.Append(wx.ID_ANY, "Open", "Open correlations file.")
         menu_item_save = file_menu.Append(wx.ID_ANY, "Save", "Save correlations file.")
         menu_item_saveas = file_menu.Append(wx.ID_ANY, "Save As", "Save correlations file.")
         file_menu.AppendSeparator()
-        menu_item_calculate = file_menu.Append(wx.ID_ANY, "Calculate", "Calculate base coefficients.")
-        file_menu.AppendSeparator()
         menu_item_settings = file_menu.Append(wx.ID_ANY, "Settings", "Change application settings.")
         file_menu.AppendSeparator()
         menu_item_exit = file_menu.Append(wx.ID_ANY, "Exit", "Close the application")
-
-        # Add file menu and set menu bar
         self.menubar.Append(file_menu, "File")
+
+        # Coefficient menu and items
+        coef_menu = wx.Menu()
+        menu_item_calculate = coef_menu.Append(wx.ID_ANY, "Calculate", "Calculate base coefficients.")
+        self.__menu_item_monitor = coef_menu.Append(wx.ID_ANY, "Monitor", "Monitor correlated pairs for changes to "
+                                                                          "coefficient.", kind=wx.ITEM_CHECK)
+        coef_menu.AppendSeparator()
+        menu_item_clear = coef_menu.Append(wx.ID_ANY, "Clear", "Clear coefficient and price history.")
+        self.menubar.Append(coef_menu, "Coefficient")
+
+        # Set menu bar
         self.SetMenuBar(self.menubar)
 
-        # Main window. We want 2 horizontal sections, the grid showing correlations and a graph. In the correlations
-        # section, we want 2 vertical sections, the monitor toggle and the correlations grid. For the toggle we want 2
-        # sections, a label and a toggle.
-        # ---------------------------------------------------------------
-        # |label  |  toggle       |                                      |
-        # |-----------------------|                                      |
-        # |                       |                                      |
-        # |Correlations Grid      |         Graphs                       |
-        # |                       |                                      |
-        # |                       |                                      |
-        # |                       |                                      |
-        # |                       |                                      |
-        # ----------------------------------------------------------------
+        # Main window. We want 2 horizontal sections, the grid showing correlations and a graph.
         panel = wx.Panel(self, wx.ID_ANY)
-        toggle_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Label and toggle
-        correlations_sizer = wx.BoxSizer(wx.VERTICAL)  # Toggle sizer and correlations grid
+        correlations_sizer = wx.BoxSizer(wx.VERTICAL)  # Correlations grid
         self.__main_sizer = wx.BoxSizer(wx.HORIZONTAL)  # Correlations sizer and graphs panel
         panel.SetSizer(self.__main_sizer)
 
-        # Create the label and toggle, populate the toggle sizer and add the toggle sizer to the correlations sizer
-        monitor_toggle_label = wx.StaticText(panel, id=wx.ID_ANY, label="Monitoring")
-        toggle_sizer.Add(monitor_toggle_label, 0, wx.ALL, 1)
-        self.monitor_toggle = wx.ToggleButton(panel, wx.ID_ANY, label="Off")
-        self.monitor_toggle.SetBackgroundColour(wx.RED)
-        toggle_sizer.Add(self.monitor_toggle, 0, wx.ALL, 1)
-        correlations_sizer.Add(toggle_sizer, 0, wx.ALL, 1)
-
         # Create the correlations grid. This is a data table using pandas dataframe for underlying data. Add the
         # correlations_grid to the correlations sizer.
         self.table = DataTable(self.__cor.filtered_coefficient_data)
@@ -120,10 +111,10 @@ class MonitorFrame(wx.Frame):
         self.grid_correlations.SetColSize(self.COLUMN_DATE_FROM, 0)  # UTC Date From. Hide
         self.grid_correlations.SetColSize(self.COLUMN_DATE_TO, 0)  # UTC Date To. Hide
         self.grid_correlations.SetColSize(self.COLUMN_TIMEFRAME, 0)  # Timeframe. Hide.
-        self.grid_correlations.SetColSize(self.COLUMN_LAST_CHECK, 100)  # Last Check
-        self.grid_correlations.SetColSize(self.COLUMN_LAST_COEFFICIENT, 100)  # Last Coefficient
-        self.grid_correlations.SetMinSize((520, 500))
-        self.grid_correlations.SetMaxSize((520, -1))
+        self.grid_correlations.SetColSize(self.COLUMN_LAST_CALCULATION, 0)  # Last Calculation. Hide
+        self.grid_correlations.SetColSize(self.COLUMN_STATUS, 100)  # Status
+        self.grid_correlations.SetMinSize((420, 500))
+        self.grid_correlations.SetMaxSize((420, -1))
         correlations_sizer.Add(self.grid_correlations, 1, wx.ALL | wx.EXPAND, 1)
 
         # Create the charts and hide as we have no data to display yet
@@ -140,9 +131,6 @@ class MonitorFrame(wx.Frame):
         # Set up timer to refresh grid
         self.timer = wx.Timer(self)
 
-        # Bind monitor button
-        self.monitor_toggle.Bind(wx.EVT_TOGGLEBUTTON, self.monitor)
-
         # Bind timer
         self.Bind(wx.EVT_TIMER, self.__timer_event, self.timer)
 
@@ -152,6 +140,8 @@ class MonitorFrame(wx.Frame):
         self.Bind(wx.EVT_MENU, self.save_file_as, menu_item_saveas)
         self.Bind(wx.EVT_MENU, self.calculate_coefficients, menu_item_calculate)
         self.Bind(wx.EVT_MENU, self.open_settings, menu_item_settings)
+        self.Bind(wx.EVT_MENU, self.__monitor, self.__menu_item_monitor)
+        self.Bind(wx.EVT_MENU, self.__clear_history, menu_item_clear)
         self.Bind(wx.EVT_MENU, self.quit, menu_item_exit)
 
         # Bind row select
@@ -170,23 +160,23 @@ class MonitorFrame(wx.Frame):
             # Load the file chosen by the user.
             self.__opened_filename = fileDialog.GetPath()
 
-            self.SetStatusText(f"Loading file {self.__opened_filename}.")
+            self.SetStatusText(f"Loading file {self.__opened_filename}.", 1)
             self.__cor.load(self.__opened_filename)
 
             # Refresh data in grid
-            self.refresh_grid()
+            self.__refresh_grid()
 
-            self.SetStatusText(f"File {self.__opened_filename} loaded.")
+            self.SetStatusText(f"File {self.__opened_filename} loaded.", 1)
 
     def save_file(self, event):
-        self.SetStatusText(f"Saving file as {self.__opened_filename}")
+        self.SetStatusText(f"Saving file as {self.__opened_filename}", 1)
 
         if self.__opened_filename is None:
             self.save_file_as(event)
         else:
             self.__cor.save(self.__opened_filename)
 
-        self.SetStatusText(f"File saved as {self.__opened_filename}")
+        self.SetStatusText(f"File saved as {self.__opened_filename}", 1)
 
     def save_file_as(self, event):
         with wx.FileDialog(self, "Save Coefficients file", wildcard="cpd (*.cpd)|*.cpd",
@@ -195,12 +185,12 @@ class MonitorFrame(wx.Frame):
                 return  # the user changed their mind
 
             # Save the file and price data file, changing opened filename so next save writes to new file
-            self.SetStatusText(f"Saving file as {self.__opened_filename}")
+            self.SetStatusText(f"Saving file as {self.__opened_filename}", 1)
 
             self.__opened_filename = fileDialog.GetPath()
             self.__cor.save(self.__opened_filename)
 
-            self.SetStatusText(f"File saved as {self.__opened_filename}")
+            self.SetStatusText(f"File saved as {self.__opened_filename}", 1)
 
     def calculate_coefficients(self, event):
         # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
@@ -209,23 +199,23 @@ class MonitorFrame(wx.Frame):
         utc_from = utc_to - timedelta(days=self.__config.get('calculate.from.days'))
 
         # Calculate
-        self.SetStatusText("Calculating coefficients.")
+        self.SetStatusText("Calculating coefficients.", 1)
         self.__cor.calculate(date_from=utc_from, date_to=utc_to,
                              timeframe=self.__config.get('calculate.timeframe'),
                              min_prices=self.__config.get('calculate.min_prices'),
                              max_set_size_diff_pct=self.__config.get('calculate.max_set_size_diff_pct'),
                              overlap_pct=self.__config.get('calculate.overlap_pct'),
                              max_p_value=self.__config.get('calculate.max_p_value'))
-        self.SetStatusText("")
+        self.SetStatusText("", 1)
 
         # Show calculated data
-        self.refresh_grid()
+        self.__refresh_grid()
 
     def quit(self, event):
         # Close
         self.Close()
 
-    def refresh_grid(self):
+    def __refresh_grid(self):
         """
         Refreshes grid. Notifies if rows have been added or deleted.
         :return:
@@ -233,17 +223,13 @@ class MonitorFrame(wx.Frame):
         self.__log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}")
 
         # Update data
-        self.table.data = self.__cor.coefficient_data.copy()
+        self.table.data = self.__cor.filtered_coefficient_data.copy()
 
         # Format
         self.table.data.loc[:, 'Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format)
-        self.table.data.loc[:, 'Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True)
-        self.table.data.loc[:, 'Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S')
-        self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format)
-
-        # Remove nans. The ones from the float column will be str nan as they have been formatted
-        self.table.data = self.table.data.fillna('')
-        self.table.data.loc[:, 'Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '')
+        self.table.data.loc[:, 'Last Calculation'] = pd.to_datetime(self.table.data['Last Calculation'], utc=True)
+        self.table.data.loc[:, 'Last Calculation'] = \
+            self.table.data['Last Calculation'].dt.strftime('%d-%m-%y %H:%M:%S')
 
         # Start refresh
         self.grid_correlations.BeginBatch()
@@ -270,33 +256,34 @@ class MonitorFrame(wx.Frame):
         # Update row count
         self.__rows = cur_rows
 
-    def monitor(self, event):
+    def __monitor(self, event):
         # Check state of toggle button. If on, then start monitoring, else stop
-        if self.monitor_toggle.GetValue():
-            self.__log.info("Starting monitoring.")
-            self.monitor_toggle.SetBackgroundColour(wx.GREEN)
-            self.monitor_toggle.SetLabelText("On")
-            self.SetStatusText("Monitoring for changes to coefficients.")
+        if self.__menu_item_monitor.IsChecked():
+            self.__log.info("Starting monitoring for changes to coefficients.")
+            self.SetStatusText("Monitoring", 0)
+            self.__statusbar.SetBackgroundColour('green')
+            self.__statusbar.Refresh()
 
             self.timer.Start(self.__config.get('monitor.interval')*1000)
 
             # Autosave filename
             filename = self.__opened_filename if self.__opened_filename is not None else 'autosave.cpd'
 
+            # Build calculation params and start monitor
+            calculation_params = [self.__config.get('monitor.calculations.long'),
+                                  self.__config.get('monitor.calculations.medium'),
+                                  self.__config.get('monitor.calculations.short')]
+
             self.__cor.start_monitor(interval=self.__config.get('monitor.interval'),
-                                     from_mins=self.__config.get('monitor.from.minutes'),
-                                     min_prices=self.__config.get('monitor.min_prices'),
-                                     max_set_size_diff_pct=self.__config.get('monitor.max_set_size_diff_pct'),
-                                     overlap_pct=self.__config.get('monitor.overlap_pct'),
-                                     max_p_value=self.__config.get('monitor.max_p_value'),
+                                     calculation_params=calculation_params,
                                      cache_time=self.__config.get('monitor.tick_cache_time'),
                                      autosave=self.__config.get('monitor.autosave'),
                                      filename=filename)
         else:
             self.__log.info("Stopping monitoring.")
-            self.monitor_toggle.SetBackgroundColour(wx.RED)
-            self.monitor_toggle.SetLabelText("Off")
-            self.SetStatusText("Monitoring stopped.")
+            self.SetStatusText("Not Monitoring", 0)
+            self.__statusbar.SetBackgroundColour('lightgray')
+            self.__statusbar.Refresh()
             self.timer.Stop()
             self.__cor.stop_monitor()
 
@@ -329,19 +316,25 @@ class MonitorFrame(wx.Frame):
                     reload_correlations = True
                 if setting.startswith('logging.'):
                     reload_logger = True
-                if setting.startswith('monitor.from.'):
+                if setting.startswith('monitor.calculations'):
                     reload_graph = True
 
             # Now perform the actions
             if restart_monitor_timer:
                 self.__log.info("Settings updated. Reloading monitoring timer.")
                 self.__cor.stop_monitor()
+
+                # Build calculation params and start monitor
+                calculation_params = [self.__config.get('monitor.calculations.long'),
+                                      self.__config.get('monitor.calculations.medium'),
+                                      self.__config.get('monitor.calculations.short')]
+
                 self.__cor.start_monitor(interval=self.__config.get('monitor.interval'),
-                                         from_mins=self.__config.get('monitor.from.minutes'),
-                                         min_prices=self.__config.get('monitor.min_prices'),
-                                         max_set_size_diff_pct=self.__config.get('monitor.max_set_size_diff_pct'),
-                                         overlap_pct=self.__config.get('monitor.overlap_pct'),
-                                         max_p_value=self.__config.get('monitor.max_p_value'))
+                                         calculation_params=calculation_params,
+                                         cache_time=self.__config.get('monitor.tick_cache_time'),
+                                         autosave=self.__config.get('monitor.autosave'),
+                                         filename=self.__opened_filename)
+
             if restart_gui_timer:
                 self.__log.info("Settings updated. Restarting gui timer.")
                 self.timer.Stop()
@@ -350,7 +343,7 @@ class MonitorFrame(wx.Frame):
             if reload_correlations:
                 self.__log.info("Settings updated. Updating monitoring threshold and reloading grid.")
                 self.__cor.monitoring_threshold = self.__config.get("monitor.monitoring_threshold")
-                self.refresh_grid()
+                self.__refresh_grid()
 
             if reload_logger:
                 self.__log.info("Settings updated. Reloading logger.")
@@ -418,14 +411,21 @@ class MonitorFrame(wx.Frame):
         symbol_2_price_data = self.__cor.get_price_data(symbol2)
         symbol_1_ticks = self.__cor.get_ticks(symbol1, cache_only=True)
         symbol_2_ticks = self.__cor.get_ticks(symbol2, cache_only=True)
-        history_data = self.__cor.get_coefficient_history(symbol1, symbol2)
-        times = history_data['UTC Date To']
-        coefficients = history_data['Coefficient']
-
+        history_data_short = \
+            self.__cor.get_coefficient_history({'Symbol 1': symbol1, 'Symbol 2': symbol2,
+                                                'Timeframe': self.__config.get('monitor.calculations.short.from')})
+        history_data_med = \
+            self.__cor.get_coefficient_history({'Symbol 1': symbol1, 'Symbol 2': symbol2,
+                                                'Timeframe': self.__config.get('monitor.calculations.medium.from')})
+        history_data_long = \
+            self.__cor.get_coefficient_history({'Symbol 1': symbol1, 'Symbol 2': symbol2,
+                                                'Timeframe': self.__config.get('monitor.calculations.long.from')})
         # Display if we have any data
         self.__log.debug(f"Refreshing history graph {symbol1}:{symbol2}.")
-        self.__graph.draw(times=times, coefficients=coefficients, prices=[symbol_1_price_data, symbol_2_price_data],
-                          ticks=[symbol_1_ticks, symbol_2_ticks], symbols=[symbol1, symbol2])
+        self.__graph.draw(prices=[symbol_1_price_data, symbol_2_price_data], ticks=[symbol_1_ticks, symbol_2_ticks],
+                          history=[history_data_short, history_data_med, history_data_long], symbols=[symbol1, symbol2],
+                          divergence_threshold=self.__cor.divergence_threshold,
+                          monitor_inverse=self.__cor.monitor_inverse)
 
         # Un-hide and layout if hidden
         if not self.__graph.IsShown():
@@ -434,13 +434,33 @@ class MonitorFrame(wx.Frame):
 
     def __timer_event(self, event):
         """
-        Called on timer event. Refreshes grid and updatates selected graph.
+        Called on timer event. Refreshes grid and updates selected graph.
         :return:
         """
-        self.refresh_grid()
+        self.__refresh_grid()
         if len(self.__selected_correlation) == 2:
             self.show_graph(symbol1=self.__selected_correlation[0], symbol2=self.__selected_correlation[1])
 
+        # Set status message
+        self.SetStatusText(f"Status updated at {self.__cor.get_last_calculation():%d-%b %H:%M:%S}.", 1)
+
+    def __clear_history(self, event):
+        """
+        Clears the calculated coefficient history and associated price data
+        :param event:
+        :return:
+        """
+        # Clear the history
+        self.__cor.clear_coefficient_history()
+
+        # Reload graph if we have a coefficient selected
+        self.__log.info("History cleared. Reloading graph.")
+        if len(self.__selected_correlation) == 2:
+            self.show_graph(symbol1=self.__selected_correlation[0], symbol2=self.__selected_correlation[1])
+
+        # Reload the table
+        self.__refresh_grid()
+
 
 class DataTable(wx.grid.GridTableBase):
     """
@@ -486,11 +506,11 @@ class DataTable(wx.grid.GridTableBase):
 
         # If column is last coefficient, get value and check against threshold. Highlight if diverged.
         threshold = Config().get('monitor.divergence_threshold')
-        if col == MonitorFrame.COLUMN_LAST_COEFFICIENT:
+        if col in [MonitorFrame.COLUMN_STATUS]:
+            # Is status one of interest
             value = self.GetValue(row, col)
             if value != "":
-                value = float(value)
-                if value <= threshold:
+                if value in [cor.STATUS_BELOW_DIVERGENCE_THRESHOLD]:
                     attr.SetBackgroundColour(wx.YELLOW)
                 else:
                     attr.SetBackgroundColour(wx.WHITE)
@@ -507,12 +527,8 @@ class GraphPanel(wx.Panel):
         # Super
         wx.Panel.__init__(self, parent)
 
-        # 3  axis, 2 price data for calculate, 2 price data for last coefficient and coefficient history.
-        # All will have axis labels and top and right boarders
-        # removed
-        self.__fig, self.__axes = plt.subplots(nrows=5, ncols=1)
-
-        # Create the canvas
+        # Fig & canvas
+        self.__fig = plt.figure()
         self.__canvas = FigureCanvas(self, -1, self.__fig)
 
         # Date format for x axes
@@ -531,71 +547,153 @@ class GraphPanel(wx.Panel):
         self.__axes = None
         self.__fig = None
 
-    def draw(self, times, coefficients, prices=None, symbols=None, ticks=None):
+    def draw(self, prices, ticks, history, symbols, divergence_threshold=None, monitor_inverse=False):
         """
         Plot the correlations.
-        :param times: Series of time values for x axis for coefficient history chart
-        :param coefficients: Series of coefficients values for y axis of coefficient history chart
         :param prices: Price data used to calculate base coefficient. List [Symbol1 Price Data, Symbol 2 Price Data]
-        :param symbols: Symbols. List [Symbol1, Symbol2]
         :param ticks: Ticks used to calculate last coefficient. List [Symbol1, Symbol2]
+        :param history: Coefficient history data. List of data for one or more timeframes.
+        :param symbols: Symbols. List [Symbol1, Symbol2]
+        :param divergence_threshold: The divergence threshold. Will be plotted on the coefficients charts if specified.
+        :param monitor_inverse: Are we monitoring inverse correlations. If so, a line for the inverse threshold will be
+            plotted if the divergence threshold is specified.
         :return:
         """
-        # Clear. We will need to redraw
-        for ax in self.__axes:
-            ax.clear()
 
-        if symbols is not None and len(symbols) == 2:
+        # Check what data we have available
+        price_data_available = prices is not None and len(prices) == 2 and \
+            prices[0] is not None and prices[1] is not None and len(prices[0]) > 0 and len(prices[1]) > 0
+        tick_data_available = ticks is not None and len(ticks) == 2 and ticks[0] is not None and ticks[1] is not None \
+            and len(ticks[0]) > 0 and len(ticks[1]) > 0
+        history_data_available = history is not None and len(history) > 0
+        symbols_selected = symbols is not None and len(symbols) == 2
+
+        # Get all plots for history. History can contain multiple plots for different timeframes. They will all be
+        # plotted on the same chart.
+        times = []
+        coefficients = []
+        if history_data_available:
+            for hist in history:
+                times.append(hist['Date To'])
+                coefficients.append(hist['Coefficient'])
+
+        if symbols_selected:
             # Axis ranges
-            price_chart_date_range = [min(min(prices[0]['time']), min(prices[1]['time'])),
-                                      max(max(prices[0]['time']), max(prices[1]['time']))]
-            tick_chart_date_range = [min(min(ticks[0]['time']), min(ticks[1]['time'])),
-                                     max(max(ticks[0]['time']), max(ticks[1]['time']))]
+            if price_data_available:
+                price_chart_date_range = [min(min(prices[0]['time']), min(prices[1]['time'])),
+                                          max(max(prices[0]['time']), max(prices[1]['time']))]
+            else:
+                price_chart_date_range = [datetime.now() - timedelta(days=1), datetime.now()]
 
-            # Chart config
-            titles = [f"Base Coefficient Price Data for {symbols[0]}", f"Base Coefficient Price Data for {symbols[1]}",
-                      f"Coefficient Tick Data for {symbols[0]}", f"Coefficient Tick Data for {symbols[1]}",
-                      f"Coefficient History for {symbols[0]}:{symbols[1]}"]
-            xlims = [price_chart_date_range, price_chart_date_range, tick_chart_date_range, tick_chart_date_range, None]
-            ylims = [None, None, None, None, [-1, 1]]
-            xlabels = [None, None, None, None, None]
-            ylabels = ['Price', 'Price', 'Price', 'Price', 'Coefficient']
-            tick_labels = [[], prices[1]['time'], [], ticks[1]['time'], times]
-            mtick_fmts = [None, self.__tick_fmt_date, None, self.__tick_fmt_time, self.__tick_fmt_time]
-            mtick_rot = [0, 45, 0, 45, 45]
-            xdata = [prices[0]['time'], prices[1]['time'], ticks[0]['time'], ticks[1]['time'], times]
-            ydata = [prices[0]['close'], prices[1]['close'], ticks[0]['ask'], ticks[1]['ask'], coefficients]
-            types = ['plot', 'plot', 'plot', 'plot', 'scatter']
+            if tick_data_available:
+                tick_chart_date_range = [min(min(ticks[0]['time']), min(ticks[1]['time'])),
+                                         max(max(ticks[0]['time']), max(ticks[1]['time']))]
+            else:
+                tick_chart_date_range = [datetime.now() - timedelta(days=1/48), datetime.now()]
+
+            # First two charts. Data used to calculate base coefficient and data used to calculate latest coefficient.
+            # Both charts will use 2 plots on a single axis and have different y ranges.
+            titles = [f"Base Coefficient Price Data for {symbols[0]}:{symbols[1]}",
+                      f"Coefficient Tick Data for {symbols[0]}:{symbols[1]}"]
+            xlims = [price_chart_date_range, tick_chart_date_range]
+
+            xdata = [[prices[0]['time'] if price_data_available else [],
+                      prices[1]['time'] if price_data_available else []],
+                     [ticks[0]['time'] if tick_data_available else [],
+                      ticks[1]['time'] if tick_data_available else []]]
+
+            ydata = [[prices[0]['close'] if price_data_available else [],
+                      prices[1]['close'] if price_data_available else []],
+                     [ticks[0]['ask'] if tick_data_available else [],
+                      ticks[1]['ask'] if tick_data_available else []]]
+
+            tick_labels = [prices[1]['time'] if price_data_available else [],
+                           ticks[1]['time'] if tick_data_available else []]
+
+            tick_formats = [self.__tick_fmt_date, self.__tick_fmt_time]
+
+            # Clear the figure then redraw the 2 charts
+            self.__fig.clf()
+            for i in range(0, 2):
+                # 2 axis. One for each symbol
+                s1ax = self.__fig.add_subplot(3, 1, i+1)
+                s2ax = s1ax.twinx()
 
-            # Draw 5 charts
-            for index in range(0, len(self.__axes)):
                 # Titles and axis labels
-                self.__axes[index].set_title(titles[index])
-                self.__axes[index].set_xlabel(xlabels[index])
-                self.__axes[index].set_ylabel(ylabels[index])
+                s1ax.set_title(titles[i])
+                s1ax.set_ylabel('Price')
 
-                # Limits
-                if xlims[index] is not None:
-                    self.__axes[index].set_xlim(xlims[index])
+                # X Limits
+                s1ax.set_xlim(xlims[i])
 
-                if ylims[index] is not None:
-                    self.__axes[index].set_ylim(ylims[index])
+                # Y Labels. Left for symbol1, right for symbol2
+                colors = ['green', 'blue']
+                s1ax.set_ylabel(f"{symbols[0]}", color=colors[0])
+                s2ax.set_ylabel(f"{symbols[1]}", color=colors[1])
 
-                # Tick labels and formats
-                self.__axes[index].xaxis.set_ticklabels(tick_labels[index])
-                if mtick_fmts[index] is not None:
-                    self.__axes[index].xaxis.set_major_formatter(mtick_fmts[index])
-                plt.setp(self.__axes[index].xaxis.get_majorticklabels(), rotation=mtick_rot[index])
+                # Plot both lines
+                s1ax.plot(xdata[i][0], ydata[i][0], color=colors[0])
+                s2ax.plot(xdata[i][1], ydata[i][1], color=colors[1])
 
-                # Remove typ and right boarders
-                self.__axes[index].spines["top"].set_visible(False)
-                self.__axes[index].spines["right"].set_visible(False)
+                # Y tick colours
+                s1ax.tick_params(axis='y', labelcolor=colors[0])
+                s2ax.tick_params(axis='y', labelcolor=colors[1])
 
-                # Plot
-                if types[index] == 'plot':
-                    self.__axes[index].plot(xdata[index], ydata[index])
-                elif types[index] == 'scatter':
-                    self.__axes[index].scatter(xdata[index], ydata[index], s=1)
+                # Ticks, labels and formats. Fixing xticks with FixedLocator but also using MaxNLocator to avoid
+                # cramped x-labels
+                if len(tick_labels[i]) > 0:
+                    s1ax.xaxis.set_major_locator(mticker.MaxNLocator(10))
+                    ticks_loc = s1ax.get_xticks().tolist()
+                    s1ax.xaxis.set_major_locator(mticker.FixedLocator(ticks_loc))
+                    s1ax.set_xticklabels(ticks_loc)
+                    if tick_formats[i] is not None:
+                        s1ax.xaxis.set_major_formatter(tick_formats[i])
+                    plt.setp(s1ax.xaxis.get_majorticklabels(), rotation=45)
+                else:
+                    s1ax.set_xticklabels([])
+
+            # Third chart showing the coefficient history and the divergence threshold lines.
+            ax = self.__fig.add_subplot(3, 1, 3)
+
+            # Titles and axis labels
+            ax.set_title(f"Coefficient History for {symbols[0]}:{symbols[1]}")
+            ax.set_ylabel('Coefficient')
+
+            # Y Limits. Coefficients range from -1 to 1
+            ax.set_ylim([-1, 1])
+
+            # Plot data if we have history data available
+            if history_data_available:
+                # Plot. There may be more than one set of data for chart. One for each coefficient date range. Convert
+                # single data to list, then loop to plot
+                xdata = times if isinstance(times, list) else [times, ]
+                ydata = coefficients if isinstance(coefficients, list) else [coefficients, ]
+
+                for i in range(0, len(xdata)):
+                    ax.scatter(xdata[i], ydata[i], s=1)
+
+                # Ticks, labels and formats. Fixing xticks with FixedLocator but also using MaxNLocator to avoid
+                # cramped x-labels
+                if len(times[0].array) > 0:
+                    ax.xaxis.set_major_locator(mticker.MaxNLocator(10))
+                    ticks_loc = ax.get_xticks().tolist()
+                    ax.xaxis.set_major_locator(mticker.FixedLocator(ticks_loc))
+                    ax.set_xticklabels(ticks_loc)
+                    ax.xaxis.set_major_formatter(self.__tick_fmt_time)
+                    plt.setp(ax.xaxis.get_majorticklabels(), rotation=45)
+                else:
+                    ax.set_xticklabels([])
+
+                # Legend
+                ax.legend([f"{Config().get('monitor.calculations.long.from')} Minutes",
+                           f"{Config().get('monitor.calculations.medium.from')} Minutes",
+                           f"{Config().get('monitor.calculations.short.from')} Minutes"])
+
+                # Lines showing divergence threshold. 2 if we are monitoring inverse correlations.
+                if divergence_threshold is not None:
+                    ax.axhline(y=divergence_threshold, color="red", label='_nolegend_', linewidth=1)
+                    if monitor_inverse:
+                        ax.axhline(y=divergence_threshold * -1, color="red", label='_nolegend_', linewidth=1)
 
             # Layout with padding between charts
             self.__fig.tight_layout(pad=0.5)

requirements.txt

@@ -5,4 +5,6 @@ pytz==2021.1
 scipy==1.6.0
 logging==0.4.9.6
 pyyaml==5.4.1
-wxpython==4.1.1
+wxpython==4.1.1
+mock==4.0.3
+numpy==1.20.0

test/test_correlation.py

@@ -0,0 +1,330 @@
+import unittest
+from unittest.mock import patch
+import time
+import mt5_correlation.correlation as correlation
+import pandas as pd
+from datetime import datetime, timedelta
+from test_mt5 import Symbol
+import random
+import os
+
+
+class TestCorrelation(unittest.TestCase):
+    # Mock symbols. 4 Symbols, 3 visible.
+    mock_symbols = [Symbol(name='SYMBOL1', visible=True),
+                    Symbol(name='SYMBOL2', visible=True),
+                    Symbol(name='SYMBOL3', visible=False),
+                    Symbol(name='SYMBOL4', visible=True),
+                    Symbol(name='SYMBOL5', visible=True)]
+
+    # Start and end date for price data and mock prices: base; correlated; and uncorrelated.
+    start_date = None
+    end_date = None
+    price_columns = None
+    mock_base_prices = None
+    mock_correlated_prices = None
+    mock_uncorrelated_prices = None
+
+    def setUp(self):
+        """
+        Creates some price data fro use in tests
+        :return:
+        """
+        # Start and end date for price data and mock price dataframes. One for: base; correlated; uncorrelated and
+        # different dates.
+        self.start_date = datetime(2021, 1, 1, 1, 5, 0)
+        self.end_date = datetime(2021, 1, 1, 11, 30, 0)
+        self.price_columns = ['time', 'close']
+        self.mock_base_prices = pd.DataFrame(columns=self.price_columns)
+        self.mock_correlated_prices = pd.DataFrame(columns=self.price_columns)
+        self.mock_uncorrelated_prices = pd.DataFrame(columns=self.price_columns)
+        self.mock_correlated_different_dates = pd.DataFrame(columns=self.price_columns)
+        self.mock_inverse_correlated_prices = pd.DataFrame(columns=self.price_columns)
+
+        # Build the price data for the test. One price every 5 minutes for 500 rows. Base will use min for price,
+        # correlated will use min + 5 and uncorrelated will use random
+        for date in (self.start_date + timedelta(minutes=m) for m in range(0, 500*5, 5)):
+            self.mock_base_prices = self.mock_base_prices.append(pd.DataFrame(columns=self.price_columns,
+                                                                              data=[[date, date.minute]]))
+            self.mock_correlated_prices = \
+                self.mock_correlated_prices.append(pd.DataFrame(columns=self.price_columns,
+                                                                data=[[date, date.minute + 5]]))
+            self.mock_uncorrelated_prices = \
+                self.mock_uncorrelated_prices.append(pd.DataFrame(columns=self.price_columns,
+                                                                  data=[[date, random.randint(0, 1000000)]]))
+
+            self.mock_correlated_different_dates = \
+                self.mock_correlated_different_dates.append(pd.DataFrame(columns=self.price_columns,
+                                                                         data=[[date + timedelta(minutes=100),
+                                                                                date.minute + 5]]))
+            self.mock_inverse_correlated_prices = \
+                self.mock_inverse_correlated_prices.append(pd.DataFrame(columns=self.price_columns,
+                                                                        data=[[date, (date.minute + 5) * -1]]))
+
+    @patch('mt5_correlation.mt5.MetaTrader5')
+    def test_calculate(self, mock):
+        """
+        Test the calculate method. Uses mock for MT5 symbols and prices.
+        :param mock:
+        :return:
+        """
+        # Mock symbol return values
+        mock.symbols_get.return_value = self.mock_symbols
+
+        # Correlation class
+        cor = correlation.Correlation(monitoring_threshold=1, monitor_inverse=True)
+
+        # Calculate for price data. We should have 100% matching dates in sets. Get prices should be called 4 times.
+        # We don't have a SYMBOL3 as this is set as not visible. Correlations should be as follows:
+        #   SYMBOL1:SYMBOL2 should be fully correlated (1)
+        #   SYMBOL1:SYMBOL4 should be uncorrelated (0)
+        #   SYMBOL1:SYMBOL5 should be negatively correlated
+        #   SYMBOL2:SYMBOL5 should be negatively correlated
+        # We will not use p_value as the last set uses random numbers so p value will not be useful.
+        mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices,
+                                             self.mock_uncorrelated_prices, self.mock_inverse_correlated_prices]
+        cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100,
+                      max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1)
+
+        # Test the output. We should have 6 rows. S1:S2 c=1, S1:S4 c<1, S1:S5 c=-1, S2:S5 c=-1. We are not checking
+        # S2:S4 or S4:S5
+        self.assertEqual(len(cor.coefficient_data.index), 6, "There should be six correlations rows calculated.")
+        self.assertEqual(cor.get_base_coefficient('SYMBOL1', 'SYMBOL2'), 1,
+                         "The correlation for SYMBOL1:SYMBOL2 should be 1.")
+        self.assertTrue(cor.get_base_coefficient('SYMBOL1', 'SYMBOL4') < 1,
+                        "The correlation for SYMBOL1:SYMBOL4 should be <1.")
+        self.assertEqual(cor.get_base_coefficient('SYMBOL1', 'SYMBOL5'), -1,
+                         "The correlation for SYMBOL1:SYMBOL5 should be -1.")
+        self.assertEqual(cor.get_base_coefficient('SYMBOL2', 'SYMBOL5'), -1,
+                         "The correlation for SYMBOL2:SYMBOL5 should be -1.")
+
+        # Monitoring threshold is 1 and we are monitoring inverse. Get filtered correlations. There should be 3 (S1:S2,
+        # S1:S5 and S2:S5)
+        self.assertEqual(len(cor.filtered_coefficient_data.index), 3,
+                         "There should be 3 rows in filtered coefficient data when we are monitoring inverse "
+                         "correlations.")
+
+        # Now aren't monitoring inverse correlations. There should only be one correlation when filtered
+        cor.monitor_inverse = False
+        self.assertEqual(len(cor.filtered_coefficient_data.index), 1,
+                         "There should be only 1 rows in filtered coefficient data when we are not monitoring  inverse "
+                         "correlations.")
+
+        # Now were going to recalculate, but this time SYMBOL1:SYMBOL2 will have non overlapping dates and coefficient
+        # should be None. There shouldn't be a row. We should have correlations for S1:S4, S1:S5 and S4:S5
+        mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_different_dates,
+                                             self.mock_correlated_prices, self.mock_correlated_prices]
+        cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100,
+                      max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1)
+        self.assertEqual(len(cor.coefficient_data.index), 3, "There should be three correlations rows calculated.")
+        self.assertEqual(cor.coefficient_data.iloc[0, 2], 1, "The correlation for SYMBOL1:SYMBOL4 should be 1.")
+        self.assertEqual(cor.coefficient_data.iloc[1, 2], 1, "The correlation for SYMBOL1:SYMBOL5 should be 1.")
+        self.assertEqual(cor.coefficient_data.iloc[2, 2], 1, "The correlation for SYMBOL4:SYMBOL5 should be 1.")
+
+        # Get the price data used to calculate the coefficients fro symbol 1. It should match mock_base_prices.
+        price_data = cor.get_price_data('SYMBOL1')
+        self.assertTrue(price_data.equals(self.mock_base_prices), "Price data returned post calculation should match "
+                                                                  "mock price data.")
+
+    def test_calculate_coefficient(self):
+        """
+        Tests the coefficient calculation.
+        :return:
+        """
+        # Correlation class
+        cor = correlation.Correlation()
+
+        # Test 2 correlated sets
+        coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_correlated_prices)
+        self.assertEqual(coefficient, 1, "Coefficient should be 1.")
+
+        # Test 2 uncorrelated sets. Set p value to 1 to force correlation to be returned.
+        coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_uncorrelated_prices, max_p_value=1)
+        self.assertTrue(coefficient < 1, "Coefficient should be < 1.")
+
+        # Test 2 sets where prices dont overlap
+        coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_correlated_different_dates)
+        self.assertTrue(coefficient < 1, "Coefficient should be None.")
+
+        # Test 2 inversely correlated sets
+        coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_inverse_correlated_prices)
+        self.assertEqual(coefficient, -1, "Coefficient should be -1.")
+
+    @patch('mt5_correlation.mt5.MetaTrader5')
+    def test_get_ticks(self, mock):
+        """
+        Test that caching works. For the purpose of this test, we can use price data rather than tick data.
+        Mock 2 different sets of prices. Get three times. Base, One within cache threshold and one outside. Set 1
+        should match set 2 but differ from set 3.
+        :param mock:
+        :return:
+        """
+
+        # Correlation class to test
+        cor = correlation.Correlation()
+
+        # Mock the tick data to contain 2 different sets. Then get twice. They should match as the data was cached.
+        mock.copy_ticks_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices]
+
+        # We need to start and stop the monitor as this will set the cache time
+        cor.start_monitor(interval=10, calculation_params={'from': 10, 'min_prices': 0, 'max_set_size_diff_pct': 0,
+                                                           'overlap_pct': 0, 'max_p_value': 1}, cache_time=3)
+        cor.stop_monitor()
+
+        # Get the ticks within cache time and check that they match
+        base_ticks = cor.get_ticks('SYMBOL1', None, None)
+        cached_ticks = cor.get_ticks('SYMBOL1', None, None)
+        self.assertTrue(base_ticks.equals(cached_ticks),
+                        "Both sets of tick data should match as set 2 came from cache.")
+
+        # Wait 3 seconds
+        time.sleep(3)
+
+        # Retrieve again. This one should be different as the cache has expired.
+        non_cached_ticks = cor.get_ticks('SYMBOL1', None, None)
+        self.assertTrue(not base_ticks.equals(non_cached_ticks),
+                        "Both sets of tick data should differ as cached data had expired.")
+
+    @patch('mt5_correlation.mt5.MetaTrader5')
+    def test_start_monitor(self, mock):
+        """
+        Test that starting the monitor and running for 2 seconds produces two sets of coefficient history when using an
+        interval of 1 second.
+        :param mock:
+        :return:
+        """
+        # Mock symbol return values
+        mock.symbols_get.return_value = self.mock_symbols
+
+        # Create correlation class. We will set a divergence threshold so that we can test status.
+        cor = correlation.Correlation(divergence_threshold=0.8, monitor_inverse=True)
+
+        # Calculate for price data. We should have 100% matching dates in sets. Get prices should be called 4 times.
+        # We dont have a SYMBOL2 as this is set as not visible. All pairs should be correlated for the purpose of this
+        # test.
+        mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices,
+                                             self.mock_correlated_prices, self.mock_inverse_correlated_prices]
+
+        cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100,
+                      max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1)
+
+        # We will build some tick data for each symbol and patch it in. Tick data will be from 10 seconds ago to now.
+        # We only need to patch in one set of tick data for each symbol as it will be cached.
+        columns = ['time', 'ask']
+        starttime = datetime.now() - timedelta(seconds=10)
+        tick_data_s1 = pd.DataFrame(columns=columns)
+        tick_data_s2 = pd.DataFrame(columns=columns)
+        tick_data_s4 = pd.DataFrame(columns=columns)
+        tick_data_s5 = pd.DataFrame(columns=columns)
+
+        now = datetime.now()
+        price_base = 1
+        while starttime < now:
+            tick_data_s1 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.5]]))
+            tick_data_s2 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.1]]))
+            tick_data_s4 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.25]]))
+            tick_data_s5 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * -0.25]]))
+            starttime = starttime + timedelta(milliseconds=10*random.randint(0, 100))
+            price_base += 1
+
+        # Patch it in
+        mock.copy_ticks_range.side_effect = [tick_data_s1, tick_data_s2, tick_data_s4, tick_data_s5]
+
+        # Start the monitor. Run every second. Use ~10 and ~5 seconds of data. Were not testing the overlap and price
+        # data quality metrics here as that is set elsewhere so these can be set to not take effect. Set cache level
+        # high and don't use autosave. Timer runs in a separate thread so test can continue after it has started.
+        cor.start_monitor(interval=1, calculation_params=[{'from': 0.66, 'min_prices': 0,
+                                                           'max_set_size_diff_pct': 0, 'overlap_pct': 0,
+                                                           'max_p_value': 1},
+                                                          {'from': 0.33, 'min_prices': 0,
+                                                           'max_set_size_diff_pct': 0, 'overlap_pct': 0,
+                                                           'max_p_value': 1}], cache_time=100, autosave=False)
+
+        # Wait 2 seconds so timer runs twice
+        time.sleep(2)
+
+        # Stop the monitor
+        cor.stop_monitor()
+
+        # We should have 2 coefficients calculated for each symbol pair (6), for each date_from value (2),
+        # for each run (2) so 24 in total.
+        self.assertEqual(len(cor.coefficient_history.index), 24)
+
+        # We should have 2 coefficients calculated for a single symbol pair and timeframe
+        self.assertEqual(len(cor.get_coefficient_history({'Symbol 1': 'SYMBOL1', 'Symbol 2': 'SYMBOL2',
+                                                          'Timeframe': 0.66})),
+                         2, "We should have 2 history records for SYMBOL1:SYMBOL2 using the 0.66 min timeframe.")
+
+        # The status should be BELOW for SYMBOL1:SYMBOL2 and ABOVE for SYMBOL1:SYMBOL4 and SYMBOL2:SYMBOL4.
+        self.assertTrue(cor.get_last_status('SYMBOL1', 'SYMBOL2') == correlation.STATUS_BELOW_DIVERGENCE_THRESHOLD)
+        self.assertTrue(cor.get_last_status('SYMBOL1', 'SYMBOL4') == correlation.STATUS_ABOVE_DIVERGENCE_THRESHOLD)
+        self.assertTrue(cor.get_last_status('SYMBOL2', 'SYMBOL4') == correlation.STATUS_ABOVE_DIVERGENCE_THRESHOLD)
+
+        # We are monitoring inverse correlations, status for SYMBOL1:SYMBOL5 should be BELOW
+        self.assertTrue(cor.get_last_status('SYMBOL2', 'SYMBOL5') == correlation.STATUS_BELOW_DIVERGENCE_THRESHOLD)
+
+    @patch('mt5_correlation.mt5.MetaTrader5')
+    def test_load_and_save(self, mock):
+        """Calculate and run monitor for a few seconds. Store the data. Save it, load it then compare against stored
+        data."""
+
+        # Correlation class
+        cor = correlation.Correlation()
+
+        # Patch symbol and price data, then calculate
+        mock.symbols_get.return_value = self.mock_symbols
+        mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices,
+                                             self.mock_correlated_prices, self.mock_inverse_correlated_prices]
+        cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100,
+                      max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1)
+
+        # Patch the tick data
+        columns = ['time', 'ask']
+        starttime = datetime.now() - timedelta(seconds=10)
+        tick_data_s1 = pd.DataFrame(columns=columns)
+        tick_data_s3 = pd.DataFrame(columns=columns)
+        tick_data_s4 = pd.DataFrame(columns=columns)
+        now = datetime.now()
+        price_base = 1
+        while starttime < now:
+            tick_data_s1 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.5]]))
+            tick_data_s3 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.1]]))
+            tick_data_s4 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.25]]))
+            starttime = starttime + timedelta(milliseconds=10 * random.randint(0, 100))
+            price_base += 1
+        mock.copy_ticks_range.side_effect = [tick_data_s1, tick_data_s3, tick_data_s4]
+
+        # Start monitor and run for a seconds with a 1 second interval to produce some coefficient history. Then stop
+        # the monitor
+        cor.start_monitor(interval=1, calculation_params={'from': 0.66, 'min_prices': 0, 'max_set_size_diff_pct': 0,
+                                                          'overlap_pct': 0, 'max_p_value': 1},
+                          cache_time=100, autosave=False)
+        time.sleep(2)
+        cor.stop_monitor()
+
+        # Get copies of data that will be saved.
+        cd_copy = cor.coefficient_data
+        pd_copy = cor.get_price_data('SYMBOL1')
+        mtd_copy = cor.get_ticks('SYMBOL1', cache_only=True)
+        ch_copy = cor.coefficient_history
+
+        # Save, reset data, then reload
+        cor.save("unittest.cpd")
+        cor.load("unittest.cpd")
+
+        # Test that the reloaded data matches the original
+        self.assertTrue(cd_copy.equals(cor.coefficient_data),
+                        "Saved and reloaded coefficient data should match original.")
+        self.assertTrue(pd_copy.equals(cor.get_price_data('SYMBOL1')),
+                        "Saved and reloaded price data should match original.")
+        self.assertTrue(mtd_copy.equals(cor.get_ticks('SYMBOL1', cache_only=True)),
+                        "Saved and reloaded tick data should match original.")
+        self.assertTrue(ch_copy.equals(cor.coefficient_history),
+                        "Saved and reloaded coefficient history should match original.")
+
+        # Cleanup. delete the file
+        os.remove("unittest.cpd")
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_mt5.py

@@ -0,0 +1,77 @@
+import unittest
+from unittest.mock import patch
+import mt5_correlation.mt5 as mt5
+import pandas as pd
+from datetime import datetime
+
+
+class Symbol:
+    """ A Mock symbol class"""
+    name = None
+    visible = None
+
+    def __init__(self, name, visible):
+        self.name = name
+        self.visible = visible
+
+
+class TestMT5(unittest.TestCase):
+    """
+    Unit test for MT5. Uses mock to mock MetaTrader5 connection.
+    """
+
+    # Mock symbols. 5 Symbols, 4 visible.
+    mock_symbols = [Symbol(name='SYMBOL1', visible=True),
+                    Symbol(name='SYMBOL2', visible=True),
+                    Symbol(name='SYMBOL3', visible=False),
+                    Symbol(name='SYMBOL4', visible=True),
+                    Symbol(name='SYMBOL5', visible=True)]
+
+    # Mock prices for symbol 1
+    mock_prices = pd.DataFrame(columns=['time', 'close'],
+                               data=[[datetime(2021, 1, 1, 1, 5, 0), 123.123],
+                                     [datetime(2021, 1, 1, 1, 10, 0), 123.124],
+                                     [datetime(2021, 1, 1, 1, 15, 0), 123.125],
+                                     [datetime(2021, 1, 1, 1, 20, 0), 125.126],
+                                     [datetime(2021, 1, 1, 1, 25, 0), 123.127],
+                                     [datetime(2021, 1, 1, 1, 30, 0), 123.128]])
+
+    @patch('mt5_correlation.mt5.MetaTrader5')
+    def test_get_symbols(self, mock):
+        # Mock return value
+        mock.symbols_get.return_value = self.mock_symbols
+
+        # Call get_symbols
+        symbols = mt5.MT5().get_symbols()
+
+        # There should be four, as one is set as not visible
+        self.assertTrue(len(symbols) == 4, "There should be 5 symbols returned from MT5.")
+
+    @patch('mt5_correlation.mt5.MetaTrader5')
+    def test_get_prices(self, mock):
+        # Mock return value
+        mock.copy_rates_range.return_value = self.mock_prices
+
+        # Call get prices
+        prices = mt5.MT5().get_prices(symbol='SYMBOL1', from_date='01-JAN-2021 01:00:00',
+                                      to_date='01-JAN-2021 01:10:25', timeframe=mt5.TIMEFRAME_M5)
+
+        # There should be 6
+        self.assertTrue(len(prices.index) == 6, "There should be 6 prices.")
+
+    @patch('mt5_correlation.mt5.MetaTrader5')
+    def test_get_ticks(self, mock):
+        # Mock return value
+        mock.copy_ticks_range.return_value = self.mock_prices
+
+        # Call get ticks
+        ticks = mt5.MT5().get_ticks(symbol='SYMBOL1', from_date='01-JAN-2021 01:00:00', to_date='01-JAN-2021 01:10:25')
+
+        # There should be 6
+        self.assertTrue(len(ticks.index) == 6, "There should be 6 prices.")
+
+
+if __name__ == '__main__':
+    unittest.main()
+
+