Improved readme and added some customisation options in correlation calculation

README.md

@@ -1,2 +1,77 @@
 # mt5-correlation
-Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch
+Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch.
+
+# Setup
+1) Set up your MetaTrader 5 environment ensuring that all symbols that you would like to assess for correlation are shown in your Market Watch window;
+2) Set up your python environment; and
+3) Install the required libraries.
+```
+pip install -r mt5-correlation\requirements.txt
+```
+
+# Usage
+If you set up a virtual environment in the Setup step, ensure this is activated. Then run the script.
+```
+python -m mt5_correlations\mt5_correlations.py
+```
+
+A .csv file containing the correlation coefficient for all combinations of sybmols from the MetaTrader market watch will be produced in the current directory.
+
+|Symbol 1    |Symbol 2    |Coefficient|UTC Date From      |UTC Date To        |Timeframe|
+|------------|------------|-----------|-------------------|-------------------|---------|
+|OIL-MAR21   |OILMn-MAR21 |1.0        |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+|EURUSD      |EURHKD      |0.99980    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+|OILMn-MAR21 |BRENT-APR21 |0.99894    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+|OIL-MAR21   |BRENT-APR21 |0.99894    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+|GSOIL-FEB21 |BRENT-APR21 |0.99605    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+|GSOIL-FEB21 |OILMn-MAR21 |0.99543    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+|GSOIL-FEB21 |OIL-MAR21   |0.99543    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+|EU50Cash    |FRA40Cash   |0.99072    |2021-01-29 11:54:29|2021-02-05 11:54:29|15       |
+
+# Customising
+Edit mt5_correlations.py to customise.
+
+The coefficients are calculated only if:
+* The smallest set of price data is no less than 90% of the size of the largest set;
+* The overlapping prices between both sets of price data contains no less than 90% of the prices in the smallest set;
+* The pearsonr p-value for the calculated coefficient is less than 0.05.
+
+These settings can all be changed in the call to Correlation.calculate_coefficient by passing values for max_set_size_diff_pct; overlap_pct; or max_p_value.
+```
+coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data,
+                                                        symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90,
+                                                        overlap_pct=90, max_p_value=0.05)
+```
+
+The price data compared is 15 minute price data for the last 7 days. This can be changed by changing the values for the following variables:
+```
+utc_to = datetime.now(tz=timezone)
+utc_from = utc_to - timedelta(days=7)
+timeframe = mt5.TIMEFRAME_M15
+```
+
+The possible values for timeframe are:
+
+|Timeframe|Description|
+|--------------|-----------|
+|TIMEFRAME_M1  |1 minute   |
+|TIMEFRAME_M2  |2 minutes  |
+|TIMEFRAME_M3  |3 minutes  |
+|TIMEFRAME_M4  |4 minutes  |
+|TIMEFRAME_M5  |5 minutes  |
+|TIMEFRAME_M6  |6 minutes  |
+|TIMEFRAME_M10 |10 minutes |
+|TIMEFRAME_M12 |12 minutes |
+|TIMEFRAME_M12 |15 minutes |
+|TIMEFRAME_M20 |20 minutes |
+|TIMEFRAME_M30 |30 minutes |
+|TIMEFRAME_H1  |1 hour     |
+|TIMEFRAME_H2  |2 hours    |
+|TIMEFRAME_H3  |3 hours    |
+|TIMEFRAME_H4  |4 hours    |
+|TIMEFRAME_H6  |6 hours    |
+|TIMEFRAME_H8  |8 hours    |
+|TIMEFRAME_H12 |12 hours   |
+|TIMEFRAME_D1  |1 day      |
+|TIMEFRAME_W1  |1 week     |
+|TIMEFRAME_MN1 |1 month    |

logging_conf.yaml

@@ -18,28 +18,12 @@ handlers:
         formatter: brief
         stream: ext://sys.stdout
 
-    file-debug:
-        level: DEBUG
-        class: logging.handlers.RotatingFileHandler
-        formatter: precice
-        filename: log\mt5-correlation-debug.log
-        mode: a
-        maxBytes: 2560000  # Max allowed for code insight features of PyCharm
-        backupCount: 2
-
-    file-info:
-        level: INFO
-        class: logging.FileHandler
-        formatter: precice
-        filename: log\mt5-correlation-info.log
-        mode: a
-
 root:
     level: DEBUG
-    handlers: [console, file-info, file-debug]
+    handlers: [console]
 
 loggers:
     mt5-correlation:
         level: DEBUG
-        handlers: [console, file-info, file-debug]
+        handlers: [console]
         propagate: 0

mt5_correlation.py

@@ -3,8 +3,8 @@ import logging.config
 import pandas as pd
 import pytz
 import yaml
-from mt5 import MT5
+from mt5_correlation.mt5 import MT5
-from correlation import Correlation
+from mt5_correlation.correlation import Correlation
 import definitions
 
 # Configure logger
@@ -24,15 +24,19 @@ timezone = pytz.timezone("Etc/UTC")
 utc_to = datetime.now(tz=timezone)
 utc_from = utc_to - timedelta(days=7)
 
+# Set timeframe
+timeframe = mt5.TIMEFRAME_M15
+
 # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
 price_data = {}
 for symbol in symbols:
-    price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to)
+    price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to,
+                                             timeframe=timeframe)
 
 # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
 # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
 # avoid duplicating. We will store all coefficients in a dataframe for export as CSV.
-columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Interval']
+columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe']
 coefficients = pd.DataFrame(columns=columns)
 
 index = 0
@@ -51,23 +55,25 @@ for i in range(0, len(symbols)):
         symbol2_price_data = price_data[symbol2.name]
 
         # Get coefficient and store if valid
-        coefficient = Correlation.calculate_coefficient(symbol1_price_data, symbol2_price_data)
+        coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data,
+                                                        symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90,
+                                                        overlap_pct=90, max_p_value=0.05)
 
         if coefficient is not None:
             coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
                                                 'Coefficient': coefficient, 'UTC Date From': utc_from,
-                                                'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True)
+                                                'UTC Date To': utc_to, 'Timeframe': timeframe}, ignore_index=True)
 
             log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of "
                      f"{coefficient}.")
         else:
-            log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not "
+            log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not"
-                     f"be calculated.")
+                     f" be calculated.")
 
 # Sort, highest correlated first
 coefficients = coefficients.sort_values('Coefficient', ascending=False)
 
 # Save as CSV
-filename = f"out/Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv"
+filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S}.csv"
 log.info(f"Saving coefficients as '{filename}'.")
 coefficients.to_csv(filename, index=False)

mt5_correlation/correlation.py

@@ -8,24 +8,29 @@ class Correlation:
     """
 
     @staticmethod
-    def calculate_coefficient(symbol1_prices, symbol2_prices):
+    def calculate_coefficient(symbol1_prices, symbol2_prices, max_set_size_diff_pct=90, overlap_pct=90,
+                              max_p_value=0.05):
         """
         Calculates the correlation coefficient between two sets of price data. Uses close price.
 
         :param symbol1_prices:
         :param symbol2_prices:
+        :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+            within this pct of each other
+        :param overlap_pct:
+        :param max_p_value: The maximum p value for the correlation to be meaningful
         :return: correlation coefficient, or None if coefficient could not be calculated.
         """
         # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
-        # correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the
+        # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of
-        # largest set and is the overlap set size at least 90% the size of the smallest set?
+        # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set?
         coefficient = None
 
         intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time']))
         len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)]))
         len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
-        similar_size = len_largest_set * .9 <= len_smallest_set
+        similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set
-        enough_overlap = len(intersect_dates) >= len_smallest_set * .9
+        enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100)
         suitable = similar_size and enough_overlap
 
         if suitable:
@@ -38,7 +43,7 @@ class Correlation:
             # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
             # hypothesis is false).
             coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close'])
-            coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0]
+            coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0]
 
             # If NaN, change to None
             if coefficient is not None and math.isnan(coefficient):

mt5_correlation/mt5.py

@@ -8,6 +8,29 @@ class MT5:
     A class to connect to and interface with MetaTrader 5
     """
 
+    # Timeframes
+    TIMEFRAME_M1 = mt5.TIMEFRAME_M1
+    TIMEFRAME_M2 = mt5.TIMEFRAME_M2
+    TIMEFRAME_M3 = mt5.TIMEFRAME_M3
+    TIMEFRAME_M4 = mt5.TIMEFRAME_M4
+    TIMEFRAME_M5 = mt5.TIMEFRAME_M5
+    TIMEFRAME_M6 = mt5.TIMEFRAME_M6
+    TIMEFRAME_M10 = mt5.TIMEFRAME_M10
+    TIMEFRAME_M12 = mt5.TIMEFRAME_M10
+    TIMEFRAME_M15 = mt5.TIMEFRAME_M15
+    TIMEFRAME_M20 = mt5.TIMEFRAME_M20
+    TIMEFRAME_M30 = mt5.TIMEFRAME_M30
+    TIMEFRAME_H1 = mt5.TIMEFRAME_H1
+    TIMEFRAME_H2 = mt5.TIMEFRAME_H2
+    TIMEFRAME_H3 = mt5.TIMEFRAME_H3
+    TIMEFRAME_H4 = mt5.TIMEFRAME_H4
+    TIMEFRAME_H6 = mt5.TIMEFRAME_H6
+    TIMEFRAME_H8 = mt5.TIMEFRAME_H8
+    TIMEFRAME_H12 = mt5.TIMEFRAME_H12
+    TIMEFRAME_D1 = mt5.TIMEFRAME_D1
+    TIMEFRAME_W1 = mt5.TIMEFRAME_W1
+    TIMEFRAME_MN1 = mt5.TIMEFRAME_MN1
+
     def __init__(self):
         # Connect to MetaTrader5. Opens if not already open.
 
@@ -48,16 +71,38 @@ class MT5:
 
         return selected_symbols
 
-    def get_prices(self, symbol, from_date, to_date):
+    def get_prices(self, symbol, from_date, to_date, timeframe):
         """
-        Gets the 1 weeks of M15 OHLC price data for the specified symbol.
+        Gets OHLC price data for the specified symbol.
         :param symbol: The MT5 symbol to get the price data for
         :param from_date: Date from when to retrieve data
         :param to_date: Date where to receive data to
+        :param timeframe: The timeframe for the candes. Possible values are:
+            TIMEFRAME_M1: 1 minute
+            TIMEFRAME_M2: 2 minutes
+            TIMEFRAME_M3: 3 minutes
+            TIMEFRAME_M4: 4 minutes
+            TIMEFRAME_M5: 5 minutes
+            TIMEFRAME_M6: 6 minutes
+            TIMEFRAME_M10: 10 minutes
+            TIMEFRAME_M12: 12 minutes
+            TIMEFRAME_M15: 15 minutes
+            TIMEFRAME_M20: 20 minutes
+            TIMEFRAME_M30: 30 minutes
+            TIMEFRAME_H1: 1 hour
+            TIMEFRAME_H2: 2 hours
+            TIMEFRAME_H3: 3 hours
+            TIMEFRAME_H4: 4 hours
+            TIMEFRAME_H6: 6 hours
+            TIMEFRAME_H8: 8 hours
+            TIMEFRAME_H12: 12 hours
+            TIMEFRAME_D1: 1 day
+            TIMEFRAME_W1: 1 week
+            TIMEFRAME_MN1: 1 month
         :return: Price data for symbol as dataframe
         """
         # Get prices from MT5
-        prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, from_date, to_date)
+        prices = mt5.copy_rates_range(symbol.name, timeframe, from_date, to_date)
         self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.")
 
         # Create dataframe from data and convert time in seconds to datetime format