From 16d8a98609296d90536c48451198b34e1d3d381a Mon Sep 17 00:00:00 2001 From: Jamie Cash Date: Fri, 5 Feb 2021 12:08:14 +0000 Subject: [PATCH] Improved readme and added some customisation options in correlation calculation --- README.md | 77 ++++++++++++++++++- logging_conf.yaml | 20 +---- .../mt5_correlation.py => mt5_correlation.py | 24 +++--- mt5_correlation/correlation.py | 17 ++-- mt5_correlation/mt5.py | 51 +++++++++++- 5 files changed, 152 insertions(+), 37 deletions(-) rename mt5_correlation/mt5_correlation.py => mt5_correlation.py (78%) diff --git a/README.md b/README.md index e30a89d..e21a33b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,77 @@ # mt5-correlation -Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch +Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch. + +# Setup +1) Set up your MetaTrader 5 environment ensuring that all symbols that you would like to assess for correlation are shown in your Market Watch window; +2) Set up your python environment; and +3) Install the required libraries. +``` +pip install -r mt5-correlation\requirements.txt +``` + +# Usage +If you set up a virtual environment in the Setup step, ensure this is activated. Then run the script. +``` +python -m mt5_correlations\mt5_correlations.py +``` + +A .csv file containing the correlation coefficient for all combinations of sybmols from the MetaTrader market watch will be produced in the current directory. + +|Symbol 1 |Symbol 2 |Coefficient|UTC Date From |UTC Date To |Timeframe| +|------------|------------|-----------|-------------------|-------------------|---------| +|OIL-MAR21 |OILMn-MAR21 |1.0 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | +|EURUSD |EURHKD |0.99980 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | +|OILMn-MAR21 |BRENT-APR21 |0.99894 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | +|OIL-MAR21 |BRENT-APR21 |0.99894 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | +|GSOIL-FEB21 |BRENT-APR21 |0.99605 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | +|GSOIL-FEB21 |OILMn-MAR21 |0.99543 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | +|GSOIL-FEB21 |OIL-MAR21 |0.99543 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | +|EU50Cash |FRA40Cash |0.99072 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | + +# Customising +Edit mt5_correlations.py to customise. + +The coefficients are calculated only if: +* The smallest set of price data is no less than 90% of the size of the largest set; +* The overlapping prices between both sets of price data contains no less than 90% of the prices in the smallest set; +* The pearsonr p-value for the calculated coefficient is less than 0.05. + +These settings can all be changed in the call to Correlation.calculate_coefficient by passing values for max_set_size_diff_pct; overlap_pct; or max_p_value. +``` +coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data, + symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90, + overlap_pct=90, max_p_value=0.05) +``` + +The price data compared is 15 minute price data for the last 7 days. This can be changed by changing the values for the following variables: +``` +utc_to = datetime.now(tz=timezone) +utc_from = utc_to - timedelta(days=7) +timeframe = mt5.TIMEFRAME_M15 +``` + +The possible values for timeframe are: + +|Timeframe|Description| +|--------------|-----------| +|TIMEFRAME_M1 |1 minute | +|TIMEFRAME_M2 |2 minutes | +|TIMEFRAME_M3 |3 minutes | +|TIMEFRAME_M4 |4 minutes | +|TIMEFRAME_M5 |5 minutes | +|TIMEFRAME_M6 |6 minutes | +|TIMEFRAME_M10 |10 minutes | +|TIMEFRAME_M12 |12 minutes | +|TIMEFRAME_M12 |15 minutes | +|TIMEFRAME_M20 |20 minutes | +|TIMEFRAME_M30 |30 minutes | +|TIMEFRAME_H1 |1 hour | +|TIMEFRAME_H2 |2 hours | +|TIMEFRAME_H3 |3 hours | +|TIMEFRAME_H4 |4 hours | +|TIMEFRAME_H6 |6 hours | +|TIMEFRAME_H8 |8 hours | +|TIMEFRAME_H12 |12 hours | +|TIMEFRAME_D1 |1 day | +|TIMEFRAME_W1 |1 week | +|TIMEFRAME_MN1 |1 month | \ No newline at end of file diff --git a/logging_conf.yaml b/logging_conf.yaml index dda05f8..176be17 100644 --- a/logging_conf.yaml +++ b/logging_conf.yaml @@ -18,28 +18,12 @@ handlers: formatter: brief stream: ext://sys.stdout - file-debug: - level: DEBUG - class: logging.handlers.RotatingFileHandler - formatter: precice - filename: log\mt5-correlation-debug.log - mode: a - maxBytes: 2560000 # Max allowed for code insight features of PyCharm - backupCount: 2 - - file-info: - level: INFO - class: logging.FileHandler - formatter: precice - filename: log\mt5-correlation-info.log - mode: a - root: level: DEBUG - handlers: [console, file-info, file-debug] + handlers: [console] loggers: mt5-correlation: level: DEBUG - handlers: [console, file-info, file-debug] + handlers: [console] propagate: 0 \ No newline at end of file diff --git a/mt5_correlation/mt5_correlation.py b/mt5_correlation.py similarity index 78% rename from mt5_correlation/mt5_correlation.py rename to mt5_correlation.py index 339f055..1c5ebb9 100644 --- a/mt5_correlation/mt5_correlation.py +++ b/mt5_correlation.py @@ -3,8 +3,8 @@ import logging.config import pandas as pd import pytz import yaml -from mt5 import MT5 -from correlation import Correlation +from mt5_correlation.mt5 import MT5 +from mt5_correlation.correlation import Correlation import definitions # Configure logger @@ -24,15 +24,19 @@ timezone = pytz.timezone("Etc/UTC") utc_to = datetime.now(tz=timezone) utc_from = utc_to - timedelta(days=7) +# Set timeframe +timeframe = mt5.TIMEFRAME_M15 + # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict. price_data = {} for symbol in symbols: - price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to) + price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to, + timeframe=timeframe) # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to # avoid duplicating. We will store all coefficients in a dataframe for export as CSV. -columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Interval'] +columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe'] coefficients = pd.DataFrame(columns=columns) index = 0 @@ -51,23 +55,25 @@ for i in range(0, len(symbols)): symbol2_price_data = price_data[symbol2.name] # Get coefficient and store if valid - coefficient = Correlation.calculate_coefficient(symbol1_price_data, symbol2_price_data) + coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data, + symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90, + overlap_pct=90, max_p_value=0.05) if coefficient is not None: coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name, 'Coefficient': coefficient, 'UTC Date From': utc_from, - 'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True) + 'UTC Date To': utc_to, 'Timeframe': timeframe}, ignore_index=True) log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of " f"{coefficient}.") else: - log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not " - f"be calculated.") + log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not" + f" be calculated.") # Sort, highest correlated first coefficients = coefficients.sort_values('Coefficient', ascending=False) # Save as CSV -filename = f"out/Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv" +filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S}.csv" log.info(f"Saving coefficients as '{filename}'.") coefficients.to_csv(filename, index=False) diff --git a/mt5_correlation/correlation.py b/mt5_correlation/correlation.py index 40dbb03..c58a70f 100644 --- a/mt5_correlation/correlation.py +++ b/mt5_correlation/correlation.py @@ -8,24 +8,29 @@ class Correlation: """ @staticmethod - def calculate_coefficient(symbol1_prices, symbol2_prices): + def calculate_coefficient(symbol1_prices, symbol2_prices, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): """ Calculates the correlation coefficient between two sets of price data. Uses close price. :param symbol1_prices: :param symbol2_prices: + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful :return: correlation coefficient, or None if coefficient could not be calculated. """ # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for - # correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the - # largest set and is the overlap set size at least 90% the size of the smallest set? + # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of + # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set? coefficient = None intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time'])) len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)])) len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)])) - similar_size = len_largest_set * .9 <= len_smallest_set - enough_overlap = len(intersect_dates) >= len_smallest_set * .9 + similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set + enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100) suitable = similar_size and enough_overlap if suitable: @@ -38,7 +43,7 @@ class Correlation: # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null # hypothesis is false). coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close']) - coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0] + coefficient = None if coefficient_with_p_value[1] >= max_p_value else coefficient_with_p_value[0] # If NaN, change to None if coefficient is not None and math.isnan(coefficient): diff --git a/mt5_correlation/mt5.py b/mt5_correlation/mt5.py index c21d060..8b53d26 100644 --- a/mt5_correlation/mt5.py +++ b/mt5_correlation/mt5.py @@ -8,6 +8,29 @@ class MT5: A class to connect to and interface with MetaTrader 5 """ + # Timeframes + TIMEFRAME_M1 = mt5.TIMEFRAME_M1 + TIMEFRAME_M2 = mt5.TIMEFRAME_M2 + TIMEFRAME_M3 = mt5.TIMEFRAME_M3 + TIMEFRAME_M4 = mt5.TIMEFRAME_M4 + TIMEFRAME_M5 = mt5.TIMEFRAME_M5 + TIMEFRAME_M6 = mt5.TIMEFRAME_M6 + TIMEFRAME_M10 = mt5.TIMEFRAME_M10 + TIMEFRAME_M12 = mt5.TIMEFRAME_M10 + TIMEFRAME_M15 = mt5.TIMEFRAME_M15 + TIMEFRAME_M20 = mt5.TIMEFRAME_M20 + TIMEFRAME_M30 = mt5.TIMEFRAME_M30 + TIMEFRAME_H1 = mt5.TIMEFRAME_H1 + TIMEFRAME_H2 = mt5.TIMEFRAME_H2 + TIMEFRAME_H3 = mt5.TIMEFRAME_H3 + TIMEFRAME_H4 = mt5.TIMEFRAME_H4 + TIMEFRAME_H6 = mt5.TIMEFRAME_H6 + TIMEFRAME_H8 = mt5.TIMEFRAME_H8 + TIMEFRAME_H12 = mt5.TIMEFRAME_H12 + TIMEFRAME_D1 = mt5.TIMEFRAME_D1 + TIMEFRAME_W1 = mt5.TIMEFRAME_W1 + TIMEFRAME_MN1 = mt5.TIMEFRAME_MN1 + def __init__(self): # Connect to MetaTrader5. Opens if not already open. @@ -48,16 +71,38 @@ class MT5: return selected_symbols - def get_prices(self, symbol, from_date, to_date): + def get_prices(self, symbol, from_date, to_date, timeframe): """ - Gets the 1 weeks of M15 OHLC price data for the specified symbol. + Gets OHLC price data for the specified symbol. :param symbol: The MT5 symbol to get the price data for :param from_date: Date from when to retrieve data :param to_date: Date where to receive data to + :param timeframe: The timeframe for the candes. Possible values are: + TIMEFRAME_M1: 1 minute + TIMEFRAME_M2: 2 minutes + TIMEFRAME_M3: 3 minutes + TIMEFRAME_M4: 4 minutes + TIMEFRAME_M5: 5 minutes + TIMEFRAME_M6: 6 minutes + TIMEFRAME_M10: 10 minutes + TIMEFRAME_M12: 12 minutes + TIMEFRAME_M15: 15 minutes + TIMEFRAME_M20: 20 minutes + TIMEFRAME_M30: 30 minutes + TIMEFRAME_H1: 1 hour + TIMEFRAME_H2: 2 hours + TIMEFRAME_H3: 3 hours + TIMEFRAME_H4: 4 hours + TIMEFRAME_H6: 6 hours + TIMEFRAME_H8: 8 hours + TIMEFRAME_H12: 12 hours + TIMEFRAME_D1: 1 day + TIMEFRAME_W1: 1 week + TIMEFRAME_MN1: 1 month :return: Price data for symbol as dataframe """ # Get prices from MT5 - prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, from_date, to_date) + prices = mt5.copy_rates_range(symbol.name, timeframe, from_date, to_date) self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.") # Create dataframe from data and convert time in seconds to datetime format