diff --git a/.gitignore b/.gitignore
index 9f21b54..a13b364 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
/venv/
+/log/
+/out/
diff --git a/.idea/mt5-correlation.iml b/.idea/mt5-correlation.iml
index d94d4c3..211e5e7 100644
--- a/.idea/mt5-correlation.iml
+++ b/.idea/mt5-correlation.iml
@@ -4,6 +4,8 @@
+
+
diff --git a/mt5_correlation.py b/mt5_correlation.py
index 7bd40de..a52e500 100644
--- a/mt5_correlation.py
+++ b/mt5_correlation.py
@@ -1,56 +1,31 @@
-# Gets all symbols from MetaTrader5 market watch and calculates correlation for all pairs
-
from datetime import datetime, timedelta
-import pytz
-import math
+import logging.config
import pandas as pd
-import MetaTrader5 as mt5
-from scipy.stats.stats import pearsonr
+import pytz
+import yaml
+from mt5_correlation.mt5_correlation import MT5Correlation
-# connect to MetaTrader 5
-if not mt5.initialize():
- print("initialize() failed")
- mt5.shutdown()
+# Configure logger
+with open(r'.\logging_conf.yaml', 'rt') as file:
+ config = yaml.safe_load(file.read())
+ logging.config.dictConfig(config)
+ log = logging.getLogger()
-# Print connection status
-print(mt5.terminal_info())
+# Create mt5 correlation class. This contains required methods for interacting with MT5 and calculating coefficients.
+mtc = MT5Correlation()
-# get data on MetaTrader 5 version
-print(mt5.version())
+# Gte all visible symbols
+symbols = mtc.get_symbols()
-# Iterate symbols and get those in market watch.
-symbols = mt5.symbols_get()
-selected_symbols = []
-for symbol in symbols:
- if symbol.visible:
- selected_symbols.append(symbol)
-
-# Print symbol counts
-total_symbols = mt5.symbols_total()
-num_selected_symbols = len(selected_symbols)
-print(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
-
-# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
-price_data = {}
# set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
timezone = pytz.timezone("Etc/UTC")
utc_to = datetime.now(tz=timezone)
utc_from = utc_to - timedelta(days=7)
-# get 15 min bars from all selected symbols for 7 days
-print(f"Getting prices for all selected symbols.")
-for symbol in selected_symbols:
- prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, utc_from, utc_to)
- print(f"{len(prices)} prices retrieved for {symbol.name}.")
-
- # Create dataframe from data and convert time in seconds to datetime format
- prices_dataframe = pd.DataFrame(prices)
- prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
-
- # Store prices in dict
- price_data[symbol.name] = prices_dataframe
-
-# Calculate correlation coefficients for all pair combinations.
+# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
+price_data = {}
+for symbol in symbols:
+ price_data[symbol.name] = mtc.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to)
# Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
# eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
@@ -60,64 +35,37 @@ coefficients = pd.DataFrame(columns=columns)
index = 0
# There will be (x^2 - x) / 2 pairs where x is number of symbols
-num_pair_combinations = int((len(selected_symbols) ** 2 - len(selected_symbols)) / 2)
+num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
-for i in range(0, len(selected_symbols)):
- symbol1 = selected_symbols[i]
+for i in range(0, len(symbols)):
+ symbol1 = symbols[i]
- for j in range(i + 1, len(selected_symbols)):
- symbol2 = selected_symbols[j]
+ for j in range(i + 1, len(symbols)):
+ symbol2 = symbols[j]
index += 1
- print(f"Calculating coefficients for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name}.")
# Get price data for both symbols
symbol1_price_data = price_data[symbol1.name]
symbol2_price_data = price_data[symbol2.name]
- # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
- # correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the
- # largest set and is the overlap set size at least 90% the size of the smallest set?
- intersect_dates = (set(symbol1_price_data['time']) & set(symbol2_price_data['time']))
- len_smallest_set = int(min([len(symbol1_price_data.index), len(symbol2_price_data.index)]))
- len_largest_set = int(max([len(symbol1_price_data.index), len(symbol2_price_data.index)]))
- similar_size = len_largest_set * .9 <= len_smallest_set
- enough_overlap = len(intersect_dates) >= len_smallest_set * .9
- suitable = similar_size and enough_overlap
+ # Get coefficient and store if valid
+ coefficient = mtc.calculate_coefficient(symbol1_price_data, symbol2_price_data)
- if suitable:
- # Calculate coefficient on close prices
+ if coefficient is not None:
+ coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
+ 'Coefficient': coefficient, 'UTC Date From': utc_from,
+ 'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True)
- # First filter prices to only include those that intersect
- symbol1_price_data_filtered = symbol1_price_data[symbol1_price_data['time'].isin(intersect_dates)]
- symbol2_price_data_filtered = symbol2_price_data[symbol2_price_data['time'].isin(intersect_dates)]
-
- # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
- # hypothesis is false).
- coefficient_with_p_value = pearsonr(symbol1_price_data_filtered['close'],
- symbol2_price_data_filtered['close'])
- coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0]
-
- # If not NaN or None round it and store in coefficients dict
- if coefficient is not None and math.isnan(coefficient):
- print("No coefficient calculated. NaN returned.")
- elif coefficient is None:
- print("No coefficient calculated. None returned.")
- else:
- coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
- 'Coefficient': coefficient, 'UTC Date From': utc_from,
- 'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True)
+ log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of "
+ f"{coefficient}.")
else:
- print(
- f"Symbol pair {symbol1.name}:{symbol2.name} is not suitable for coefficient calculation. "
- f"Min: {len_smallest_set} Max: {len_largest_set} Overlap {len(intersect_dates)}")
+ log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not "
+ f"be calculated.")
# Sort, highest correlated first
coefficients = coefficients.sort_values('Coefficient', ascending=False)
# Save as CSV
-filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv"
-print(f"Saving coefficients as '{filename}'.")
+filename = f"out/Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv"
+log.info(f"Saving coefficients as '{filename}'.")
coefficients.to_csv(filename, index=False)
-
-# shut down connection to the MetaTrader 5 terminal
-mt5.shutdown()
diff --git a/mt5_correlation/__init__.py b/mt5_correlation/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/mt5_correlation/mt5_correlation.py b/mt5_correlation/mt5_correlation.py
new file mode 100644
index 0000000..125b5ab
--- /dev/null
+++ b/mt5_correlation/mt5_correlation.py
@@ -0,0 +1,110 @@
+import math
+import pandas as pd
+import MetaTrader5 as mt5
+import logging
+from scipy.stats.stats import pearsonr
+
+
+class MT5Correlation:
+ """
+ A class to connect to MetaTrader 5 and calculate correlation coefficients between all pairs of symbols in MarketView
+ """
+
+ def __init__(self):
+ # Connect to MetaTrader5. Opens if not already open.
+
+ # Logger
+ self.log = logging.getLogger(__name__)
+
+ # Open MT5 and log error if it could not open
+ if not mt5.initialize():
+ self.log.error("initialize() failed")
+ mt5.shutdown()
+
+ # Print connection status
+ self.log.debug(mt5.terminal_info())
+
+ # Print data on MetaTrader 5 version
+ self.log.debug(mt5.version())
+
+ def __del__(self):
+ # shut down connection to the MetaTrader 5 terminal
+ mt5.shutdown()
+
+ def get_symbols(self):
+ """
+ Gets list of symbols open in MT5 market watch.
+ :return: list of symbols
+ """
+ # Iterate symbols and get those in market watch.
+ symbols = mt5.symbols_get()
+ selected_symbols = []
+ for symbol in symbols:
+ if symbol.visible:
+ selected_symbols.append(symbol)
+
+ # Log symbol counts
+ total_symbols = mt5.symbols_total()
+ num_selected_symbols = len(selected_symbols)
+ self.log.info(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
+
+ return selected_symbols
+
+ def get_prices(self, symbol, from_date, to_date):
+ """
+ Gets the 1 weeks of M15 OHLC price data for the specified symbol.
+ :param symbol: The MT5 symbol to get the price data for
+ :param from_date: Date from when to retrieve data
+ :param to_date: Date where to receive data to
+ :return: Price data for symbol as dataframe
+ """
+ # Get prices from MT5
+ prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, from_date, to_date)
+ self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.")
+
+ # Create dataframe from data and convert time in seconds to datetime format
+ prices_dataframe = pd.DataFrame(prices)
+ prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
+
+ return prices_dataframe
+
+ def calculate_coefficient(self, symbol1_prices, symbol2_prices):
+ """
+ Calculates the correlation coefficient between two sets of price data. Uses close price.
+
+ :param symbol1_prices:
+ :param symbol2_prices:
+ :return: correlation coefficient, or None if coefficient could not be calculated.
+ """
+ # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
+ # correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the
+ # largest set and is the overlap set size at least 90% the size of the smallest set?
+ coefficient = None
+
+ intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time']))
+ len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)]))
+ len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
+ similar_size = len_largest_set * .9 <= len_smallest_set
+ enough_overlap = len(intersect_dates) >= len_smallest_set * .9
+ suitable = similar_size and enough_overlap
+
+ if suitable:
+ # Calculate coefficient on close prices
+
+ # First filter prices to only include those that intersect
+ symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)]
+ symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)]
+
+ # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
+ # hypothesis is false).
+ coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close'])
+ coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0]
+
+ # If NaN, change to None
+ if coefficient is not None and math.isnan(coefficient):
+ coefficient = None
+
+ return coefficient
+
+
+
diff --git a/requirements.txt b/requirements.txt
index 3a7a3e5..a561556 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,7 @@
pandas==1.2.1
matplotlib==3.3.4
-metatrader5==5.0.34
+MetaTrader5==5.0.34
pytz==2021.1
-scipy==1.6.0
\ No newline at end of file
+scipy==1.6.0
+logging==0.4.9.6
+pyyaml==5.4.1
\ No newline at end of file