From 7ee3bfd0e2b56bd12aadb2391248d96e88fddee7 Mon Sep 17 00:00:00 2001 From: Jamie Cash Date: Tue, 2 Feb 2021 16:13:49 +0000 Subject: [PATCH] Added .idea project files --- .gitignore | 2 + .idea/mt5-correlation.iml | 2 + mt5_correlation.py | 120 ++++++++--------------------- mt5_correlation/__init__.py | 0 mt5_correlation/mt5_correlation.py | 110 ++++++++++++++++++++++++++ requirements.txt | 6 +- 6 files changed, 152 insertions(+), 88 deletions(-) create mode 100644 mt5_correlation/__init__.py create mode 100644 mt5_correlation/mt5_correlation.py diff --git a/.gitignore b/.gitignore index 9f21b54..a13b364 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /venv/ +/log/ +/out/ diff --git a/.idea/mt5-correlation.iml b/.idea/mt5-correlation.iml index d94d4c3..211e5e7 100644 --- a/.idea/mt5-correlation.iml +++ b/.idea/mt5-correlation.iml @@ -4,6 +4,8 @@ + + diff --git a/mt5_correlation.py b/mt5_correlation.py index 7bd40de..a52e500 100644 --- a/mt5_correlation.py +++ b/mt5_correlation.py @@ -1,56 +1,31 @@ -# Gets all symbols from MetaTrader5 market watch and calculates correlation for all pairs - from datetime import datetime, timedelta -import pytz -import math +import logging.config import pandas as pd -import MetaTrader5 as mt5 -from scipy.stats.stats import pearsonr +import pytz +import yaml +from mt5_correlation.mt5_correlation import MT5Correlation -# connect to MetaTrader 5 -if not mt5.initialize(): - print("initialize() failed") - mt5.shutdown() +# Configure logger +with open(r'.\logging_conf.yaml', 'rt') as file: + config = yaml.safe_load(file.read()) + logging.config.dictConfig(config) + log = logging.getLogger() -# Print connection status -print(mt5.terminal_info()) +# Create mt5 correlation class. This contains required methods for interacting with MT5 and calculating coefficients. +mtc = MT5Correlation() -# get data on MetaTrader 5 version -print(mt5.version()) +# Gte all visible symbols +symbols = mtc.get_symbols() -# Iterate symbols and get those in market watch. -symbols = mt5.symbols_get() -selected_symbols = [] -for symbol in symbols: - if symbol.visible: - selected_symbols.append(symbol) - -# Print symbol counts -total_symbols = mt5.symbols_total() -num_selected_symbols = len(selected_symbols) -print(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.") - -# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict. -price_data = {} # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today) timezone = pytz.timezone("Etc/UTC") utc_to = datetime.now(tz=timezone) utc_from = utc_to - timedelta(days=7) -# get 15 min bars from all selected symbols for 7 days -print(f"Getting prices for all selected symbols.") -for symbol in selected_symbols: - prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, utc_from, utc_to) - print(f"{len(prices)} prices retrieved for {symbol.name}.") - - # Create dataframe from data and convert time in seconds to datetime format - prices_dataframe = pd.DataFrame(prices) - prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s') - - # Store prices in dict - price_data[symbol.name] = prices_dataframe - -# Calculate correlation coefficients for all pair combinations. +# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict. +price_data = {} +for symbol in symbols: + price_data[symbol.name] = mtc.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to) # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to @@ -60,64 +35,37 @@ coefficients = pd.DataFrame(columns=columns) index = 0 # There will be (x^2 - x) / 2 pairs where x is number of symbols -num_pair_combinations = int((len(selected_symbols) ** 2 - len(selected_symbols)) / 2) +num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2) -for i in range(0, len(selected_symbols)): - symbol1 = selected_symbols[i] +for i in range(0, len(symbols)): + symbol1 = symbols[i] - for j in range(i + 1, len(selected_symbols)): - symbol2 = selected_symbols[j] + for j in range(i + 1, len(symbols)): + symbol2 = symbols[j] index += 1 - print(f"Calculating coefficients for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name}.") # Get price data for both symbols symbol1_price_data = price_data[symbol1.name] symbol2_price_data = price_data[symbol2.name] - # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for - # correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the - # largest set and is the overlap set size at least 90% the size of the smallest set? - intersect_dates = (set(symbol1_price_data['time']) & set(symbol2_price_data['time'])) - len_smallest_set = int(min([len(symbol1_price_data.index), len(symbol2_price_data.index)])) - len_largest_set = int(max([len(symbol1_price_data.index), len(symbol2_price_data.index)])) - similar_size = len_largest_set * .9 <= len_smallest_set - enough_overlap = len(intersect_dates) >= len_smallest_set * .9 - suitable = similar_size and enough_overlap + # Get coefficient and store if valid + coefficient = mtc.calculate_coefficient(symbol1_price_data, symbol2_price_data) - if suitable: - # Calculate coefficient on close prices + if coefficient is not None: + coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name, + 'Coefficient': coefficient, 'UTC Date From': utc_from, + 'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True) - # First filter prices to only include those that intersect - symbol1_price_data_filtered = symbol1_price_data[symbol1_price_data['time'].isin(intersect_dates)] - symbol2_price_data_filtered = symbol2_price_data[symbol2_price_data['time'].isin(intersect_dates)] - - # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null - # hypothesis is false). - coefficient_with_p_value = pearsonr(symbol1_price_data_filtered['close'], - symbol2_price_data_filtered['close']) - coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0] - - # If not NaN or None round it and store in coefficients dict - if coefficient is not None and math.isnan(coefficient): - print("No coefficient calculated. NaN returned.") - elif coefficient is None: - print("No coefficient calculated. None returned.") - else: - coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name, - 'Coefficient': coefficient, 'UTC Date From': utc_from, - 'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True) + log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of " + f"{coefficient}.") else: - print( - f"Symbol pair {symbol1.name}:{symbol2.name} is not suitable for coefficient calculation. " - f"Min: {len_smallest_set} Max: {len_largest_set} Overlap {len(intersect_dates)}") + log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not " + f"be calculated.") # Sort, highest correlated first coefficients = coefficients.sort_values('Coefficient', ascending=False) # Save as CSV -filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv" -print(f"Saving coefficients as '{filename}'.") +filename = f"out/Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv" +log.info(f"Saving coefficients as '{filename}'.") coefficients.to_csv(filename, index=False) - -# shut down connection to the MetaTrader 5 terminal -mt5.shutdown() diff --git a/mt5_correlation/__init__.py b/mt5_correlation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mt5_correlation/mt5_correlation.py b/mt5_correlation/mt5_correlation.py new file mode 100644 index 0000000..125b5ab --- /dev/null +++ b/mt5_correlation/mt5_correlation.py @@ -0,0 +1,110 @@ +import math +import pandas as pd +import MetaTrader5 as mt5 +import logging +from scipy.stats.stats import pearsonr + + +class MT5Correlation: + """ + A class to connect to MetaTrader 5 and calculate correlation coefficients between all pairs of symbols in MarketView + """ + + def __init__(self): + # Connect to MetaTrader5. Opens if not already open. + + # Logger + self.log = logging.getLogger(__name__) + + # Open MT5 and log error if it could not open + if not mt5.initialize(): + self.log.error("initialize() failed") + mt5.shutdown() + + # Print connection status + self.log.debug(mt5.terminal_info()) + + # Print data on MetaTrader 5 version + self.log.debug(mt5.version()) + + def __del__(self): + # shut down connection to the MetaTrader 5 terminal + mt5.shutdown() + + def get_symbols(self): + """ + Gets list of symbols open in MT5 market watch. + :return: list of symbols + """ + # Iterate symbols and get those in market watch. + symbols = mt5.symbols_get() + selected_symbols = [] + for symbol in symbols: + if symbol.visible: + selected_symbols.append(symbol) + + # Log symbol counts + total_symbols = mt5.symbols_total() + num_selected_symbols = len(selected_symbols) + self.log.info(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.") + + return selected_symbols + + def get_prices(self, symbol, from_date, to_date): + """ + Gets the 1 weeks of M15 OHLC price data for the specified symbol. + :param symbol: The MT5 symbol to get the price data for + :param from_date: Date from when to retrieve data + :param to_date: Date where to receive data to + :return: Price data for symbol as dataframe + """ + # Get prices from MT5 + prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, from_date, to_date) + self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.") + + # Create dataframe from data and convert time in seconds to datetime format + prices_dataframe = pd.DataFrame(prices) + prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s') + + return prices_dataframe + + def calculate_coefficient(self, symbol1_prices, symbol2_prices): + """ + Calculates the correlation coefficient between two sets of price data. Uses close price. + + :param symbol1_prices: + :param symbol2_prices: + :return: correlation coefficient, or None if coefficient could not be calculated. + """ + # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for + # correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the + # largest set and is the overlap set size at least 90% the size of the smallest set? + coefficient = None + + intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time'])) + len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)])) + len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)])) + similar_size = len_largest_set * .9 <= len_smallest_set + enough_overlap = len(intersect_dates) >= len_smallest_set * .9 + suitable = similar_size and enough_overlap + + if suitable: + # Calculate coefficient on close prices + + # First filter prices to only include those that intersect + symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)] + symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)] + + # Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null + # hypothesis is false). + coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close']) + coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0] + + # If NaN, change to None + if coefficient is not None and math.isnan(coefficient): + coefficient = None + + return coefficient + + + diff --git a/requirements.txt b/requirements.txt index 3a7a3e5..a561556 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ pandas==1.2.1 matplotlib==3.3.4 -metatrader5==5.0.34 +MetaTrader5==5.0.34 pytz==2021.1 -scipy==1.6.0 \ No newline at end of file +scipy==1.6.0 +logging==0.4.9.6 +pyyaml==5.4.1 \ No newline at end of file