gavindiaz/mt5-correlation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added .idea project files

Browse Source
This commit is contained in:
Jamie Cash
2021-02-02 16:13:49 +00:00
parent 91a8e950a5
commit 7ee3bfd0e2
6 changed files with 152 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -1 +1,3 @@
/venv/
/log/
/out/
.idea/mt5-correlation.iml
Generated
+2
View File
@@ -4,6 +4,8 @@
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
<excludeFolder url="file://$MODULE_DIR$/.idea" />
<excludeFolder url="file://$MODULE_DIR$/log" />
<excludeFolder url="file://$MODULE_DIR$/out" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
mt5_correlation.py
+34 -86
View File
@@ -1,56 +1,31 @@
# Gets all symbols from MetaTrader5 market watch and calculates correlation for all pairs
from datetime import datetime, timedelta
import pytz
import math
import logging.config
import pandas as pd
import MetaTrader5 as mt5
from scipy.stats.stats import pearsonr
import pytz
import yaml
from mt5_correlation.mt5_correlation import MT5Correlation
# connect to MetaTrader 5
if not mt5.initialize():
print("initialize() failed")
mt5.shutdown()
# Configure logger
with open(r'.\logging_conf.yaml', 'rt') as file:
config = yaml.safe_load(file.read())
logging.config.dictConfig(config)
log = logging.getLogger()
# Print connection status
print(mt5.terminal_info())
# Create mt5 correlation class. This contains required methods for interacting with MT5 and calculating coefficients.
mtc = MT5Correlation()
# get data on MetaTrader 5 version
print(mt5.version())
# Gte all visible symbols
symbols = mtc.get_symbols()
# Iterate symbols and get those in market watch.
symbols = mt5.symbols_get()
selected_symbols = []
for symbol in symbols:
if symbol.visible:
selected_symbols.append(symbol)
# Print symbol counts
total_symbols = mt5.symbols_total()
num_selected_symbols = len(selected_symbols)
print(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
price_data = {}
# set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
timezone = pytz.timezone("Etc/UTC")
utc_to = datetime.now(tz=timezone)
utc_from = utc_to - timedelta(days=7)
# get 15 min bars from all selected symbols for 7 days
print(f"Getting prices for all selected symbols.")
for symbol in selected_symbols:
prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, utc_from, utc_to)
print(f"{len(prices)} prices retrieved for {symbol.name}.")
# Create dataframe from data and convert time in seconds to datetime format
prices_dataframe = pd.DataFrame(prices)
prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
# Store prices in dict
price_data[symbol.name] = prices_dataframe
# Calculate correlation coefficients for all pair combinations.
# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
price_data = {}
for symbol in symbols:
price_data[symbol.name] = mtc.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to)
# Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
# eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
@@ -60,64 +35,37 @@ coefficients = pd.DataFrame(columns=columns)
index = 0
# There will be (x^2 - x) / 2 pairs where x is number of symbols
num_pair_combinations = int((len(selected_symbols) ** 2 - len(selected_symbols)) / 2)
num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
for i in range(0, len(selected_symbols)):
symbol1 = selected_symbols[i]
for i in range(0, len(symbols)):
symbol1 = symbols[i]
for j in range(i + 1, len(selected_symbols)):
symbol2 = selected_symbols[j]
for j in range(i + 1, len(symbols)):
symbol2 = symbols[j]
index += 1
print(f"Calculating coefficients for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name}.")
# Get price data for both symbols
symbol1_price_data = price_data[symbol1.name]
symbol2_price_data = price_data[symbol2.name]
# Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
# correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the
# largest set and is the overlap set size at least 90% the size of the smallest set?
intersect_dates = (set(symbol1_price_data['time']) & set(symbol2_price_data['time']))
len_smallest_set = int(min([len(symbol1_price_data.index), len(symbol2_price_data.index)]))
len_largest_set = int(max([len(symbol1_price_data.index), len(symbol2_price_data.index)]))
similar_size = len_largest_set * .9 <= len_smallest_set
enough_overlap = len(intersect_dates) >= len_smallest_set * .9
suitable = similar_size and enough_overlap
# Get coefficient and store if valid
coefficient = mtc.calculate_coefficient(symbol1_price_data, symbol2_price_data)
if suitable:
# Calculate coefficient on close prices
if coefficient is not None:
coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
'Coefficient': coefficient, 'UTC Date From': utc_from,
'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True)
# First filter prices to only include those that intersect
symbol1_price_data_filtered = symbol1_price_data[symbol1_price_data['time'].isin(intersect_dates)]
symbol2_price_data_filtered = symbol2_price_data[symbol2_price_data['time'].isin(intersect_dates)]
# Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
# hypothesis is false).
coefficient_with_p_value = pearsonr(symbol1_price_data_filtered['close'],
symbol2_price_data_filtered['close'])
coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0]
# If not NaN or None round it and store in coefficients dict
if coefficient is not None and math.isnan(coefficient):
print("No coefficient calculated. NaN returned.")
elif coefficient is None:
print("No coefficient calculated. None returned.")
else:
coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
'Coefficient': coefficient, 'UTC Date From': utc_from,
'UTC Date To': utc_to, 'Interval': 'M15'}, ignore_index=True)
log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of "
f"{coefficient}.")
else:
print(
f"Symbol pair {symbol1.name}:{symbol2.name} is not suitable for coefficient calculation. "
f"Min: {len_smallest_set} Max: {len_largest_set} Overlap {len(intersect_dates)}")
log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not "
f"be calculated.")
# Sort, highest correlated first
coefficients = coefficients.sort_values('Coefficient', ascending=False)
# Save as CSV
filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv"
print(f"Saving coefficients as '{filename}'.")
filename = f"out/Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S} at M15.csv"
log.info(f"Saving coefficients as '{filename}'.")
coefficients.to_csv(filename, index=False)
# shut down connection to the MetaTrader 5 terminal
mt5.shutdown()
mt5_correlation/__init__.py
View File
mt5_correlation/mt5_correlation.py
+110
View File
@@ -0,0 +1,110 @@
import math
import pandas as pd
import MetaTrader5 as mt5
import logging
from scipy.stats.stats import pearsonr
class MT5Correlation:
"""
A class to connect to MetaTrader 5 and calculate correlation coefficients between all pairs of symbols in MarketView
"""
def __init__(self):
# Connect to MetaTrader5. Opens if not already open.
# Logger
self.log = logging.getLogger(__name__)
# Open MT5 and log error if it could not open
if not mt5.initialize():
self.log.error("initialize() failed")
mt5.shutdown()
# Print connection status
self.log.debug(mt5.terminal_info())
# Print data on MetaTrader 5 version
self.log.debug(mt5.version())
def __del__(self):
# shut down connection to the MetaTrader 5 terminal
mt5.shutdown()
def get_symbols(self):
"""
Gets list of symbols open in MT5 market watch.
:return: list of symbols
"""
# Iterate symbols and get those in market watch.
symbols = mt5.symbols_get()
selected_symbols = []
for symbol in symbols:
if symbol.visible:
selected_symbols.append(symbol)
# Log symbol counts
total_symbols = mt5.symbols_total()
num_selected_symbols = len(selected_symbols)
self.log.info(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
return selected_symbols
def get_prices(self, symbol, from_date, to_date):
"""
Gets the 1 weeks of M15 OHLC price data for the specified symbol.
:param symbol: The MT5 symbol to get the price data for
:param from_date: Date from when to retrieve data
:param to_date: Date where to receive data to
:return: Price data for symbol as dataframe
"""
# Get prices from MT5
prices = mt5.copy_rates_range(symbol.name, mt5.TIMEFRAME_M15, from_date, to_date)
self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.")
# Create dataframe from data and convert time in seconds to datetime format
prices_dataframe = pd.DataFrame(prices)
prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s')
return prices_dataframe
def calculate_coefficient(self, symbol1_prices, symbol2_prices):
"""
Calculates the correlation coefficient between two sets of price data. Uses close price.
:param symbol1_prices:
:param symbol2_prices:
:return: correlation coefficient, or None if coefficient could not be calculated.
"""
# Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
# correlation coefficient calculation to be meaningful. Is the smallest set at least 90% of the size of the
# largest set and is the overlap set size at least 90% the size of the smallest set?
coefficient = None
intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time']))
len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)]))
len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
similar_size = len_largest_set * .9 <= len_smallest_set
enough_overlap = len(intersect_dates) >= len_smallest_set * .9
suitable = similar_size and enough_overlap
if suitable:
# Calculate coefficient on close prices
# First filter prices to only include those that intersect
symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)]
symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)]
# Calculate coefficient. Only use if p value is < 0.01 (highly likely that coefficient is valid and null
# hypothesis is false).
coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close'])
coefficient = None if coefficient_with_p_value[1] > 0.01 else coefficient_with_p_value[0]
# If NaN, change to None
if coefficient is not None and math.isnan(coefficient):
coefficient = None
return coefficient
requirements.txt
+4 -2
View File
@@ -1,5 +1,7 @@
pandas==1.2.1
matplotlib==3.3.4
metatrader5==5.0.34
MetaTrader5==5.0.34
pytz==2021.1
scipy==1.6.0
scipy==1.6.0
logging==0.4.9.6
pyyaml==5.4.1