gavindiaz/mt5-correlation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Autosave, more charts & layout.

Browse Source
This commit is contained in:
Jamie Cash
2021-03-01 17:59:25 +00:00
parent 7ca46b1be2
commit c85ff8f79e
3 changed files with 148 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.yaml
+2 -1
View File
@@ -3,7 +3,7 @@ calculate:
from:
days: 10
timeframe: 15
min_prices: 600
min_prices: 400
max_set_size_diff_pct: 90
overlap_pct: 90
max_p_value: 0.05
@@ -18,6 +18,7 @@ monitor:
monitoring_threshold: 0.9
divergence_threshold: 0.8
tick_cache_time: 10
autosave: true
logging:
version: 1
disable_existing_loggers: false
mt5_correlation/correlation.py
+79 -59
View File
@@ -7,7 +7,6 @@ import sched
import threading
import pytz
from scipy.stats.stats import pearsonr
import yaml
import pickle
from mt5_correlation.mt5 import MT5
@@ -36,8 +35,9 @@ class Correlation:
coefficient_data = None
coefficient_history = None
# Cache for ticks. Dict: {Symbol: [retrieved datetime, ticks dataframe]}
__ticks = {}
# Stores tick data used to calculate coefficient during Monitor.
# Dict: {Symbol: [retrieved datetime, ticks dataframe]}
__monitor_tick_data = {}
def __init__(self):
# Logger
@@ -62,37 +62,34 @@ class Correlation:
else:
return None
def load(self, filename, price_data_filename=None):
def load(self, filename):
"""
Loads a csv file containing calculated coefficients, and optionally the price data used to calculate those
Loads calculated coefficients, price data used to calculate them and tick data used during monitoring.
coefficients
:param filename: The filename for the coefficient data to load.
:param price_data_filename: The filename for the price data to load.
:return:
"""
# Load coefficients file
self.coefficient_data = pd.read_csv(filename)
# Load data
with open(filename, 'rb') as file:
loaded_dict = pickle.load(file)
# If specified, load price data yaml file
if price_data_filename is not None:
self.__price_data = {} # Clear
with open(price_data_filename, 'rb') as file:
self.__price_data = pickle.load(file)
# Get data from loaded dict and save
self.coefficient_data = loaded_dict["coefficient_data"]
self.__price_data = loaded_dict["price_data"]
self.__monitor_tick_data = loaded_dict["monitor_tick_data"]
self.coefficient_history = loaded_dict["coefficient_history"]
def save(self, filename, price_data_filename=None):
def save(self, filename):
"""
Saves the calculated coefficients as a csv file
:param filename: The filename for the coefficient data to save to.
:param price_data_filename: The filename for the price data to save to.
Saves the calculated coefficients, the price data used to calculate and the tick data for monitoring to a file.
:param filename: The filename to save the data to.
:return:
"""
# Save the coefficient data
self.coefficient_data.to_csv(filename, index=False)
# Save the price data if required
if price_data_filename is not None:
with open(price_data_filename, 'wb') as file:
pickle.dump(self.__price_data, file, protocol=pickle.HIGHEST_PROTOCOL)
# Add data to dict then use pickle to save
save_dict = {"coefficient_data": self.coefficient_data, "price_data": self.__price_data,
"monitor_tick_data": self.__monitor_tick_data, "coefficient_history": self.coefficient_history}
with open(filename, 'wb') as file:
pickle.dump(save_dict, file, protocol=pickle.HIGHEST_PROTOCOL)
def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
max_p_value=0.05):
@@ -196,7 +193,7 @@ class Correlation:
return price_data
def start_monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
max_p_value=0.05, cache_time=10):
max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'):
"""
Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
threshold.
@@ -211,6 +208,9 @@ class Correlation:
:param max_p_value: The maximum p value for the correlation to be meaningful
:param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
the tick data. Number of seconds to cache tick data for before it becomes stale.
:param autosave: Whether to autosave after every monitor run. If there is no filename specified then will
create one named autosave.cpd
:param filename: Filename for autosave. Default is autosave.cpd.
:return: correlation coefficient, or None if coefficient could not be calculated.
"""
@@ -228,7 +228,8 @@ class Correlation:
# to stop and restart the monitor. Note, this happens during calculate
self.__monitoring_params = {'interval': interval, 'from_mins': from_mins,
'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct,
'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time}
'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time,
'autosave': autosave, 'filename': filename}
thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params)
thread.start()
@@ -312,8 +313,45 @@ class Correlation:
(self.coefficient_history['Symbol 2'] == symbol2)]
return history
def get_ticks(self, symbol, date_from=None, date_to=None, cache_time=0, cache_only=False):
"""
Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe.
:param symbol: Name of symbol to get ticks for.
:param date_from: Date to get ticks from. Can only be None if getting from cache (cache_only=True)
:param date_to:Date to get ticks to. Can only be None if getting from cache (cache_only=True)
:param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed,
get data from source and refresh cache.
:param cache_only: Only retrieve from cache. cache_time is ignored. Returns None if symbol is not available in
cache.
:return:
"""
timezone = pytz.timezone("Etc/UTC")
utc_now = datetime.now(tz=timezone)
ticks = None
# Cache only
if cache_only:
if symbol in self.__monitor_tick_data:
ticks = self.__monitor_tick_data[symbol][1]
# Check if we already have it and it is not stale
elif symbol in self.__monitor_tick_data and utc_now < \
self.__monitor_tick_data[symbol][0] + timedelta(seconds=cache_time):
# Cached ticks are not stale. Get them
ticks = self.__monitor_tick_data[symbol][1]
self.__log.debug(f"Ticks for {symbol} retrieved from cache.")
else:
# Data does not exist in cache or cached data is stale. Retrieve from source and cache.
ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to)
self.__monitor_tick_data[symbol] = [utc_now, ticks]
self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.")
return ticks
def __monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
max_p_value=0.05, cache_time=10):
max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'):
"""
The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
stop_monitoring.
@@ -328,6 +366,9 @@ class Correlation:
:param max_p_value: The maximum p value for the correlation to be meaningful
:param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
the tick data. Number of seconds to cache tick data for before it becomes stale.
:param autosave: Whether to autosave after every monitor run. If there is no filename specified then will
create one named autosave.cpd
:param filename: Filename for autosave. Default is autosave.cpd.
:return: correlation coefficient, or None if coefficient could not be calculated.
"""
@@ -340,10 +381,15 @@ class Correlation:
max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
max_p_value=max_p_value, cache_time=cache_time)
# Autosave
if autosave:
self.save(filename=filename)
# Schedule the timer to run again
params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices,
'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
'max_p_value': max_p_value, "cache_time": cache_time}
'max_p_value': max_p_value, "cache_time": cache_time, 'autosave': autosave,
'filename': filename}
self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
self.__scheduler.run()
@@ -374,8 +420,8 @@ class Correlation:
date_from = date_to - timedelta(minutes=from_mins)
# Get the tick data
symbol1ticks = self.__get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time)
symbol2ticks = self.__get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time)
symbol1ticks = self.get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time)
symbol2ticks = self.get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time)
# Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one
# tick per second and ensuring that times can match. We will need to set the index to time for the resample
@@ -414,7 +460,7 @@ class Correlation:
return coefficient
def __update_all_coefficients(self, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
max_p_value=0.05, cache_time=10):
max_p_value=0.05, cache_time=10, autosave=False):
"""
Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
the threshold can be accessed through the filtered_coefficient_data property.
@@ -428,6 +474,8 @@ class Correlation:
:param max_p_value: The maximum p value for the correlation to be meaningful
:param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse
the tick data. Number of seconds to cache tick data for before it becomes stale.
:param autosave: Whether to autosave after every monitor run. If there is no filename set then will create one
named autosave.cpd
:return: correlation coefficient, or None if coefficient could not be calculated.
"""
@@ -439,34 +487,6 @@ class Correlation:
min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
overlap_pct=overlap_pct, max_p_value=max_p_value, cache_time=cache_time)
def __get_ticks(self, symbol, date_from, date_to, cache_time):
"""
Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe.
:param symbol: Name of symbol to get ticks for.
:param date_from:
:param date_to:
:param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed,
get data from source and refresh cache.
:return:
"""
timezone = pytz.timezone("Etc/UTC")
utc_now = datetime.now(tz=timezone)
# Check if in cache and not stale
if symbol in self.__ticks and utc_now < self.__ticks[symbol][0] + timedelta(seconds=cache_time):
# Cached ticks are not stale. Get them
ticks = self.__ticks[symbol][1]
self.__log.debug(f"Ticks for {symbol} retrieved from cache.")
else:
# Data does not exist in cache or cached data is stale. Retrieve from source and cache.
ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to)
self.__ticks[symbol] = [utc_now, ticks]
self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.")
return ticks
def __reset_coefficient_data(self):
"""
Clears coefficient data and history.
mt5_correlation/gui.py
+67 -37
View File
@@ -12,7 +12,6 @@ import pytz
import pandas as pd
import logging
import logging.config
import os
matplotlib.use('WXAgg')
@@ -158,19 +157,17 @@ class MonitorFrame(wx.Frame):
self.Bind(wx.EVT_CLOSE, self.on_close, self)
def open_file(self, event):
with wx.FileDialog(self, "Open Coefficients file", wildcard="CSV (*.csv)|*.csv",
with wx.FileDialog(self, "Open Coefficients file", wildcard="cpd (*.cpd)|*.cpd",
style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog:
if fileDialog.ShowModal() == wx.ID_CANCEL:
return # the user changed their mind
# Load the file chosen by the user. Also load the corresponding data file if there is one.
# Load the file chosen by the user.
self.__opened_filename = fileDialog.GetPath()
data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data"
if os.path.isfile(data_filename):
self.cor.load(self.__opened_filename, price_data_filename=data_filename)
else:
self.cor.load(self.__opened_filename)
self.SetStatusText(f"Loading file {self.__opened_filename}.")
self.cor.load(self.__opened_filename)
# Refresh data in grid
self.refresh_grid()
@@ -188,7 +185,7 @@ class MonitorFrame(wx.Frame):
self.SetStatusText(f"File saved as {self.__opened_filename}")
def save_file_as(self, event):
with wx.FileDialog(self, "Save Coefficients file", wildcard="CSV (*.csv)|*.csv",
with wx.FileDialog(self, "Save Coefficients file", wildcard="cpd (*.cpd)|*.cpd",
style=wx.FD_SAVE) as fileDialog:
if fileDialog.ShowModal() == wx.ID_CANCEL:
return # the user changed their mind
@@ -197,8 +194,7 @@ class MonitorFrame(wx.Frame):
self.SetStatusText(f"Saving file as {self.__opened_filename}")
self.__opened_filename = fileDialog.GetPath()
data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data"
self.cor.save(self.__opened_filename, price_data_filename=data_filename)
self.cor.save(self.__opened_filename)
self.SetStatusText(f"File saved as {self.__opened_filename}")
@@ -278,13 +274,19 @@ class MonitorFrame(wx.Frame):
self.SetStatusText("Monitoring for changes to coefficients.")
self.timer.Start(self.config.get('monitor.interval')*1000)
# Autosave filename
filename = self.__opened_filename if self.__opened_filename is not None else 'autosave.cpd'
self.cor.start_monitor(interval=self.config.get('monitor.interval'),
from_mins=self.config.get('monitor.from.minutes'),
min_prices=self.config.get('monitor.min_prices'),
max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'),
overlap_pct=self.config.get('monitor.overlap_pct'),
max_p_value=self.config.get('monitor.max_p_value'),
cache_time=self.config.get('monitor.tick_cache_time'))
cache_time=self.config.get('monitor.tick_cache_time'),
autosave=self.config.get('monitor.autosave'),
filename=filename)
else:
self.__log.info("Stopping monitoring.")
self.monitor_toggle.SetBackgroundColour(wx.RED)
@@ -389,9 +391,12 @@ class MonitorFrame(wx.Frame):
:param symbol2:
:return:
"""
# Get the data price data for the base coefficient calculation and the coefficient history data
# Get the price data for the base coefficient calculation, tick data to calculate last coefficient and and the
# coefficient history data
symbol_1_price_data = self.cor.get_price_data(symbol1)
symbol_2_price_data = self.cor.get_price_data(symbol2)
symbol_1_ticks = self.cor.get_ticks(symbol1, cache_only=True)
symbol_2_ticks = self.cor.get_ticks(symbol2, cache_only=True)
history_data = self.cor.get_coefficient_history(symbol1, symbol2)
times = history_data['UTC Date To']
coefficients = history_data['Coefficient']
@@ -399,7 +404,7 @@ class MonitorFrame(wx.Frame):
# Display if we have any data
self.__log.debug(f"Refreshing history graph {symbol1}:{symbol2}.")
self.__graph.draw(times=times, coefficients=coefficients, prices=[symbol_1_price_data, symbol_2_price_data],
symbols=[symbol1, symbol2])
ticks=[symbol_1_ticks, symbol_2_ticks], symbols=[symbol1, symbol2])
# Un-hide and layout if hidden
if not self.__graph.IsShown():
@@ -477,9 +482,10 @@ class GraphPanel(wx.Panel):
# Super
wx.Panel.__init__(self, parent)
# 3 axis, price data 1, price data 2 and coefficient data. All will have axis labels and top and right boarders
# 3 axis, 2 price data for calculate, 2 price data for last coefficient and coefficient history.
# All will have axis labels and top and right boarders
# removed
self.__fig, self.__axes = plt.subplots(nrows=3, ncols=1)
self.__fig, self.__axes = plt.subplots(nrows=5, ncols=1)
# Create the canvas
self.__canvas = FigureCanvas(self, -1, self.__fig)
@@ -500,13 +506,14 @@ class GraphPanel(wx.Panel):
self.__axes = None
self.__fig = None
def draw(self, times, coefficients, prices=None, symbols=None):
def draw(self, times, coefficients, prices=None, symbols=None, ticks=None):
"""
Plot the correlations.
:param times: Series of time values for x axis
:param coefficients: Series of coefficients values for y axis
:param times: Series of time values for x axis for coefficient history chart
:param coefficients: Series of coefficients values for y axis of coefficient history chart
:param prices: Price data used to calculate base coefficient. List [Symbol1 Price Data, Symbol 2 Price Data]
:param symbols: Symbols. List [Symbol1, Symbol2]
:param ticks: Ticks used to calculate last coefficient. List [Symbol1, Symbol2]
:return:
"""
# Clear. We will need to redraw
@@ -514,25 +521,48 @@ class GraphPanel(wx.Panel):
ax.clear()
if symbols is not None and len(symbols) == 2:
# Price history chart for both symbols
for i in range(0, 2):
if symbols[i] is not None and prices is not None and len(prices) == 2 and prices[i] is not None:
self.__axes[i].set_title(f"Base Coefficient Price Data for {symbols[i]}")
self.__axes[i].set_xlabel('Time')
self.__axes[i].set_ylabel('Price')
self.__axes[i].plot(prices[i]['time'], prices[i]['close'])
self.__axes[i].xaxis.set_major_formatter(self.__tick_fmt_date)
self.__axes[i].xaxis.set_minor_formatter(self.__tick_fmt_time)
plt.setp(self.__axes[i].xaxis.get_majorticklabels(), rotation=45)
# Axis ranges
price_chart_date_range = [min(min(prices[0]['time']), min(prices[1]['time'])),
max(max(prices[0]['time']), max(prices[1]['time']))]
tick_chart_date_range = [min(min(ticks[0]['time']), min(ticks[1]['time'])),
max(max(ticks[0]['time']), max(ticks[1]['time']))]
# Coefficient history chart
self.__axes[2].set_title(f"Coefficient History for {symbols[0]}:{symbols[1]}")
self.__axes[2].set_xlabel('Time')
self.__axes[2].set_ylabel('Coefficient')
self.__axes[2].plot(times, coefficients)
self.__axes[2].set_ylim([-1, 1])
self.__axes[2].xaxis.set_major_formatter(self.__tick_fmt_time)
plt.setp(self.__axes[2].xaxis.get_majorticklabels(), rotation=45)
# Chart config
titles = [f"Base Coefficient Price Data for {symbols[0]}", f"Base Coefficient Price Data for {symbols[1]}",
f"Coefficient Tick Data for {symbols[0]}", f"Coefficient Tick Data for {symbols[1]}",
f"Coefficient History for {symbols[0]}:{symbols[1]}"]
xlims = [price_chart_date_range, price_chart_date_range, tick_chart_date_range, tick_chart_date_range, None]
ylims = [None, None, None, None, [-1, 1]]
xlabels = [None, None, None, None, None]
ylabels = ['Price', 'Price', 'Price', 'Price', 'Coefficient']
tick_labels = [[], prices[1]['time'], [], ticks[1]['time'], times]
mtick_fmts = [None, self.__tick_fmt_date, None, self.__tick_fmt_time, self.__tick_fmt_time]
mtick_rot = [0, 45, 0, 45, 45]
xdata = [prices[0]['time'], prices[1]['time'], ticks[0]['time'], ticks[1]['time'], times]
ydata = [prices[0]['close'], prices[1]['close'], ticks[0]['ask'], ticks[1]['ask'], coefficients]
# Draw 5 charts
for index in range(0, len(self.__axes)):
# Titles and axis labels
self.__axes[index].set_title(titles[index])
self.__axes[index].set_xlabel(xlabels[index])
self.__axes[index].set_ylabel(ylabels[index])
# Limits
if xlims[index] is not None:
self.__axes[index].set_xlim(xlims[index])
if ylims[index] is not None:
self.__axes[index].set_ylim(ylims[index])
# Tick labels and formats
self.__axes[index].xaxis.set_ticklabels(tick_labels[index])
if mtick_fmts[index] is not None:
self.__axes[index].xaxis.set_major_formatter(mtick_fmts[index])
plt.setp(self.__axes[index].xaxis.get_majorticklabels(), rotation=mtick_rot[index])
# Plot
self.__axes[index].plot(xdata[index], ydata[index])
# Layout with padding between charts
self.__fig.tight_layout(pad=0.5)