diff --git a/config.yaml b/config.yaml index 2a006f1..f894902 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,7 @@ calculate: from: days: 10 timeframe: 15 - min_prices: 600 + min_prices: 400 max_set_size_diff_pct: 90 overlap_pct: 90 max_p_value: 0.05 @@ -18,6 +18,7 @@ monitor: monitoring_threshold: 0.9 divergence_threshold: 0.8 tick_cache_time: 10 + autosave: true logging: version: 1 disable_existing_loggers: false diff --git a/mt5_correlation/correlation.py b/mt5_correlation/correlation.py index 860e170..b997b8f 100644 --- a/mt5_correlation/correlation.py +++ b/mt5_correlation/correlation.py @@ -7,7 +7,6 @@ import sched import threading import pytz from scipy.stats.stats import pearsonr -import yaml import pickle from mt5_correlation.mt5 import MT5 @@ -36,8 +35,9 @@ class Correlation: coefficient_data = None coefficient_history = None - # Cache for ticks. Dict: {Symbol: [retrieved datetime, ticks dataframe]} - __ticks = {} + # Stores tick data used to calculate coefficient during Monitor. + # Dict: {Symbol: [retrieved datetime, ticks dataframe]} + __monitor_tick_data = {} def __init__(self): # Logger @@ -62,37 +62,34 @@ class Correlation: else: return None - def load(self, filename, price_data_filename=None): + def load(self, filename): """ - Loads a csv file containing calculated coefficients, and optionally the price data used to calculate those + Loads calculated coefficients, price data used to calculate them and tick data used during monitoring. coefficients :param filename: The filename for the coefficient data to load. - :param price_data_filename: The filename for the price data to load. :return: """ - # Load coefficients file - self.coefficient_data = pd.read_csv(filename) + # Load data + with open(filename, 'rb') as file: + loaded_dict = pickle.load(file) - # If specified, load price data yaml file - if price_data_filename is not None: - self.__price_data = {} # Clear - with open(price_data_filename, 'rb') as file: - self.__price_data = pickle.load(file) + # Get data from loaded dict and save + self.coefficient_data = loaded_dict["coefficient_data"] + self.__price_data = loaded_dict["price_data"] + self.__monitor_tick_data = loaded_dict["monitor_tick_data"] + self.coefficient_history = loaded_dict["coefficient_history"] - def save(self, filename, price_data_filename=None): + def save(self, filename): """ - Saves the calculated coefficients as a csv file - :param filename: The filename for the coefficient data to save to. - :param price_data_filename: The filename for the price data to save to. + Saves the calculated coefficients, the price data used to calculate and the tick data for monitoring to a file. + :param filename: The filename to save the data to. :return: """ - # Save the coefficient data - self.coefficient_data.to_csv(filename, index=False) - - # Save the price data if required - if price_data_filename is not None: - with open(price_data_filename, 'wb') as file: - pickle.dump(self.__price_data, file, protocol=pickle.HIGHEST_PROTOCOL) + # Add data to dict then use pickle to save + save_dict = {"coefficient_data": self.coefficient_data, "price_data": self.__price_data, + "monitor_tick_data": self.__monitor_tick_data, "coefficient_history": self.coefficient_history} + with open(filename, 'wb') as file: + pickle.dump(save_dict, file, protocol=pickle.HIGHEST_PROTOCOL) def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, max_p_value=0.05): @@ -196,7 +193,7 @@ class Correlation: return price_data def start_monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05, cache_time=10): + max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'): """ Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient threshold. @@ -211,6 +208,9 @@ class Correlation: :param max_p_value: The maximum p value for the correlation to be meaningful :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse the tick data. Number of seconds to cache tick data for before it becomes stale. + :param autosave: Whether to autosave after every monitor run. If there is no filename specified then will + create one named autosave.cpd + :param filename: Filename for autosave. Default is autosave.cpd. :return: correlation coefficient, or None if coefficient could not be calculated. """ @@ -228,7 +228,8 @@ class Correlation: # to stop and restart the monitor. Note, this happens during calculate self.__monitoring_params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct, - 'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time} + 'overlap_pct': overlap_pct, 'max_p_value': max_p_value, 'cache_time': cache_time, + 'autosave': autosave, 'filename': filename} thread = threading.Thread(target=self.__monitor, kwargs=self.__monitoring_params) thread.start() @@ -312,8 +313,45 @@ class Correlation: (self.coefficient_history['Symbol 2'] == symbol2)] return history + def get_ticks(self, symbol, date_from=None, date_to=None, cache_time=0, cache_only=False): + """ + Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe. + + :param symbol: Name of symbol to get ticks for. + :param date_from: Date to get ticks from. Can only be None if getting from cache (cache_only=True) + :param date_to:Date to get ticks to. Can only be None if getting from cache (cache_only=True) + :param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed, + get data from source and refresh cache. + :param cache_only: Only retrieve from cache. cache_time is ignored. Returns None if symbol is not available in + cache. + + :return: + """ + + timezone = pytz.timezone("Etc/UTC") + utc_now = datetime.now(tz=timezone) + + ticks = None + + # Cache only + if cache_only: + if symbol in self.__monitor_tick_data: + ticks = self.__monitor_tick_data[symbol][1] + # Check if we already have it and it is not stale + elif symbol in self.__monitor_tick_data and utc_now < \ + self.__monitor_tick_data[symbol][0] + timedelta(seconds=cache_time): + # Cached ticks are not stale. Get them + ticks = self.__monitor_tick_data[symbol][1] + self.__log.debug(f"Ticks for {symbol} retrieved from cache.") + else: + # Data does not exist in cache or cached data is stale. Retrieve from source and cache. + ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to) + self.__monitor_tick_data[symbol] = [utc_now, ticks] + self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.") + return ticks + def __monitor(self, interval, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05, cache_time=10): + max_p_value=0.05, cache_time=10, autosave=False, filename='autosave.cpd'): """ The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and stop_monitoring. @@ -328,6 +366,9 @@ class Correlation: :param max_p_value: The maximum p value for the correlation to be meaningful :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse the tick data. Number of seconds to cache tick data for before it becomes stale. + :param autosave: Whether to autosave after every monitor run. If there is no filename specified then will + create one named autosave.cpd + :param filename: Filename for autosave. Default is autosave.cpd. :return: correlation coefficient, or None if coefficient could not be calculated. """ @@ -340,10 +381,15 @@ class Correlation: max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct, max_p_value=max_p_value, cache_time=cache_time) + # Autosave + if autosave: + self.save(filename=filename) + # Schedule the timer to run again params = {'interval': interval, 'from_mins': from_mins, 'min_prices': min_prices, 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct, - 'max_p_value': max_p_value, "cache_time": cache_time} + 'max_p_value': max_p_value, "cache_time": cache_time, 'autosave': autosave, + 'filename': filename} self.__scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params) self.__scheduler.run() @@ -374,8 +420,8 @@ class Correlation: date_from = date_to - timedelta(minutes=from_mins) # Get the tick data - symbol1ticks = self.__get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time) - symbol2ticks = self.__get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time) + symbol1ticks = self.get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to, cache_time=cache_time) + symbol2ticks = self.get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to, cache_time=cache_time) # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one # tick per second and ensuring that times can match. We will need to set the index to time for the resample @@ -414,7 +460,7 @@ class Correlation: return coefficient def __update_all_coefficients(self, from_mins, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05, cache_time=10): + max_p_value=0.05, cache_time=10, autosave=False): """ Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet the threshold can be accessed through the filtered_coefficient_data property. @@ -428,6 +474,8 @@ class Correlation: :param max_p_value: The maximum p value for the correlation to be meaningful :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse the tick data. Number of seconds to cache tick data for before it becomes stale. + :param autosave: Whether to autosave after every monitor run. If there is no filename set then will create one + named autosave.cpd :return: correlation coefficient, or None if coefficient could not be calculated. """ @@ -439,34 +487,6 @@ class Correlation: min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct, max_p_value=max_p_value, cache_time=cache_time) - def __get_ticks(self, symbol, date_from, date_to, cache_time): - """ - Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe. - - :param symbol: Name of symbol to get ticks for. - :param date_from: - :param date_to: - :param cache_time: Number of seconds before cached data is stale. If > than this number of seconds has elapsed, - get data from source and refresh cache. - - :return: - """ - - timezone = pytz.timezone("Etc/UTC") - utc_now = datetime.now(tz=timezone) - - # Check if in cache and not stale - if symbol in self.__ticks and utc_now < self.__ticks[symbol][0] + timedelta(seconds=cache_time): - # Cached ticks are not stale. Get them - ticks = self.__ticks[symbol][1] - self.__log.debug(f"Ticks for {symbol} retrieved from cache.") - else: - # Data does not exist in cache or cached data is stale. Retrieve from source and cache. - ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to) - self.__ticks[symbol] = [utc_now, ticks] - self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.") - return ticks - def __reset_coefficient_data(self): """ Clears coefficient data and history. diff --git a/mt5_correlation/gui.py b/mt5_correlation/gui.py index d3d8671..b84f6d6 100644 --- a/mt5_correlation/gui.py +++ b/mt5_correlation/gui.py @@ -12,7 +12,6 @@ import pytz import pandas as pd import logging import logging.config -import os matplotlib.use('WXAgg') @@ -158,19 +157,17 @@ class MonitorFrame(wx.Frame): self.Bind(wx.EVT_CLOSE, self.on_close, self) def open_file(self, event): - with wx.FileDialog(self, "Open Coefficients file", wildcard="CSV (*.csv)|*.csv", + with wx.FileDialog(self, "Open Coefficients file", wildcard="cpd (*.cpd)|*.cpd", style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog: if fileDialog.ShowModal() == wx.ID_CANCEL: return # the user changed their mind - # Load the file chosen by the user. Also load the corresponding data file if there is one. + # Load the file chosen by the user. self.__opened_filename = fileDialog.GetPath() - data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data" - if os.path.isfile(data_filename): - self.cor.load(self.__opened_filename, price_data_filename=data_filename) - else: - self.cor.load(self.__opened_filename) + + self.SetStatusText(f"Loading file {self.__opened_filename}.") + self.cor.load(self.__opened_filename) # Refresh data in grid self.refresh_grid() @@ -188,7 +185,7 @@ class MonitorFrame(wx.Frame): self.SetStatusText(f"File saved as {self.__opened_filename}") def save_file_as(self, event): - with wx.FileDialog(self, "Save Coefficients file", wildcard="CSV (*.csv)|*.csv", + with wx.FileDialog(self, "Save Coefficients file", wildcard="cpd (*.cpd)|*.cpd", style=wx.FD_SAVE) as fileDialog: if fileDialog.ShowModal() == wx.ID_CANCEL: return # the user changed their mind @@ -197,8 +194,7 @@ class MonitorFrame(wx.Frame): self.SetStatusText(f"Saving file as {self.__opened_filename}") self.__opened_filename = fileDialog.GetPath() - data_filename = f"{os.path.splitext(self.__opened_filename)[0]}.price.data" - self.cor.save(self.__opened_filename, price_data_filename=data_filename) + self.cor.save(self.__opened_filename) self.SetStatusText(f"File saved as {self.__opened_filename}") @@ -278,13 +274,19 @@ class MonitorFrame(wx.Frame): self.SetStatusText("Monitoring for changes to coefficients.") self.timer.Start(self.config.get('monitor.interval')*1000) + + # Autosave filename + filename = self.__opened_filename if self.__opened_filename is not None else 'autosave.cpd' + self.cor.start_monitor(interval=self.config.get('monitor.interval'), from_mins=self.config.get('monitor.from.minutes'), min_prices=self.config.get('monitor.min_prices'), max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'), overlap_pct=self.config.get('monitor.overlap_pct'), max_p_value=self.config.get('monitor.max_p_value'), - cache_time=self.config.get('monitor.tick_cache_time')) + cache_time=self.config.get('monitor.tick_cache_time'), + autosave=self.config.get('monitor.autosave'), + filename=filename) else: self.__log.info("Stopping monitoring.") self.monitor_toggle.SetBackgroundColour(wx.RED) @@ -389,9 +391,12 @@ class MonitorFrame(wx.Frame): :param symbol2: :return: """ - # Get the data price data for the base coefficient calculation and the coefficient history data + # Get the price data for the base coefficient calculation, tick data to calculate last coefficient and and the + # coefficient history data symbol_1_price_data = self.cor.get_price_data(symbol1) symbol_2_price_data = self.cor.get_price_data(symbol2) + symbol_1_ticks = self.cor.get_ticks(symbol1, cache_only=True) + symbol_2_ticks = self.cor.get_ticks(symbol2, cache_only=True) history_data = self.cor.get_coefficient_history(symbol1, symbol2) times = history_data['UTC Date To'] coefficients = history_data['Coefficient'] @@ -399,7 +404,7 @@ class MonitorFrame(wx.Frame): # Display if we have any data self.__log.debug(f"Refreshing history graph {symbol1}:{symbol2}.") self.__graph.draw(times=times, coefficients=coefficients, prices=[symbol_1_price_data, symbol_2_price_data], - symbols=[symbol1, symbol2]) + ticks=[symbol_1_ticks, symbol_2_ticks], symbols=[symbol1, symbol2]) # Un-hide and layout if hidden if not self.__graph.IsShown(): @@ -477,9 +482,10 @@ class GraphPanel(wx.Panel): # Super wx.Panel.__init__(self, parent) - # 3 axis, price data 1, price data 2 and coefficient data. All will have axis labels and top and right boarders + # 3 axis, 2 price data for calculate, 2 price data for last coefficient and coefficient history. + # All will have axis labels and top and right boarders # removed - self.__fig, self.__axes = plt.subplots(nrows=3, ncols=1) + self.__fig, self.__axes = plt.subplots(nrows=5, ncols=1) # Create the canvas self.__canvas = FigureCanvas(self, -1, self.__fig) @@ -500,13 +506,14 @@ class GraphPanel(wx.Panel): self.__axes = None self.__fig = None - def draw(self, times, coefficients, prices=None, symbols=None): + def draw(self, times, coefficients, prices=None, symbols=None, ticks=None): """ Plot the correlations. - :param times: Series of time values for x axis - :param coefficients: Series of coefficients values for y axis + :param times: Series of time values for x axis for coefficient history chart + :param coefficients: Series of coefficients values for y axis of coefficient history chart :param prices: Price data used to calculate base coefficient. List [Symbol1 Price Data, Symbol 2 Price Data] :param symbols: Symbols. List [Symbol1, Symbol2] + :param ticks: Ticks used to calculate last coefficient. List [Symbol1, Symbol2] :return: """ # Clear. We will need to redraw @@ -514,25 +521,48 @@ class GraphPanel(wx.Panel): ax.clear() if symbols is not None and len(symbols) == 2: - # Price history chart for both symbols - for i in range(0, 2): - if symbols[i] is not None and prices is not None and len(prices) == 2 and prices[i] is not None: - self.__axes[i].set_title(f"Base Coefficient Price Data for {symbols[i]}") - self.__axes[i].set_xlabel('Time') - self.__axes[i].set_ylabel('Price') - self.__axes[i].plot(prices[i]['time'], prices[i]['close']) - self.__axes[i].xaxis.set_major_formatter(self.__tick_fmt_date) - self.__axes[i].xaxis.set_minor_formatter(self.__tick_fmt_time) - plt.setp(self.__axes[i].xaxis.get_majorticklabels(), rotation=45) + # Axis ranges + price_chart_date_range = [min(min(prices[0]['time']), min(prices[1]['time'])), + max(max(prices[0]['time']), max(prices[1]['time']))] + tick_chart_date_range = [min(min(ticks[0]['time']), min(ticks[1]['time'])), + max(max(ticks[0]['time']), max(ticks[1]['time']))] - # Coefficient history chart - self.__axes[2].set_title(f"Coefficient History for {symbols[0]}:{symbols[1]}") - self.__axes[2].set_xlabel('Time') - self.__axes[2].set_ylabel('Coefficient') - self.__axes[2].plot(times, coefficients) - self.__axes[2].set_ylim([-1, 1]) - self.__axes[2].xaxis.set_major_formatter(self.__tick_fmt_time) - plt.setp(self.__axes[2].xaxis.get_majorticklabels(), rotation=45) + # Chart config + titles = [f"Base Coefficient Price Data for {symbols[0]}", f"Base Coefficient Price Data for {symbols[1]}", + f"Coefficient Tick Data for {symbols[0]}", f"Coefficient Tick Data for {symbols[1]}", + f"Coefficient History for {symbols[0]}:{symbols[1]}"] + xlims = [price_chart_date_range, price_chart_date_range, tick_chart_date_range, tick_chart_date_range, None] + ylims = [None, None, None, None, [-1, 1]] + xlabels = [None, None, None, None, None] + ylabels = ['Price', 'Price', 'Price', 'Price', 'Coefficient'] + tick_labels = [[], prices[1]['time'], [], ticks[1]['time'], times] + mtick_fmts = [None, self.__tick_fmt_date, None, self.__tick_fmt_time, self.__tick_fmt_time] + mtick_rot = [0, 45, 0, 45, 45] + xdata = [prices[0]['time'], prices[1]['time'], ticks[0]['time'], ticks[1]['time'], times] + ydata = [prices[0]['close'], prices[1]['close'], ticks[0]['ask'], ticks[1]['ask'], coefficients] + + # Draw 5 charts + for index in range(0, len(self.__axes)): + # Titles and axis labels + self.__axes[index].set_title(titles[index]) + self.__axes[index].set_xlabel(xlabels[index]) + self.__axes[index].set_ylabel(ylabels[index]) + + # Limits + if xlims[index] is not None: + self.__axes[index].set_xlim(xlims[index]) + + if ylims[index] is not None: + self.__axes[index].set_ylim(ylims[index]) + + # Tick labels and formats + self.__axes[index].xaxis.set_ticklabels(tick_labels[index]) + if mtick_fmts[index] is not None: + self.__axes[index].xaxis.set_major_formatter(mtick_fmts[index]) + plt.setp(self.__axes[index].xaxis.get_majorticklabels(), rotation=mtick_rot[index]) + + # Plot + self.__axes[index].plot(xdata[index], ydata[index]) # Layout with padding between charts self.__fig.tight_layout(pad=0.5)