From ef28f80a859904c9b16e6c452a53669c7d829649 Mon Sep 17 00:00:00 2001 From: Jamie Cash Date: Wed, 10 Feb 2021 18:21:49 +0000 Subject: [PATCH] Added UI and basic monitoring --- .gitignore | 7 +- .idea/mt5-correlation.iml | 3 + README.md | 8 +- config.yaml | 19 ++ logging_conf.yaml | 14 +- mt5_correlation.py | 96 ++------- mt5_correlation/config.py | 84 ++++++++ mt5_correlation/correlation.py | 301 +++++++++++++++++++++++++++-- mt5_correlation/gui.py | 343 +++++++++++++++++++++++++++++++++ mt5_correlation/mt5.py | 94 +++++---- requirements.txt | 3 +- test/__init__.py | 0 test/test_config.py | 35 ++++ test/testconfig.yaml | 20 ++ 14 files changed, 896 insertions(+), 131 deletions(-) create mode 100644 config.yaml create mode 100644 mt5_correlation/config.py create mode 100644 mt5_correlation/gui.py create mode 100644 test/__init__.py create mode 100644 test/test_config.py create mode 100644 test/testconfig.yaml diff --git a/.gitignore b/.gitignore index 08e827a..3dc1979 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ /venv/ -/log/ -/out/ /.idea/workspace.xml +/gui/*.bak +/gui/*.py +/gui/#~wxg.autosave~app_design.wxg# +/*.log +/*.log.? diff --git a/.idea/mt5-correlation.iml b/.idea/mt5-correlation.iml index 211e5e7..cdf40ae 100644 --- a/.idea/mt5-correlation.iml +++ b/.idea/mt5-correlation.iml @@ -10,4 +10,7 @@ + + \ No newline at end of file diff --git a/README.md b/README.md index e21a33b..80928f5 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,16 @@ Calculates correlation coefficient between all symbols in MetaTrader5 Market Wat 1) Set up your MetaTrader 5 environment ensuring that all symbols that you would like to assess for correlation are shown in your Market Watch window; 2) Set up your python environment; and 3) Install the required libraries. + ``` -pip install -r mt5-correlation\requirements.txt +pip install -r mt5-correlation/requirements.txt ``` # Usage If you set up a virtual environment in the Setup step, ensure this is activated. Then run the script. + ``` -python -m mt5_correlations\mt5_correlations.py +python -m mt5_correlations/get_correlations.py ``` A .csv file containing the correlation coefficient for all combinations of sybmols from the MetaTrader market watch will be produced in the current directory. @@ -29,7 +31,7 @@ A .csv file containing the correlation coefficient for all combinations of sybmo |EU50Cash |FRA40Cash |0.99072 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 | # Customising -Edit mt5_correlations.py to customise. +Edit get_correlations.py to customise. The coefficients are calculated only if: * The smallest set of price data is no less than 90% of the size of the largest set; diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..ba10243 --- /dev/null +++ b/config.yaml @@ -0,0 +1,19 @@ +--- +calculate: + from: + days: 7 + timeframe: 15 + min_prices: 400 + max_set_size_diff_pct: 90 + overlap_pct: 90 + max_p_value: 0.05 +monitor: + from: + minutes: 10 + interval: 10 + min_prices: 400 + max_set_size_diff_pct: 50 + overlap_pct: 50 + max_p_value: 0.05 + divergence_threshold: 0.8 +... \ No newline at end of file diff --git a/logging_conf.yaml b/logging_conf.yaml index 176be17..b314cd7 100644 --- a/logging_conf.yaml +++ b/logging_conf.yaml @@ -13,17 +13,25 @@ formatters: handlers: console: - level: WARNING + level: INFO class: logging.StreamHandler formatter: brief stream: ext://sys.stdout + file: + level: DEBUG + class: logging.handlers.RotatingFileHandler + formatter: precice + filename: debug.log + mode: a + maxBytes: 2560000 + backupCount: 1 root: level: DEBUG - handlers: [console] + handlers: [console, file] loggers: mt5-correlation: level: DEBUG - handlers: [console] + handlers: [console, file] propagate: 0 \ No newline at end of file diff --git a/mt5_correlation.py b/mt5_correlation.py index 1c5ebb9..2cdff55 100644 --- a/mt5_correlation.py +++ b/mt5_correlation.py @@ -1,79 +1,25 @@ -from datetime import datetime, timedelta -import logging.config -import pandas as pd -import pytz -import yaml -from mt5_correlation.mt5 import MT5 -from mt5_correlation.correlation import Correlation +""" +Application to monitor previously correlated symbol pairs for correlation divergence. +""" import definitions +import yaml +import logging.config +from mt5_correlation.gui import MonitorFrame +from mt5_correlation.config import Config +import wx -# Configure logger -with open(fr'{definitions.ROOT_DIR}\logging_conf.yaml', 'rt') as file: - config = yaml.safe_load(file.read()) - logging.config.dictConfig(config) - log = logging.getLogger() +if __name__ == "__main__": + # Configure the logger + with open(fr'{definitions.ROOT_DIR}\logging_conf.yaml', 'rt') as file: + config = yaml.safe_load(file.read()) + logging.config.dictConfig(config) -# Create mt5 class. This contains required methods for interacting with MT5. -mt5 = MT5() + # Load the config + config = Config.instance() + config.load(fr"{definitions.ROOT_DIR}\config.yaml") -# Gte all visible symbols -symbols = mt5.get_symbols() - -# set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today) -timezone = pytz.timezone("Etc/UTC") -utc_to = datetime.now(tz=timezone) -utc_from = utc_to - timedelta(days=7) - -# Set timeframe -timeframe = mt5.TIMEFRAME_M15 - -# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict. -price_data = {} -for symbol in symbols: - price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to, - timeframe=timeframe) - -# Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs -# eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to -# avoid duplicating. We will store all coefficients in a dataframe for export as CSV. -columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe'] -coefficients = pd.DataFrame(columns=columns) - -index = 0 -# There will be (x^2 - x) / 2 pairs where x is number of symbols -num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2) - -for i in range(0, len(symbols)): - symbol1 = symbols[i] - - for j in range(i + 1, len(symbols)): - symbol2 = symbols[j] - index += 1 - - # Get price data for both symbols - symbol1_price_data = price_data[symbol1.name] - symbol2_price_data = price_data[symbol2.name] - - # Get coefficient and store if valid - coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data, - symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90, - overlap_pct=90, max_p_value=0.05) - - if coefficient is not None: - coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name, - 'Coefficient': coefficient, 'UTC Date From': utc_from, - 'UTC Date To': utc_to, 'Timeframe': timeframe}, ignore_index=True) - - log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of " - f"{coefficient}.") - else: - log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not" - f" be calculated.") - -# Sort, highest correlated first -coefficients = coefficients.sort_values('Coefficient', ascending=False) - -# Save as CSV -filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S}.csv" -log.info(f"Saving coefficients as '{filename}'.") -coefficients.to_csv(filename, index=False) + # Start the app + app = wx.App(False) + frame = MonitorFrame(None, wx.ID_ANY, "") + frame.Show() + app.MainLoop() diff --git a/mt5_correlation/config.py b/mt5_correlation/config.py new file mode 100644 index 0000000..320f4d6 --- /dev/null +++ b/mt5_correlation/config.py @@ -0,0 +1,84 @@ +import yaml +import definitions + + +class Config(object): + """ + Provides access to application configuration parameters stored in config.yaml. + """ + + _config = None + _path = None + _instance = None + + def __init__(self): + """ + Singleton. Raise runtime error + """ + raise RuntimeError('Call instance() instead') + + @classmethod + def instance(cls): + """ + Singleton. Get instance of this class. Create if not already created. + :return: + """ + if cls._instance is None: + cls._instance = cls.__new__(cls) + return cls._instance + + def load(self, path): + """ + Loads the applications config file + :param path: Path to config file + :return: + """ + with open(path, 'r') as yamlfile: + self._config = yaml.safe_load(yamlfile) + + # Store path so that we can save later + self._path = path + + def save(self): + """ + Saves config file + :return: + """ + + with open(self._path, 'w') as file: + file.write("---\n") + yaml.dump(self._config, file, sort_keys=False) + file.write("...") + + def get(self, path): + """ + Gets a config property value. + :param path: path to property. Path separated by . + :return: property value + """ + + elements = path.split('.') + last = None + + for element in elements: + if last is None: + last = self._config[element] + else: + last = last[element] + + return last + + def set(self, path, value): + """ + Sets a config property value + :param path: path to property. Path separated by . + :param value: Value to set property to + :return: + """ + obj = self._config + key_list = path.split(".") + + for k in key_list[:-1]: + obj = obj[k] + + obj[key_list[-1]] = value diff --git a/mt5_correlation/correlation.py b/mt5_correlation/correlation.py index c58a70f..595e3ea 100644 --- a/mt5_correlation/correlation.py +++ b/mt5_correlation/correlation.py @@ -1,26 +1,305 @@ import math +import logging +import pandas as pd +from datetime import datetime +import time +import sched +import threading +import pytz from scipy.stats.stats import pearsonr +from mt5_correlation.mt5 import MT5 + class Correlation: """ - A class to calculate the correlation coefficient between two sets of price data + A class to maintain the state of the calculated correlation coefficients. """ - @staticmethod - def calculate_coefficient(symbol1_prices, symbol2_prices, max_set_size_diff_pct=90, overlap_pct=90, - max_p_value=0.05): - """ - Calculates the correlation coefficient between two sets of price data. Uses close price. + min_coefficient = 0.9 + monitoring = False # Monitoring cor correlations - :param symbol1_prices: - :param symbol2_prices: + def __init__(self): + self.log = logging.getLogger(__name__) + + # Create dataframe + columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe', + 'Last Check', 'Last Coefficient'] + self.coefficient_data = pd.DataFrame(columns=columns) + + # Create timer for continuous monitoring + self.scheduler = sched.scheduler(time.time, time.sleep) + + def load(self, filename): + """ + Loads a csv file containing calculated coefficients + :param filename: + :return: + """ + self.coefficient_data = pd.read_csv(filename) + + def save(self, filename): + """ + Saves the calculated coefficients as a csv file + :param filename: + :return: + """ + self.coefficient_data.to_csv(filename, index=False) + + def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): + """ + Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch. Updates coefficient data. + + :param date_from: From date for price data from which to calculate correlation coefficients + :param date_to: To date for price data from which to calculate correlation coefficients + :param timeframe: Timeframe for price data from which to calculate correlation coefficients + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + + :return: + """ + + # Create mt5 class. This contains required methods for interacting with MT5. + mt5 = MT5() + + # Gte all visible symbols + symbols = mt5.get_symbols() + + # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict. + price_data = {} + for symbol in symbols: + price_data[symbol] = mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to, + timeframe=timeframe) + + # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs + # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to + # avoid duplicating. We will store all coefficients in a dataframe for export as CSV. + index = 0 + # There will be (x^2 - x) / 2 pairs where x is number of symbols + num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2) + + for i in range(0, len(symbols)): + symbol1 = symbols[i] + + for j in range(i + 1, len(symbols)): + symbol2 = symbols[j] + index += 1 + + # Get price data for both symbols + symbol1_price_data = price_data[symbol1] + symbol2_price_data = price_data[symbol2] + + # Get coefficient and store if valid + coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data, + symbol2_prices=symbol2_price_data, + min_prices=min_prices, + max_set_size_diff_pct=max_set_size_diff_pct, + overlap_pct=overlap_pct, max_p_value=max_p_value) + + if coefficient is not None: + + self.coefficient_data = \ + self.coefficient_data.append({'Symbol 1': symbol1, 'Symbol 2': symbol2, + 'Base Coefficient': coefficient, 'UTC Date From': date_from, + 'UTC Date To': date_to, 'Timeframe': timeframe}, + ignore_index=True) + self.log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a " + f"coefficient of {coefficient}.") + else: + self.log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:" + f"{symbol2} could no be calculated.") + + # Sort, highest correlated first + self.coefficient_data = self.coefficient_data.sort_values('Base Coefficient', ascending=False) + + def update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, + overlap_pct=90, max_p_value=0.05): + """ + Updates the coefficient for the specified symbol pair + :param symbol1: Name of symbol to calculate coefficient for. + :param symbol2: Name of symbol to calculate coefficient for. + :param date_from: From date for tick data from which to calculate correlation coefficients + :param date_to: To date for tick data from which to calculate correlation coefficients + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are within this pct of each other :param overlap_pct: :param max_p_value: The maximum p value for the correlation to be meaningful :return: correlation coefficient, or None if coefficient could not be calculated. """ + + # Get the tick data + mt5 = MT5() + symbol1ticks = mt5.get_ticks(symbol=symbol1, from_date=date_from, to_date=date_to) + symbol2ticks = mt5.get_ticks(symbol=symbol2, from_date=date_from, to_date=date_to) + + # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one + # tick per second and ensuring that times can match. We will need to set the index to time for the resample + # then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price + if symbol1ticks is not None and symbol2ticks is not None and \ + len(symbol1ticks.index) > 0 and len(symbol2ticks.index) > 0: + symbol1ticks.set_index('time', inplace=True) + symbol2ticks.set_index('time', inplace=True) + symbol1prices = symbol1ticks['ask'].resample('1S').ohlc() + symbol2prices = symbol2ticks['ask'].resample('1S').ohlc() + symbol1prices.reset_index(inplace=True) + symbol2prices.reset_index(inplace=True) + symbol1prices = symbol1prices[symbol1prices['close'].notna()] + symbol2prices = symbol2prices[symbol2prices['close'].notna()] + + # Calculate the coefficient + coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices, + min_prices=min_prices, + max_set_size_diff_pct=max_set_size_diff_pct, + overlap_pct=overlap_pct, max_p_value=max_p_value) + else: + coefficient = None + + # Find the correct row in the coefficient data and update with calculation date and calculated coefficient + timezone = pytz.timezone("Etc/UTC") + now = datetime.now(tz=timezone) + + # Update data if we have a coefficient + if coefficient is not None: + self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) & + (self.coefficient_data['Symbol 2'] == symbol2), + 'Last Check'] = now + + self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) & + (self.coefficient_data['Symbol 2'] == symbol2), + 'Last Coefficient'] = coefficient + + return coefficient + + def update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): + """ + Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet + the threshold can be accessed through the filtered_coefficient_data property. + + :param date_from: From date for tick data from which to calculate correlation coefficients + :param date_to: To date for tick data from which to calculate correlation coefficients + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + :return: correlation coefficient, or None if coefficient could not be calculated. + """ + # Update latest coefficient for every pair + for index, row in self.filtered_coefficient_data.iterrows(): + symbol1 = row['Symbol 1'] + symbol2 = row['Symbol 2'] + self.update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to, + min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct, + overlap_pct=overlap_pct, max_p_value=max_p_value) + + def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): + """ + Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient + threshold. + + :param interval: How often to check in seconds + :param date_from: From date for tick data from which to calculate correlation coefficients + :param date_to: To date for tick data from which to calculate correlation coefficients + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + :return: correlation coefficient, or None if coefficient could not be calculated. + + :return: + """ + self.log.debug(f"Starting monitor.") + self.monitoring = True + + # Create thread to run monitoring This will call private __monitor method that will run the calculation and + # keep scheduling itself while self.monitoring is True + params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices, + 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct, + 'max_p_value': max_p_value} + thread = threading.Thread(target=self.__monitor, kwargs=params) + thread.start() + + def stop_monitor(self): + """ + Stops monitoring symbol pairs for correlation. + :return: + """ + self.log.debug(f"Stopping monitor.") + self.monitoring = False + + def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, + max_p_value=0.05): + """ + The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and + stop_monitoring. + + :param interval: How often to check in seconds + :param date_from: From date for tick data from which to calculate correlation coefficients + :param date_to: To date for tick data from which to calculate correlation coefficients + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + :return: correlation coefficient, or None if coefficient could not be calculated. + :return: + """ + self.log.debug(f"In monitor event. Monitoring: {self.monitoring}.") + + # Only run if monitor is not stopped + if self.monitoring: + # Update all coefficients + self.update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices, + max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct, + max_p_value=max_p_value) + + # Schedule the timer to run again + params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices, + 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct, + 'max_p_value': max_p_value} + self.scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params) + self.scheduler.run() + + @property + def filtered_coefficient_data(self): + """ + :return: Coefficient data filtered so that all base coefficients >= min coefficient + """ + if self.coefficient_data is not None: + return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.min_coefficient] + else: + return None + + @staticmethod + def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90, + overlap_pct=90, max_p_value=0.05): + """ + Calculates the correlation coefficient between two sets of price data. Uses close price. + + :param symbol1_prices: prices or ticks for symbol 1 + :param symbol2_prices: prices or ticks for symbol 2 + :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold + is not met then returned coefficient will be None + :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are + within this pct of each other + :param overlap_pct: + :param max_p_value: The maximum p value for the correlation to be meaningful + :return: correlation coefficient, or None if coefficient could not be calculated. + """ + # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set? @@ -31,7 +310,8 @@ class Correlation: len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)])) similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100) - suitable = similar_size and enough_overlap + enough_prices = len_smallest_set >= min_prices + suitable = similar_size and enough_overlap and enough_prices if suitable: # Calculate coefficient on close prices @@ -50,6 +330,3 @@ class Correlation: coefficient = None return coefficient - - - diff --git a/mt5_correlation/gui.py b/mt5_correlation/gui.py new file mode 100644 index 0000000..72d6649 --- /dev/null +++ b/mt5_correlation/gui.py @@ -0,0 +1,343 @@ +import wx +import wx.grid +import wx.lib.masked as masked +from mt5_correlation.correlation import Correlation +from mt5_correlation.config import Config +from datetime import datetime, timedelta +import pytz +import pandas as pd +import logging +import definitions + + +class MonitorFrame(wx.Frame): + + cor = None + rows = 0 # Need to track as we need to notify grid if row count changes. + opened_filename = None # So we can save to same file as we opened + config = None # The applications config + + COLUMN_INDEX = 0 + COLUMN_SYMBOL1 = 1 + COLUMN_SYMBOL2 = 2 + COLUMN_BASE_COEFFICIENT = 3 + COLUMN_DATE_FROM = 4 + COLUMN_DATE_TO = 5 + COLUMN_TIMEFRAME = 6 + COLUMN_LAST_CHECK = 7 + COLUMN_LAST_COEFFICIENT = 8 + + def __init__(self, *args, **kwds): + self.log = logging.getLogger(__name__) + self.config = Config.instance() + + # Create correlation instance to maintain state of calculated coefficients + self.cor = Correlation() + + kwds["style"] = kwds.get("style", 0) | wx.DEFAULT_FRAME_STYLE + wx.Frame.__init__(self, *args, **kwds) + self.SetSize((1235, 800)) + self.SetTitle("Monitor for Divergence") + + # Status bar + self.statusbar = self.CreateStatusBar(1) + + # Menu Bar + self.menubar = wx.MenuBar() + file_menu = wx.Menu() + + # Open and save + menu_item_open = file_menu.Append(wx.ID_ANY, "Open", "Open correlations file.") + menu_item_save = file_menu.Append(wx.ID_ANY, "Save", "Save correlations file.") + menu_item_saveas = file_menu.Append(wx.ID_ANY, "Save As", "Save correlations file.") + + # Calculate + file_menu.AppendSeparator() + menu_item_calculate = file_menu.Append(wx.ID_ANY, "Calculate", "Calculate base coefficients.") + + # Close + file_menu.AppendSeparator() + menu_item_exit = file_menu.Append(wx.ID_ANY, "Exit", "Close the application") + + # Add file menu and set menu bar + self.menubar.Append(file_menu, "File") + self.SetMenuBar(self.menubar) + + # Main window + self.window = wx.SplitterWindow(self, wx.ID_ANY) + self.window.SetMinimumPaneSize(20) + + self.correlations_pane = wx.Panel(self.window, wx.ID_ANY) + + sizer_grid = wx.BoxSizer(wx.VERTICAL) + + # Filter label, input box and button + sizer_coefficient = wx.BoxSizer(wx.HORIZONTAL) + sizer_grid.Add(sizer_coefficient, 0, 0, 0) + + label_min_coefficient = wx.StaticText(self.correlations_pane, wx.ID_ANY, "Min Coefficient (Range -1 - 1)") + sizer_coefficient.Add(label_min_coefficient, 0, wx.ALL, 0) + + self.edit_ctrl_min_coefficient = masked.NumCtrl(self.correlations_pane, value=self.cor.min_coefficient, + allowNegative=True, min=-1, max=1, integerWidth=1, + fractionWidth=5) + sizer_coefficient.Add(self.edit_ctrl_min_coefficient, 0, wx.ALL, 0) + + self.filter_button = wx.Button(self.correlations_pane, wx.ID_ANY, label="Filter") + sizer_coefficient.Add(self.filter_button, 0, wx.ALL, 0) + + self.monitor_toggle = wx.ToggleButton(self.correlations_pane, wx.ID_ANY, label="Monitoring") + sizer_coefficient.Add(self.monitor_toggle, 0, wx.ALL, 0) + + # Data table using pandas dataframe for underlying data + self.table = DataTable(self.cor.filtered_coefficient_data) + self.grid_correlations = wx.grid.Grid(self.correlations_pane, wx.ID_ANY, size=(1, 1)) + self.grid_correlations.SetTable(self.table, takeOwnership=True) + self.grid_correlations.EnableEditing(0) + self.grid_correlations.EnableDragRowSize(0) + self.grid_correlations.EnableDragGridSize(0) + self.grid_correlations.SetSelectionMode(wx.grid.Grid.SelectRows) + self.grid_correlations.SetColSize(self.COLUMN_INDEX, 0) # Index. Hide + self.grid_correlations.SetColSize(self.COLUMN_SYMBOL1, 100) # Symbol 1 + self.grid_correlations.SetColSize(self.COLUMN_SYMBOL2, 100) # Symbol 2 + self.grid_correlations.SetColSize(self.COLUMN_BASE_COEFFICIENT, 100) # Base Coefficient + self.grid_correlations.SetColSize(self.COLUMN_DATE_FROM, 0) # UTC Date From. Hide + self.grid_correlations.SetColSize(self.COLUMN_DATE_TO, 0) # UTC Date To. Hide + self.grid_correlations.SetColSize(self.COLUMN_TIMEFRAME, 0) # Timeframe. Hide. + self.grid_correlations.SetColSize(self.COLUMN_LAST_CHECK, 100) # Last Check + self.grid_correlations.SetColSize(self.COLUMN_LAST_COEFFICIENT, 100) # Last Coefficient + sizer_grid.Add(self.grid_correlations, 1, wx.ALL | wx.EXPAND, 0) + + self.charts_pane = wx.Panel(self.window, wx.ID_ANY) + + # Charts + sizer_chart = wx.BoxSizer(wx.VERTICAL) + + label_2 = wx.StaticText(self.charts_pane, wx.ID_ANY, "Charts Go Here", style=wx.ALIGN_CENTER_HORIZONTAL) + sizer_chart.Add(label_2, 0, wx.ALIGN_CENTER_HORIZONTAL, 0) + + # Add the sizers to the panes + self.charts_pane.SetSizer(sizer_chart) + self.correlations_pane.SetSizer(sizer_grid) + + # Set window split to 2 panes and layout + self.window.SplitVertically(self.correlations_pane, self.charts_pane) + self.Layout() + + # Set up timer to refresh grid + self.timer = wx.Timer(self) + + # Bind my buttons, timer, menu + self.filter_button.Bind(wx.EVT_BUTTON, self.change_min_coefficient) + self.monitor_toggle.Bind(wx.EVT_TOGGLEBUTTON, self.monitor) + self.Bind(wx.EVT_TIMER, self.refresh_grid, self.timer) + self.Bind(wx.EVT_MENU, self.open_file, menu_item_open) + self.Bind(wx.EVT_MENU, self.save_file, menu_item_save) + self.Bind(wx.EVT_MENU, self.save_file_as, menu_item_saveas) + self.Bind(wx.EVT_MENU, self.calculate_coefficients, menu_item_calculate) + self.Bind(wx.EVT_MENU, self.quit, menu_item_exit) + + # Bind window close event + self.Bind(wx.EVT_CLOSE, self.on_close, self) + + def open_file(self, event): + with wx.FileDialog(self, "Open Coefficients file", wildcard="CSV (*.csv)|*.csv", + style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog: + + if fileDialog.ShowModal() == wx.ID_CANCEL: + return # the user changed their mind + + # Load the file chosen by the user + self.opened_filename = fileDialog.GetPath() + self.cor.load(self.opened_filename) + + # Refresh data in grid + self.refresh_grid(event) + + self.SetStatusText(f"File {self.opened_filename} loaded.") + + def save_file(self, event): + self.cor.save(self.opened_filename) + self.SetStatusText(f"File saved as {self.opened_filename}") + + def save_file_as(self, event): + with wx.FileDialog(self, "Save Coefficients file", wildcard="CSV (*.csv)|*.csv", + style=wx.FD_SAVE) as fileDialog: + if fileDialog.ShowModal() == wx.ID_CANCEL: + return # the user changed their mind + + # Save the file, changing opened filename so next save writes to new file + self.opened_filename = fileDialog.GetPath() + self.cor.save(self.opened_filename) + + self.SetStatusText(f"File saved as {self.opened_filename}") + + def calculate_coefficients(self, event): + # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today) + timezone = pytz.timezone("Etc/UTC") + utc_to = datetime.now(tz=timezone) + utc_from = utc_to - timedelta(days=self.config.get('calculate.from.days')) + + # Set timeframe + timeframe = self.config.get('calculate.timeframe') + + # Calculate + self.SetStatusText("Calculating coefficients.") + self.cor.calculate(date_from=utc_from, date_to=utc_to, + timeframe=self.config.get('calculate.timeframe'), + min_prices=self.config.get('calculate.min_prices'), + max_set_size_diff_pct=self.config.get('calculate.max_set_size_diff_pct'), + overlap_pct=self.config.get('calculate.overlap_pct'), + max_p_value=self.config.get('calculate.max_p_value')) + self.SetStatusText("") + + # Show calculated data + self.refresh_grid(event) + + def quit(self, event): + self.Close() + + def change_min_coefficient(self, event): + self.cor.min_coefficient = self.edit_ctrl_min_coefficient.GetValue() + self.refresh_grid(event) + + def refresh_grid(self, event): + """ + Refreshes grid. Notifies if rows have been added or deleted. + :return: + """ + self.log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}") + + # Update data + self.table.data = self.cor.coefficient_data.copy() + + # Format + self.table.data['Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format) + self.table.data['Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True) + self.table.data['Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S') + self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format) + + # Remove nans. The ones from the float column wil be str nan as they have been formatted + self.table.data = self.table.data.fillna('') + self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '') + + # Start refresh + self.grid_correlations.BeginBatch() + + # Check if num rows in dataframe has changed, and send appropriate APPEND or DELETE messages + cur_rows = len(self.cor.filtered_coefficient_data.index) + if cur_rows < self.rows: + # Data has been deleted. Send message + msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_DELETED, + self.rows - cur_rows, self.rows - cur_rows) + self.grid_correlations.ProcessTableMessage(msg) + elif cur_rows > self.rows: + # Data has been added. Send message + msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_APPENDED, + cur_rows - self.rows) # how many + self.grid_correlations.ProcessTableMessage(msg) + + self.grid_correlations.EndBatch() + + # Send updated message + msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_REQUEST_VIEW_GET_VALUES) + self.grid_correlations.ProcessTableMessage(msg) + + # Update row count + self.rows = cur_rows + + def monitor(self, event): + # Check state of toggle button. If on, then start monitoring, else stop + if self.monitor_toggle.GetValue(): + self.log.debug("Starting monitoring.") + self.SetStatusText("Monitoring for changes to coefficients.") + + # Calculate correlations fro last 10 mins + timezone = pytz.timezone("Etc/UTC") + utc_to = datetime.now(tz=timezone) + utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes')) + + self.timer.Start(10000) + self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to, + min_prices=self.config.get('monitor.min_prices'), + max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'), + overlap_pct=self.config.get('monitor.overlap_pct'), + max_p_value=self.config.get('monitor.max_p_value')) + else: + self.log.debug("Stopping monitoring.") + self.SetStatusText("Monitoring stopped.") + self.timer.Stop() + self.cor.stop_monitor() + + def on_close(self, event): + """ + Window closing. Save coefficients and stop monitoring. + :param event: + :return: + """ + if self.opened_filename is not None: + self.cor.save(self.opened_filename) + + self.cor.stop_monitor() + + event.Skip() + + +class DataTable(wx.grid.GridTableBase): + """ + A data table that holds data in a pandas dataframe + """ + def __init__(self, data=None): + wx.grid.GridTableBase.__init__(self) + self.headerRows = 1 + if data is None: + data = pd.DataFrame() + self.data = data + + # Get divergence threshold from app config + # Get application config + self.config = Config.instance() + + # Get divergence threshold. This will be used by DataTable to highlight cells + self.divergence_threshold = self.config.get('monitor.divergence_threshold') + + def GetNumberRows(self): + return len(self.data) + + def GetNumberCols(self): + return len(self.data.columns) + 1 + + def GetValue(self, row, col): + if col == 0: + return self.data.index[row] + return self.data.iloc[row, col - 1] + + def SetValue(self, row, col, value): + self.data.iloc[row, col - 1] = value + + def GetColLabelValue(self, col): + if col == 0: + if self.data.index.name is None: + return 'Index' + else: + return self.data.index.name + return str(self.data.columns[col - 1]) + + def GetTypeName(self, row, col): + return wx.grid.GRID_VALUE_STRING + + def GetAttr(self, row, col, prop): + attr = wx.grid.GridCellAttr() + + # If column is last coefficient, get value and check against threshold. Highlight if diverged. + threshold = self.config.get('monitor.divergence_threshold') + if col == MonitorFrame.COLUMN_LAST_COEFFICIENT: + value = self.GetValue(row, col) + if value != "": + value = float(value) + if value <= threshold: + attr.SetBackgroundColour(wx.YELLOW) + else: + attr.SetBackgroundColour(wx.WHITE) + + return attr diff --git a/mt5_correlation/mt5.py b/mt5_correlation/mt5.py index 8b53d26..184126f 100644 --- a/mt5_correlation/mt5.py +++ b/mt5_correlation/mt5.py @@ -1,5 +1,5 @@ import pandas as pd -import MetaTrader5 as mt5 +import MetaTrader5 import logging @@ -9,27 +9,27 @@ class MT5: """ # Timeframes - TIMEFRAME_M1 = mt5.TIMEFRAME_M1 - TIMEFRAME_M2 = mt5.TIMEFRAME_M2 - TIMEFRAME_M3 = mt5.TIMEFRAME_M3 - TIMEFRAME_M4 = mt5.TIMEFRAME_M4 - TIMEFRAME_M5 = mt5.TIMEFRAME_M5 - TIMEFRAME_M6 = mt5.TIMEFRAME_M6 - TIMEFRAME_M10 = mt5.TIMEFRAME_M10 - TIMEFRAME_M12 = mt5.TIMEFRAME_M10 - TIMEFRAME_M15 = mt5.TIMEFRAME_M15 - TIMEFRAME_M20 = mt5.TIMEFRAME_M20 - TIMEFRAME_M30 = mt5.TIMEFRAME_M30 - TIMEFRAME_H1 = mt5.TIMEFRAME_H1 - TIMEFRAME_H2 = mt5.TIMEFRAME_H2 - TIMEFRAME_H3 = mt5.TIMEFRAME_H3 - TIMEFRAME_H4 = mt5.TIMEFRAME_H4 - TIMEFRAME_H6 = mt5.TIMEFRAME_H6 - TIMEFRAME_H8 = mt5.TIMEFRAME_H8 - TIMEFRAME_H12 = mt5.TIMEFRAME_H12 - TIMEFRAME_D1 = mt5.TIMEFRAME_D1 - TIMEFRAME_W1 = mt5.TIMEFRAME_W1 - TIMEFRAME_MN1 = mt5.TIMEFRAME_MN1 + TIMEFRAME_M1 = MetaTrader5.TIMEFRAME_M1 + TIMEFRAME_M2 = MetaTrader5.TIMEFRAME_M2 + TIMEFRAME_M3 = MetaTrader5.TIMEFRAME_M3 + TIMEFRAME_M4 = MetaTrader5.TIMEFRAME_M4 + TIMEFRAME_M5 = MetaTrader5.TIMEFRAME_M5 + TIMEFRAME_M6 = MetaTrader5.TIMEFRAME_M6 + TIMEFRAME_M10 = MetaTrader5.TIMEFRAME_M10 + TIMEFRAME_M12 = MetaTrader5.TIMEFRAME_M10 + TIMEFRAME_M15 = MetaTrader5.TIMEFRAME_M15 + TIMEFRAME_M20 = MetaTrader5.TIMEFRAME_M20 + TIMEFRAME_M30 = MetaTrader5.TIMEFRAME_M30 + TIMEFRAME_H1 = MetaTrader5.TIMEFRAME_H1 + TIMEFRAME_H2 = MetaTrader5.TIMEFRAME_H2 + TIMEFRAME_H3 = MetaTrader5.TIMEFRAME_H3 + TIMEFRAME_H4 = MetaTrader5.TIMEFRAME_H4 + TIMEFRAME_H6 = MetaTrader5.TIMEFRAME_H6 + TIMEFRAME_H8 = MetaTrader5.TIMEFRAME_H8 + TIMEFRAME_H12 = MetaTrader5.TIMEFRAME_H12 + TIMEFRAME_D1 = MetaTrader5.TIMEFRAME_D1 + TIMEFRAME_W1 = MetaTrader5.TIMEFRAME_W1 + TIMEFRAME_MN1 = MetaTrader5.TIMEFRAME_MN1 def __init__(self): # Connect to MetaTrader5. Opens if not already open. @@ -38,43 +38,43 @@ class MT5: self.log = logging.getLogger(__name__) # Open MT5 and log error if it could not open - if not mt5.initialize(): + if not MetaTrader5.initialize(): self.log.error("initialize() failed") - mt5.shutdown() + MetaTrader5.shutdown() # Print connection status - self.log.debug(mt5.terminal_info()) + self.log.debug(MetaTrader5.terminal_info()) # Print data on MetaTrader 5 version - self.log.debug(mt5.version()) + self.log.debug(MetaTrader5.version()) def __del__(self): # shut down connection to the MetaTrader 5 terminal - mt5.shutdown() + MetaTrader5.shutdown() def get_symbols(self): """ Gets list of symbols open in MT5 market watch. - :return: list of symbols + :return: list of symbol names """ # Iterate symbols and get those in market watch. - symbols = mt5.symbols_get() + symbols = MetaTrader5.symbols_get() selected_symbols = [] for symbol in symbols: if symbol.visible: - selected_symbols.append(symbol) + selected_symbols.append(symbol.name) # Log symbol counts - total_symbols = mt5.symbols_total() + total_symbols = MetaTrader5.symbols_total() num_selected_symbols = len(selected_symbols) - self.log.info(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.") + self.log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.") return selected_symbols def get_prices(self, symbol, from_date, to_date, timeframe): """ Gets OHLC price data for the specified symbol. - :param symbol: The MT5 symbol to get the price data for + :param symbol: The name of the symbol to get the price data for. :param from_date: Date from when to retrieve data :param to_date: Date where to receive data to :param timeframe: The timeframe for the candes. Possible values are: @@ -101,9 +101,10 @@ class MT5: TIMEFRAME_MN1: 1 month :return: Price data for symbol as dataframe """ + # Get prices from MT5 - prices = mt5.copy_rates_range(symbol.name, timeframe, from_date, to_date) - self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.") + prices = MetaTrader5.copy_rates_range(symbol, timeframe, from_date, to_date) + self.log.debug(f"{len(prices)} prices retrieved for {symbol}.") # Create dataframe from data and convert time in seconds to datetime format prices_dataframe = pd.DataFrame(prices) @@ -111,5 +112,28 @@ class MT5: return prices_dataframe + def get_ticks(self, symbol, from_date, to_date): + """ + Gets OHLC price data for the specified symbol. + :param symbol: The name of the symbol to get the price data for. + :param from_date: Date from when to retrieve data + :param to_date: Date where to receive data to + :return: Tick data for symbol as dataframe + """ + # Get ticks from MT5 + ticks = MetaTrader5.copy_ticks_range(symbol, from_date, to_date, MetaTrader5.COPY_TICKS_ALL) + # If ticks is None, there was an error + if ticks is None: + error = MetaTrader5.last_error() + self.log.error(f"Error retrieving ticks for {symbol}: {error}") + return None + else: + self.log.debug(f"{len(ticks)} ticks retrieved for {symbol}.") + + # Create dataframe from data and convert time in seconds to datetime format + ticks_dataframe = pd.DataFrame(ticks) + ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s') + + return ticks_dataframe diff --git a/requirements.txt b/requirements.txt index a561556..5eb187d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ MetaTrader5==5.0.34 pytz==2021.1 scipy==1.6.0 logging==0.4.9.6 -pyyaml==5.4.1 \ No newline at end of file +pyyaml==5.4.1 +wxpython==4.1.1 \ No newline at end of file diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 0000000..7c73c90 --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,35 @@ +import unittest +from mt5_correlation.config import Config + + +class TestConfig(unittest.TestCase): + def test_load_and_get(self): + config = Config.instance() + config.load("testconfig.yaml") + val121 = config.get('test1.test1_2.val1_2_1') + self.assertEqual(val121, 'val1_2_1', "Get returned incorrect value.") + + def test_set_and_save(self): + config = Config.instance() + config.load("testconfig.yaml") + + path = 'test1.test1_2.val1_2_1' + + # Save new value, storing orig so we can restore later + orig_value = config.get(path) + config.set(path, "newval") + config.save() + + # Reopen config and get value to see if it is previously saved value + config = Config.instance() + config.load("testconfig.yaml") + saved_value = config.get(path) + self.assertEqual(saved_value, 'newval', "New value was not saved and returned.") + + # Restore and save file + config.set(path, orig_value) + config.save() + + +if __name__ == '__main__': + unittest.main() diff --git a/test/testconfig.yaml b/test/testconfig.yaml new file mode 100644 index 0000000..44a7a7c --- /dev/null +++ b/test/testconfig.yaml @@ -0,0 +1,20 @@ +--- +test1: + test1_1: + val1_1_1: val1_1_1 + val1_1_2: val1_1_2 + val1_1_3: val1_1_3 + test1_2: + val1_2_1: val1_2_1 + val1_2_2: val1_2_2 + val1_2_3: val1_2_3 +test2: + test2_1: + val2_1_1: val1_1_1 + val2_1_2: val1_1_2 + val2_1_3: val1_1_3 + test2_2: + val2_2_1: val1_2_1 + val2_2_2: val1_2_2 + val2_2_3: val1_2_3 +... \ No newline at end of file