diff --git a/.gitignore b/.gitignore
index 08e827a..3dc1979 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,7 @@
/venv/
-/log/
-/out/
/.idea/workspace.xml
+/gui/*.bak
+/gui/*.py
+/gui/#~wxg.autosave~app_design.wxg#
+/*.log
+/*.log.?
diff --git a/.idea/mt5-correlation.iml b/.idea/mt5-correlation.iml
index 211e5e7..cdf40ae 100644
--- a/.idea/mt5-correlation.iml
+++ b/.idea/mt5-correlation.iml
@@ -10,4 +10,7 @@
+
+
+
\ No newline at end of file
diff --git a/README.md b/README.md
index e21a33b..80928f5 100644
--- a/README.md
+++ b/README.md
@@ -5,14 +5,16 @@ Calculates correlation coefficient between all symbols in MetaTrader5 Market Wat
1) Set up your MetaTrader 5 environment ensuring that all symbols that you would like to assess for correlation are shown in your Market Watch window;
2) Set up your python environment; and
3) Install the required libraries.
+
```
-pip install -r mt5-correlation\requirements.txt
+pip install -r mt5-correlation/requirements.txt
```
# Usage
If you set up a virtual environment in the Setup step, ensure this is activated. Then run the script.
+
```
-python -m mt5_correlations\mt5_correlations.py
+python -m mt5_correlations/get_correlations.py
```
A .csv file containing the correlation coefficient for all combinations of sybmols from the MetaTrader market watch will be produced in the current directory.
@@ -29,7 +31,7 @@ A .csv file containing the correlation coefficient for all combinations of sybmo
|EU50Cash |FRA40Cash |0.99072 |2021-01-29 11:54:29|2021-02-05 11:54:29|15 |
# Customising
-Edit mt5_correlations.py to customise.
+Edit get_correlations.py to customise.
The coefficients are calculated only if:
* The smallest set of price data is no less than 90% of the size of the largest set;
diff --git a/config.yaml b/config.yaml
new file mode 100644
index 0000000..ba10243
--- /dev/null
+++ b/config.yaml
@@ -0,0 +1,19 @@
+---
+calculate:
+ from:
+ days: 7
+ timeframe: 15
+ min_prices: 400
+ max_set_size_diff_pct: 90
+ overlap_pct: 90
+ max_p_value: 0.05
+monitor:
+ from:
+ minutes: 10
+ interval: 10
+ min_prices: 400
+ max_set_size_diff_pct: 50
+ overlap_pct: 50
+ max_p_value: 0.05
+ divergence_threshold: 0.8
+...
\ No newline at end of file
diff --git a/logging_conf.yaml b/logging_conf.yaml
index 176be17..b314cd7 100644
--- a/logging_conf.yaml
+++ b/logging_conf.yaml
@@ -13,17 +13,25 @@ formatters:
handlers:
console:
- level: WARNING
+ level: INFO
class: logging.StreamHandler
formatter: brief
stream: ext://sys.stdout
+ file:
+ level: DEBUG
+ class: logging.handlers.RotatingFileHandler
+ formatter: precice
+ filename: debug.log
+ mode: a
+ maxBytes: 2560000
+ backupCount: 1
root:
level: DEBUG
- handlers: [console]
+ handlers: [console, file]
loggers:
mt5-correlation:
level: DEBUG
- handlers: [console]
+ handlers: [console, file]
propagate: 0
\ No newline at end of file
diff --git a/mt5_correlation.py b/mt5_correlation.py
index 1c5ebb9..2cdff55 100644
--- a/mt5_correlation.py
+++ b/mt5_correlation.py
@@ -1,79 +1,25 @@
-from datetime import datetime, timedelta
-import logging.config
-import pandas as pd
-import pytz
-import yaml
-from mt5_correlation.mt5 import MT5
-from mt5_correlation.correlation import Correlation
+"""
+Application to monitor previously correlated symbol pairs for correlation divergence.
+"""
import definitions
+import yaml
+import logging.config
+from mt5_correlation.gui import MonitorFrame
+from mt5_correlation.config import Config
+import wx
-# Configure logger
-with open(fr'{definitions.ROOT_DIR}\logging_conf.yaml', 'rt') as file:
- config = yaml.safe_load(file.read())
- logging.config.dictConfig(config)
- log = logging.getLogger()
+if __name__ == "__main__":
+ # Configure the logger
+ with open(fr'{definitions.ROOT_DIR}\logging_conf.yaml', 'rt') as file:
+ config = yaml.safe_load(file.read())
+ logging.config.dictConfig(config)
-# Create mt5 class. This contains required methods for interacting with MT5.
-mt5 = MT5()
+ # Load the config
+ config = Config.instance()
+ config.load(fr"{definitions.ROOT_DIR}\config.yaml")
-# Gte all visible symbols
-symbols = mt5.get_symbols()
-
-# set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
-timezone = pytz.timezone("Etc/UTC")
-utc_to = datetime.now(tz=timezone)
-utc_from = utc_to - timedelta(days=7)
-
-# Set timeframe
-timeframe = mt5.TIMEFRAME_M15
-
-# Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
-price_data = {}
-for symbol in symbols:
- price_data[symbol.name] = mt5.get_prices(symbol=symbol, from_date=utc_from, to_date=utc_to,
- timeframe=timeframe)
-
-# Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
-# eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
-# avoid duplicating. We will store all coefficients in a dataframe for export as CSV.
-columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe']
-coefficients = pd.DataFrame(columns=columns)
-
-index = 0
-# There will be (x^2 - x) / 2 pairs where x is number of symbols
-num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
-
-for i in range(0, len(symbols)):
- symbol1 = symbols[i]
-
- for j in range(i + 1, len(symbols)):
- symbol2 = symbols[j]
- index += 1
-
- # Get price data for both symbols
- symbol1_price_data = price_data[symbol1.name]
- symbol2_price_data = price_data[symbol2.name]
-
- # Get coefficient and store if valid
- coefficient = Correlation.calculate_coefficient(symbol1_prices=symbol1_price_data,
- symbol2_prices=symbol2_price_data, max_set_size_diff_pct=90,
- overlap_pct=90, max_p_value=0.05)
-
- if coefficient is not None:
- coefficients = coefficients.append({'Symbol 1': symbol1.name, 'Symbol 2': symbol2.name,
- 'Coefficient': coefficient, 'UTC Date From': utc_from,
- 'UTC Date To': utc_to, 'Timeframe': timeframe}, ignore_index=True)
-
- log.info(f"Pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} has a coefficient of "
- f"{coefficient}.")
- else:
- log.info(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1.name}:{symbol2.name} could not"
- f" be calculated.")
-
-# Sort, highest correlated first
-coefficients = coefficients.sort_values('Coefficient', ascending=False)
-
-# Save as CSV
-filename = f"Coefficients from {utc_from:%Y%m%d %H%M%S} to {utc_to:%Y%m%d %H%M%S}.csv"
-log.info(f"Saving coefficients as '{filename}'.")
-coefficients.to_csv(filename, index=False)
+ # Start the app
+ app = wx.App(False)
+ frame = MonitorFrame(None, wx.ID_ANY, "")
+ frame.Show()
+ app.MainLoop()
diff --git a/mt5_correlation/config.py b/mt5_correlation/config.py
new file mode 100644
index 0000000..320f4d6
--- /dev/null
+++ b/mt5_correlation/config.py
@@ -0,0 +1,84 @@
+import yaml
+import definitions
+
+
+class Config(object):
+ """
+ Provides access to application configuration parameters stored in config.yaml.
+ """
+
+ _config = None
+ _path = None
+ _instance = None
+
+ def __init__(self):
+ """
+ Singleton. Raise runtime error
+ """
+ raise RuntimeError('Call instance() instead')
+
+ @classmethod
+ def instance(cls):
+ """
+ Singleton. Get instance of this class. Create if not already created.
+ :return:
+ """
+ if cls._instance is None:
+ cls._instance = cls.__new__(cls)
+ return cls._instance
+
+ def load(self, path):
+ """
+ Loads the applications config file
+ :param path: Path to config file
+ :return:
+ """
+ with open(path, 'r') as yamlfile:
+ self._config = yaml.safe_load(yamlfile)
+
+ # Store path so that we can save later
+ self._path = path
+
+ def save(self):
+ """
+ Saves config file
+ :return:
+ """
+
+ with open(self._path, 'w') as file:
+ file.write("---\n")
+ yaml.dump(self._config, file, sort_keys=False)
+ file.write("...")
+
+ def get(self, path):
+ """
+ Gets a config property value.
+ :param path: path to property. Path separated by .
+ :return: property value
+ """
+
+ elements = path.split('.')
+ last = None
+
+ for element in elements:
+ if last is None:
+ last = self._config[element]
+ else:
+ last = last[element]
+
+ return last
+
+ def set(self, path, value):
+ """
+ Sets a config property value
+ :param path: path to property. Path separated by .
+ :param value: Value to set property to
+ :return:
+ """
+ obj = self._config
+ key_list = path.split(".")
+
+ for k in key_list[:-1]:
+ obj = obj[k]
+
+ obj[key_list[-1]] = value
diff --git a/mt5_correlation/correlation.py b/mt5_correlation/correlation.py
index c58a70f..595e3ea 100644
--- a/mt5_correlation/correlation.py
+++ b/mt5_correlation/correlation.py
@@ -1,26 +1,305 @@
import math
+import logging
+import pandas as pd
+from datetime import datetime
+import time
+import sched
+import threading
+import pytz
from scipy.stats.stats import pearsonr
+from mt5_correlation.mt5 import MT5
+
class Correlation:
"""
- A class to calculate the correlation coefficient between two sets of price data
+ A class to maintain the state of the calculated correlation coefficients.
"""
- @staticmethod
- def calculate_coefficient(symbol1_prices, symbol2_prices, max_set_size_diff_pct=90, overlap_pct=90,
- max_p_value=0.05):
- """
- Calculates the correlation coefficient between two sets of price data. Uses close price.
+ min_coefficient = 0.9
+ monitoring = False # Monitoring cor correlations
- :param symbol1_prices:
- :param symbol2_prices:
+ def __init__(self):
+ self.log = logging.getLogger(__name__)
+
+ # Create dataframe
+ columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 'Timeframe',
+ 'Last Check', 'Last Coefficient']
+ self.coefficient_data = pd.DataFrame(columns=columns)
+
+ # Create timer for continuous monitoring
+ self.scheduler = sched.scheduler(time.time, time.sleep)
+
+ def load(self, filename):
+ """
+ Loads a csv file containing calculated coefficients
+ :param filename:
+ :return:
+ """
+ self.coefficient_data = pd.read_csv(filename)
+
+ def save(self, filename):
+ """
+ Saves the calculated coefficients as a csv file
+ :param filename:
+ :return:
+ """
+ self.coefficient_data.to_csv(filename, index=False)
+
+ def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+ max_p_value=0.05):
+ """
+ Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch. Updates coefficient data.
+
+ :param date_from: From date for price data from which to calculate correlation coefficients
+ :param date_to: To date for price data from which to calculate correlation coefficients
+ :param timeframe: Timeframe for price data from which to calculate correlation coefficients
+ :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+ is not met then returned coefficient will be None
+ :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+ within this pct of each other
+ :param overlap_pct:
+ :param max_p_value: The maximum p value for the correlation to be meaningful
+
+ :return:
+ """
+
+ # Create mt5 class. This contains required methods for interacting with MT5.
+ mt5 = MT5()
+
+ # Gte all visible symbols
+ symbols = mt5.get_symbols()
+
+ # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict.
+ price_data = {}
+ for symbol in symbols:
+ price_data[symbol] = mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to,
+ timeframe=timeframe)
+
+ # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs
+ # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to
+ # avoid duplicating. We will store all coefficients in a dataframe for export as CSV.
+ index = 0
+ # There will be (x^2 - x) / 2 pairs where x is number of symbols
+ num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2)
+
+ for i in range(0, len(symbols)):
+ symbol1 = symbols[i]
+
+ for j in range(i + 1, len(symbols)):
+ symbol2 = symbols[j]
+ index += 1
+
+ # Get price data for both symbols
+ symbol1_price_data = price_data[symbol1]
+ symbol2_price_data = price_data[symbol2]
+
+ # Get coefficient and store if valid
+ coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data,
+ symbol2_prices=symbol2_price_data,
+ min_prices=min_prices,
+ max_set_size_diff_pct=max_set_size_diff_pct,
+ overlap_pct=overlap_pct, max_p_value=max_p_value)
+
+ if coefficient is not None:
+
+ self.coefficient_data = \
+ self.coefficient_data.append({'Symbol 1': symbol1, 'Symbol 2': symbol2,
+ 'Base Coefficient': coefficient, 'UTC Date From': date_from,
+ 'UTC Date To': date_to, 'Timeframe': timeframe},
+ ignore_index=True)
+ self.log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a "
+ f"coefficient of {coefficient}.")
+ else:
+ self.log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:"
+ f"{symbol2} could no be calculated.")
+
+ # Sort, highest correlated first
+ self.coefficient_data = self.coefficient_data.sort_values('Base Coefficient', ascending=False)
+
+ def update_coefficient(self, symbol1, symbol2, date_from, date_to, min_prices=100, max_set_size_diff_pct=90,
+ overlap_pct=90, max_p_value=0.05):
+ """
+ Updates the coefficient for the specified symbol pair
+ :param symbol1: Name of symbol to calculate coefficient for.
+ :param symbol2: Name of symbol to calculate coefficient for.
+ :param date_from: From date for tick data from which to calculate correlation coefficients
+ :param date_to: To date for tick data from which to calculate correlation coefficients
+ :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+ is not met then returned coefficient will be None
:param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
within this pct of each other
:param overlap_pct:
:param max_p_value: The maximum p value for the correlation to be meaningful
:return: correlation coefficient, or None if coefficient could not be calculated.
"""
+
+ # Get the tick data
+ mt5 = MT5()
+ symbol1ticks = mt5.get_ticks(symbol=symbol1, from_date=date_from, to_date=date_to)
+ symbol2ticks = mt5.get_ticks(symbol=symbol2, from_date=date_from, to_date=date_to)
+
+ # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than one
+ # tick per second and ensuring that times can match. We will need to set the index to time for the resample
+ # then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price
+ if symbol1ticks is not None and symbol2ticks is not None and \
+ len(symbol1ticks.index) > 0 and len(symbol2ticks.index) > 0:
+ symbol1ticks.set_index('time', inplace=True)
+ symbol2ticks.set_index('time', inplace=True)
+ symbol1prices = symbol1ticks['ask'].resample('1S').ohlc()
+ symbol2prices = symbol2ticks['ask'].resample('1S').ohlc()
+ symbol1prices.reset_index(inplace=True)
+ symbol2prices.reset_index(inplace=True)
+ symbol1prices = symbol1prices[symbol1prices['close'].notna()]
+ symbol2prices = symbol2prices[symbol2prices['close'].notna()]
+
+ # Calculate the coefficient
+ coefficient = self.calculate_coefficient(symbol1_prices=symbol1prices, symbol2_prices=symbol2prices,
+ min_prices=min_prices,
+ max_set_size_diff_pct=max_set_size_diff_pct,
+ overlap_pct=overlap_pct, max_p_value=max_p_value)
+ else:
+ coefficient = None
+
+ # Find the correct row in the coefficient data and update with calculation date and calculated coefficient
+ timezone = pytz.timezone("Etc/UTC")
+ now = datetime.now(tz=timezone)
+
+ # Update data if we have a coefficient
+ if coefficient is not None:
+ self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
+ (self.coefficient_data['Symbol 2'] == symbol2),
+ 'Last Check'] = now
+
+ self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) &
+ (self.coefficient_data['Symbol 2'] == symbol2),
+ 'Last Coefficient'] = coefficient
+
+ return coefficient
+
+ def update_all_coefficients(self, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+ max_p_value=0.05):
+ """
+ Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet
+ the threshold can be accessed through the filtered_coefficient_data property.
+
+ :param date_from: From date for tick data from which to calculate correlation coefficients
+ :param date_to: To date for tick data from which to calculate correlation coefficients
+ :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+ is not met then returned coefficient will be None
+ :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+ within this pct of each other
+ :param overlap_pct:
+ :param max_p_value: The maximum p value for the correlation to be meaningful
+ :return: correlation coefficient, or None if coefficient could not be calculated.
+ """
+ # Update latest coefficient for every pair
+ for index, row in self.filtered_coefficient_data.iterrows():
+ symbol1 = row['Symbol 1']
+ symbol2 = row['Symbol 2']
+ self.update_coefficient(symbol1=symbol1, symbol2=symbol2, date_from=date_from, date_to=date_to,
+ min_prices=min_prices, max_set_size_diff_pct=max_set_size_diff_pct,
+ overlap_pct=overlap_pct, max_p_value=max_p_value)
+
+ def start_monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+ max_p_value=0.05):
+ """
+ Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient
+ threshold.
+
+ :param interval: How often to check in seconds
+ :param date_from: From date for tick data from which to calculate correlation coefficients
+ :param date_to: To date for tick data from which to calculate correlation coefficients
+ :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+ is not met then returned coefficient will be None
+ :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+ within this pct of each other
+ :param overlap_pct:
+ :param max_p_value: The maximum p value for the correlation to be meaningful
+ :return: correlation coefficient, or None if coefficient could not be calculated.
+
+ :return:
+ """
+ self.log.debug(f"Starting monitor.")
+ self.monitoring = True
+
+ # Create thread to run monitoring This will call private __monitor method that will run the calculation and
+ # keep scheduling itself while self.monitoring is True
+ params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
+ 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
+ 'max_p_value': max_p_value}
+ thread = threading.Thread(target=self.__monitor, kwargs=params)
+ thread.start()
+
+ def stop_monitor(self):
+ """
+ Stops monitoring symbol pairs for correlation.
+ :return:
+ """
+ self.log.debug(f"Stopping monitor.")
+ self.monitoring = False
+
+ def __monitor(self, interval, date_from, date_to, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90,
+ max_p_value=0.05):
+ """
+ The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and
+ stop_monitoring.
+
+ :param interval: How often to check in seconds
+ :param date_from: From date for tick data from which to calculate correlation coefficients
+ :param date_to: To date for tick data from which to calculate correlation coefficients
+ :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+ is not met then returned coefficient will be None
+ :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+ within this pct of each other
+ :param overlap_pct:
+ :param max_p_value: The maximum p value for the correlation to be meaningful
+ :return: correlation coefficient, or None if coefficient could not be calculated.
+ :return:
+ """
+ self.log.debug(f"In monitor event. Monitoring: {self.monitoring}.")
+
+ # Only run if monitor is not stopped
+ if self.monitoring:
+ # Update all coefficients
+ self.update_all_coefficients(date_from=date_from, date_to=date_to, min_prices=min_prices,
+ max_set_size_diff_pct=max_set_size_diff_pct, overlap_pct=overlap_pct,
+ max_p_value=max_p_value)
+
+ # Schedule the timer to run again
+ params = {'interval': interval, 'date_from': date_from, 'date_to': date_to, 'min_prices': min_prices,
+ 'max_set_size_diff_pct': max_set_size_diff_pct, 'overlap_pct': overlap_pct,
+ 'max_p_value': max_p_value}
+ self.scheduler.enter(delay=interval, priority=1, action=self.__monitor, kwargs=params)
+ self.scheduler.run()
+
+ @property
+ def filtered_coefficient_data(self):
+ """
+ :return: Coefficient data filtered so that all base coefficients >= min coefficient
+ """
+ if self.coefficient_data is not None:
+ return self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= self.min_coefficient]
+ else:
+ return None
+
+ @staticmethod
+ def calculate_coefficient(symbol1_prices, symbol2_prices, min_prices=100, max_set_size_diff_pct=90,
+ overlap_pct=90, max_p_value=0.05):
+ """
+ Calculates the correlation coefficient between two sets of price data. Uses close price.
+
+ :param symbol1_prices: prices or ticks for symbol 1
+ :param symbol2_prices: prices or ticks for symbol 2
+ :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold
+ is not met then returned coefficient will be None
+ :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are
+ within this pct of each other
+ :param overlap_pct:
+ :param max_p_value: The maximum p value for the correlation to be meaningful
+ :return: correlation coefficient, or None if coefficient could not be calculated.
+ """
+
# Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for
# correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of
# the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set?
@@ -31,7 +310,8 @@ class Correlation:
len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)]))
similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set
enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100)
- suitable = similar_size and enough_overlap
+ enough_prices = len_smallest_set >= min_prices
+ suitable = similar_size and enough_overlap and enough_prices
if suitable:
# Calculate coefficient on close prices
@@ -50,6 +330,3 @@ class Correlation:
coefficient = None
return coefficient
-
-
-
diff --git a/mt5_correlation/gui.py b/mt5_correlation/gui.py
new file mode 100644
index 0000000..72d6649
--- /dev/null
+++ b/mt5_correlation/gui.py
@@ -0,0 +1,343 @@
+import wx
+import wx.grid
+import wx.lib.masked as masked
+from mt5_correlation.correlation import Correlation
+from mt5_correlation.config import Config
+from datetime import datetime, timedelta
+import pytz
+import pandas as pd
+import logging
+import definitions
+
+
+class MonitorFrame(wx.Frame):
+
+ cor = None
+ rows = 0 # Need to track as we need to notify grid if row count changes.
+ opened_filename = None # So we can save to same file as we opened
+ config = None # The applications config
+
+ COLUMN_INDEX = 0
+ COLUMN_SYMBOL1 = 1
+ COLUMN_SYMBOL2 = 2
+ COLUMN_BASE_COEFFICIENT = 3
+ COLUMN_DATE_FROM = 4
+ COLUMN_DATE_TO = 5
+ COLUMN_TIMEFRAME = 6
+ COLUMN_LAST_CHECK = 7
+ COLUMN_LAST_COEFFICIENT = 8
+
+ def __init__(self, *args, **kwds):
+ self.log = logging.getLogger(__name__)
+ self.config = Config.instance()
+
+ # Create correlation instance to maintain state of calculated coefficients
+ self.cor = Correlation()
+
+ kwds["style"] = kwds.get("style", 0) | wx.DEFAULT_FRAME_STYLE
+ wx.Frame.__init__(self, *args, **kwds)
+ self.SetSize((1235, 800))
+ self.SetTitle("Monitor for Divergence")
+
+ # Status bar
+ self.statusbar = self.CreateStatusBar(1)
+
+ # Menu Bar
+ self.menubar = wx.MenuBar()
+ file_menu = wx.Menu()
+
+ # Open and save
+ menu_item_open = file_menu.Append(wx.ID_ANY, "Open", "Open correlations file.")
+ menu_item_save = file_menu.Append(wx.ID_ANY, "Save", "Save correlations file.")
+ menu_item_saveas = file_menu.Append(wx.ID_ANY, "Save As", "Save correlations file.")
+
+ # Calculate
+ file_menu.AppendSeparator()
+ menu_item_calculate = file_menu.Append(wx.ID_ANY, "Calculate", "Calculate base coefficients.")
+
+ # Close
+ file_menu.AppendSeparator()
+ menu_item_exit = file_menu.Append(wx.ID_ANY, "Exit", "Close the application")
+
+ # Add file menu and set menu bar
+ self.menubar.Append(file_menu, "File")
+ self.SetMenuBar(self.menubar)
+
+ # Main window
+ self.window = wx.SplitterWindow(self, wx.ID_ANY)
+ self.window.SetMinimumPaneSize(20)
+
+ self.correlations_pane = wx.Panel(self.window, wx.ID_ANY)
+
+ sizer_grid = wx.BoxSizer(wx.VERTICAL)
+
+ # Filter label, input box and button
+ sizer_coefficient = wx.BoxSizer(wx.HORIZONTAL)
+ sizer_grid.Add(sizer_coefficient, 0, 0, 0)
+
+ label_min_coefficient = wx.StaticText(self.correlations_pane, wx.ID_ANY, "Min Coefficient (Range -1 - 1)")
+ sizer_coefficient.Add(label_min_coefficient, 0, wx.ALL, 0)
+
+ self.edit_ctrl_min_coefficient = masked.NumCtrl(self.correlations_pane, value=self.cor.min_coefficient,
+ allowNegative=True, min=-1, max=1, integerWidth=1,
+ fractionWidth=5)
+ sizer_coefficient.Add(self.edit_ctrl_min_coefficient, 0, wx.ALL, 0)
+
+ self.filter_button = wx.Button(self.correlations_pane, wx.ID_ANY, label="Filter")
+ sizer_coefficient.Add(self.filter_button, 0, wx.ALL, 0)
+
+ self.monitor_toggle = wx.ToggleButton(self.correlations_pane, wx.ID_ANY, label="Monitoring")
+ sizer_coefficient.Add(self.monitor_toggle, 0, wx.ALL, 0)
+
+ # Data table using pandas dataframe for underlying data
+ self.table = DataTable(self.cor.filtered_coefficient_data)
+ self.grid_correlations = wx.grid.Grid(self.correlations_pane, wx.ID_ANY, size=(1, 1))
+ self.grid_correlations.SetTable(self.table, takeOwnership=True)
+ self.grid_correlations.EnableEditing(0)
+ self.grid_correlations.EnableDragRowSize(0)
+ self.grid_correlations.EnableDragGridSize(0)
+ self.grid_correlations.SetSelectionMode(wx.grid.Grid.SelectRows)
+ self.grid_correlations.SetColSize(self.COLUMN_INDEX, 0) # Index. Hide
+ self.grid_correlations.SetColSize(self.COLUMN_SYMBOL1, 100) # Symbol 1
+ self.grid_correlations.SetColSize(self.COLUMN_SYMBOL2, 100) # Symbol 2
+ self.grid_correlations.SetColSize(self.COLUMN_BASE_COEFFICIENT, 100) # Base Coefficient
+ self.grid_correlations.SetColSize(self.COLUMN_DATE_FROM, 0) # UTC Date From. Hide
+ self.grid_correlations.SetColSize(self.COLUMN_DATE_TO, 0) # UTC Date To. Hide
+ self.grid_correlations.SetColSize(self.COLUMN_TIMEFRAME, 0) # Timeframe. Hide.
+ self.grid_correlations.SetColSize(self.COLUMN_LAST_CHECK, 100) # Last Check
+ self.grid_correlations.SetColSize(self.COLUMN_LAST_COEFFICIENT, 100) # Last Coefficient
+ sizer_grid.Add(self.grid_correlations, 1, wx.ALL | wx.EXPAND, 0)
+
+ self.charts_pane = wx.Panel(self.window, wx.ID_ANY)
+
+ # Charts
+ sizer_chart = wx.BoxSizer(wx.VERTICAL)
+
+ label_2 = wx.StaticText(self.charts_pane, wx.ID_ANY, "Charts Go Here", style=wx.ALIGN_CENTER_HORIZONTAL)
+ sizer_chart.Add(label_2, 0, wx.ALIGN_CENTER_HORIZONTAL, 0)
+
+ # Add the sizers to the panes
+ self.charts_pane.SetSizer(sizer_chart)
+ self.correlations_pane.SetSizer(sizer_grid)
+
+ # Set window split to 2 panes and layout
+ self.window.SplitVertically(self.correlations_pane, self.charts_pane)
+ self.Layout()
+
+ # Set up timer to refresh grid
+ self.timer = wx.Timer(self)
+
+ # Bind my buttons, timer, menu
+ self.filter_button.Bind(wx.EVT_BUTTON, self.change_min_coefficient)
+ self.monitor_toggle.Bind(wx.EVT_TOGGLEBUTTON, self.monitor)
+ self.Bind(wx.EVT_TIMER, self.refresh_grid, self.timer)
+ self.Bind(wx.EVT_MENU, self.open_file, menu_item_open)
+ self.Bind(wx.EVT_MENU, self.save_file, menu_item_save)
+ self.Bind(wx.EVT_MENU, self.save_file_as, menu_item_saveas)
+ self.Bind(wx.EVT_MENU, self.calculate_coefficients, menu_item_calculate)
+ self.Bind(wx.EVT_MENU, self.quit, menu_item_exit)
+
+ # Bind window close event
+ self.Bind(wx.EVT_CLOSE, self.on_close, self)
+
+ def open_file(self, event):
+ with wx.FileDialog(self, "Open Coefficients file", wildcard="CSV (*.csv)|*.csv",
+ style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog:
+
+ if fileDialog.ShowModal() == wx.ID_CANCEL:
+ return # the user changed their mind
+
+ # Load the file chosen by the user
+ self.opened_filename = fileDialog.GetPath()
+ self.cor.load(self.opened_filename)
+
+ # Refresh data in grid
+ self.refresh_grid(event)
+
+ self.SetStatusText(f"File {self.opened_filename} loaded.")
+
+ def save_file(self, event):
+ self.cor.save(self.opened_filename)
+ self.SetStatusText(f"File saved as {self.opened_filename}")
+
+ def save_file_as(self, event):
+ with wx.FileDialog(self, "Save Coefficients file", wildcard="CSV (*.csv)|*.csv",
+ style=wx.FD_SAVE) as fileDialog:
+ if fileDialog.ShowModal() == wx.ID_CANCEL:
+ return # the user changed their mind
+
+ # Save the file, changing opened filename so next save writes to new file
+ self.opened_filename = fileDialog.GetPath()
+ self.cor.save(self.opened_filename)
+
+ self.SetStatusText(f"File saved as {self.opened_filename}")
+
+ def calculate_coefficients(self, event):
+ # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today)
+ timezone = pytz.timezone("Etc/UTC")
+ utc_to = datetime.now(tz=timezone)
+ utc_from = utc_to - timedelta(days=self.config.get('calculate.from.days'))
+
+ # Set timeframe
+ timeframe = self.config.get('calculate.timeframe')
+
+ # Calculate
+ self.SetStatusText("Calculating coefficients.")
+ self.cor.calculate(date_from=utc_from, date_to=utc_to,
+ timeframe=self.config.get('calculate.timeframe'),
+ min_prices=self.config.get('calculate.min_prices'),
+ max_set_size_diff_pct=self.config.get('calculate.max_set_size_diff_pct'),
+ overlap_pct=self.config.get('calculate.overlap_pct'),
+ max_p_value=self.config.get('calculate.max_p_value'))
+ self.SetStatusText("")
+
+ # Show calculated data
+ self.refresh_grid(event)
+
+ def quit(self, event):
+ self.Close()
+
+ def change_min_coefficient(self, event):
+ self.cor.min_coefficient = self.edit_ctrl_min_coefficient.GetValue()
+ self.refresh_grid(event)
+
+ def refresh_grid(self, event):
+ """
+ Refreshes grid. Notifies if rows have been added or deleted.
+ :return:
+ """
+ self.log.debug(f"Refreshing grid. Timer running: {self.timer.IsRunning()}")
+
+ # Update data
+ self.table.data = self.cor.coefficient_data.copy()
+
+ # Format
+ self.table.data['Base Coefficient'] = self.table.data['Base Coefficient'].map('{:.5f}'.format)
+ self.table.data['Last Check'] = pd.to_datetime(self.table.data['Last Check'], utc=True)
+ self.table.data['Last Check'] = self.table.data['Last Check'].dt.strftime('%d-%m-%y %H:%M:%S')
+ self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].map('{:.5f}'.format)
+
+ # Remove nans. The ones from the float column wil be str nan as they have been formatted
+ self.table.data = self.table.data.fillna('')
+ self.table.data['Last Coefficient'] = self.table.data['Last Coefficient'].replace('nan', '')
+
+ # Start refresh
+ self.grid_correlations.BeginBatch()
+
+ # Check if num rows in dataframe has changed, and send appropriate APPEND or DELETE messages
+ cur_rows = len(self.cor.filtered_coefficient_data.index)
+ if cur_rows < self.rows:
+ # Data has been deleted. Send message
+ msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_DELETED,
+ self.rows - cur_rows, self.rows - cur_rows)
+ self.grid_correlations.ProcessTableMessage(msg)
+ elif cur_rows > self.rows:
+ # Data has been added. Send message
+ msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_NOTIFY_ROWS_APPENDED,
+ cur_rows - self.rows) # how many
+ self.grid_correlations.ProcessTableMessage(msg)
+
+ self.grid_correlations.EndBatch()
+
+ # Send updated message
+ msg = wx.grid.GridTableMessage(self.table, wx.grid.GRIDTABLE_REQUEST_VIEW_GET_VALUES)
+ self.grid_correlations.ProcessTableMessage(msg)
+
+ # Update row count
+ self.rows = cur_rows
+
+ def monitor(self, event):
+ # Check state of toggle button. If on, then start monitoring, else stop
+ if self.monitor_toggle.GetValue():
+ self.log.debug("Starting monitoring.")
+ self.SetStatusText("Monitoring for changes to coefficients.")
+
+ # Calculate correlations fro last 10 mins
+ timezone = pytz.timezone("Etc/UTC")
+ utc_to = datetime.now(tz=timezone)
+ utc_from = utc_to - timedelta(minutes=self.config.get('monitor.from.minutes'))
+
+ self.timer.Start(10000)
+ self.cor.start_monitor(interval=self.config.get('monitor.interval'), date_from=utc_from, date_to=utc_to,
+ min_prices=self.config.get('monitor.min_prices'),
+ max_set_size_diff_pct=self.config.get('monitor.max_set_size_diff_pct'),
+ overlap_pct=self.config.get('monitor.overlap_pct'),
+ max_p_value=self.config.get('monitor.max_p_value'))
+ else:
+ self.log.debug("Stopping monitoring.")
+ self.SetStatusText("Monitoring stopped.")
+ self.timer.Stop()
+ self.cor.stop_monitor()
+
+ def on_close(self, event):
+ """
+ Window closing. Save coefficients and stop monitoring.
+ :param event:
+ :return:
+ """
+ if self.opened_filename is not None:
+ self.cor.save(self.opened_filename)
+
+ self.cor.stop_monitor()
+
+ event.Skip()
+
+
+class DataTable(wx.grid.GridTableBase):
+ """
+ A data table that holds data in a pandas dataframe
+ """
+ def __init__(self, data=None):
+ wx.grid.GridTableBase.__init__(self)
+ self.headerRows = 1
+ if data is None:
+ data = pd.DataFrame()
+ self.data = data
+
+ # Get divergence threshold from app config
+ # Get application config
+ self.config = Config.instance()
+
+ # Get divergence threshold. This will be used by DataTable to highlight cells
+ self.divergence_threshold = self.config.get('monitor.divergence_threshold')
+
+ def GetNumberRows(self):
+ return len(self.data)
+
+ def GetNumberCols(self):
+ return len(self.data.columns) + 1
+
+ def GetValue(self, row, col):
+ if col == 0:
+ return self.data.index[row]
+ return self.data.iloc[row, col - 1]
+
+ def SetValue(self, row, col, value):
+ self.data.iloc[row, col - 1] = value
+
+ def GetColLabelValue(self, col):
+ if col == 0:
+ if self.data.index.name is None:
+ return 'Index'
+ else:
+ return self.data.index.name
+ return str(self.data.columns[col - 1])
+
+ def GetTypeName(self, row, col):
+ return wx.grid.GRID_VALUE_STRING
+
+ def GetAttr(self, row, col, prop):
+ attr = wx.grid.GridCellAttr()
+
+ # If column is last coefficient, get value and check against threshold. Highlight if diverged.
+ threshold = self.config.get('monitor.divergence_threshold')
+ if col == MonitorFrame.COLUMN_LAST_COEFFICIENT:
+ value = self.GetValue(row, col)
+ if value != "":
+ value = float(value)
+ if value <= threshold:
+ attr.SetBackgroundColour(wx.YELLOW)
+ else:
+ attr.SetBackgroundColour(wx.WHITE)
+
+ return attr
diff --git a/mt5_correlation/mt5.py b/mt5_correlation/mt5.py
index 8b53d26..184126f 100644
--- a/mt5_correlation/mt5.py
+++ b/mt5_correlation/mt5.py
@@ -1,5 +1,5 @@
import pandas as pd
-import MetaTrader5 as mt5
+import MetaTrader5
import logging
@@ -9,27 +9,27 @@ class MT5:
"""
# Timeframes
- TIMEFRAME_M1 = mt5.TIMEFRAME_M1
- TIMEFRAME_M2 = mt5.TIMEFRAME_M2
- TIMEFRAME_M3 = mt5.TIMEFRAME_M3
- TIMEFRAME_M4 = mt5.TIMEFRAME_M4
- TIMEFRAME_M5 = mt5.TIMEFRAME_M5
- TIMEFRAME_M6 = mt5.TIMEFRAME_M6
- TIMEFRAME_M10 = mt5.TIMEFRAME_M10
- TIMEFRAME_M12 = mt5.TIMEFRAME_M10
- TIMEFRAME_M15 = mt5.TIMEFRAME_M15
- TIMEFRAME_M20 = mt5.TIMEFRAME_M20
- TIMEFRAME_M30 = mt5.TIMEFRAME_M30
- TIMEFRAME_H1 = mt5.TIMEFRAME_H1
- TIMEFRAME_H2 = mt5.TIMEFRAME_H2
- TIMEFRAME_H3 = mt5.TIMEFRAME_H3
- TIMEFRAME_H4 = mt5.TIMEFRAME_H4
- TIMEFRAME_H6 = mt5.TIMEFRAME_H6
- TIMEFRAME_H8 = mt5.TIMEFRAME_H8
- TIMEFRAME_H12 = mt5.TIMEFRAME_H12
- TIMEFRAME_D1 = mt5.TIMEFRAME_D1
- TIMEFRAME_W1 = mt5.TIMEFRAME_W1
- TIMEFRAME_MN1 = mt5.TIMEFRAME_MN1
+ TIMEFRAME_M1 = MetaTrader5.TIMEFRAME_M1
+ TIMEFRAME_M2 = MetaTrader5.TIMEFRAME_M2
+ TIMEFRAME_M3 = MetaTrader5.TIMEFRAME_M3
+ TIMEFRAME_M4 = MetaTrader5.TIMEFRAME_M4
+ TIMEFRAME_M5 = MetaTrader5.TIMEFRAME_M5
+ TIMEFRAME_M6 = MetaTrader5.TIMEFRAME_M6
+ TIMEFRAME_M10 = MetaTrader5.TIMEFRAME_M10
+ TIMEFRAME_M12 = MetaTrader5.TIMEFRAME_M10
+ TIMEFRAME_M15 = MetaTrader5.TIMEFRAME_M15
+ TIMEFRAME_M20 = MetaTrader5.TIMEFRAME_M20
+ TIMEFRAME_M30 = MetaTrader5.TIMEFRAME_M30
+ TIMEFRAME_H1 = MetaTrader5.TIMEFRAME_H1
+ TIMEFRAME_H2 = MetaTrader5.TIMEFRAME_H2
+ TIMEFRAME_H3 = MetaTrader5.TIMEFRAME_H3
+ TIMEFRAME_H4 = MetaTrader5.TIMEFRAME_H4
+ TIMEFRAME_H6 = MetaTrader5.TIMEFRAME_H6
+ TIMEFRAME_H8 = MetaTrader5.TIMEFRAME_H8
+ TIMEFRAME_H12 = MetaTrader5.TIMEFRAME_H12
+ TIMEFRAME_D1 = MetaTrader5.TIMEFRAME_D1
+ TIMEFRAME_W1 = MetaTrader5.TIMEFRAME_W1
+ TIMEFRAME_MN1 = MetaTrader5.TIMEFRAME_MN1
def __init__(self):
# Connect to MetaTrader5. Opens if not already open.
@@ -38,43 +38,43 @@ class MT5:
self.log = logging.getLogger(__name__)
# Open MT5 and log error if it could not open
- if not mt5.initialize():
+ if not MetaTrader5.initialize():
self.log.error("initialize() failed")
- mt5.shutdown()
+ MetaTrader5.shutdown()
# Print connection status
- self.log.debug(mt5.terminal_info())
+ self.log.debug(MetaTrader5.terminal_info())
# Print data on MetaTrader 5 version
- self.log.debug(mt5.version())
+ self.log.debug(MetaTrader5.version())
def __del__(self):
# shut down connection to the MetaTrader 5 terminal
- mt5.shutdown()
+ MetaTrader5.shutdown()
def get_symbols(self):
"""
Gets list of symbols open in MT5 market watch.
- :return: list of symbols
+ :return: list of symbol names
"""
# Iterate symbols and get those in market watch.
- symbols = mt5.symbols_get()
+ symbols = MetaTrader5.symbols_get()
selected_symbols = []
for symbol in symbols:
if symbol.visible:
- selected_symbols.append(symbol)
+ selected_symbols.append(symbol.name)
# Log symbol counts
- total_symbols = mt5.symbols_total()
+ total_symbols = MetaTrader5.symbols_total()
num_selected_symbols = len(selected_symbols)
- self.log.info(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
+ self.log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.")
return selected_symbols
def get_prices(self, symbol, from_date, to_date, timeframe):
"""
Gets OHLC price data for the specified symbol.
- :param symbol: The MT5 symbol to get the price data for
+ :param symbol: The name of the symbol to get the price data for.
:param from_date: Date from when to retrieve data
:param to_date: Date where to receive data to
:param timeframe: The timeframe for the candes. Possible values are:
@@ -101,9 +101,10 @@ class MT5:
TIMEFRAME_MN1: 1 month
:return: Price data for symbol as dataframe
"""
+
# Get prices from MT5
- prices = mt5.copy_rates_range(symbol.name, timeframe, from_date, to_date)
- self.log.info(f"{len(prices)} prices retrieved for {symbol.name}.")
+ prices = MetaTrader5.copy_rates_range(symbol, timeframe, from_date, to_date)
+ self.log.debug(f"{len(prices)} prices retrieved for {symbol}.")
# Create dataframe from data and convert time in seconds to datetime format
prices_dataframe = pd.DataFrame(prices)
@@ -111,5 +112,28 @@ class MT5:
return prices_dataframe
+ def get_ticks(self, symbol, from_date, to_date):
+ """
+ Gets OHLC price data for the specified symbol.
+ :param symbol: The name of the symbol to get the price data for.
+ :param from_date: Date from when to retrieve data
+ :param to_date: Date where to receive data to
+ :return: Tick data for symbol as dataframe
+ """
+ # Get ticks from MT5
+ ticks = MetaTrader5.copy_ticks_range(symbol, from_date, to_date, MetaTrader5.COPY_TICKS_ALL)
+ # If ticks is None, there was an error
+ if ticks is None:
+ error = MetaTrader5.last_error()
+ self.log.error(f"Error retrieving ticks for {symbol}: {error}")
+ return None
+ else:
+ self.log.debug(f"{len(ticks)} ticks retrieved for {symbol}.")
+
+ # Create dataframe from data and convert time in seconds to datetime format
+ ticks_dataframe = pd.DataFrame(ticks)
+ ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s')
+
+ return ticks_dataframe
diff --git a/requirements.txt b/requirements.txt
index a561556..5eb187d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,4 +4,5 @@ MetaTrader5==5.0.34
pytz==2021.1
scipy==1.6.0
logging==0.4.9.6
-pyyaml==5.4.1
\ No newline at end of file
+pyyaml==5.4.1
+wxpython==4.1.1
\ No newline at end of file
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/test/test_config.py b/test/test_config.py
new file mode 100644
index 0000000..7c73c90
--- /dev/null
+++ b/test/test_config.py
@@ -0,0 +1,35 @@
+import unittest
+from mt5_correlation.config import Config
+
+
+class TestConfig(unittest.TestCase):
+ def test_load_and_get(self):
+ config = Config.instance()
+ config.load("testconfig.yaml")
+ val121 = config.get('test1.test1_2.val1_2_1')
+ self.assertEqual(val121, 'val1_2_1', "Get returned incorrect value.")
+
+ def test_set_and_save(self):
+ config = Config.instance()
+ config.load("testconfig.yaml")
+
+ path = 'test1.test1_2.val1_2_1'
+
+ # Save new value, storing orig so we can restore later
+ orig_value = config.get(path)
+ config.set(path, "newval")
+ config.save()
+
+ # Reopen config and get value to see if it is previously saved value
+ config = Config.instance()
+ config.load("testconfig.yaml")
+ saved_value = config.get(path)
+ self.assertEqual(saved_value, 'newval', "New value was not saved and returned.")
+
+ # Restore and save file
+ config.set(path, orig_value)
+ config.save()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/testconfig.yaml b/test/testconfig.yaml
new file mode 100644
index 0000000..44a7a7c
--- /dev/null
+++ b/test/testconfig.yaml
@@ -0,0 +1,20 @@
+---
+test1:
+ test1_1:
+ val1_1_1: val1_1_1
+ val1_1_2: val1_1_2
+ val1_1_3: val1_1_3
+ test1_2:
+ val1_2_1: val1_2_1
+ val1_2_2: val1_2_2
+ val1_2_3: val1_2_3
+test2:
+ test2_1:
+ val2_1_1: val1_1_1
+ val2_1_2: val1_1_2
+ val2_1_3: val1_1_3
+ test2_2:
+ val2_2_1: val1_2_1
+ val2_2_2: val1_2_2
+ val2_2_3: val1_2_3
+...
\ No newline at end of file