From 1222236ded328127c72ea3d13a58f8dfb18b8648 Mon Sep 17 00:00:00 2001 From: Matt Corcoran Date: Sat, 12 Jul 2025 15:53:31 +0200 Subject: [PATCH] added in MyLibs from private Mt5 repo --- My_MQL5_Libs/CalculatePositionData.mqh | 282 +++++++ My_MQL5_Libs/DrawdownControl.mqh | 209 ++++++ My_MQL5_Libs/MyFunctions.mqh | 216 ++++++ My_MQL5_Libs/OrderManagement.mqh | 560 ++++++++++++++ My_MQL5_Libs/TimeZones.mqh | 148 ++++ .../USDGBP_exchange_rate.txt | 1 + .../binance_api_stake_generator.py | 183 +++++ .../Live_trading_updates/process_data.py | 263 +++++++ .../Live_trading_updates/stake_amount.txt | 1 + Python_freqtrade/walkforward_optimisation.py | 699 ++++++++++++++++++ README.md | 55 ++ post_process_test.py | 223 ++++++ pre_process.py | 81 ++ process_entry_indicators.py | 436 +++++++++++ 14 files changed, 3357 insertions(+) create mode 100644 My_MQL5_Libs/CalculatePositionData.mqh create mode 100644 My_MQL5_Libs/DrawdownControl.mqh create mode 100644 My_MQL5_Libs/MyFunctions.mqh create mode 100644 My_MQL5_Libs/OrderManagement.mqh create mode 100644 My_MQL5_Libs/TimeZones.mqh create mode 100644 Python_freqtrade/Live_trading_updates/USDGBP_exchange_rate.txt create mode 100644 Python_freqtrade/Live_trading_updates/binance_api_stake_generator.py create mode 100644 Python_freqtrade/Live_trading_updates/process_data.py create mode 100644 Python_freqtrade/Live_trading_updates/stake_amount.txt create mode 100644 Python_freqtrade/walkforward_optimisation.py create mode 100644 README.md create mode 100644 post_process_test.py create mode 100644 pre_process.py create mode 100644 process_entry_indicators.py diff --git a/My_MQL5_Libs/CalculatePositionData.mqh b/My_MQL5_Libs/CalculatePositionData.mqh new file mode 100644 index 0000000..34a3325 --- /dev/null +++ b/My_MQL5_Libs/CalculatePositionData.mqh @@ -0,0 +1,282 @@ +//+------------------------------------------------------------------+ +//| CalculatePositionData.mqh | +//| xMattC | +//+------------------------------------------------------------------+ +#property library +#include +#include +#include + +class CalculatePositionData : public CObject{ + + protected: + CTrade trade; + TimeZones tz; + CPositionInfo position; + MyFunctions mf; + + bool check_lots(double &lots, string symbol); + bool normalise_price(double price, double &normalizedPrice, string symbol); + // double adjusted_point(string symbol); + + public: + + double calculate_stoploss(string symbol, double price, int order_side, string _sl_mode, double sl_var, ENUM_TIMEFRAMES atr_period); + double calculate_take_profit(string symbol, double price, double stoploss, int order_side, string mode_tp, double tp_var, ENUM_TIMEFRAMES atr_period); + double calculate_lots(string symbol, double sl_distance, double price, string mode_lot, double lot_var); + double calculate_trading_cost(string symbol, ulong position_ticket); + +}; + +double CalculatePositionData::calculate_stoploss(string symbol, double price, int order_side, string mode_sl, double sl_var, ENUM_TIMEFRAMES atr_period){ + // order_side int must be 1 for BUY or 2 for + + double sl=0; + + if(mode_sl=="NO_STOPLOSS"){ + sl=0; + } + + if(mode_sl=="SL_BREAKEVEN"){ + // https://www.youtube.com/watch?v=idPulZ3_iR0 + Alert("Not implemented yet yet"); + } + + if(mode_sl=="SL_FIXED_PIPS"){ + // pips/poins = https://www.mql5.com/en/forum/187757 + double adj_point = mf.adjusted_point(symbol); + + if(order_side == 1){ + sl = price - sl_var * adj_point; + if(!normalise_price(sl,sl,symbol)){return false;} + } + if(order_side == 2){ + sl = price + sl_var * adj_point; + if(!normalise_price(sl,sl,symbol)){return false;} + } + } + + if(mode_sl=="SL_FIXED_PERCENT"){ + if(order_side == 1){ + sl = (-1.0 * sl_var * price / 100.00) + price; + if(!normalise_price(sl,sl,symbol)){return false;} + } + if(order_side == 2){ + sl = sl_var * price / 100.00 + price; + if(!normalise_price(sl,sl,symbol)){return false;} + } + } + + if(mode_sl=="SL_ATR_MULTIPLE"){ + + int atr_handle = iATR(symbol,atr_period,14); + double atr[]; + ArraySetAsSeries(atr,true); + CopyBuffer(atr_handle,MAIN_LINE,1,1,atr); + + if(order_side == 1){ + sl = price - (atr[0] * sl_var); + if(!normalise_price(sl,sl,symbol)){return false;} + + } + if(order_side == 2){ + sl = price + (atr[0] * sl_var); + if(!normalise_price(sl,sl,symbol)){return false;} + } + } + + if(mode_sl=="SL_SPECIFIED_VALUE"){ + + double adj_point = mf.adjusted_point(symbol); + + if(order_side == 1){ + + double pip_50_sl = price - 10 * adj_point; + if(sl_var >= pip_50_sl){ + sl = pip_50_sl; + } + else sl = sl_var; + + if(!normalise_price(sl,sl,symbol)){return false;} + } + if(order_side == 2){ + double pip_50_sl = price + 10 * adj_point; + if(sl_var <= pip_50_sl){ + sl = pip_50_sl; + } + else sl = sl_var; + + sl = sl = sl_var; + if(!normalise_price(sl,sl,symbol)){return false;} + } + } + + return sl; +} + +double CalculatePositionData::calculate_take_profit(string symbol, double price, double stoploss, int order_side, string mode_tp, double _tp_var, ENUM_TIMEFRAMES atr_period){ + // order_side int must be 1 for BUY or 2 for SELL + + double tp=0; + + if(mode_tp=="NO_TAKE_PROFIT"){ + tp=0; + } + + if(mode_tp=="TP_FIXED_PIPS"){ + + double adj_point = mf.adjusted_point(symbol); + if(order_side == 1){ + tp = price + _tp_var * adj_point; + if(!normalise_price(tp,tp,symbol)){return false;} + } + if(order_side == 2){ + tp = price - _tp_var * adj_point; + if(!normalise_price(tp,tp,symbol)){return false;} + } + } + + if(mode_tp=="TP_FIXED_PERCENT"){ + if(order_side == 1){ + tp = _tp_var * price / 100.00 + price; + if(!normalise_price(tp,tp,symbol)){return false;} + } + if(order_side == 2){ + tp = (-1 * _tp_var * price / 100.00) + price; + if(!normalise_price(tp,tp,symbol)){return false;} + } + } + + if(mode_tp=="TP_ATR_MULTIPLE"){ + + int atr_handle = iATR(symbol,atr_period,14); + double atr[]; + ArraySetAsSeries(atr,true); + CopyBuffer(atr_handle,MAIN_LINE,1,1,atr); + + if(order_side == 1){ + tp = price + (atr[0] * _tp_var); + if(!normalise_price(tp,tp,symbol)){return false;} + } + if(order_side == 2){ + tp = price - (atr[0] * _tp_var); + if(!normalise_price(tp,tp,symbol)){return false;} + } + } + + if(mode_tp=="TP_SL_MULTIPLE"){ + if(order_side == 1){ + double sl_size = price - stoploss; + tp = price + (_tp_var * sl_size); + if(!normalise_price(tp,tp,symbol)){return false;} + } + if(order_side == 2){ + double sl_size = stoploss - price; + tp = price - (_tp_var * sl_size); + if(!normalise_price(tp,tp,symbol)){return false;} + } + } + + if(mode_tp=="TP_SPECIFIED_VALUE"){ + + if(_tp_var!=0){ + double adj_point = mf.adjusted_point(symbol); + + if(order_side == 1){ + double pip_limit = price + 10 * adj_point; + if(_tp_var <= pip_limit){ + tp = pip_limit; + } + else tp = _tp_var; + + if(!normalise_price(tp,tp,symbol)){return false;} + } + if(order_side == 2){ + double pip_limit = price - 10 * adj_point; + if(_tp_var >= pip_limit){ + tp = pip_limit; + } + else tp = _tp_var; + tp = tp = _tp_var; + if(!normalise_price(tp,tp,symbol)){return false;} + } + } + } + return tp; + +} + +double CalculatePositionData::calculate_lots(string symbol, double sl_distance, double price, string mode_lot, double lot_var){ + + double lots = 0; + double tick_size = SymbolInfoDouble(symbol, SYMBOL_TRADE_TICK_SIZE); + double tick_value = SymbolInfoDouble(symbol, SYMBOL_TRADE_TICK_VALUE); + double volume_step = SymbolInfoDouble(symbol, SYMBOL_VOLUME_STEP); + + double account_value = fmin(fmin(AccountInfoDouble(ACCOUNT_EQUITY),AccountInfoDouble(ACCOUNT_BALANCE)),AccountInfoDouble(ACCOUNT_MARGIN_FREE)); + double risk_money = account_value * lot_var / 100; + + if(mode_lot=="LOT_MODE_FIXED"){ + lots = lot_var; + } + + if(mode_lot=="LOT_MODE_PCT_RISK"){ + double money_lot_step = (sl_distance / tick_size) * tick_value * volume_step; + lots = MathFloor(risk_money/money_lot_step) * volume_step; + } + + if(mode_lot=="LOT_MODE_PCT_ACCOUNT"){ + double money_lot_step = (price / tick_size) * tick_value * volume_step; + lots = MathFloor(risk_money/money_lot_step) * volume_step; + } + + if(!check_lots(lots, symbol)){return false;} + return lots; +} + +bool CalculatePositionData::check_lots(double &lots, string symbol){ + + double min = SymbolInfoDouble(symbol, SYMBOL_VOLUME_MIN); + double max = SymbolInfoDouble(symbol, SYMBOL_VOLUME_MAX); + double step = SymbolInfoDouble(symbol, SYMBOL_VOLUME_STEP); + + if(lotsmax){ + Print("Lot size greater than maximum allowed volume. lots:",lots,"max:",max); + return false; + } + + lots = (int)MathFloor(lots/step) * step; + return true; +} + +bool CalculatePositionData::normalise_price(double price, double &normalizedPrice, string symbol){ + double tickSize; + if(!SymbolInfoDouble(symbol,SYMBOL_TRADE_TICK_SIZE,tickSize)){ + Print("Failed to get tick size"); + return false; + } + int symbol_digits = (int)SymbolInfoInteger(symbol, SYMBOL_DIGITS); + normalizedPrice = NormalizeDouble(MathRound(price/tickSize)*tickSize, symbol_digits); + return true; +} + +double CalculatePositionData::calculate_trading_cost(string symbol, ulong position_ticket){ + + position.SelectByTicket(position_ticket); + + double swap = PositionGetDouble(POSITION_SWAP); + double commission = PositionGetDouble(POSITION_COMMISSION); + double tick_size = SymbolInfoDouble(symbol, SYMBOL_TRADE_TICK_SIZE); + double tick_value = SymbolInfoDouble(symbol, SYMBOL_TRADE_TICK_VALUE); + double lot_step = SymbolInfoDouble(symbol, SYMBOL_VOLUME_STEP); + double lots = PositionGetDouble(POSITION_VOLUME); + double trading_cost = -1 * ((commission + swap) / tick_value * tick_size / lots); + + return trading_cost; +} \ No newline at end of file diff --git a/My_MQL5_Libs/DrawdownControl.mqh b/My_MQL5_Libs/DrawdownControl.mqh new file mode 100644 index 0000000..a31b978 --- /dev/null +++ b/My_MQL5_Libs/DrawdownControl.mqh @@ -0,0 +1,209 @@ +//+------------------------------------------------------------------+ +//| DrawdownControl.mqh | +//| xMattC | +//+------------------------------------------------------------------+ +#property library +#include +#include + +class DrawdownControl : public CObject { + protected: + CTrade trade; + MyFunctions mf; + + string data_file; + double daily_max_dd_per; + string daily_reset_time; + bool print_statments; + + double acc_max_dd_per; + double equaty_control_high; + double equaty_control_low; + + + double daily_equity_start; + double daily_max_dd_target; + bool daily_dd_limit_reached; + + bool write_global_var_data(); + bool print_messages(); + + public: + void init_dd_control(string inp_data_file, double inp_acc_max_dd_per, double inp_daily_max_dd_per, string inp_daily_reset_time, bool inp_print_statments = true); + bool determine_daily_dd_limit(); + double lot_correction_factor(double acc_equity_start, double min_lot_factor, double max_lot_factor, bool dynm_lot_factor=false, double dlf_trail_per=20); + double lot_correction_dynamic(double acc_dd_percent, double min_lot_factor, double max_lot_factor); +}; + +void DrawdownControl::init_dd_control(string inp_data_file, double inp_acc_max_dd_per, double inp_daily_max_dd_per, string inp_daily_reset_time, bool inp_print_statments = true) { + + data_file = inp_data_file; + acc_max_dd_per = inp_acc_max_dd_per; + daily_max_dd_per = inp_daily_max_dd_per; + daily_reset_time = inp_daily_reset_time; + print_statments = inp_print_statments; + + // If no data file exisits, create one and set global vairiables: + if(FileIsExist(data_file) == false) { + daily_equity_start = AccountInfoDouble(ACCOUNT_EQUITY); + daily_max_dd_target = daily_equity_start - (daily_equity_start * (daily_max_dd_per / 100)); + daily_dd_limit_reached = false; + equaty_control_high = 9999999; + equaty_control_low = 0; + write_global_var_data(); + } + // If file exisits read file: + if(FileIsExist(data_file) == true) { + + int file_handle = FileOpen(data_file, FILE_READ | FILE_ANSI | FILE_TXT); + if(file_handle == INVALID_HANDLE) { + Print("Error opening file: ", data_file); + } + + // If data file is older than 24h 10min create a new file and reset global vars: + long modifided_date = FileGetInteger(file_handle, FILE_MODIFY_DATE); + long time_delta = ((long)TimeCurrent() - modifided_date) / 60; + + if(time_delta >= 1450) { + daily_equity_start = AccountInfoDouble(ACCOUNT_EQUITY); + daily_max_dd_target = daily_equity_start - (daily_equity_start * (daily_max_dd_per / 100)); + daily_dd_limit_reached = false; + equaty_control_high = equaty_control_high; + equaty_control_low = equaty_control_low; + write_global_var_data(); + Print(data_file, " is older than 24h and 10min; global vars reset!"); + } + // If data file is younger than 24h+10 min read data and set global vars: + else { + daily_equity_start = (double)FileReadString(file_handle, 0); + daily_max_dd_target = (double)FileReadString(file_handle, 1); + daily_dd_limit_reached = FileReadBool(file_handle); + equaty_control_high = (double)FileReadString(file_handle, 3); + equaty_control_low = (double)FileReadString(file_handle, 4);; + } + FileClose(file_handle); + } + print_messages(); +} + +bool DrawdownControl::determine_daily_dd_limit() { + + // Reset max equity at the start of each day: + string ct = TimeToString(TimeCurrent(), TIME_MINUTES); + if(ct == daily_reset_time) { + daily_equity_start = AccountInfoDouble(ACCOUNT_EQUITY); + daily_max_dd_target = (daily_equity_start - (daily_equity_start * (daily_max_dd_per / 100))); + daily_dd_limit_reached = false; + write_global_var_data(); + print_messages(); + } + + // If in drawdown close all positions and delete orders + if(daily_dd_limit_reached || AccountInfoDouble(ACCOUNT_EQUITY) <= daily_max_dd_target) { + + if(daily_dd_limit_reached == false) { + daily_dd_limit_reached = true; + write_global_var_data(); + print_messages(); + } + + for(int i = PositionsTotal() - 1; i >= 0; i--) { + ulong ticket = PositionGetTicket(i); + trade.PositionClose(ticket); + } + + for(int i = OrdersTotal() - 1; i >= 0; i--) { + ulong ticket = OrderGetTicket(i); + trade.OrderDelete(ticket); + } + } + return daily_dd_limit_reached; +} + +// Reduces lot size as account apporchaes max allowed drawdown limit. +double DrawdownControl::lot_correction_factor(double acc_equity_start, double min_lot_factor, double max_lot_factor, bool dynm_lot_factor=false, double dlf_trail_per=20) { + + double account_value = fmin(AccountInfoDouble(ACCOUNT_EQUITY), AccountInfoDouble(ACCOUNT_BALANCE)); + double lot_factor; + + // Interpolate to find lot factor between given min and max values. + if (account_value < acc_equity_start){ + + double acc_equity_min = acc_equity_start - (acc_equity_start * (acc_max_dd_per / 100)); + double y1 = min_lot_factor; + double y2 = max_lot_factor; + double x1 = acc_equity_min; + double x = account_value; + double x2 = acc_equity_start; + lot_factor = y1 + (x - x1) * ((y2 - y1) / (x2 - x1)); + } + + else if(account_value >= acc_equity_start) { + + if(dynm_lot_factor=true){ + lot_factor = lot_correction_dynamic(dlf_trail_per, min_lot_factor, max_lot_factor); + } + + else { + lot_factor = max_lot_factor; + } + } + return max_lot_factor; +} + + +double DrawdownControl::lot_correction_dynamic(double acc_dd_percent, double min_lot_factor, double max_lot_factor) { + + double account_value = fmin(AccountInfoDouble(ACCOUNT_EQUITY), AccountInfoDouble(ACCOUNT_BALANCE)); + double trail_point = account_value - (account_value * (acc_dd_percent / 100)); + + if(equaty_control_low < trail_point){ + equaty_control_low = trail_point; + } + + if(equaty_control_high < account_value){ + equaty_control_high = account_value; + } + + if(account_value < equaty_control_low){ + equaty_control_low = account_value; + equaty_control_high = account_value + (account_value * (acc_dd_percent / 100)); + } + + // back-up to file every hour: + if(mf.is_new_bar(_Symbol, PERIOD_H1) == true){ + write_global_var_data(); + } + + // Linear interpolation: + double y1 = min_lot_factor; + double y2 = max_lot_factor; + double x1 = equaty_control_low; + double x = account_value; + double x2 = equaty_control_high; + + double y = y1 + (x - x1) * ((y2 - y1) / (x2 - x1)); + + return y; +} + +bool DrawdownControl::write_global_var_data() { + int file_handle = FileOpen(data_file, FILE_WRITE | FILE_ANSI | FILE_TXT); + FileWrite(file_handle, daily_equity_start); + FileWrite(file_handle, daily_max_dd_target); + FileWrite(file_handle, daily_dd_limit_reached); + FileClose(file_handle); + Print(data_file, " written"); + return true; +} + +bool DrawdownControl::print_messages() { + if(print_statments == true) { + Print("TimeCurrent(): ", TimeToString(TimeCurrent())); + Print("Daily Equity Start: ", (int)daily_equity_start); + Print("Current Equity: ", (int)AccountInfoDouble(ACCOUNT_EQUITY)); + Print("Daily Drawdown Limit: ", (int)daily_max_dd_target, " (", daily_max_dd_per, "%) of DES"); + Print("Daily Drawdown Limit Hit: ", daily_dd_limit_reached); + } + return true; +} diff --git a/My_MQL5_Libs/MyFunctions.mqh b/My_MQL5_Libs/MyFunctions.mqh new file mode 100644 index 0000000..1e3ca45 --- /dev/null +++ b/My_MQL5_Libs/MyFunctions.mqh @@ -0,0 +1,216 @@ +//+------------------------------------------------------------------+ +//| MyFunctions.mqh | +//| xMattC | +//+------------------------------------------------------------------+ +#property library +#include +#include +#include + +class MyFunctions : public CObject{ + + protected: + CTrade trade; + TradingWindow tw; + datetime previousTime; + datetime bar_open_time; + + public: + void draw_line(double value, string name,color clr); + bool check_indicator_handles(int &indicator_handles[]); + double adjusted_point(string symbol); + double get_bid_ask_price(string symbol, int price_side); + bool is_new_bar(string symbol, ENUM_TIMEFRAMES time_frame, string daily_start_time="00:10"); + bool trade_window(string t1, string t2, string time_zone, bool plot_range_inp=true); + bool in_test_period(MODE_SPLIT_DATA data_period); + void get_white_list(MULTI_SYM_MODE mode, string& DataArray[]); + +}; + +void MyFunctions::get_white_list(MULTI_SYM_MODE mode, string& DataArray[]){ + + if(mode==MULTI_SYM_CHART){ + string a[] = {_Symbol}; + ArrayResize(DataArray, ArraySize(a)); + for(int i = 0; i < ArraySize(DataArray); i++){ + DataArray[i]=a[i]; + } + } + if(mode==MULTI_SYM_FX_B5){ + string a[] = {"EURUSD", "AUDNZD", "EURGBP", "AUDCAD", "CHFJPY"}; + ArrayResize(DataArray, ArraySize(a)); + for(int i = 0; i < ArraySize(DataArray); i++){ + DataArray[i]=a[i]; + } + } + if(mode==MULTI_SYM_FX_28){ + string a[] = {"EURUSD","AUDNZD","AUDUSD","AUDJPY","EURCHF","EURGBP","EURJPY","GBPCHF","GBPJPY","GBPUSD","NZDUSD","USDCAD","USDCHF","USDJPY","CADJPY","EURAUD","CHFJPY","EURCAD","AUDCAD","AUDCHF","CADCHF","EURNZD","GBPAUD","GBPCAD","GBPNZD","NZDCAD","NZDCHF","NZDJPY",}; + ArrayResize(DataArray, ArraySize(a)); + for(int i = 0; i < ArraySize(DataArray); i++){ + DataArray[i]=a[i]; + } + } +} + +bool MyFunctions::trade_window(string t1, string t2, string time_zone="Broker", bool plot_range_inp=true){ + bool in_window = tw.define_window(t1, t2, time_zone, plot_range_inp); + return in_window; +} + +//if(!mf.is_new_bar(symbol, PERIOD_D1, "00:06")){return;} +bool MyFunctions::is_new_bar(string symbol, ENUM_TIMEFRAMES time_frame, string daily_start_time="00:10"){ + + bar_open_time = iTime(symbol, time_frame, 0); + if(previousTime!=bar_open_time){ + + if(PeriodSeconds(time_frame)==PeriodSeconds(PERIOD_D1)){ + if(TimeCurrent() > StringToTime(daily_start_time)){ + previousTime=bar_open_time; + return true; + } + } + + else{ + previousTime=bar_open_time; + return true; + } + + } + return false; +} + +//if(!mf.in_test_period(data_split_method){return;} +bool MyFunctions::in_test_period(MODE_SPLIT_DATA data_split_method){ + + string result[]; + string string_tc = TimeToString(TimeCurrent()); + ushort u_sep = StringGetCharacter(".",0); + int split_string = StringSplit(string_tc, u_sep, result); + bool odd_year = int(result[0]) % 2; + bool odd_month = int(result[1]) % 2; + + // get week of the year. rough estimate can be late the first week of jan: + MqlDateTime dt; + TimeToStruct(TimeCurrent(),dt); + int iDay = (dt.day_of_week + 6 ) % 7 + 1; // convert day to standard index (1=Mon,...,7=Sun) + int iWeek = (dt.day_of_year - iDay + 10 ) / 7; // calculate standard week number + + bool odd_week = iWeek % 2; + + + if(data_split_method==NO_SPLIT){ + return true; + } + + if(data_split_method==ODD_YEARS){ + if (odd_year){ + return true; + } + } + + if(data_split_method==EVEN_YEARS){ + if (!odd_year){ + return true; + } + } + + if(data_split_method==ODD_MONTHS){ + if (odd_month){ + return true; + } + } + + if(data_split_method==EVEN_MONTHS){ + + if (!odd_month){ + return true; + } + } + + if(data_split_method==ODD_WEEKS){ + if (odd_week){ + return true; + } + } + + if(data_split_method==EVEN_WEEKS){ + + if (!odd_week){ + return true; + } + } + + return false; +} + +void MyFunctions::draw_line(double value, string name,color clr=clrBlack){ + // EG: + // ArrayResize(bar,1000); + // ArraySetAsSeries(bar, true); + // CopyRates(symbol,PERIOD_CURRENT,1,1000,bar); + // double close = bar[0].close; + // draw_line(close,"CLOSE",clrBlue); + + if(ObjectFind(0,name)<0){ + ResetLastError(); + + if(!ObjectCreate(0,name,OBJ_HLINE,0,0,value)){ + Print(__FUNCTION__,": failed to create a horizontal line! Error code = ",GetLastError()); + return; + } + + ObjectSetInteger(0,name,OBJPROP_COLOR,clr); + ObjectSetInteger(0,name,OBJPROP_STYLE,STYLE_SOLID); + ObjectSetInteger(0,name,OBJPROP_WIDTH,1); + } + + ResetLastError(); + + if(!ObjectMove(0,name,0,0,value)){ + Print(__FUNCTION__,": failed to move the horizontal line! Error code = ",GetLastError()); + return; + } + + ChartRedraw(); +} + +double MyFunctions::adjusted_point(string symbol){ + + int symbol_digits = (int)SymbolInfoInteger(symbol, SYMBOL_DIGITS); + int digits_adjust=1; + if(symbol_digits==3 || symbol_digits==5){ + digits_adjust=10; + } + + double symbol_point_val = SymbolInfoDouble(symbol,SYMBOL_POINT); + double m_adjusted_point; + m_adjusted_point = symbol_point_val * digits_adjust; + + return m_adjusted_point; + +} +// price side - 1 for the ask price and 2 for the bid price +double MyFunctions::get_bid_ask_price(string symbol, int price_side){ + + int symbol_digits = (int)SymbolInfoInteger(symbol, SYMBOL_DIGITS); + double symbol_point = SymbolInfoDouble(symbol, SYMBOL_POINT); + + double ask = SymbolInfoDouble(symbol, SYMBOL_ASK); + ask = NormalizeDouble(ask, symbol_digits); + + double bid = SymbolInfoDouble(symbol, SYMBOL_BID); + bid = NormalizeDouble(bid, symbol_digits); + + double price = 0; + + if(price_side==1){ + price = ask; + } + + else if(price_side==2){ + price = bid; + } + + return price; + +} \ No newline at end of file diff --git a/My_MQL5_Libs/OrderManagement.mqh b/My_MQL5_Libs/OrderManagement.mqh new file mode 100644 index 0000000..e03ccd9 --- /dev/null +++ b/My_MQL5_Libs/OrderManagement.mqh @@ -0,0 +1,560 @@ +//+------------------------------------------------------------------+ +//| OrderManagement.mqh | +//| xMattC | +//+------------------------------------------------------------------+ +#property library +#include +#include +#include +#include +#include +#include +#include + +class OrderManagment : public CObject{ + + protected: + CTrade trade; + TimeZones tz; + CalculatePositionData cpd; + CPositionInfo m_position; + COrderInfo m_order; + + double stop_loss; + double take_profit; + ulong posTicket; + int time_difference; + int total_open_buy_orders; + int total_open_sell_orders; + double current_price; + int total_pos; + long position_open_time; + long first_allowed_close_time; + datetime current_bar_open_time; + + public: + bool open_buy_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number); + bool open_sell_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number); + bool open_nnfx_buy_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number); + bool open_nnfx_sell_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number); + + bool open_buy_stop_order(string symbol, bool condition, double entry_price, datetime experation, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number); + bool open_sell_stop_order(string symbol, bool condition, double entry_price, datetime experation, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number); + bool close_buy_orders(string symbol, bool buy_out, int close_bars, ENUM_TIMEFRAMES close_bar_period, long magic_number); + bool close_sell_orders(string symbol, bool sell_out, int close_bars, ENUM_TIMEFRAMES close_bar_period, long magic_number); + bool first_profitable_close_exit(string symbol, ENUM_TIMEFRAMES close_bar_period, long magic_number); + bool daily_timed_exit(string symbol, datetime exit_time, int delay_days, long magic_number); + bool daily_timed_profit_exit(string symbol, ENUM_TIMEFRAMES close_bar_period, string exit_time, string tz, int delay_days, long magic_number); + int count_all_positions(string symbol, long magic_number); + int count_pending_orders(string symbol, ENUM_ORDER_TYPE pendingType, long magic); + double sl_specified_value_switch(string _sl_mode, double _inp_sl_var, double value); + double tp_specified_value_switch(string _tp_mode, double _inp_tp_var, double value); + int count_open_positions(string symbol,int order_side, long magic_number); + void break_even_stop(string symbol, ulong magic_number, int be_trigger_points, int be_puffer); + void nnfx_trailing_stop(string symbol, double sl_var, double tp_var, double atr_value, ulong magic_number); + }; + +bool OrderManagment::open_buy_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var, long magic_number){ + + if(condition == true){ + current_price = SymbolInfoDouble(symbol, SYMBOL_ASK); // ask for buy side + + total_open_buy_orders = count_open_positions(symbol, 1, magic_number); + if(total_open_buy_orders == 0){ + + stop_loss = cpd.calculate_stoploss(symbol, current_price, 1, _sl_mode, sl_var, atr_period); + take_profit = cpd.calculate_take_profit(symbol, current_price, stop_loss, 1, _tp_mode, tp_var, atr_period); + + double sl_distance = current_price-stop_loss; + double lots = cpd.calculate_lots(symbol, sl_distance, current_price, _lot_mode, lot_var); + + trade.SetExpertMagicNumber(magic_number); + string comment = "Magic Number: " + IntegerToString(magic_number); + trade.PositionOpen(symbol,ORDER_TYPE_BUY,lots,current_price,stop_loss,take_profit,comment); + } + } + return true; +} + + +bool OrderManagment::open_sell_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number){ + + if(condition == true){ + + // if(!SymbolInfoTick(symbol,currentTick)){Print("FAILED TO GET TICK:", symbol);return false;} + current_price = SymbolInfoDouble(symbol, SYMBOL_BID); // bid for sell side + + total_open_sell_orders = count_open_positions(symbol, 2, magic_number); + if(total_open_sell_orders == 0){ + + stop_loss = cpd.calculate_stoploss(symbol, current_price, 2, _sl_mode, sl_var, atr_period); + take_profit = cpd.calculate_take_profit(symbol, current_price, stop_loss, 2, _tp_mode, tp_var, atr_period); + + double sl_distance = stop_loss-current_price; + double lots = cpd.calculate_lots(symbol, sl_distance, current_price, _lot_mode, lot_var); + + trade.SetExpertMagicNumber(magic_number); + string comment = "Magic Number: " + IntegerToString(magic_number); + trade.PositionOpen(symbol,ORDER_TYPE_SELL,lots,current_price,stop_loss,take_profit,comment); + } + } + return true; +} + +bool OrderManagment::open_nnfx_buy_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var, long magic_number){ + + if(condition == true){ + current_price = SymbolInfoDouble(symbol, SYMBOL_ASK); // ask for buy side + + total_open_buy_orders = count_open_positions(symbol, 1, magic_number); + if(total_open_buy_orders == 0){ + + stop_loss = cpd.calculate_stoploss(symbol, current_price, 1, _sl_mode, sl_var, atr_period); + take_profit = cpd.calculate_take_profit(symbol, current_price, stop_loss, 1, _tp_mode, tp_var, atr_period); + + double sl_distance = current_price-stop_loss; + double lots = cpd.calculate_lots(symbol, sl_distance, current_price, _lot_mode, lot_var/2); + + trade.SetExpertMagicNumber(magic_number); + string comment = "Magic Number: " + IntegerToString(magic_number); + trade.PositionOpen(symbol,ORDER_TYPE_BUY,lots,current_price,stop_loss,take_profit,comment); + trade.PositionOpen(symbol,ORDER_TYPE_BUY,lots,current_price,stop_loss,0,comment); + } + } + return true; +} + + +bool OrderManagment::open_nnfx_sell_orders(string symbol, bool condition, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number){ + + if(condition == true){ + + // if(!SymbolInfoTick(symbol,currentTick)){Print("FAILED TO GET TICK:", symbol);return false;} + current_price = SymbolInfoDouble(symbol, SYMBOL_BID); // bid for sell side + + total_open_sell_orders = count_open_positions(symbol, 2, magic_number); + if(total_open_sell_orders == 0){ + + stop_loss = cpd.calculate_stoploss(symbol, current_price, 2, _sl_mode, sl_var, atr_period); + take_profit = cpd.calculate_take_profit(symbol, current_price, stop_loss, 2, _tp_mode, tp_var, atr_period); + + double sl_distance = stop_loss-current_price; + double lots = cpd.calculate_lots(symbol, sl_distance, current_price, _lot_mode, lot_var); + + trade.SetExpertMagicNumber(magic_number); + string comment = "Magic Number: " + IntegerToString(magic_number); + trade.PositionOpen(symbol,ORDER_TYPE_SELL,lots,current_price,stop_loss,take_profit,comment); + trade.PositionOpen(symbol,ORDER_TYPE_SELL,lots,current_price,stop_loss,0,comment); + } + } + return true; +} +// some usfull comment here +bool OrderManagment::open_buy_stop_order(string symbol, bool condition, double entry_price, datetime experation, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var,string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number){ + + if(condition == true){ + + total_open_buy_orders = count_open_positions(symbol, 1, magic_number); + if(total_open_buy_orders == 0){ + + stop_loss = cpd.calculate_stoploss(symbol, entry_price, 1, _sl_mode, sl_var, atr_period); + take_profit = cpd.calculate_take_profit(symbol, entry_price, stop_loss, 1, _tp_mode, tp_var, atr_period); + + double sl_distance = entry_price-stop_loss; + double lots = cpd.calculate_lots(symbol, sl_distance, entry_price, _lot_mode, lot_var); + + trade.SetExpertMagicNumber(magic_number); + string comment = "Magic Number: " + IntegerToString(magic_number); + trade.BuyStop(lots, entry_price, symbol, stop_loss, take_profit, ORDER_TIME_SPECIFIED, experation, comment); + } + } + return true; +} + + +bool OrderManagment::open_sell_stop_order(string symbol, bool condition, double entry_price, datetime experation, ENUM_TIMEFRAMES atr_period, string _sl_mode, double sl_var, string _tp_mode, double tp_var, string _lot_mode, double lot_var,long magic_number){ + + if(condition == true){ + + total_open_sell_orders = count_open_positions(symbol, 2, magic_number); + if(total_open_sell_orders == 0){ + + stop_loss = cpd.calculate_stoploss(symbol, entry_price, 2, _sl_mode, sl_var, atr_period); + take_profit = cpd.calculate_take_profit(symbol, entry_price, stop_loss, 2, _tp_mode, tp_var, atr_period); + + double sl_distance = stop_loss-entry_price; + double lots = cpd.calculate_lots(symbol, sl_distance, entry_price, _lot_mode, lot_var); + + trade.SetExpertMagicNumber(magic_number); + string comment = "Magic Number: " + IntegerToString(magic_number); + trade.SellStop(lots, entry_price, symbol, stop_loss, take_profit, ORDER_TIME_SPECIFIED, experation, comment); + } + } + return true; +} + +bool OrderManagment::close_buy_orders(string symbol, bool condition, int close_bars, ENUM_TIMEFRAMES close_bar_period, long magic_number){ + + for(int i = PositionsTotal()-1; i >=0; i--){ + posTicket = PositionGetTicket(i); + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC) == magic_number){ + + time_difference = Bars(symbol, close_bar_period, PositionGetInteger(POSITION_TIME), TimeCurrent()) - 1; + + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY){ + + if(condition){ + trade.PositionClose(posTicket); + } + + if(close_bars > 0){ + if(time_difference >= close_bars){ + trade.PositionClose(posTicket); + } + } + } + } + } + return true; +} + +bool OrderManagment::close_sell_orders(string symbol, bool condition, int close_bars, ENUM_TIMEFRAMES close_bar_period, long magic_number){ + + for(int i = PositionsTotal()-1; i >=0; i--){ + posTicket = PositionGetTicket(i); + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC) == magic_number){ + + time_difference = Bars(symbol, close_bar_period, PositionGetInteger(POSITION_TIME), TimeCurrent()) - 1; + + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_SELL){ + + if(condition){trade.PositionClose(posTicket);} + + if(close_bars > 0){ + if(time_difference >= close_bars){ + trade.PositionClose(posTicket); + } + } + } + } + } + return true; +} + +// order_side int must be 1 for BUY or 2 for SELL +int OrderManagment::count_open_positions(string symbol,int order_side, long magic_number){ + + + int count = 0; + bool match = (PositionGetInteger(POSITION_MAGIC)==magic_number); + + for(int i = PositionsTotal()-1; i >=0; i--){ + ulong ticket = PositionGetTicket(i); + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC)==magic_number){ + + // Count only Buy orders: + if(order_side == 1){ + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY){ + count = count + 1; + } + } + + // Count only Sell orders: + if(order_side == 2){ + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_SELL){ + count = count + 1; + } + } + } + } + return count; +} + +int OrderManagment::count_all_positions(string symbol, long magic_number){ + + int count = 0; + for(int i = PositionsTotal()-1; i >=0; i--){ + ulong ticket = PositionGetTicket(i); + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC)==magic_number){ + count = count + 1; + } + } + return count; +} + +bool OrderManagment::daily_timed_exit(string symbol, datetime exit_time, int delay_days, long magic_number){ + + for(int i = PositionsTotal()-1; i >=0; i--){ + posTicket = PositionGetTicket(i); + position_open_time = PositionGetInteger(POSITION_TIME); + + if((int)position_open_time>0){ + + first_allowed_close_time = position_open_time + (delay_days * PeriodSeconds(PERIOD_D1)); + if(TimeCurrent() > first_allowed_close_time){ + + // datetime broker_close_time = tz.timezone_conversions(cw_tzone, StringToTime(exit_time), "Broker"); + if(TimeCurrent()>= exit_time){ + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC) == magic_number){ + + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY){ + trade.PositionClose(posTicket); + } + + // Sell orders: + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_SELL){ + trade.PositionClose(posTicket); + } + } + } + } + } + } +return true; +} + +bool OrderManagment::daily_timed_profit_exit(string symbol, ENUM_TIMEFRAMES close_bar_period, string exit_time, string cw_tzone, int delay_days, long magic_number){ + + // om.daily_timed_profit_exit(_Symbol, PERIOD_CURRENT, "16:45", "17:00", "NY", 1, inp_magic); + + for(int i = PositionsTotal()-1; i >=0; i--){ + posTicket = PositionGetTicket(i); + position_open_time = PositionGetInteger(POSITION_TIME); + + if((int)position_open_time>0){ + + first_allowed_close_time = position_open_time + (delay_days * PeriodSeconds(PERIOD_D1)); + if(TimeCurrent() > first_allowed_close_time){ + + + datetime broker_close_time = tz.timezone_conversions(cw_tzone, StringToTime(exit_time), "Broker"); + if(TimeCurrent()>= broker_close_time){ + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC) == magic_number){ + + double position_open_price = PositionGetDouble(POSITION_PRICE_OPEN); + double spread = SymbolInfoDouble(symbol,SYMBOL_ASK) - SymbolInfoDouble(symbol,SYMBOL_BID); + double bar_close = iClose(_Symbol, close_bar_period, 1); // shift 1 because 0 = live candle. + double trading_cost = cpd.calculate_trading_cost(symbol, posTicket); + + + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY){ + if(bar_close > (position_open_price + spread + trading_cost)){ + trade.PositionClose(posTicket); + + } + } + + // Sell orders: + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_SELL){ + if(bar_close < position_open_price - spread - trading_cost){ + trade.PositionClose(posTicket); + } + } + } + } + } + } + } +return true; +} + + +bool OrderManagment::first_profitable_close_exit(string symbol, ENUM_TIMEFRAMES close_bar_period, long magic_number){ + // om.first_profitable_close_exit(_Symbol, PERIOD_CURRENT, inp_magic); + + position_open_time = PositionGetInteger(POSITION_TIME); + first_allowed_close_time = position_open_time + PeriodSeconds(close_bar_period); + + if((int)position_open_time>0){ + + if(TimeCurrent() > first_allowed_close_time){ + for(int i = PositionsTotal()-1; i >=0; i--){ + posTicket = PositionGetTicket(i); + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC) == magic_number){ + + double position_open_price = PositionGetDouble(POSITION_PRICE_OPEN); + double spread = SymbolInfoDouble(symbol,SYMBOL_ASK) - SymbolInfoDouble(symbol,SYMBOL_BID); + double bar_close = iClose(_Symbol,close_bar_period, 1); // shift 1 because 0 = live candle. + double trading_cost = cpd.calculate_trading_cost(symbol, posTicket); + + + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY){ + if(bar_close > (position_open_price + spread + trading_cost)){ + trade.PositionClose(posTicket); + + } + } + + // Sell orders: + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_SELL){ + if(bar_close < position_open_price - spread - trading_cost){ + trade.PositionClose(posTicket); + } + } + } + } + } + } +return true; +} + +// e.g. int buy_stop_count = om.count_pending_orders(symbol, ORDER_TYPE_BUY_STOP, inp_magic); +// order types: ORDER_TYPE_BUY_LIMIT, ORDER_TYPE_SELL_LIMIT, ORDER_TYPE_BUY_STOP, ORDER_TYPE_SELL_STOP +int OrderManagment::count_pending_orders(string symbol, ENUM_ORDER_TYPE order_type, long magic){ + int count = 0; + + for(int i=OrdersTotal()-1;i>=0;i--) { + + if(m_order.SelectByIndex(i)){ + if( OrderGetInteger(ORDER_MAGIC) == magic && OrderGetString(ORDER_SYMBOL) == symbol){ + + if(m_order.OrderType()==order_type){ + count++; + } + } + } + } + return(count); +} + +void OrderManagment::break_even_stop(string symbol, ulong magic_number, int be_trigger_points, int be_puffer){ + + for(int i = PositionsTotal()-1; i >=0; i--){ + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC) == magic_number){ + + int symbol_digits = (int)SymbolInfoInteger(symbol, SYMBOL_DIGITS); + double symbol_point = SymbolInfoDouble(symbol, SYMBOL_POINT); + + double ask = SymbolInfoDouble(symbol, SYMBOL_ASK); + ask = NormalizeDouble(ask, symbol_digits); + + double bid = SymbolInfoDouble(symbol, SYMBOL_BID); + bid = NormalizeDouble(bid, symbol_digits); + + if(be_trigger_points !=0){ + + + ulong ticket = PositionGetTicket(i); + if(PositionSelectByTicket(ticket)){ + + double position_open_price = PositionGetDouble(POSITION_PRICE_OPEN); + double position_volume = PositionGetDouble(POSITION_VOLUME); + double position_sl = PositionGetDouble(POSITION_SL); + double position_tp = PositionGetDouble(POSITION_TP); + ENUM_POSITION_TYPE position_type = (ENUM_POSITION_TYPE)PositionGetInteger(POSITION_TYPE); + + if(position_type == POSITION_TYPE_BUY){ + + if(bid > position_open_price + be_trigger_points * symbol_point){ + + double sl = position_open_price + be_puffer * symbol_point; + sl = NormalizeDouble(sl, symbol_digits); + if(sl > position_sl){ + + if(trade.PositionModify(ticket, sl, position_tp)){ + Print("-----------------------------------Stop moved to break even"); + } + } + } + } + else if(position_type == POSITION_TYPE_SELL){ + + if(ask < position_open_price - be_trigger_points * symbol_point){ + + double sl = position_open_price - be_puffer * symbol_point; + sl = NormalizeDouble(sl, symbol_digits); + if(sl < position_sl){ + + if(trade.PositionModify(ticket, sl, position_tp)){ + Print("-----------------------------------Stop moved to break even"); + } + } + } + } + } + } + } + } +} + + +void OrderManagment::nnfx_trailing_stop(string symbol, double sl_var, double tp_var, double atr_value, ulong magic_number){ + + MyFunctions mf3; + + for(int i = PositionsTotal()-1; i >=0; i--){ + + ulong ticket = PositionGetTicket(i); + if(PositionSelectByTicket(ticket)){ + + if(PositionGetString(POSITION_SYMBOL) == symbol && PositionGetInteger(POSITION_MAGIC) == magic_number){ + + int symbol_digits = (int)SymbolInfoInteger(symbol, SYMBOL_DIGITS); + double symbol_point = SymbolInfoDouble(symbol, SYMBOL_POINT); + + double ask = SymbolInfoDouble(symbol, SYMBOL_ASK); + ask = NormalizeDouble(ask, symbol_digits); + + double bid = SymbolInfoDouble(symbol, SYMBOL_BID); + bid = NormalizeDouble(bid, symbol_digits); + + double position_open_price = PositionGetDouble(POSITION_PRICE_OPEN); + double position_sl = PositionGetDouble(POSITION_SL); + double position_tp = PositionGetDouble(POSITION_TP); + ENUM_POSITION_TYPE position_type = (ENUM_POSITION_TYPE)PositionGetInteger(POSITION_TYPE); + + if(position_type == POSITION_TYPE_BUY){ + + if(bid > position_open_price + (atr_value * tp_var)){ + + double sl = bid - (atr_value * sl_var); + sl = NormalizeDouble(sl, symbol_digits); + if(sl > (position_sl + (atr_value * 0.5))){ + + if(trade.PositionModify(ticket, sl, position_tp)){ + + } + } + } + } + + + else if(position_type == POSITION_TYPE_SELL){ + + if(ask < position_open_price - (atr_value * tp_var)){ + + double sl = ask + (atr_value * sl_var); + sl = NormalizeDouble(sl, symbol_digits); + if(sl < (position_sl + (atr_value * 0.5))){ + + if(trade.PositionModify(ticket, sl, position_tp)){ + } + } + } + } + } + } + + + } +} + +double OrderManagment::sl_specified_value_switch(string _sl_mode, double _inp_sl_var, double value){ + double sl = 0; + if(_sl_mode=="SL_SPECIFIED_VALUE"){sl = value;} + if(_sl_mode!="SL_SPECIFIED_VALUE"){sl = _inp_sl_var;} + return sl; +} +double OrderManagment::tp_specified_value_switch(string _tp_mode, double _inp_tp_var, double value){ + double tp = 0; + if(_tp_mode=="SL_SPECIFIED_VALUE"){tp = value;} + if(_tp_mode!="SL_SPECIFIED_VALUE"){tp = _inp_tp_var;} + return tp; +} \ No newline at end of file diff --git a/My_MQL5_Libs/TimeZones.mqh b/My_MQL5_Libs/TimeZones.mqh new file mode 100644 index 0000000..91454dc --- /dev/null +++ b/My_MQL5_Libs/TimeZones.mqh @@ -0,0 +1,148 @@ +//+------------------------------------------------------------------+ +//| TimeZones.mqh | +//| xMattC | +//+------------------------------------------------------------------+ +#property library +#include +#include + +class TimeZones: public CObject{ + + protected: + string dt_s; + int len; + string dt_string; + datetime tC, tGMT, tNY, tLon, tFfm, tMosc, tSyd, tTok; + datetime tz_time; + string tz_date; + datetime time_start; + datetime time_end; + bool is_time; + datetime tGIVEN; + datetime tREQ; + datetime tzt; + datetime tz_req; + double ny_daily_close_protected(string symbol, int shift_days, bool print_data=false); + double required_close; + + public: + string get_date_string_from_datetime(datetime dt); + datetime get_timezone_time(string time_zone, bool print_time); + datetime timezone_conversions(string time_zone_known, datetime time_given, string time_zone_required); + double ny_daily_close(string symbol, int shift_days, bool print_data=false); +}; + +string TimeZones::get_date_string_from_datetime(datetime dt){ + dt_s = TimeToString(dt); + len = StringLen(dt_s); + dt_string = StringSubstr(dt_s, 0, len-5); + return dt_string; +} + + +datetime TimeZones::get_timezone_time(string time_zone, bool print_time){ + // https://www.mql5.com/en/code/45287 + // https://www.mql5.com/en/articles/9926 + // https://www.mql5.com/en/articles/9929 + + checkTimeOffset(TimeCurrent()); // check changes of DST + // cto(); + + tC = TimeCurrent(); + tGMT = TimeCurrent() + OffsetBroker.actOffset; // GMT + tNY = tGMT - (NYShift+DST_USD); // time in New York (EST) + tLon = tGMT - (LondonShift+DST_EUR); // time in London + tFfm = tGMT - (FfmShift+DST_EUR); // time in Frankfurt + tSyd = tGMT - (SidneyShift+DST_AUD); // time in Sidney + tMosc = tGMT - (MoskwaShift+DST_RUS); // time in Moscow + tTok = tGMT - (TokyoShift); // time in Tokyo - no DST + + if(print_time==true){ + Print("----------------------------------"); + Print("Broker: ", tC); + Print("GMT: ", tGMT); + Print("time in New York: ", tNY); + Print("time in London: ", tLon); + Print("time in Frankfurt: ", tFfm); + Print("time in Sidney: ", tSyd); + Print("time in Moscow: ", tMosc); + Print("time in Tokyo: ", tTok); + } + + if(time_zone=="NY"){return tNY;} + if(time_zone=="Lon"){return tLon;} + if(time_zone=="Ffm"){return tFfm;} + if(time_zone=="Syd"){return tSyd;} + if(time_zone=="Mosc"){return tMosc;} + if(time_zone=="Tok"){return tTok;} + + return NULL; +} + + +datetime TimeZones::timezone_conversions(string time_zone_known, datetime time_given, string time_zone_required){ + // https://www.mql5.com/en/code/45287 + // https://www.mql5.com/en/articles/9926 + // https://www.mql5.com/en/articles/9929 + + tGIVEN = time_given; //StringToTime(time_given); + + checkTimeOffset(tGIVEN); // check changes of DST + + // Get GMT: + if(time_zone_known=="GMT" ){tGMT = tGIVEN;} + if(time_zone_known=="Broker" ){tGMT = tGIVEN + OffsetBroker.actOffset;} + if(time_zone_known=="NY" ){tGMT = tGIVEN + (NYShift+DST_USD);} + if(time_zone_known=="Lon" ){tGMT = tGIVEN + (LondonShift+DST_EUR);} + if(time_zone_known=="Ffm" ){tGMT = tGIVEN + (FfmShift+DST_EUR);} + if(time_zone_known=="Syd" ){tGMT = tGIVEN + (SidneyShift+DST_AUD);} + if(time_zone_known=="Mosc" ){tGMT = tGIVEN + (MoskwaShift+DST_RUS);} + if(time_zone_known=="Tok" ){tGMT = tGIVEN + (TokyoShift);} + + // define the required time: + tREQ = NULL; + if(time_zone_required=="GMT" ){tREQ = tGMT;} + if(time_zone_required=="Broker" ){tREQ = tGMT - OffsetBroker.actOffset;} + if(time_zone_required=="NY" ){tREQ = tGMT - (NYShift+DST_USD);} + if(time_zone_required=="Lon" ){tREQ = tGMT - (LondonShift+DST_EUR);} + if(time_zone_required=="Ffm" ){tREQ = tGMT - (FfmShift+DST_EUR);} + if(time_zone_required=="Syd" ){tREQ = tGMT - (SidneyShift+DST_AUD) ;} + if(time_zone_required=="Mosc" ){tREQ = tGMT - (MoskwaShift+DST_RUS);} + if(time_zone_required=="Tok" ){tREQ = tGMT - (TokyoShift);} + + return tREQ; +} + +// Calculte NY close time: +double TimeZones::ny_daily_close(string symbol, int shift_days, bool print_data=false){ + required_close = ny_daily_close_protected(symbol, shift_days, print_data); + return required_close; +} +double TimeZones::ny_daily_close_protected(string symbol, int shift_days, bool print_data=false){ + + // Get the brokers times for when NY openend today and tomorrow: + datetime time_5pm = iTime(symbol, PERIOD_D1 , 0) - (PeriodSeconds(PERIOD_H1) * 7); + datetime ny_close_in_brokers_time = timezone_conversions("NY", time_5pm, "Broker"); + datetime ny_close_time = ny_close_in_brokers_time + PeriodSeconds(PERIOD_D1); // ny close tomorrow + + if(TimeCurrent()= cut_before] + store["df"] = df + + logging.info("Trimmed DataFrame to the last 24 hours.") + except Exception as e: + logging.error(f"Could not trim DataFrame: {e}") + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, "Could not trim DataFrame.") + + +def update_exchange_rate(): + """ + Fetch the latest USD to GBP exchange rate and store it. + """ + cr = CurrencyRates() + file_path = os.path.join(DATA_DIR, "USDGBP_exchange_rate.txt") + + try: + exchange_rate = cr.get_rate("USD", "GBP") + with open(file_path, "w") as f: + f.write(str(exchange_rate)) + logging.info("Updated USD to GBP exchange rate.") + except Exception as e: + logging.error(f"Could not update exchange rate: {e}") + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, "Could not update exchange rate.") + + +def convert_usd_to_gbp(usd): + """ + Convert USD to GBP using the stored exchange rate. + """ + file_path = os.path.join(DATA_DIR, "USDGBP_exchange_rate.txt") + + try: + with open(file_path, "r") as f: + exchange_rate = float(f.read()) + gbp = usd * exchange_rate + return gbp, exchange_rate + except Exception as e: + logging.error(f"Could not convert USD to GBP: {e}") + return usd, 1.0 # Fallback to 1:1 conversion + + +def get_balance(): + """ + Retrieve account balances from Binance and update the stake amount. + """ + try: + client = Client(API_KEY, API_SECRET) + account_info = client.get_account() + balances = account_info["balances"] + + usdt = 0.0 + for balance in balances: + asset = balance["asset"] + free = float(balance["free"]) + locked = float(balance["locked"]) + total = free + locked + + if total > 0: + if asset == "USDT": + usdt += total + else: + try: + price = float(client.get_symbol_ticker(symbol=f"{asset}USDT")["price"]) + usdt += total * price + except Exception: + pass + + btc_price = float(client.get_symbol_ticker(symbol="BTCUSDT")["price"]) + btc = usdt / btc_price + gbp, exchange_rate = convert_usd_to_gbp(usdt) + stake_amount = round(usdt / MAX_TRADES) + + with open(os.path.join(DATA_DIR, "stake_amount.txt"), "w") as f: + f.write(str(stake_amount)) + + update_df(gbp, usdt, btc, exchange_rate) + except Exception as e: + logging.error(f"Could not fetch balances: {e}") + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, "Could not fetch Binance balances.") + + +def run_code(): + """ + Main function to schedule tasks and run the bot. + """ + update_exchange_rate() + get_balance() + schedule.every(RESET_STAKE_AMOUNT).minutes.do(get_balance) + schedule.every().day.at("11:45").do(update_exchange_rate) + schedule.every(4).hours.do(trim_df) + + while True: + schedule.run_pending() + sleep(1) + + +if __name__ == "__main__": + run_code() diff --git a/Python_freqtrade/Live_trading_updates/process_data.py b/Python_freqtrade/Live_trading_updates/process_data.py new file mode 100644 index 0000000..3aec8e6 --- /dev/null +++ b/Python_freqtrade/Live_trading_updates/process_data.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +import os +import time +import shutil +import warnings +import pandas as pd +import matplotlib.pyplot as plt # pip install matplotlib +import matplotlib.dates as mdates +from datetime import datetime, timedelta, date +import telegram +import schedule # pip install schedule + +warnings.simplefilter(action='ignore', category=FutureWarning) + +# Constants +LONG_PLOT_DAYS = -1 # 60 # -1 all days in df +LONG_PLOT_CURRENCY = '£' +PLOT_CURRENCY = '$' + +# Environment variables for sensitive information +API_KEY = os.getenv("BINANCE_API_KEY") +API_SECRET = os.getenv("BINANCE_API_SECRET") +BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN") +BOT_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID") + + +date_today = date.today() +TODAY = date_today.strftime("%Y_%m_%d") + + +def telegram_send_image(bot_token, bot_chat_id, image_path): + """Send an image to the specified Telegram chat.""" + bot = telegram.Bot(token=bot_token) + with open(image_path, 'rb') as photo: + bot.send_photo(chat_id=bot_chat_id, photo=photo) + return () + + +def telegram_send_message(bot_token, bot_chat_id, message): + """Send a message to the specified Telegram chat.""" + bot = telegram.Bot(token=bot_token) + bot.send_message(chat_id=bot_chat_id, text=message) + return () + + +def plot_and_send_image(df, currency, plot_title, file_name): + """Generate plot and send it as an image to Telegram.""" + plt.rcParams.update({'font.size': 14, 'font.family': 'STIXGeneral', 'mathtext.fontset': 'stix'}) + fig, axs = plt.subplots(figsize=(7, 4)) + axs.xaxis.set_major_formatter(mdates.DateFormatter("%d %b")) + df[currency].plot.line(ax=axs, color="darkgreen", linewidth=1.50) + delta_y = int(df[currency].max()) - int(df[currency].min()) + y_min = int(df[currency].min()) - (delta_y * 0.05) + y_max = int(df[currency].max()) + (delta_y * 0.05) + x_max = datetime.now() + x_min = datetime.now() - timedelta(days=len(df)) + axs.set_title(plot_title) + axs.set_ylim(y_min, y_max) + axs.set_xlim(x_min, x_max) + axs.set_ylabel("") + axs.set_xlabel("") + axs.grid(color='grey', alpha=0.5, linestyle='dashed', linewidth=0.5) + axs.yaxis.set_major_formatter(f"{currency} {{x:1.0f}}") + + plt.savefig(file_name) + plt.cla() + plt.close(fig) + + # Send image to Telegram + telegram_send_image(BOT_TOKEN, BOT_CHAT_ID, file_name) + return () + + +def plot_long(period=LONG_PLOT_DAYS, currency=LONG_PLOT_CURRENCY): + """Plot long-term data.""" + try: + if os.path.exists('balances_24h.h5'): + balances_24h = pd.HDFStore('balances_24h.h5') + df = balances_24h['df_24h'].iloc[1:, :] + balances_24h.close() + if period == -1: + no_of_days = len(df) + else: + no_of_days = period + df = df.tail(no_of_days) + plot_and_send_image(df, currency, f"{no_of_days} days plot", "plot_long.png") + else: + message = "No balances_24h.h5 file" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + except Exception: + message = "Could not generate long plot" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +def plot_30days(currency=PLOT_CURRENCY): + """Plot 30 days data.""" + try: + if os.path.exists('balances_4h.h5'): + balances_4h = pd.HDFStore('balances_4h.h5') + df = balances_4h['df_4h'].iloc[1:, :] + balances_4h.close() + df = df.tail(180) + plot_and_send_image(df, currency, "30 Day Balances", "30_days.png") + else: + message = "No balances_4h.h5 file" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + except Exception: + message = "Could not generate 30 day plot" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +def plot_7days(currency=PLOT_CURRENCY): + """Plot 7 days data.""" + try: + if os.path.exists('balances_1h.h5'): + balances_1h = pd.HDFStore('balances_1h.h5') + df = balances_1h['df_1h'] + end_date = datetime.now().replace(microsecond=0) + cut_before_date = end_date - timedelta(days=7) + df = df.loc[df.index >= cut_before_date] + plot_and_send_image(df, currency, "7 Day Balances", "7_days.png") + else: + message = "No balances_1h.h5 file" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + except Exception: + message = "Could not generate 7 day plot" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +def plot_24h(currency=PLOT_CURRENCY): + """Plot 24 hours data.""" + try: + if os.path.exists('balances.h5'): + balances = pd.HDFStore('balances.h5') + df = balances['df'].iloc[1:, :] + df['date_time'] = pd.to_datetime(df['date_time']) + balances.close() + df = df.set_index('date_time') + plot_and_send_image(df, currency, TODAY, "24_hour.png") + else: + message = "No balances.h5 file" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + except Exception: + message = "Could not generate 24h plot" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +def send_plots(): + """Send all generated plots to Telegram.""" + try: + balances = pd.HDFStore('balances.h5') + df = balances['df'] + balances.close() + GBP = df['£'].iloc[-1] + USDT = df['$'].iloc[-1] + BTC = df['BTC'].iloc[-1] + + message = f"Balance:\n GBP £ {round(GBP, 2)}\n USD $ {round(USDT, 2)}\n BTC {round(BTC, 6)}" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + + # Send all plots + for file_name in ["plot_long.png", "30_days.png", "7_days.png", "24_hour.png"]: + if os.path.exists(file_name): + telegram_send_image(BOT_TOKEN, BOT_CHAT_ID, file_name) + else: + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, f"Could not find {file_name}") + + print(f"{datetime.now().replace(microsecond=0)} - Sent plots to Telegram.") + except Exception: + message = "Could not send plots." + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +def resample_data(): + """Resample data to 1h, 4h, and 24h.""" + try: + if not os.path.exists("balances_1h.h5"): + balances = pd.HDFStore('balances.h5') + df = balances['df'] + df['date_time'] = pd.to_datetime(df['date_time']) + df = df.set_index('date_time') + balances.close() + + # Resample to 1h + data_1h = df.resample("H").mean() + balances_1h = pd.HDFStore('balances_1h.h5') + balances_1h['df_1h'] = data_1h + balances_1h.close() + + if not os.path.exists("balances_4h.h5"): + balances_1h = pd.HDFStore('balances_1h.h5') + df_1h = balances_1h['df_1h'] + balances_1h.close() + + # Resample to 4h + data_4h = df_1h.resample("4H").mean() + balances_4h = pd.HDFStore('balances_4h.h5') + balances_4h['df_4h'] = data_4h + balances_4h.close() + + if not os.path.exists("balances_24h.h5"): + balances_4h = pd.HDFStore('balances_4h.h5') + df_4h = balances_4h['df_4h'] + balances_4h.close() + + # Resample to 24h + data_24h = df_4h.resample("24H").mean() + balances_24h = pd.HDFStore('balances_24h.h5') + balances_24h['df_24h'] = data_24h + balances_24h.close() + + except Exception: + message = "Error in resampling data" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +def delete_old_files(): + """Delete old files in the directory.""" + try: + directory = "path_to_your_directory" + for file_name in os.listdir(directory): + file_path = os.path.join(directory, file_name) + if os.path.getmtime(file_path) < time.time() - 7 * 86400: + os.remove(file_path) + except Exception: + message = "Error in deleting old files" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +def archive_data(): + """Archive old data to a zip file.""" + try: + archive_name = f"archive_data_{TODAY}.zip" + if not os.path.exists('archive_data'): + os.makedirs('archive_data') + shutil.make_archive(f'archive_data/{archive_name}', 'zip', 'path_to_your_directory') + except Exception: + message = "Error in archiving data" + telegram_send_message(BOT_TOKEN, BOT_CHAT_ID, message) + return () + + +# Scheduling tasks +schedule.every().day.at("00:00").do(archive_data) +schedule.every().day.at("01:00").do(delete_old_files) +schedule.every().day.at("02:00").do(resample_data) +schedule.every().day.at("02:30").do(plot_long) +schedule.every().day.at("03:00").do(plot_30days) +schedule.every().day.at("03:30").do(plot_7days) +schedule.every().day.at("04:00").do(plot_24h) +schedule.every().day.at("05:00").do(send_plots) + +# Main loop +while True: + schedule.run_pending() + time.sleep(1) diff --git a/Python_freqtrade/Live_trading_updates/stake_amount.txt b/Python_freqtrade/Live_trading_updates/stake_amount.txt new file mode 100644 index 0000000..1018216 --- /dev/null +++ b/Python_freqtrade/Live_trading_updates/stake_amount.txt @@ -0,0 +1 @@ +4164 \ No newline at end of file diff --git a/Python_freqtrade/walkforward_optimisation.py b/Python_freqtrade/walkforward_optimisation.py new file mode 100644 index 0000000..f5984b0 --- /dev/null +++ b/Python_freqtrade/walkforward_optimisation.py @@ -0,0 +1,699 @@ +""" +MattC - 2025 +This code is a working prototype and is intended for initial testing and development purposes. Some Python +standards, including but not limited to PEP 8 compliance, error handling, and code optimization, are yet to be fully +implemented. Further refactoring and enhancements are planned to improve readability, maintainability, +and efficiency. +""" + +import os +from os import walk +import json +from datetime import datetime, timedelta +import pandas as pd +import shutil +from pathlib import Path +import logging +import matplotlib.pyplot as plt # pip install matplotlib +import matplotlib.dates as mdates +import warnings +import urllib +from urllib.request import urlopen +import time + +warnings.simplefilter(action='ignore', category=FutureWarning) +logger = logging.getLogger(__name__) + + +# TODO sort out download data +class WalkForward(object): + + def __init__(self, strategy_path, strategy, config, output_dir, wf_start, wf_finish, anchored_start, min_trades, + in_sample_days, out_sample_days, loss_function, cpu, epochs, wallet="2500", fee="0.002", + pre_live=False, re_opt=False, re_opt_t="5m 1h 1d"): + + self.in_sample_days = in_sample_days + self.out_sample_days = out_sample_days + self.loss_function = loss_function + self.re_opt = re_opt + self.output_dir = self.create_run_dir(output_dir) + self.config = self.copy_input_config(config) + self.strategy_path = strategy_path + self.strategy_name = strategy + self.strategy = self.copy_input_strategy(strategy_path, self.strategy_name) + self.wf_start = wf_start + self.wf_finish = wf_finish + self.anchored_start = anchored_start + self.is_start = self.in_sample_start() + self.min_trades = min_trades + self.cpu = cpu + self.epochs = epochs + self.wallet = wallet + self.fee = fee + self.pre_live = pre_live + self.re_opt_t = re_opt_t + self.start_log() + + def run_walk_forward(self): + + stages = self.generate_wf_stages() + no_stages = len(stages) + start_wallet = self.wallet + + start_time = datetime.now() + logger.info(f'Running Walk-forward for {no_stages} stages') + + for count, value in enumerate(stages, 1): + stage_start_time = datetime.now() + logger.info(f'{"-" * 79}') + hyperopt_time = value[0] + backtest_time = value[1] + full_time_period = f"{hyperopt_time.split('-')[0]}-{backtest_time.split('-')[1]}" + logger.info(f'Walk-forward optimization for stage {count} of {no_stages}') + stage_dir = self.create_stage_dir(self.output_dir, count, full_time_period) + + # Run Hyperopt and process required data + logger.info(f'Hyperopting for {hyperopt_time}') + cpu = self.set_cpu(hyperopt_time) + self.run_hyperopt(hyperopt_time, stage_dir, self.epochs, cpu) + hy_start = datetime.strptime(hyperopt_time.split('-')[0], "%Y%m%d") + hy_finish = datetime.strptime(hyperopt_time.split('-')[1], "%Y%m%d") + hy_delta = hy_start - hy_finish + hy_days = int(hy_delta.days) + logger.info(f'Running Hyperopt Backtest for period {hyperopt_time} ({hy_days} days)') + result_op, bt_file_op = self.run_backtest(hyperopt_time, "op_bt", start_wallet, stage_dir) + self.save_bt_file_data(stage_dir, bt_file_op, "op_bt") + df_op = self.update_df(result_op, "op_bt", count, hyperopt_time, start_wallet) + + # Walk forward testing constant starting wallet: + logger.info(f'Running WF Backtest for period {backtest_time} ({self.out_sample_days} days)') + result_bt, bt_file_bt = self.run_backtest(backtest_time, "wf_bt", start_wallet, stage_dir) + self.save_bt_file_data(stage_dir, bt_file_bt, "wf_bt") + df_wf = self.update_df(result_bt, "wf_bt", count, backtest_time, start_wallet) + plot_equity_curve(df_wf, self.output_dir, save_fig=True) + + self.combine_data(hyperopt_time, df_op, df_wf) + stage_run_time = str(datetime.now() - stage_start_time) + logger.info(f'Stage {count} walk-forward analysis Duration: {stage_run_time.split(".")[0]}') + + if self.pre_live: + self.pre_live_optimise() + + if self.re_opt: + self.re_optimise() + + run_time = str(datetime.now() - start_time) + logger.info(f'Total walk-forward analysis Duration: {run_time.split(".")[0]}') + + def re_optimise(self): + # Download data + # time_periods = "5m 15m 1h 4h 12h 1d" + time_periods = self.re_opt_t + self.download_data(time_periods) + + end_date = datetime.strptime(self.wf_finish, "%Y%m%d") + start_date = end_date - timedelta(int(self.in_sample_days)) + t1 = start_date.strftime("%Y%m%d") + t2 = end_date.strftime("%Y%m%d") + hyperopt_time = t1 + "-" + t2 + full_time_period = f"{hyperopt_time}" + + stage_dir = self.create_stage_dir(self.output_dir, "re_optimise", full_time_period) + epochs = f"{int(self.epochs)}" + self.run_hyperopt(hyperopt_time, stage_dir, epochs, self.cpu) + logger.info(f'Running Hyperopt Backtest for period {hyperopt_time} ({self.in_sample_days} days)') + _ = self.run_backtest(hyperopt_time, "re_optimise", self.wallet, stage_dir) + + def save_bt_file_data(self, stage_dir, bt_file_op, file_id): + + with open(f'{stage_dir}/{bt_file_op}') as f: + data1 = json.load(f) + df1 = pd.DataFrame.from_dict(data1["strategy"][self.strategy_name]["results_per_pair"]) + csv_filepath = f"{stage_dir}/{file_id}_bt_results.csv" + df1.to_csv(csv_filepath) + + def update_df(self, result, file_id, wf_stage, time_range, start_wallet): + + h5_string = f"{self.output_dir}/{file_id}.h5" + if not os.path.exists(h5_string): + cols = ["profit_mean", "profit_mean_pct", "profit_sum", "profit_sum_pct", "profit_total_abs", + "profit_total_pct" "profit_total", "wins", "draws", "losses", "wf-stage", "bt_time_period", + "start-balance", "final-balance", "%_profit_pa", "acc-start", "acc-finish"] + df = pd.DataFrame(columns=cols) + df.to_hdf(h5_string, 'data') + + df = pd.read_hdf(h5_string, 'data') + + if file_id == "op_bt": + is_start = datetime.strptime(time_range.split('-')[0], "%Y%m%d") + is_finish = datetime.strptime(time_range.split('-')[1], "%Y%m%d") + delta = is_start - is_finish + sample_days = int(delta.days) + else: + sample_days = self.out_sample_days + + data = result[0] + data.pop('key') + data["%_profit_pa"] = data["profit_total_pct"] / float(sample_days) * 365 # percent profit year + data["wf-stage"] = wf_stage + data["start-balance"] = start_wallet + data["final-balance"] = float(start_wallet) + float(data["profit_total_abs"]) + data["bt_time_period"] = time_range + + if len(df) == 0: + data["acc-start"] = float(self.wallet) + else: + data["acc-start"] = df["acc-finish"].iloc[-1] + + data["acc-finish"] = (data["acc-start"] * data["profit_total_pct"] / 100) + data["acc-start"] + df2 = pd.DataFrame(data, index=[0]) + df_new = df.copy() + df_new = df_new.append([df2], ignore_index=True) + + csv_filepath = f"{self.output_dir}/{file_id}.csv" + df_new.to_csv(csv_filepath) + df_new.to_hdf(h5_string, 'data') + + return df_new + + def pre_live_optimise(self): + + end_date = datetime.strptime(self.wf_finish, "%Y%m%d") + start_date = end_date - timedelta(int(self.in_sample_days)) + t1 = start_date.strftime("%Y%m%d") + t2 = end_date.strftime("%Y%m%d") + hyperopt_time = t1 + "-" + t2 + full_time_period = f"{hyperopt_time}" + + stage_dir = self.create_stage_dir(self.output_dir, "pre_live", full_time_period) + epochs = f"{int(self.epochs) * 2}" + self.run_hyperopt(hyperopt_time, stage_dir, epochs, self.cpu) + logger.info(f'Running Hyperopt Backtest for period {hyperopt_time} ({self.in_sample_days} days)') + result_op = self.run_backtest(hyperopt_time, "pre_live", self.wallet, stage_dir) + + def combine_data(self, hyp_time_range, df_op, df_wf): + + is_start = datetime.strptime(hyp_time_range.split('-')[0], "%Y%m%d") + is_finish = datetime.strptime(hyp_time_range.split('-')[1], "%Y%m%d") + delta = is_start - is_finish + days_in = int(delta.days) + + days_out = int(self.out_sample_days) + + try: + a = pd.Series(df_op["profit_mean_pct"], name='op_profit_av') + b = pd.Series(df_wf["profit_mean_pct"], name='wf_profit_av') + c = pd.Series(df_op["wins"] / df_op["losses"], name='op_wl%') + d = pd.Series(df_wf["wins"] / df_wf["losses"], name='wf_wl%') + e = pd.Series(df_op['trades'].apply(lambda x: x / days_in), name='op_trades_per_day') + f = pd.Series(df_wf['trades'].apply(lambda x: x / days_out), name='wf_trades_per_day') + g = pd.Series(df_op["profit_total"].apply(lambda x: (x / days_in) * 365), name='op_ppa') + h = pd.Series(df_wf["profit_total"].apply(lambda x: (x / days_out) * 365), name='wf_ppa') + i = pd.Series(df_op["%_profit_pa"], name='op_%ppa') + j = pd.Series(df_wf["%_profit_pa"], name='wf_%ppa') + + df_combined = pd.concat([a, b, c, d, e, f, g, h, i, j], axis=1) + filepath = f"{self.output_dir}/combined.csv" + df_combined.to_csv(filepath) + + except: + pass + + return + + def generate_wf_stages(self): + + is_start = datetime.strptime(self.is_start, "%Y%m%d") + oos_start = datetime.strptime(self.wf_start, "%Y%m%d") + end_date = datetime.strptime(self.wf_finish, "%Y%m%d") + oos_end = oos_start + timedelta(int(self.out_sample_days)) + stages = [] + while True: + + t1 = is_start.strftime("%Y%m%d") + t2 = oos_start.strftime("%Y%m%d") + t3 = oos_end.strftime("%Y%m%d") + + # define stage: + insample_timframe = t1 + "-" + t2 + outsample_timframe = t2 + "-" + t3 + stage = [insample_timframe, outsample_timframe] + stages.append(stage) + + oos_start = oos_start + timedelta(int(self.out_sample_days)) + oos_end = oos_start + timedelta(int(self.out_sample_days)) + + if not self.in_sample_days == "anchored": + is_start = is_start + timedelta(int(self.out_sample_days)) + + if oos_end > end_date: + break + + return stages + + def run_hyperopt(self, time_range, stage_dir, epochs, cpu): + """ + :param stage_number: + :param time_range: + :param epochs: + :param loss_function: SortinoHyperOptLoss, + :param fee: + :param cpu: + :return: + """ + self.wait_for_internet_connection() + start_time = datetime.now() + os.system( + "freqtrade hyperopt" + + " --min-trades " + self.min_trades + + " -j " + cpu + + " -e " + epochs + + " --spaces buy " + + " --fee " + self.fee + + " --logfile " + stage_dir + "/op_log" + + " --timerange " + time_range + + " --hyperopt-loss " + self.loss_function + + " --strategy " + self.strategy + + " --strategy-path " + self.output_dir + + " --config " + self.config + + " --dry-run-wallet " + self.wallet + ) + + self.wait_for_internet_connection() + os.system( + "freqtrade hyperopt-list" + + " --no-details " + + " --export-csv " + stage_dir + "/op.csv" + ) + + # # Copy the best optimisation results to "stage output directory": + src = Path(f"{self.output_dir}/{self.strategy}.json") + dst = f"{stage_dir}/op_result.json" + shutil.copyfile(str(src), dst) + + run_time = str(datetime.now() - start_time) + logger.info(f'Hyperopt Duration: {run_time.split(".")[0]}') + + with open(dst) as f: + data = json.load(f) + + results = data["params"]["buy"] + for i in results: + logger.info(f'Hyperopt result: {i}: {results[i]}') + + def run_backtest(self, time_range, file_id, wallet, stage_dir): + self.wait_for_internet_connection() + start_time = datetime.now() + os.system( + "freqtrade backtesting" + + " --export trades " + + " --fee " + self.fee + + f" --logfile {stage_dir}/{file_id}_log.txt" + " --timerange " + time_range + + " --strategy " + self.strategy + + " --strategy-path " + self.output_dir + + " --config " + self.config + + " --dry-run-wallet " + wallet + + f" --export-filename {stage_dir}/{file_id}_result.json" + ) + + result, bt_file = self.get_backtest_data(stage_dir, file_id) + run_time = str(datetime.now() - start_time) + if not file_id == "wf_acc_bt": + logger.info(f'Backtest Duration: {run_time.split(".")[0]}') + + self.log_bt_results(time_range, result, wallet, file_id) + self.plot_bt_profit(time_range, stage_dir, bt_file, file_id) + + return result, bt_file + + def download_data(self, time_periods, days="4000"): + self.wait_for_internet_connection() + logger.info(f'downloading data') + os.system( + "freqtrade download-data" + + " -t " + time_periods + + " --exchange binance " + + " --pairs .*/USDT " + + " --new-pairs-days " + days + + " --include-inactive-pairs " + ) + logger.info(f'finished downloading data') + + return + + @staticmethod + def wait_for_internet_connection(): + start_time = datetime.now() + switch = True + while True: + try: + urlopen('https://www.google.com', timeout=1) + if not switch: + offline_time = str(datetime.now() - start_time) + logger.warning(f"Disconnected time: {offline_time.split('.')[0]}") + logger.warning("#############################") + return + + except urllib.error.URLError: + + if switch: + logger.warning("#############################") + logger.warning("NO INTERNET") + + switch = False + time.sleep(2) + + pass + + def log_bt_results(self, time_range, result, wallet, file_id): + + data = result[0].copy() + + final_balance = float(wallet) + data["profit_total_abs"] + w_start = round(float(wallet), 2) + w_finish = round(final_balance, 2) + percent_prof = round(data["profit_total_pct"], 1) + + if file_id == "op_bt": + + is_start = datetime.strptime(time_range.split('-')[0], "%Y%m%d") + is_finish = datetime.strptime(time_range.split('-')[1], "%Y%m%d") + delta = is_start - is_finish + sample_days = int(delta.days) + + else: + sample_days = self.out_sample_days + + pppa = round((percent_prof / float(sample_days) * 365), 2) # percent profit per year + logger.info(f'Backtest result - Balance £{w_start} --> £{w_finish} ({percent_prof}%): {pppa} %profit pa') + + # Trade stats + win = data['wins'] + loss = data['losses'] + draw = data['draws'] + trades = data['trades'] + logger.info(f"Backtest result - Wins: {win}, Draws: {draw}, Losses: {loss}, trades: {trades}") + + # Average tade profits: + mean_p = round(float(data['profit_mean']), 2) + mp_percent = round(float(data['profit_mean_pct']), 2) + logger.info(f"Backtest result - mean trade profit £{mean_p}, {mp_percent}%") + + # Account draw-down: + dd_percent = round((data['max_drawdown_account'] * 100), 2) + dd_abs = round(float(data['max_drawdown_abs']), 2) + logger.info(f"Backtest result - Max dd: {dd_percent}%, £{dd_abs}") + + def plot_bt_profit(self, time_range, stage_dir, bt_file, file_id): + os.system( + "freqtrade plot-profit " + " --timeframe 1d " + " --timerange " + time_range + + " --strategy " + self.strategy + + " --strategy-path " + self.output_dir + + " --config " + self.config + + f" --export-filename {stage_dir}/{bt_file}" + ) + + # TODO relative path required: + src = Path(f"/home/matt/freqtrade/user_data/plot/freqtrade-profit-plot.html") + dst = f"{stage_dir}/{file_id}_profit-plot.html" + shutil.copyfile(str(src), dst) + + @staticmethod + def get_backtest_data(stage_dir, file_id): + f = [] + for (dirpath, dirnames, filenames) in walk(stage_dir): + f.extend(filenames) + break + + # Find the backtest file for wf stage + for file in f: + + # not meta.json: + if file[-9:-5] != "meta": + + my_file = file_id + "_result" + if file.split("-")[0] == my_file: + # change to while open: + f = open(stage_dir + "/" + file) + data = json.load(f) + result = data["strategy_comparison"] + backtest_file = file + + return result, backtest_file + + def create_run_dir(self, path): + + _time = datetime.now().strftime('%Y%m%d_%I:%M%p') + if self.re_opt: + directory = f"{path}/{_time}_{self.loss_function}_re-optimise_{self.in_sample_days}" + else: + directory = f"{path}/{_time}_{self.loss_function}_in_{self.in_sample_days}_out_{self.out_sample_days}" + + if not os.path.exists(path): + os.makedirs(path) + + try: + os.makedirs(directory) + + except: + pass + + return directory + + @staticmethod + def create_stage_dir(path, stage, full_time_period): + dir_string = f"{path}/stage_{stage}_{full_time_period}" + if not os.path.exists(path): + os.makedirs(path) + + try: + os.makedirs(dir_string) + + except Exception as e: + pass + + return dir_string + + def copy_input_config(self, in_config): + # Copy config: + config = f"{in_config}" + dst_config = f"{self.output_dir}/{config.split('/')[-1]}" + shutil.copyfile(config, dst_config) + return dst_config + + def copy_input_strategy(self, in_strategy_path, strategy): + # Copy Strategy + src = f"{in_strategy_path}/{strategy}.py" + dst_strategy = f"{self.output_dir}/{strategy}.py" + shutil.copyfile(src, dst_strategy) + return strategy + + def in_sample_start(self): + + oos_start = datetime.strptime(self.wf_start, "%Y%m%d") + + if self.in_sample_days == "anchored": + is_start = datetime.strptime(self.anchored_start, "%Y%m%d") # + oos days + + else: + is_start = oos_start - timedelta(int(self.in_sample_days)) + + is_start = is_start.strftime("%Y%m%d") + + return is_start + + def set_cpu(self, hyperopt_time): + + is_start = datetime.strptime(hyperopt_time.split('-')[0], "%Y%m%d") + is_finish = datetime.strptime(hyperopt_time.split('-')[1], "%Y%m%d") + delta = is_finish - is_start + days = int(delta.days) + + if self.in_sample_days == "anchored": + if days < 50: + cpu = "-1" + if days > 50: + cpu = "-2" + if days > 100: + cpu = "-4" + if days > 150: + cpu = "-6" + if days > 200: + cpu = "-8" + if days > 250: + cpu = "-10" + if days > 300: + cpu = "-11" + if days > 350: + cpu = "-12" + if days > 400: + cpu = "-13" + if days > 450: + cpu = "-14" + if days > 500: + cpu = "-14" + if days > 550: + cpu = "-16" + if days > 600: + cpu = "-16" + if days > 650: + cpu = "-17" + if days > 700: + cpu = "-17" + if days > 750: + cpu = "-17" + if days > 800: + cpu = "-18" + if days > 850: + cpu = "-18" + if days > 900: + cpu = "-18" + if days > 950: + cpu = "-19" + if days > 1000: + cpu = "-19" + if days > 1100: + cpu = "-20" + else: + d1 = datetime.strptime("20220601", "%Y%m%d") + d2 = datetime.strptime("20210101", "%Y%m%d") + d3 = datetime.strptime("20200101", "%Y%m%d") + d4 = datetime.strptime("20190101", "%Y%m%d") + d5 = datetime.strptime("20180101", "%Y%m%d") + + if is_finish > d1: + cpu = self.cpu + + if d2 < is_finish < d1: + cpu = int(self.cpu) + 1 + + if d3 < is_finish < d2: + cpu = int(self.cpu) + 2 + + if d4 < is_finish < d3: + cpu = int(self.cpu) + 3 + + if d5 < is_finish < d4: + cpu = int(self.cpu) + 4 + + if is_finish < d5: + cpu = int(self.cpu) + 5 + + return str(cpu) + + def start_log(self): + # Set format and level: + logger.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - WF - %(levelname)s - %(message)s') + + # Remove old handlers: + while logger.handlers: + logger.handlers.pop() + + # Define file handler: + file_handler = logging.FileHandler(f'{self.output_dir}/wf.log') + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # Define console handler: + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + # Startup logs: + _cpu = 20 + int(self.cpu) + 1 + start_date = datetime.strptime(self.is_start, '%Y%m%d').date() + end_date = datetime.strptime(self.wf_finish, '%Y%m%d').date() + run_time = str(end_date - start_date).split(",")[0] + logger.info('Code Initiated') + logger.info(f'Strategy:{self.strategy_path}/{self.strategy}') + logger.info(f'Config:{self.config}') + logger.info(f'Loss Function: {self.loss_function}') + logger.info(f'Time Frame:{self.is_start}-{self.wf_finish} ({run_time})') + logger.info(f'IS-days:{self.in_sample_days}, OOS-days:{self.out_sample_days}') + logger.info(f'CPUs:{_cpu}, Epochs:{self.epochs}, Wallet:{"2500"}, Fee:{"0.002"}, Min-trades:{self.min_trades}') + + +def plot_equity_curve(df, output_dir, save_fig=False): + df = df.copy() + pd.set_option('display.max_columns', None) + fig, axs = plt.subplots(figsize=(7, 4)) + axs.xaxis.set_major_formatter(mdates.DateFormatter("%d %b")) + df['dates'] = df["bt_time_period"].apply(lambda i: i.split("-")[1]) + df['dt'] = df['dates'].apply(lambda i: datetime.strptime(i, '%Y%m%d')) + df.set_index('dt') + df.plot(ax=axs, x="dt", y="acc-finish") + dela_y = int(df["acc-finish"].max()) - int(df["acc-finish"].min()) + y_min = int(df["acc-finish"].min()) - (dela_y * 0.05) + y_max = int(df["acc-finish"].max()) + (dela_y * 0.05) + axs.set_title('Walk-Forward equity curve') + axs.set_ylim(y_min, y_max) + axs.set_ylabel("") + axs.set_xlabel("") + axs.grid(color='grey', alpha=0.5, linestyle='dashed', linewidth=0.5) + axs.yaxis.set_major_formatter("£" + '{x:1.0f}') + # plt.show() + + if save_fig: + try: + plt.savefig(f"{output_dir}/wf_equity_curve.png") + except: + pass + + +def walk_forward(path, strategy, config, output_dir, wf_start, wf_finish, anchored_start, pre_live=False, re_opt=False): + is_list = ["730"] + oos_list = ["30"] + n_trades = ["100"] + cpu_list = ["-15"] + + ep = "100" + loss_f = "SharpeHyperOptLoss" + + for count, is_days in enumerate(is_list): + oos_days = oos_list[count] + nt = n_trades[count] + cores = cpu_list[count] + + wf = WalkForward(strategy_path=path, strategy=strategy, config=config, output_dir=output_dir, + wf_start=wf_start, wf_finish=wf_finish, anchored_start=anchored_start, epochs=ep, + loss_function=loss_f, in_sample_days=is_days, out_sample_days=oos_days, min_trades=nt, + cpu=cores, pre_live=pre_live, re_opt=re_opt) + + wf.run_walk_forward() + return + + +def re_optimise(path, strategy, config, output_dir, cpu="-19"): + today = datetime.now().strftime("%Y%m%d") + loss_f = "SortinoHyperOptLoss" + in_sample_days = "730" + n_trades = "100" + ep = "200" + download_data_t = "5m 1h 1d" + + wf = WalkForward(strategy_path=path, strategy=strategy, config=config, output_dir=output_dir, wf_start=today, + wf_finish=today, anchored_start=today, epochs=ep, loss_function=loss_f, + in_sample_days=in_sample_days, out_sample_days="1", min_trades=n_trades, cpu=cpu, + pre_live=False, re_opt=True, re_opt_t=download_data_t) + + wf.re_optimise() + + +if __name__ == "__main__": + WF_START = "20200101" # 2023-03-25 + WF_END = "20230910" + ANCHORED_START = "20200101" # only used if in_sample_days == "anchored" + STRATEGY_PATH_THOR = "/home/matt/freqtrade/user_data/strategies/Thor" + STRATEGY_THOR = "Optimise_Thor_BuySig_RiskReward" + CONFIG_THOR = "/home/matt/freqtrade/user_data/strategies/Thor/config_Thor_WF.json" + OUTPUT_DIR_THOR = "/home/matt/freqtrade/user_data/strategies/Thor/walk_forward" + + # ------------------------------------------------------------- + walk_forward(STRATEGY_PATH_THOR, STRATEGY_THOR, CONFIG_THOR, OUTPUT_DIR_THOR, WF_START, WF_END, ANCHORED_START) + + # ------------------------------------------------------------- + re_optimise(STRATEGY_PATH_THOR, STRATEGY_THOR, CONFIG_THOR, OUTPUT_DIR_THOR) diff --git a/README.md b/README.md new file mode 100644 index 0000000..28f7bea --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# MT5_Python_Strategy_Framework + +This project provides an experimental framework for integrating MetaTrader 5 (MT5) custom indicators and trading logic with Python-based data processing and strategy testing using [Freqtrade](https://www.freqtrade.io/). + +## Features + +- 🧠 **Custom MT5 Libraries**: Modular `.mqh` files to handle position sizing, drawdown control, order management, and utility functions. +- 🐍 **Python Scripts**: + - `pre_process.py`: Prepares or cleans data before indicator processing. + - `process_entry_indicators.py`: Extracts and processes entry signals. + - `post_process_test.py`: Analyses backtest output or result data. +- 📦 **Freqtrade-Compatible Module**: Python strategies and helpers located in `Python_freqtrade/` for integration with the Freqtrade framework. +- 🛠️ **Project Structure Support**: Includes `.idea/` and `.vscode/` folders for JetBrains and VSCode IDE configurations. + +## Project Structure + +``` +MT5_Python_Strategy_Framework/ +├── My_MQL5_Libs/ # Custom MQL5 include files +├── Python_freqtrade/ # Freqtrade strategy components +├── pre_process.py # Data pre-processing script +├── process_entry_indicators.py # Entry signal extraction logic +├── post_process_test.py # Backtest result post-processing +├── .idea/, .vscode/ # IDE configs (optional) +└── README.md # Project documentation +``` + +## Getting Started + +### Requirements + +- MetaTrader 5 with access to `terminal64.exe` +- Python 3.8+ +- Optional: Freqtrade installed (`pip install freqtrade`) + +### Running Scripts + +```bash +python pre_process.py +python process_entry_indicators.py +python post_process_test.py +``` + +### MT5 Library Usage + +Place the `.mqh` files from `My_MQL5_Libs/` into your `MQL5/Include` folder to use them in your Expert Advisors or custom indicators. + +## Notes + +- This project is a scaffold for connecting MQL5 strategies to Python-based optimisation and analysis tools. +- Actual EA logic, data formats, and strategy specifics should be customised to your use case. + +## License + +This project is provided for educational and prototyping purposes. Please adapt and extend as needed for production environments. diff --git a/post_process_test.py b/post_process_test.py new file mode 100644 index 0000000..70f1e3e --- /dev/null +++ b/post_process_test.py @@ -0,0 +1,223 @@ +import pandas as pd +from pathlib import Path +import logging +from xml.sax import ContentHandler, parse +from typing import List, Tuple + +from pandas import DataFrame + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + + +class PostProcessData: + """ + Class for post-processing and combining results from multiple indicator testing files. + """ + + def __init__(self, results_dir: Path, print_outputs: bool = True, output_file: str = '1_combined_results.csv'): + """ + Initializes the PostProcessData class with the provided directory and output file. + + Args: + results_dir (Path): The directory where the result files are located. + print_outputs (bool): Whether to print the final output. + output_file (str): The name of the combined output CSV file. + """ + self.results_dir = results_dir + self.print_outputs = print_outputs + self.output_file = output_file + self.run() + + def run(self): + """ + Processes result files, calculates statistics, and saves the combined results. + """ + df_combined = self.process_results_files() + + if df_combined is not None: + # Save the combined results as a CSV file + output_path = self.results_dir / self.output_file + df_combined.to_csv(output_path, index=False) + self.combine_opt_results() + + if self.print_outputs: + print(df_combined) + else: + logging.info("Results processing complete.") + else: + logging.warning("No valid results to process.") + + def process_results_files(self): + """ + Processes each indicator result file (ins.xml and out.xml) and computes the statistics. + + Returns: + pd.DataFrame: Combined results DataFrame. + """ + df_combined = [] + failed_post_process_list = [] + + for file in self.results_dir.iterdir(): + if file.suffix == ".xml" and file.name.endswith("ins.xml"): + file_prefix = file.stem[:-4] # Remove '_ins' suffix + try: + df_combined.append(self.process_indicator_file(file_prefix)) + except Exception as e: + failed_post_process_list.append(file_prefix) + logging.error(f"Failed to process {file_prefix}: {e}") + + # Return combined DataFrame + if df_combined: + return pd.DataFrame(df_combined) + else: + return + + def process_indicator_file(self, file_prefix: str) -> dict: + """ + Processes a pair of `ins.xml` and `out.xml` files for an indicator, calculates statistics, + and returns a dictionary of the results. + + Args: + file_prefix (str): The base name of the indicator files (without extensions). + + Returns: + dict: Dictionary of indicator statistics. + """ + # Load data from XML files + result_in, p_fac_in, trades_in = self.load_xml_data(file_prefix, "ins") + result_out, p_fac_out, trades_out = self.load_xml_data(file_prefix, "out") + + # Calculate result statistics + result_mean = (result_in + result_out) / 2 + pc_result = self.calc_percent_diff(result_in, result_out) + pc_p_fac = self.calc_percent_diff(p_fac_in, p_fac_out) + pc_trades = self.calc_percent_diff(trades_in, trades_out) + + # Return a dictionary with computed data + return { + 'Indicator': file_prefix, + 'R_ins': result_in, + 'R_outs': result_out, + 'R_dif': pc_result, + 'R_mean': result_mean, + 'P_fac_in': p_fac_in, + 'P_fac_out': p_fac_out, + 'P_fac_dif': pc_p_fac, + 'trades_in': trades_in, + 'trades_out': trades_out, + 'trades_dif': pc_trades + } + + def load_xml_data(self, file_prefix: str, file_type: str): + """ + Loads data from an XML file (either 'ins' or 'out'), and extracts the result, profit factor, and trades. + + Args: + file_prefix (str): The prefix of the file (without extension). + file_type (str): The type of the file ('ins' or 'out'). + + Returns: + tuple: Contains result, profit factor, and trades values as floats. + """ + try: + df = self.load_data_from_xml(f"{file_prefix}_{file_type}.xml") + return float(df["Result"][0]), float(df["Profit Factor"][0]), float(df["Trades"][0]) + except Exception as e: + logging.error(f"Error loading {file_prefix}_{file_type}.xml: {e}") + return + + @staticmethod + def load_data_from_xml(file: str) -> pd.DataFrame: + """ + Loads data from an XML file and converts it into a Pandas DataFrame. + + Args: + file (str): The name of the XML file to load. + + Returns: + pd.DataFrame: Data extracted from the XML file. + """ + excel_handler = ExcelHandler() + parse(file, excel_handler) + df = pd.DataFrame(excel_handler.tables[0][1:], columns=excel_handler.tables[0][0]) + return df + + @staticmethod + def calc_percent_diff(in_sample: float, out_sample: float) -> float: + """ + Calculates the percentage difference between two values. + + Args: + in_sample (float): The "in-sample" value. + out_sample (float): The "out-sample" value. + + Returns: + float: The percentage difference between the two values. + """ + try: + return round(abs(in_sample - out_sample) / out_sample * 100.0, 2) + except ZeroDivisionError: + return 0.0 + + def combine_opt_results(self): + """ + Combines all 'opt_results.txt' files in the results directory into one combined file. + """ + combined_file_path = self.results_dir / "2_combined_opt_results.txt" + if combined_file_path.exists(): + combined_file_path.unlink() + + opt_results_list = [file for file in self.results_dir.iterdir() if + file.suffix == ".txt" and "opt_results" in file.name] + new_lines = [] + for file_path in opt_results_list: + new_lines.append(file_path.read_text()) + new_lines.append("\n\n") + + combined_file_path.write_text("".join(new_lines)) + + +class ExcelHandler(ContentHandler): + """ + Custom handler to parse XML files and extract table data. + """ + + def __init__(self): + self.rows = None + self.cells = None + self.chars = [] + self.tables = [] + + def characters(self, content: str): + """ + Collects characters in XML elements. + """ + self.chars.append(content) + + def start_element(self, name: str, attrs): + """ + Handle the start of XML elements. + """ + if name == "Table": + self.rows = [] + elif name == "Row": + self.cells = [] + elif name == "Data": + self.chars = [] + + def end_element(self, name: str): + """ + Handle the end of XML elements. + """ + if name == "Table": + self.tables.append(self.rows) + elif name == "Row": + self.rows.append(self.cells) + elif name == "Data": + self.cells.append("".join(self.chars)) + + +if __name__ == "__main__": + # Example usage with a results directory path + post_processor = PostProcessData(Path(r'path_to_results')) diff --git a/pre_process.py b/pre_process.py new file mode 100644 index 0000000..99dfdfe --- /dev/null +++ b/pre_process.py @@ -0,0 +1,81 @@ +from pathlib import Path +import os + +DEFAULTS_DIR = Path( + r'C:\Users\mkcor\AppData\Roaming\MetaQuotes\Terminal\49CDDEAA95A409ED22BD2287BB67CB9C\MQL5\Experts\My_Experts\NNFX\Entry_testing\default\indicators') +OPTIMISE_DIR = Path( + r'C:\Users\mkcor\AppData\Roaming\MetaQuotes\Terminal\49CDDEAA95A409ED22BD2287BB67CB9C\MQL5\Experts\My_Experts\NNFX\Entry_testing\optimise\indicators') +MASTER_CONFIG_DIR = Path( + r'C:\Users\mkcor\AppData\Roaming\MetaQuotes\Terminal\49CDDEAA95A409ED22BD2287BB67CB9C\MQL5\Experts\My_Experts\NNFX\Entry_testing\optimise\master_config_files') + + +def pre_process_checks(): + """ + Pre-process check that verifies the presence of valid indicator files and generates necessary configuration templates. + + This function: + - Checks if the indicator files in the default and optimizable directories have the correct suffixes. + - Creates master configuration template files if required in the master config directory. + """ + print("---------- Checking: default/indicators ---------------") + check_indicators_suffix(DEFAULTS_DIR) + + print("---------- Checking: optimise/indicators --------------") + check_indicators_suffix(OPTIMISE_DIR) + + print("---------- Checking: optimise/master_config_files -----") + create_cp_templates(OPTIMISE_DIR, MASTER_CONFIG_DIR) + + +def check_indicators_suffix(dir): + """ + Verifies if the indicator files in the specified directory have valid suffixes. + + Args: + dir (Path): The directory containing the indicator files. + + This function checks that all .mq5 and .ex5 files have one of the following suffixes: + - "clc", "cbc", "clx", "hcc", "hlx", "lcc", "0lx", "2lx" + + If a file does not have one of these suffixes, a warning is printed. + """ + suffex_list = ["clc", "cbc", "clx", "hcc", "hlx", "lcc", "0lx", "2lx"] + for file in os.listdir(dir): + filename, file_extension = os.path.splitext(file) + + if file_extension in [".mq5", ".ex5"]: + file_suffix = filename[-3:] + if file_suffix not in suffex_list: + print(f"{file} - INCORRECT FILE SUFFIX") + print("Complete.") + + +def create_cp_templates(opt_dir, master_config_dir): + """ + Creates master configuration template files in the specified master config directory. + + Args: + opt_dir (Path): The directory containing the optimizable indicator files. + master_config_dir (Path): The directory where the master configuration files should be created. + + This function generates a new template file for each .ex5 file in the optimizable directory, + creating a template file with the same name in the master config directory. If a corresponding + .ini file already exists, it skips the creation. + """ + for file in os.listdir(opt_dir): + filename, file_extension = os.path.splitext(file) + if file_extension == ".ex5": + template_file = master_config_dir / filename + ini_file = master_config_dir / f"{filename}.ini" + + if ini_file.exists() and template_file.exists(): + template_file.unlink() # Remove existing template file if it exists + else: + with open(template_file, 'w') as f: + f.write("Template content here") # Add content to the template + print(f"Master config for required - {filename}") + print("Complete.") + + +if __name__ == "__main__": + pre_process_checks() diff --git a/process_entry_indicators.py b/process_entry_indicators.py new file mode 100644 index 0000000..1848e88 --- /dev/null +++ b/process_entry_indicators.py @@ -0,0 +1,436 @@ +""" +MattC - 2025 +This code is a working prototype and is intended for initial testing and development purposes. Some Python +standards, including but not limited to PEP 8 compliance, error handling, and code optimization, are yet to be fully +implemented. Further refactoring and enhancements are planned to improve readability, maintainability, +and efficiency. +""" + +from pathlib import Path +import configparser +import os +import pandas as pd +from subprocess import call +from post_process_test import PostProcessData as ppd + +MT5_TERM_EXE = Path(r'C:\Program Files\FTMO MetaTrader 5\terminal64.exe') +MT5_DIRECTORY = Path(r"C:\Users\mkcor\AppData\Roaming\MetaQuotes\Terminal\49CDDEAA95A409ED22BD2287BB67CB9C") + + +## + +class TestParent: + + def __init__(self, name, start_date, end_date, chart_period, custom_loss_function, symbol_mode, data_split): + """ + @param name: test name e.g. "name" + @param start_date: backtest start e.g. "2010.10.01" + @param end_date: backtest end e.g. "2010.10.01" + @param chart_period: "Daily", "H4", "H1", "M15", etc + @param custom_loss_function: "0"= W/L ratio, "1"= W percent, "2"=W percent (min 200 trades) .... + @param symbol_mode: "0"=Chart sym only, "1"= Multi sym FX5, "2"=Multi sym 28FX pairs + @param data_split: "year" or "month" + """ + self.name = name + self.start_date = start_date + self.end_date = end_date + self.chart_period = chart_period + self.custom_loss_function = custom_loss_function + self.symbol_mode = symbol_mode + self.data_split = data_split + + self.mt5_term = MT5_TERM_EXE + self.mt5_dir = MT5_DIRECTORY + self.mq5_test_cash = Path.joinpath(self.mt5_dir, r"Tester\cache") + self.test_folder = Path.joinpath(self.mt5_dir, r'MQL5\Experts\My_Experts\NNFX\Entry_testing') + self.indi_dir = None + self.indi_rel_path = None + self.output_dir = None + self.results_dir = None + self.master_config_loc = None + self.pct_risk = 2 + self.tp_atr = 1 + self.sl_atr = 1.5 + + def create_test_name(self): + start = self.start_date.replace('.', '') + end = self.end_date.replace('.', '') + test_name = f'{self.name}_{start}_{end}_cl{self.custom_loss_function}_sm{self.symbol_mode}_ds-{self.data_split}_cp-{self.chart_period}' + return test_name + + def check_results_df(self, results_dir): + + file_list = [] + for file in os.listdir(results_dir): + file_list.append(file) + + combined_resuts_file = '1_combined_results.csv' + df_path = Path.joinpath(results_dir, combined_resuts_file) + + if combined_resuts_file not in file_list: + df = pd.DataFrame( + columns=['Indicator', 'Type', 'R_ins', 'R_outs', 'R_dif', 'R_mean', 'P_fac_in', 'P_fac_out', + 'P_fac_dif', 'trades_in', 'trades_out', 'trades_dif']) + df.to_csv(df_path, index=False) + + return df_path + + def create_indi_optimisation_ini(self, config_paser, indicator, config_files_dir, sample_data, force_optimisation, + opt_os=False): + + config = config_paser + config['Tester']['Expert'] = str(Path.joinpath(self.indi_rel_path, indicator)) + ".ex5" + config['Tester']['Symbol'] = "EURUSD" + config['Tester']['Period'] = f"{self.chart_period}" + config['Tester']['Optimization'] = "2" + config['Tester']['Model'] = "1" + config['Tester']['FromDate'] = f'{self.start_date}' + config['Tester']['ToDate'] = f'{self.end_date}' + config['Tester']['ForwardMode'] = "0" + config['Tester']['Deposit'] = "100000" + config['Tester']['Currency'] = "USD" + config['Tester']['ProfitInPips'] = "0" + config['Tester']['Leverage'] = "100" + config['Tester']['ExecutionMode'] = "0" + config['Tester']['OptimizationCriterion'] = "6" + config['Tester']['Visual'] = "0" + config['Tester']['ReplaceReport'] = "1" + config['Tester']['ShutdownTerminal'] = "1" + config['TesterInputs']['inp_lot_mode'] = "2||0||0||2||N" + config['TesterInputs']['inp_lot_var'] = f"{self.pct_risk}||2.0||0.2||20||N" + config['TesterInputs']['inp_sl_mode'] = "2||0||0||5||N" + config['TesterInputs']['inp_sl_var'] = f"{self.sl_atr}||1.0||0.1||10||N" + config['TesterInputs']['inp_tp_mode'] = "2||0||0||5||N" + config['TesterInputs']['inp_tp_var'] = f"{self.tp_atr}||1.5||0.15||15||N" + config['TesterInputs']['inp_custom_criteria'] = f"{self.custom_loss_function}||0||0||1||N" + config['TesterInputs']['inp_sym_mode'] = f"{self.symbol_mode}||0||0||2||N" + config['TesterInputs']['inp_force_opt'] = f"1||1||1||2||{force_optimisation}" + + if sample_data == "in": + + config['Tester']['report'] = f"{indicator}_ins" + + if self.data_split == "year": + config['TesterInputs']['inp_data_split_method'] = f"1||0||0||3||N" + + if self.data_split == "month": + config['TesterInputs']['inp_data_split_method'] = f"3||0||0||3||N" + + if sample_data == "out": + + config['Tester']['report'] = f"{indicator}_out" + + if self.data_split == "year": + config['TesterInputs']['inp_data_split_method'] = f"2||0||0||3||N" + + if self.data_split == "month": + config['TesterInputs']['inp_data_split_method'] = f"4||0||0||3||N" + + if opt_os: + + test_inp_list = list(config.items('TesterInputs')) + test_inp_key = [] + for x in test_inp_list: + test_inp_key.append(x[0]) + + removal_list = ['inp_lot_mode', 'inp_lot_var', 'inp_sl_mode', 'inp_sl_var', 'inp_tp_mode', 'inp_tp_var', + 'inp_custom_criteria', 'inp_sym_mode', 'inp_force_opt', 'inp_data_split_method'] + keys_to_mod = [x for x in test_inp_key if (x not in removal_list)] + + opt_results = self.get_opt_results_from_xml(indicator) + + lines = [] + lines2 = [] + + for key in keys_to_mod: + + for j in test_inp_list: + if key == j[0]: + + string_value = j[1] + string_value = string_value.split("||", 1)[ + 1] # removes the first int from e.g f"2||0||0||3||N" + + for k in opt_results: + + if key == k[0]: + result_val = k[1] + string_value = f'{result_val}||{string_value}' + string_value = string_value[:-1] + string_value = f'{string_value}N' + + config['TesterInputs'][key] = string_value + + value = config['TesterInputs'][key].split("||", 1)[0] + lines.append(f"{key}={value}, ") + lines2.append(f"{value}, ") + + self.save_in_sample_opt_results_to_file(indicator, lines, lines2) + + new_file = indicator + ".ini" + new_file_path = Path.joinpath(config_files_dir, new_file) + + with open(new_file_path, 'w', encoding='utf-16') as configfile: + config.write(configfile) + + def save_in_sample_opt_results_to_file(self, indicator, lines, lines2): + + file_path = Path.joinpath(self.results_dir, f"{indicator}_opt_results.txt") + + if file_path.is_file(): + file_path.unlink() # delete old files. + + lines_mod = '' + for str in lines2: + lines_mod = lines_mod + str + + print(lines_mod[:-2]) + + f = open(file_path, "a") + f.writelines(indicator + "\n") + f.writelines(lines) + f.writelines("\n, ") + f.write(lines_mod[:-2]) + f.close() + + def get_opt_results_from_xml(self, indicator): + + ins_results = f"{indicator}_ins.xml" + df = ppd.load_data_from_xml(self.results_dir) + df = df.drop(['Pass', 'Result', 'Profit', 'Profit Factor', 'Custom', 'Expected Payoff', 'Recovery Factor', + 'Sharpe Ratio', 'Equity DD %', 'Trades'], axis=1) + column_names = list(df.columns.values) + + opt_result = [] + for count, value in enumerate(column_names): + param_results = df[column_names[count]][0] + tup = (value.lower(), param_results) # Convert string to lower case. + opt_result.append(tup) + + return opt_result + + def create_indicator_list(self, df_path, indicator_dir, optimisation=False): + + # Create list + indi_list = [] + for file in os.listdir(os.fsencode(indicator_dir)): + filename = os.fsdecode(file) + if filename.endswith(".ex5"): + indi_name = os.path.splitext(filename)[0] + indi_list.append(str(indi_name)) + + # remove previously processed indicators: + ti_list = [] + for i in indi_list: + + df = pd.read_csv(df_path) + if i in df["Indicator"].tolist(): + # ti_list.append(i) + + print(f"Indicator - {i} - already processed") + print("-" * 60) + + indi_list2 = [x for x in indi_list if x not in ti_list] + conf_ini_list = [] + if optimisation: + + for file in os.listdir(os.fsencode(self.master_config_loc)): + filename = os.fsdecode(file) + if filename.endswith(".ini"): + indi_name = os.path.splitext(filename)[0] + conf_ini_list.append(str(indi_name)) + + removed_list = [x for x in indi_list2 if x not in conf_ini_list] + for i in removed_list: + print(f"Indicator - {i} - NO MASTER CONFIG!") + print("-" * 60) + + indi_list3 = [x for x in indi_list2 if x in conf_ini_list] + + return_list = [] + if optimisation: + return_list = indi_list3 + + else: + return_list = indi_list2 + + for i in return_list: + print(f"Indicator - {i} - To be tested.") + + return return_list + + def create_dir(self, dir_name): + dir_string = Path.joinpath(self.output_dir, dir_name) + if not os.path.exists(dir_string): + os.makedirs(dir_string) + + try: + os.makedirs(dir_string) + + except Exception as e: + pass + + return dir_string + + @staticmethod + def delete_files_in_directory(directory_path): + try: + files = os.listdir(directory_path) + for file in files: + file_path = os.path.join(directory_path, file) + if os.path.isfile(file_path): + os.remove(file_path) + + except OSError: + print("Error occurred while deleting files.") + + +class TestIndicators(TestParent): + + def __init__(self, name, start_date, end_date, chart_period, custom_loss_function, symbol_mode, data_split): + super().__init__(name, start_date, end_date, chart_period, custom_loss_function, symbol_mode, data_split) + + self.template_file = Path.joinpath(self.test_folder, "template.ini") + self.indi_dir = Path.joinpath(self.test_folder, "default\indicators") + self.indi_rel_path = Path(str(self.indi_dir).split(r'\MQL5\Experts')[1][1:]) + self.output_dir = Path.joinpath(Path(self.indi_dir).parents[0], f'Testing\{self.name}') + self.results_dir = self.create_dir("results") + self.run() + + def run(self): + print("~" * 80) + print(f" DEFAULT INDICATOR TEST - {self.name}\n") + + # Create in-sample dir/ delete its content: + input_files_is_dir = self.create_dir("config_files_in_sample") + self.delete_files_in_directory(input_files_is_dir) + + # Create out sample dir/ delete its content: + input_files_os_dir = self.create_dir("config_files_out_sample") + self.delete_files_in_directory(input_files_os_dir) + + df_path = self.check_results_df(self.results_dir) + indicator_list = self.create_indicator_list(df_path, self.indi_dir) + + print("-" * 25 + " STARTING TEST " + "-" * 25) + for indicator in indicator_list: + config_paser = self.load_config_paser() + + # Create run input files: + self.create_indi_optimisation_ini(config_paser, indicator, input_files_is_dir, "in", "Y") + self.create_indi_optimisation_ini(config_paser, indicator, input_files_os_dir, "out", "Y") + + # Delete MQL5 Tester Cash: + line = f'del /F /Q {self.mq5_test_cash}' + call(line, shell=True) + + # Run the in-sample test: + print(f"Running defalts in-sample test for {indicator}") + line = f'"{self.mt5_term}" /config: {input_files_is_dir}\{indicator}.ini' + call(line, shell=True) + + # Copy output to run results dir: + line = f'copy {self.mt5_dir}\{indicator}_ins.xml {self.results_dir}\{indicator}_ins.xml' + call(line, shell=True) + + # Run the out of sample test: + print(f"Running defalts out-of-sample test for {indicator}") + line = f'"{self.mt5_term}" /config: {input_files_os_dir}\{indicator}.ini' + call(line, shell=True) + + # Copy output to run results dir: + line = f'copy {self.mt5_dir}\{indicator}_out.xml {self.results_dir}\{indicator}_out.xml' + call(line, shell=True) + + def load_config_paser(self): + config_paser = configparser.ConfigParser() + config_paser.read(self.template_file, encoding='utf-16') + return config_paser + + +class OptimiseIndicators(TestParent): + + def __init__(self, name, start_date, end_date, chart_period, custom_loss_function, symbol_mode, data_split): + super().__init__(name, start_date, end_date, chart_period, custom_loss_function, symbol_mode, data_split) + + self.indi_dir = Path.joinpath(self.test_folder, "optimise\indicators") + self.master_config_loc = Path.joinpath(self.test_folder, "optimise\master_config_files") + self.indi_rel_path = Path(str(self.indi_dir).split(r'\MQL5\Experts')[1][1:]) + self.output_dir = Path.joinpath(Path(self.indi_dir).parents[0], f'Testing\{self.name}') + self.results_dir = self.create_dir("results") + self.run() + + def run(self): + print("~" * 80) + print(f"\n INDICATOR OPTIMISATION - {self.name}\n") + + # Create in-sample dir/ delete its content: + input_files_is_dir = self.create_dir("config_files_in_sample") + self.delete_files_in_directory(input_files_is_dir) + + # Create out sample dir/ delete its content: + input_files_os_dir = self.create_dir("config_files_out_sample") + self.delete_files_in_directory(input_files_os_dir) + + df_path = self.check_results_df(self.results_dir) + indicator_list = self.create_indicator_list(df_path, self.indi_dir, True) + + print("-" * 25 + " STARTING TEST " + "-" * 25) + for indicator in indicator_list: + # Delete MQL5 Tester Cash: + line = f'del /F /Q {self.mq5_test_cash}' + call(line, shell=True) + + # Load the MQL5 .ini for the current indicator: + config_paser = self.load_config_paser(indicator) + + # Create in-sample input file: + self.create_indi_optimisation_ini(config_paser, indicator, input_files_is_dir, "in", "N") + + # Run the in-sample test: + print(f"Running in-sample optimisation for {indicator}") + line = f'"{self.mt5_term}" /config: {input_files_is_dir}\{indicator}.ini' + call(line, shell=True) + + # Copy output to run results dir: + line = f'copy {self.mt5_dir}\{indicator}_ins.xml {self.results_dir}\{indicator}_ins.xml' + call(line, shell=True) + + # Create OOS test input file for optimisation results: + self.create_indi_optimisation_ini(config_paser, indicator, input_files_os_dir, "out", "Y", opt_os=True) + + # Run the out of sample test: + print(f"Running out-of-sample optimisation for {indicator}") + line = f'"{self.mt5_term}" /config: {input_files_os_dir}\{indicator}.ini' + call(line, shell=True) + + # Copy output to run results dir: + line = f'copy {self.mt5_dir}\{indicator}_out.xml {self.results_dir}\{indicator}_out.xml' + call(line, shell=True) + + def load_config_paser(self, indicator): + config_paser = configparser.ConfigParser() + inp_file = f'{self.master_config_loc}\{indicator}.ini' + config_paser.read(inp_file, encoding='utf-16') + return config_paser + + +if __name__ == "__main__": + # Testing on 12 years data. in/out sample data split: year + TestIndicators(name="Apollo-dftest", + start_date="2012.01.01", + end_date="2022.01.01", + chart_period="Daily", + custom_loss_function="1", # 1 = no trade limit + symbol_mode="1", + data_split="month" + ) + + OptimiseIndicators(name="Apollo-opt", + start_date="2012.01.01", + end_date="2022.01.01", + chart_period="Daily", + custom_loss_function="4", # 400 trades min + symbol_mode="1", + data_split="month" + )