""" MattC - 2025 This code is a working prototype and is intended for initial testing and development purposes. Some Python standards, including but not limited to PEP 8 compliance, error handling, and code optimization, are yet to be fully implemented. Further refactoring and enhancements are planned to improve readability, maintainability, and efficiency. """ import os from os import walk import json from datetime import datetime, timedelta import pandas as pd import shutil from pathlib import Path import logging import matplotlib.pyplot as plt # pip install matplotlib import matplotlib.dates as mdates import warnings import urllib from urllib.request import urlopen import time warnings.simplefilter(action='ignore', category=FutureWarning) logger = logging.getLogger(__name__) # TODO sort out download data class WalkForward(object): def __init__(self, strategy_path, strategy, config, output_dir, wf_start, wf_finish, anchored_start, min_trades, in_sample_days, out_sample_days, loss_function, cpu, epochs, wallet="2500", fee="0.002", pre_live=False, re_opt=False, re_opt_t="5m 1h 1d"): self.in_sample_days = in_sample_days self.out_sample_days = out_sample_days self.loss_function = loss_function self.re_opt = re_opt self.output_dir = self.create_run_dir(output_dir) self.config = self.copy_input_config(config) self.strategy_path = strategy_path self.strategy_name = strategy self.strategy = self.copy_input_strategy(strategy_path, self.strategy_name) self.wf_start = wf_start self.wf_finish = wf_finish self.anchored_start = anchored_start self.is_start = self.in_sample_start() self.min_trades = min_trades self.cpu = cpu self.epochs = epochs self.wallet = wallet self.fee = fee self.pre_live = pre_live self.re_opt_t = re_opt_t self.start_log() def run_walk_forward(self): stages = self.generate_wf_stages() no_stages = len(stages) start_wallet = self.wallet start_time = datetime.now() logger.info(f'Running Walk-forward for {no_stages} stages') for count, value in enumerate(stages, 1): stage_start_time = datetime.now() logger.info(f'{"-" * 79}') hyperopt_time = value[0] backtest_time = value[1] full_time_period = f"{hyperopt_time.split('-')[0]}-{backtest_time.split('-')[1]}" logger.info(f'Walk-forward optimization for stage {count} of {no_stages}') stage_dir = self.create_stage_dir(self.output_dir, count, full_time_period) # Run Hyperopt and process required data logger.info(f'Hyperopting for {hyperopt_time}') cpu = self.set_cpu(hyperopt_time) self.run_hyperopt(hyperopt_time, stage_dir, self.epochs, cpu) hy_start = datetime.strptime(hyperopt_time.split('-')[0], "%Y%m%d") hy_finish = datetime.strptime(hyperopt_time.split('-')[1], "%Y%m%d") hy_delta = hy_start - hy_finish hy_days = int(hy_delta.days) logger.info(f'Running Hyperopt Backtest for period {hyperopt_time} ({hy_days} days)') result_op, bt_file_op = self.run_backtest(hyperopt_time, "op_bt", start_wallet, stage_dir) self.save_bt_file_data(stage_dir, bt_file_op, "op_bt") df_op = self.update_df(result_op, "op_bt", count, hyperopt_time, start_wallet) # Walk forward testing constant starting wallet: logger.info(f'Running WF Backtest for period {backtest_time} ({self.out_sample_days} days)') result_bt, bt_file_bt = self.run_backtest(backtest_time, "wf_bt", start_wallet, stage_dir) self.save_bt_file_data(stage_dir, bt_file_bt, "wf_bt") df_wf = self.update_df(result_bt, "wf_bt", count, backtest_time, start_wallet) plot_equity_curve(df_wf, self.output_dir, save_fig=True) self.combine_data(hyperopt_time, df_op, df_wf) stage_run_time = str(datetime.now() - stage_start_time) logger.info(f'Stage {count} walk-forward analysis Duration: {stage_run_time.split(".")[0]}') if self.pre_live: self.pre_live_optimise() if self.re_opt: self.re_optimise() run_time = str(datetime.now() - start_time) logger.info(f'Total walk-forward analysis Duration: {run_time.split(".")[0]}') def re_optimise(self): # Download data # time_periods = "5m 15m 1h 4h 12h 1d" time_periods = self.re_opt_t self.download_data(time_periods) end_date = datetime.strptime(self.wf_finish, "%Y%m%d") start_date = end_date - timedelta(int(self.in_sample_days)) t1 = start_date.strftime("%Y%m%d") t2 = end_date.strftime("%Y%m%d") hyperopt_time = t1 + "-" + t2 full_time_period = f"{hyperopt_time}" stage_dir = self.create_stage_dir(self.output_dir, "re_optimise", full_time_period) epochs = f"{int(self.epochs)}" self.run_hyperopt(hyperopt_time, stage_dir, epochs, self.cpu) logger.info(f'Running Hyperopt Backtest for period {hyperopt_time} ({self.in_sample_days} days)') _ = self.run_backtest(hyperopt_time, "re_optimise", self.wallet, stage_dir) def save_bt_file_data(self, stage_dir, bt_file_op, file_id): with open(f'{stage_dir}/{bt_file_op}') as f: data1 = json.load(f) df1 = pd.DataFrame.from_dict(data1["strategy"][self.strategy_name]["results_per_pair"]) csv_filepath = f"{stage_dir}/{file_id}_bt_results.csv" df1.to_csv(csv_filepath) def update_df(self, result, file_id, wf_stage, time_range, start_wallet): h5_string = f"{self.output_dir}/{file_id}.h5" if not os.path.exists(h5_string): cols = ["profit_mean", "profit_mean_pct", "profit_sum", "profit_sum_pct", "profit_total_abs", "profit_total_pct" "profit_total", "wins", "draws", "losses", "wf-stage", "bt_time_period", "start-balance", "final-balance", "%_profit_pa", "acc-start", "acc-finish"] df = pd.DataFrame(columns=cols) df.to_hdf(h5_string, 'data') df = pd.read_hdf(h5_string, 'data') if file_id == "op_bt": is_start = datetime.strptime(time_range.split('-')[0], "%Y%m%d") is_finish = datetime.strptime(time_range.split('-')[1], "%Y%m%d") delta = is_start - is_finish sample_days = int(delta.days) else: sample_days = self.out_sample_days data = result[0] data.pop('key') data["%_profit_pa"] = data["profit_total_pct"] / float(sample_days) * 365 # percent profit year data["wf-stage"] = wf_stage data["start-balance"] = start_wallet data["final-balance"] = float(start_wallet) + float(data["profit_total_abs"]) data["bt_time_period"] = time_range if len(df) == 0: data["acc-start"] = float(self.wallet) else: data["acc-start"] = df["acc-finish"].iloc[-1] data["acc-finish"] = (data["acc-start"] * data["profit_total_pct"] / 100) + data["acc-start"] df2 = pd.DataFrame(data, index=[0]) df_new = df.copy() df_new = df_new.append([df2], ignore_index=True) csv_filepath = f"{self.output_dir}/{file_id}.csv" df_new.to_csv(csv_filepath) df_new.to_hdf(h5_string, 'data') return df_new def pre_live_optimise(self): end_date = datetime.strptime(self.wf_finish, "%Y%m%d") start_date = end_date - timedelta(int(self.in_sample_days)) t1 = start_date.strftime("%Y%m%d") t2 = end_date.strftime("%Y%m%d") hyperopt_time = t1 + "-" + t2 full_time_period = f"{hyperopt_time}" stage_dir = self.create_stage_dir(self.output_dir, "pre_live", full_time_period) epochs = f"{int(self.epochs) * 2}" self.run_hyperopt(hyperopt_time, stage_dir, epochs, self.cpu) logger.info(f'Running Hyperopt Backtest for period {hyperopt_time} ({self.in_sample_days} days)') result_op = self.run_backtest(hyperopt_time, "pre_live", self.wallet, stage_dir) def combine_data(self, hyp_time_range, df_op, df_wf): is_start = datetime.strptime(hyp_time_range.split('-')[0], "%Y%m%d") is_finish = datetime.strptime(hyp_time_range.split('-')[1], "%Y%m%d") delta = is_start - is_finish days_in = int(delta.days) days_out = int(self.out_sample_days) try: a = pd.Series(df_op["profit_mean_pct"], name='op_profit_av') b = pd.Series(df_wf["profit_mean_pct"], name='wf_profit_av') c = pd.Series(df_op["wins"] / df_op["losses"], name='op_wl%') d = pd.Series(df_wf["wins"] / df_wf["losses"], name='wf_wl%') e = pd.Series(df_op['trades'].apply(lambda x: x / days_in), name='op_trades_per_day') f = pd.Series(df_wf['trades'].apply(lambda x: x / days_out), name='wf_trades_per_day') g = pd.Series(df_op["profit_total"].apply(lambda x: (x / days_in) * 365), name='op_ppa') h = pd.Series(df_wf["profit_total"].apply(lambda x: (x / days_out) * 365), name='wf_ppa') i = pd.Series(df_op["%_profit_pa"], name='op_%ppa') j = pd.Series(df_wf["%_profit_pa"], name='wf_%ppa') df_combined = pd.concat([a, b, c, d, e, f, g, h, i, j], axis=1) filepath = f"{self.output_dir}/combined.csv" df_combined.to_csv(filepath) except: pass return def generate_wf_stages(self): is_start = datetime.strptime(self.is_start, "%Y%m%d") oos_start = datetime.strptime(self.wf_start, "%Y%m%d") end_date = datetime.strptime(self.wf_finish, "%Y%m%d") oos_end = oos_start + timedelta(int(self.out_sample_days)) stages = [] while True: t1 = is_start.strftime("%Y%m%d") t2 = oos_start.strftime("%Y%m%d") t3 = oos_end.strftime("%Y%m%d") # define stage: insample_timframe = t1 + "-" + t2 outsample_timframe = t2 + "-" + t3 stage = [insample_timframe, outsample_timframe] stages.append(stage) oos_start = oos_start + timedelta(int(self.out_sample_days)) oos_end = oos_start + timedelta(int(self.out_sample_days)) if not self.in_sample_days == "anchored": is_start = is_start + timedelta(int(self.out_sample_days)) if oos_end > end_date: break return stages def run_hyperopt(self, time_range, stage_dir, epochs, cpu): """ :param stage_number: :param time_range: :param epochs: :param loss_function: SortinoHyperOptLoss, :param fee: :param cpu: :return: """ self.wait_for_internet_connection() start_time = datetime.now() os.system( "freqtrade hyperopt" + " --min-trades " + self.min_trades + " -j " + cpu + " -e " + epochs + " --spaces buy " + " --fee " + self.fee + " --logfile " + stage_dir + "/op_log" + " --timerange " + time_range + " --hyperopt-loss " + self.loss_function + " --strategy " + self.strategy + " --strategy-path " + self.output_dir + " --config " + self.config + " --dry-run-wallet " + self.wallet ) self.wait_for_internet_connection() os.system( "freqtrade hyperopt-list" + " --no-details " + " --export-csv " + stage_dir + "/op.csv" ) # # Copy the best optimisation results to "stage output directory": src = Path(f"{self.output_dir}/{self.strategy}.json") dst = f"{stage_dir}/op_result.json" shutil.copyfile(str(src), dst) run_time = str(datetime.now() - start_time) logger.info(f'Hyperopt Duration: {run_time.split(".")[0]}') with open(dst) as f: data = json.load(f) results = data["params"]["buy"] for i in results: logger.info(f'Hyperopt result: {i}: {results[i]}') def run_backtest(self, time_range, file_id, wallet, stage_dir): self.wait_for_internet_connection() start_time = datetime.now() os.system( "freqtrade backtesting" + " --export trades " + " --fee " + self.fee + f" --logfile {stage_dir}/{file_id}_log.txt" " --timerange " + time_range + " --strategy " + self.strategy + " --strategy-path " + self.output_dir + " --config " + self.config + " --dry-run-wallet " + wallet + f" --export-filename {stage_dir}/{file_id}_result.json" ) result, bt_file = self.get_backtest_data(stage_dir, file_id) run_time = str(datetime.now() - start_time) if not file_id == "wf_acc_bt": logger.info(f'Backtest Duration: {run_time.split(".")[0]}') self.log_bt_results(time_range, result, wallet, file_id) self.plot_bt_profit(time_range, stage_dir, bt_file, file_id) return result, bt_file def download_data(self, time_periods, days="4000"): self.wait_for_internet_connection() logger.info(f'downloading data') os.system( "freqtrade download-data" + " -t " + time_periods + " --exchange binance " + " --pairs .*/USDT " + " --new-pairs-days " + days + " --include-inactive-pairs " ) logger.info(f'finished downloading data') return @staticmethod def wait_for_internet_connection(): start_time = datetime.now() switch = True while True: try: urlopen('https://www.google.com', timeout=1) if not switch: offline_time = str(datetime.now() - start_time) logger.warning(f"Disconnected time: {offline_time.split('.')[0]}") logger.warning("#############################") return except urllib.error.URLError: if switch: logger.warning("#############################") logger.warning("NO INTERNET") switch = False time.sleep(2) pass def log_bt_results(self, time_range, result, wallet, file_id): data = result[0].copy() final_balance = float(wallet) + data["profit_total_abs"] w_start = round(float(wallet), 2) w_finish = round(final_balance, 2) percent_prof = round(data["profit_total_pct"], 1) if file_id == "op_bt": is_start = datetime.strptime(time_range.split('-')[0], "%Y%m%d") is_finish = datetime.strptime(time_range.split('-')[1], "%Y%m%d") delta = is_start - is_finish sample_days = int(delta.days) else: sample_days = self.out_sample_days pppa = round((percent_prof / float(sample_days) * 365), 2) # percent profit per year logger.info(f'Backtest result - Balance £{w_start} --> £{w_finish} ({percent_prof}%): {pppa} %profit pa') # Trade stats win = data['wins'] loss = data['losses'] draw = data['draws'] trades = data['trades'] logger.info(f"Backtest result - Wins: {win}, Draws: {draw}, Losses: {loss}, trades: {trades}") # Average tade profits: mean_p = round(float(data['profit_mean']), 2) mp_percent = round(float(data['profit_mean_pct']), 2) logger.info(f"Backtest result - mean trade profit £{mean_p}, {mp_percent}%") # Account draw-down: dd_percent = round((data['max_drawdown_account'] * 100), 2) dd_abs = round(float(data['max_drawdown_abs']), 2) logger.info(f"Backtest result - Max dd: {dd_percent}%, £{dd_abs}") def plot_bt_profit(self, time_range, stage_dir, bt_file, file_id): os.system( "freqtrade plot-profit " " --timeframe 1d " " --timerange " + time_range + " --strategy " + self.strategy + " --strategy-path " + self.output_dir + " --config " + self.config + f" --export-filename {stage_dir}/{bt_file}" ) # TODO relative path required: src = Path(f"/home/matt/freqtrade/user_data/plot/freqtrade-profit-plot.html") dst = f"{stage_dir}/{file_id}_profit-plot.html" shutil.copyfile(str(src), dst) @staticmethod def get_backtest_data(stage_dir, file_id): f = [] for (dirpath, dirnames, filenames) in walk(stage_dir): f.extend(filenames) break # Find the backtest file for wf stage for file in f: # not meta.json: if file[-9:-5] != "meta": my_file = file_id + "_result" if file.split("-")[0] == my_file: # change to while open: f = open(stage_dir + "/" + file) data = json.load(f) result = data["strategy_comparison"] backtest_file = file return result, backtest_file def create_run_dir(self, path): _time = datetime.now().strftime('%Y%m%d_%I:%M%p') if self.re_opt: directory = f"{path}/{_time}_{self.loss_function}_re-optimise_{self.in_sample_days}" else: directory = f"{path}/{_time}_{self.loss_function}_in_{self.in_sample_days}_out_{self.out_sample_days}" if not os.path.exists(path): os.makedirs(path) try: os.makedirs(directory) except: pass return directory @staticmethod def create_stage_dir(path, stage, full_time_period): dir_string = f"{path}/stage_{stage}_{full_time_period}" if not os.path.exists(path): os.makedirs(path) try: os.makedirs(dir_string) except Exception as e: pass return dir_string def copy_input_config(self, in_config): # Copy config: config = f"{in_config}" dst_config = f"{self.output_dir}/{config.split('/')[-1]}" shutil.copyfile(config, dst_config) return dst_config def copy_input_strategy(self, in_strategy_path, strategy): # Copy Strategy src = f"{in_strategy_path}/{strategy}.py" dst_strategy = f"{self.output_dir}/{strategy}.py" shutil.copyfile(src, dst_strategy) return strategy def in_sample_start(self): oos_start = datetime.strptime(self.wf_start, "%Y%m%d") if self.in_sample_days == "anchored": is_start = datetime.strptime(self.anchored_start, "%Y%m%d") # + oos days else: is_start = oos_start - timedelta(int(self.in_sample_days)) is_start = is_start.strftime("%Y%m%d") return is_start def set_cpu(self, hyperopt_time): is_start = datetime.strptime(hyperopt_time.split('-')[0], "%Y%m%d") is_finish = datetime.strptime(hyperopt_time.split('-')[1], "%Y%m%d") delta = is_finish - is_start days = int(delta.days) if self.in_sample_days == "anchored": if days < 50: cpu = "-1" if days > 50: cpu = "-2" if days > 100: cpu = "-4" if days > 150: cpu = "-6" if days > 200: cpu = "-8" if days > 250: cpu = "-10" if days > 300: cpu = "-11" if days > 350: cpu = "-12" if days > 400: cpu = "-13" if days > 450: cpu = "-14" if days > 500: cpu = "-14" if days > 550: cpu = "-16" if days > 600: cpu = "-16" if days > 650: cpu = "-17" if days > 700: cpu = "-17" if days > 750: cpu = "-17" if days > 800: cpu = "-18" if days > 850: cpu = "-18" if days > 900: cpu = "-18" if days > 950: cpu = "-19" if days > 1000: cpu = "-19" if days > 1100: cpu = "-20" else: d1 = datetime.strptime("20220601", "%Y%m%d") d2 = datetime.strptime("20210101", "%Y%m%d") d3 = datetime.strptime("20200101", "%Y%m%d") d4 = datetime.strptime("20190101", "%Y%m%d") d5 = datetime.strptime("20180101", "%Y%m%d") if is_finish > d1: cpu = self.cpu if d2 < is_finish < d1: cpu = int(self.cpu) + 1 if d3 < is_finish < d2: cpu = int(self.cpu) + 2 if d4 < is_finish < d3: cpu = int(self.cpu) + 3 if d5 < is_finish < d4: cpu = int(self.cpu) + 4 if is_finish < d5: cpu = int(self.cpu) + 5 return str(cpu) def start_log(self): # Set format and level: logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - WF - %(levelname)s - %(message)s') # Remove old handlers: while logger.handlers: logger.handlers.pop() # Define file handler: file_handler = logging.FileHandler(f'{self.output_dir}/wf.log') file_handler.setFormatter(formatter) logger.addHandler(file_handler) # Define console handler: console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) logger.addHandler(console_handler) # Startup logs: _cpu = 20 + int(self.cpu) + 1 start_date = datetime.strptime(self.is_start, '%Y%m%d').date() end_date = datetime.strptime(self.wf_finish, '%Y%m%d').date() run_time = str(end_date - start_date).split(",")[0] logger.info('Code Initiated') logger.info(f'Strategy:{self.strategy_path}/{self.strategy}') logger.info(f'Config:{self.config}') logger.info(f'Loss Function: {self.loss_function}') logger.info(f'Time Frame:{self.is_start}-{self.wf_finish} ({run_time})') logger.info(f'IS-days:{self.in_sample_days}, OOS-days:{self.out_sample_days}') logger.info(f'CPUs:{_cpu}, Epochs:{self.epochs}, Wallet:{"2500"}, Fee:{"0.002"}, Min-trades:{self.min_trades}') def plot_equity_curve(df, output_dir, save_fig=False): df = df.copy() pd.set_option('display.max_columns', None) fig, axs = plt.subplots(figsize=(7, 4)) axs.xaxis.set_major_formatter(mdates.DateFormatter("%d %b")) df['dates'] = df["bt_time_period"].apply(lambda i: i.split("-")[1]) df['dt'] = df['dates'].apply(lambda i: datetime.strptime(i, '%Y%m%d')) df.set_index('dt') df.plot(ax=axs, x="dt", y="acc-finish") dela_y = int(df["acc-finish"].max()) - int(df["acc-finish"].min()) y_min = int(df["acc-finish"].min()) - (dela_y * 0.05) y_max = int(df["acc-finish"].max()) + (dela_y * 0.05) axs.set_title('Walk-Forward equity curve') axs.set_ylim(y_min, y_max) axs.set_ylabel("") axs.set_xlabel("") axs.grid(color='grey', alpha=0.5, linestyle='dashed', linewidth=0.5) axs.yaxis.set_major_formatter("£" + '{x:1.0f}') # plt.show() if save_fig: try: plt.savefig(f"{output_dir}/wf_equity_curve.png") except: pass def walk_forward(path, strategy, config, output_dir, wf_start, wf_finish, anchored_start, pre_live=False, re_opt=False): is_list = ["730"] oos_list = ["30"] n_trades = ["100"] cpu_list = ["-15"] ep = "100" loss_f = "SharpeHyperOptLoss" for count, is_days in enumerate(is_list): oos_days = oos_list[count] nt = n_trades[count] cores = cpu_list[count] wf = WalkForward(strategy_path=path, strategy=strategy, config=config, output_dir=output_dir, wf_start=wf_start, wf_finish=wf_finish, anchored_start=anchored_start, epochs=ep, loss_function=loss_f, in_sample_days=is_days, out_sample_days=oos_days, min_trades=nt, cpu=cores, pre_live=pre_live, re_opt=re_opt) wf.run_walk_forward() return def re_optimise(path, strategy, config, output_dir, cpu="-19"): today = datetime.now().strftime("%Y%m%d") loss_f = "SortinoHyperOptLoss" in_sample_days = "730" n_trades = "100" ep = "200" download_data_t = "5m 1h 1d" wf = WalkForward(strategy_path=path, strategy=strategy, config=config, output_dir=output_dir, wf_start=today, wf_finish=today, anchored_start=today, epochs=ep, loss_function=loss_f, in_sample_days=in_sample_days, out_sample_days="1", min_trades=n_trades, cpu=cpu, pre_live=False, re_opt=True, re_opt_t=download_data_t) wf.re_optimise() if __name__ == "__main__": WF_START = "20200101" # 2023-03-25 WF_END = "20230910" ANCHORED_START = "20200101" # only used if in_sample_days == "anchored" STRATEGY_PATH_THOR = "/home/matt/freqtrade/user_data/strategies/Thor" STRATEGY_THOR = "Optimise_Thor_BuySig_RiskReward" CONFIG_THOR = "/home/matt/freqtrade/user_data/strategies/Thor/config_Thor_WF.json" OUTPUT_DIR_THOR = "/home/matt/freqtrade/user_data/strategies/Thor/walk_forward" # ------------------------------------------------------------- walk_forward(STRATEGY_PATH_THOR, STRATEGY_THOR, CONFIG_THOR, OUTPUT_DIR_THOR, WF_START, WF_END, ANCHORED_START) # ------------------------------------------------------------- re_optimise(STRATEGY_PATH_THOR, STRATEGY_THOR, CONFIG_THOR, OUTPUT_DIR_THOR)