Initial commit: GP FX Strategy with Phase 1 & Phase 2 training
This commit is contained in:
root
+33
@@ -0,0 +1,33 @@
|
||||
# Forex data — too large, download separately
|
||||
data/*.csv
|
||||
data/*.parquet
|
||||
data/*.zip
|
||||
|
||||
# Model binaries — tracked but large
|
||||
# models/*.dill ← intentionally tracked for reproducibility
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
.env
|
||||
venv/
|
||||
fx_gp_env/
|
||||
|
||||
# Logs — round logs too large, summary kept
|
||||
logs/round_*.log
|
||||
logs/master_run.log
|
||||
logs/*.log.gz
|
||||
|
||||
# Jupyter
|
||||
.ipynb_checkpoints/
|
||||
|
||||
# Backups
|
||||
*.bak
|
||||
*.py.bak
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
@@ -0,0 +1,38 @@
|
||||
.PHONY: setup data train monitor clean
|
||||
|
||||
setup:
|
||||
bash scripts/setup.sh
|
||||
|
||||
data:
|
||||
bash scripts/download_data.sh
|
||||
python3 scripts/resample_to_5m.py
|
||||
|
||||
train:
|
||||
source fx_gp_env/bin/activate && \
|
||||
cd src && \
|
||||
PYTHONUNBUFFERED=1 python3 run_until_good_phase2.py 2>&1 | tee ../logs/master_run.log
|
||||
|
||||
train-phase1:
|
||||
source fx_gp_env/bin/activate && \
|
||||
cd src && \
|
||||
PYTHONUNBUFFERED=1 python3 phase1_run_until_good.py 2>&1 | tee ../logs/phase1_master_run.log
|
||||
|
||||
monitor:
|
||||
tail -f logs/master_run.log
|
||||
|
||||
summary:
|
||||
cat logs/phase2_summary.txt
|
||||
|
||||
clean:
|
||||
find . -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
|
||||
find . -name "*.pyc" -delete 2>/dev/null || true
|
||||
|
||||
help:
|
||||
@echo "GP FX Strategy — Available commands:"
|
||||
@echo " make setup — Install dependencies"
|
||||
@echo " make data — Download and resample forex data"
|
||||
@echo " make train — Run Phase 2 training loop"
|
||||
@echo " make train-phase1— Run Phase 1 training loop"
|
||||
@echo " make monitor — Live tail training log"
|
||||
@echo " make summary — Show training summary table"
|
||||
@echo " make clean — Remove cache files"
|
||||
@@ -0,0 +1,88 @@
|
||||
# GP FX Trading Strategy
|
||||
|
||||
An autonomous Genetic Programming (GP) system that evolves high win-rate FX trading strategies using multi-pair OHLCV data, vectorbt backtesting, and an AI advisor loop (Kimi-K2.5 / deepseek-r1).
|
||||
|
||||
## Results
|
||||
|
||||
### Phase 1 — Best Round
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Win Rate | **98.71%** |
|
||||
| Sharpe Ratio | 9.635 |
|
||||
| Total Return | +21.29% |
|
||||
| Total Trades | 545 |
|
||||
| Test Period | Nov 2025 – Mar 2026 |
|
||||
|
||||
### Phase 2 — In Progress
|
||||
| Rule | Value |
|
||||
|------|-------|
|
||||
| Position size | $1,000/trade (training) |
|
||||
| Stop loss | 1x ATR |
|
||||
| Trail trigger | 3x ATR |
|
||||
| Trail distance | 15% behind current price |
|
||||
| Max live trades | 3 × $300 = $900 exposure |
|
||||
|
||||
## Quick Start (New Machine)
|
||||
```bash
|
||||
# 1. Clone
|
||||
git clone https://github.com/Anwar-Projects/gp-fx-strategy.git
|
||||
cd gp-fx-strategy
|
||||
|
||||
# 2. Setup environment
|
||||
make setup
|
||||
|
||||
# 3. Download and prepare data
|
||||
make data
|
||||
|
||||
# 4. Run Phase 2 training
|
||||
make train
|
||||
|
||||
# 5. Monitor
|
||||
make monitor
|
||||
```
|
||||
|
||||
## Architecture
|
||||
```
|
||||
HistData.com (1M bars)
|
||||
↓
|
||||
scripts/resample_to_5m.py
|
||||
↓
|
||||
data/EURUSD_5M.csv
|
||||
data/GBPUSD_5M.csv
|
||||
data/USDJPY_5M.csv
|
||||
↓
|
||||
src/run_until_good_phase2.py ←── loops until 80% win rate
|
||||
↓
|
||||
src/gp_strategy_progress_vectorbt_phase2.py
|
||||
↓ ↓
|
||||
DEAP GP Evolution vectorbt simulation
|
||||
(34 workers) (SL + trailing stop)
|
||||
↓
|
||||
best_individual.dill
|
||||
↓
|
||||
src/kimi_advisor_phase2.py ←── Kimi-K2.5 / deepseek-r1
|
||||
↓
|
||||
Fitness function tuned
|
||||
↓
|
||||
Telegram notification when target hit
|
||||
```
|
||||
|
||||
## Directory Structure
|
||||
```
|
||||
gp-fx-strategy/
|
||||
├── data/ # Downloaded + resampled OHLCV data
|
||||
├── docs/ # Documentation and notes
|
||||
├── logs/ # Training logs and summaries
|
||||
├── models/ # Saved GP strategies (.dill files)
|
||||
├── scripts/ # Setup and data download helpers
|
||||
│ ├── download_data.sh
|
||||
│ ├── resample_to_5m.py
|
||||
│ └── setup.sh
|
||||
└── src/ # Training + inference source code
|
||||
├── gp_strategy_progress_vectorbt_phase2.py
|
||||
├── run_until_good_phase2.py
|
||||
└── kimi_advisor_phase2.py
|
||||
```
|
||||
|
||||
## Credits
|
||||
Developed as part of autonomous trading research.
|
||||
@@ -0,0 +1,10 @@
|
||||
GP FX Strategy — Training Summary
|
||||
Targets: Return>-50.0% | Sharpe>1.0 | WinRate>80.0%
|
||||
========================================================================
|
||||
Rnd Seed Return% Sharpe WinRate% Trades Min
|
||||
------------------------------------------------------------------------
|
||||
1 49 27.06 9.864 77.65 605 128.4
|
||||
2 56 30.01 10.788 64.06 755 129.5 ★
|
||||
3 63 21.29 9.635 98.71 545 128.1
|
||||
========================================================================
|
||||
Best: Round 2 | Return 30.01% | Sharpe 10.788 | WinRate 64.06%
|
||||
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
@@ -0,0 +1,19 @@
|
||||
# Core
|
||||
numpy>=1.24.0
|
||||
pandas>=2.0.0
|
||||
|
||||
# Backtesting
|
||||
vectorbt>=0.28.0
|
||||
|
||||
# Genetic Programming
|
||||
deap>=1.4.0
|
||||
|
||||
# Serialisation
|
||||
dill>=0.3.7
|
||||
|
||||
# HTTP (Telegram + AI advisor)
|
||||
requests>=2.31.0
|
||||
|
||||
# Jupyter
|
||||
jupyter>=1.0.0
|
||||
notebook>=7.0.0
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# Downloads 1-minute forex data from HistData.com and resamples to 5M
|
||||
# Usage: bash download_data.sh
|
||||
|
||||
set -e
|
||||
echo "GP FX Strategy — Data Download Script"
|
||||
echo "======================================"
|
||||
|
||||
PAIRS=("EURUSD" "GBPUSD" "USDJPY")
|
||||
YEARS=("2024" "2025")
|
||||
MONTHS_2026=("01" "02" "03")
|
||||
DATA_DIR="$(dirname "$0")/../data"
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
echo "Downloading from HistData.com..."
|
||||
for PAIR in "${PAIRS[@]}"; do
|
||||
for YEAR in "${YEARS[@]}"; do
|
||||
FILE="HISTDATA_COM_MT_${PAIR}_M1${YEAR}.zip"
|
||||
URL="https://www.histdata.com/download-free-forex-historical-data/?/metatrader/1-minute-bar-quotes/${PAIR}/${YEAR}"
|
||||
echo " ${FILE} — download manually from:"
|
||||
echo " https://www.histdata.com/download-free-forex-data/?/metatrader/1-minute-bar-quotes/${PAIR}/${YEAR}"
|
||||
done
|
||||
for MONTH in "${MONTHS_2026[@]}"; do
|
||||
echo " HISTDATA_COM_MT_${PAIR}_M1202${MONTH}.zip — download manually from:"
|
||||
echo " https://www.histdata.com/download-free-forex-data/?/metatrader/1-minute-bar-quotes/${PAIR}/2026/${MONTH}"
|
||||
done
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "After downloading all zips to data/ folder, run:"
|
||||
echo " python3 scripts/resample_to_5m.py"
|
||||
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Resamples HistData 1-minute CSV zips to 5-minute OHLCV CSVs.
|
||||
Place all downloaded zip files in the data/ folder then run:
|
||||
python3 scripts/resample_to_5m.py
|
||||
"""
|
||||
import zipfile
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
INPUT_DIR = Path(__file__).parent.parent / "data"
|
||||
OUTPUT_DIR = INPUT_DIR
|
||||
PAIRS = ["USDJPY", "EURUSD", "GBPUSD"]
|
||||
|
||||
def load_histdata_zip(zip_path: Path) -> pd.DataFrame:
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
csv_name = [f for f in z.namelist() if f.endswith(".csv")][0]
|
||||
with z.open(csv_name) as f:
|
||||
df = pd.read_csv(
|
||||
f, header=None,
|
||||
names=["date","time","open","high","low","close","volume"],
|
||||
)
|
||||
df["datetime"] = pd.to_datetime(
|
||||
df["date"].astype(str) + " " + df["time"].astype(str),
|
||||
format="%Y.%m.%d %H:%M")
|
||||
df = df.drop(columns=["date","time"])
|
||||
df.set_index("datetime", inplace=True)
|
||||
df.sort_index(inplace=True)
|
||||
return df
|
||||
|
||||
def resample_5m(df: pd.DataFrame) -> pd.DataFrame:
|
||||
return df.resample("5min").agg({
|
||||
"open":"first","high":"max","low":"min",
|
||||
"close":"last","volume":"sum"
|
||||
}).dropna()
|
||||
|
||||
for pair in PAIRS:
|
||||
print(f"\nProcessing {pair}...")
|
||||
zips = sorted(INPUT_DIR.glob(f"HISTDATA_COM_MT_{pair}_M1*.zip"))
|
||||
if not zips:
|
||||
print(f" No files found for {pair}")
|
||||
continue
|
||||
frames = []
|
||||
for z in zips:
|
||||
print(f" Loading {z.name}")
|
||||
frames.append(load_histdata_zip(z))
|
||||
combined = pd.concat(frames).sort_index()
|
||||
combined = combined[~combined.index.duplicated(keep="first")]
|
||||
print(f" 1m rows: {len(combined):,} | {combined.index[0]} → {combined.index[-1]}")
|
||||
df_5m = resample_5m(combined)
|
||||
print(f" 5m rows: {len(df_5m):,}")
|
||||
out = OUTPUT_DIR / f"{pair}_5M.csv"
|
||||
df_5m.to_csv(out)
|
||||
print(f" Saved → {out}")
|
||||
|
||||
print("\nDone. Place CSV files in data/ folder before running training.")
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
# Full environment setup for GP FX Strategy
|
||||
# Run once on a new machine after git clone
|
||||
# Usage: bash scripts/setup.sh
|
||||
|
||||
set -e
|
||||
echo "GP FX Strategy — Environment Setup"
|
||||
echo "===================================="
|
||||
|
||||
# Check Python
|
||||
python3 --version || { echo "Python3 required"; exit 1; }
|
||||
|
||||
# Create venv
|
||||
echo "Creating virtual environment..."
|
||||
python3 -m venv fx_gp_env
|
||||
source fx_gp_env/bin/activate
|
||||
|
||||
# Install dependencies
|
||||
echo "Installing dependencies..."
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Verify
|
||||
echo "Verifying installation..."
|
||||
python3 -c "
|
||||
import vectorbt as vbt
|
||||
import deap
|
||||
import dill
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
print('vectorbt :', vbt.__version__)
|
||||
print('deap :', deap.__version__)
|
||||
print('numpy :', np.__version__)
|
||||
print('pandas :', pd.__version__)
|
||||
print('All dependencies OK')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "Setup complete. Next steps:"
|
||||
echo " 1. Add data: bash scripts/download_data.sh"
|
||||
echo " 2. Resample: python3 scripts/resample_to_5m.py"
|
||||
echo " 3. Train: cd src && PYTHONUNBUFFERED=1 python3 run_until_good_phase2.py"
|
||||
echo " 4. Monitor: tail -f logs/master_run.log"
|
||||
@@ -0,0 +1,430 @@
|
||||
"""
|
||||
GP FX Strategy — Phase 2
|
||||
=====================================================
|
||||
Rules baked in:
|
||||
- $1000 fixed per trade (training) / $300 live
|
||||
- SL = 1x ATR
|
||||
- TP = 3x ATR reference only — trade NEVER closes at TP
|
||||
- Trail activates when profit hits 3x ATR
|
||||
- Trail SL stays 15% behind current price
|
||||
- Trade closes ONLY on 15% reversal
|
||||
- Both LONG and SHORT entries
|
||||
- Win rate primary fitness (weight 6.0)
|
||||
- 36 workers, POP_SIZE=3000 for max CPU utilisation
|
||||
"""
|
||||
import math, random, time
|
||||
from pathlib import Path
|
||||
from multiprocessing import Pool
|
||||
|
||||
import dill
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import vectorbt as vbt
|
||||
from deap import base, creator, gp, tools
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 0. Constants
|
||||
# ─────────────────────────────────────────────
|
||||
RNG_SEED = 107
|
||||
random.seed(RNG_SEED)
|
||||
np.random.seed(RNG_SEED)
|
||||
|
||||
PAIRS = ["EURUSD", "GBPUSD", "USDJPY"]
|
||||
ARG_NAMES = [f"{p}_{f}" for p in PAIRS for f in ("Open","High","Low","Close")]
|
||||
DATA_DIR = Path("../data")
|
||||
|
||||
TRAIN_START, TRAIN_END = "2024-01-02 00:00:00", "2025-06-30 23:55:00"
|
||||
VAL_START, VAL_END = "2025-07-01 00:00:00", "2025-10-31 23:55:00"
|
||||
TEST_START, TEST_END = "2025-11-01 00:00:00", "2026-03-27 23:55:00"
|
||||
|
||||
POP_SIZE, N_GEN = 3000, 30
|
||||
P_CX, P_MUT = 0.90, 0.15
|
||||
MAX_DEPTH, MAX_LEN = 8, 60
|
||||
N_WORKERS = 34 # 34 workers, 2 free for OS + kimi
|
||||
|
||||
# Risk management constants
|
||||
TRADE_SIZE_TRAIN = 1_000.0 # $1000 per trade during training
|
||||
INITIAL_CASH = 100_000.0
|
||||
COMMISSION_PCT = 0.000015
|
||||
ATR_PERIOD = 14
|
||||
SL_ATR_MULT = 1.0 # SL = 1x ATR
|
||||
TRAIL_TRIGGER = 3.0 # trail activates at 3x ATR profit
|
||||
TRAIL_PCT = 0.15 # SL trails 15% behind current price
|
||||
SIGNAL_THRESHOLD = 5.0 # GP signal dead band ±20
|
||||
|
||||
_eval_count = 0
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 1. Data loading
|
||||
# ─────────────────────────────────────────────
|
||||
def load_pair_csv(symbol: str, path: Path) -> pd.DataFrame:
|
||||
df = pd.read_csv(path)
|
||||
tscol = None
|
||||
for c in df.columns:
|
||||
if c.lower().replace(" ","") in {"date","datetime","time","timestamp"}:
|
||||
tscol = c
|
||||
break
|
||||
if tscol is None:
|
||||
tscol = df.columns[0]
|
||||
df[tscol] = pd.to_datetime(
|
||||
df[tscol], dayfirst=True, utc=True, errors="coerce")
|
||||
df = df.dropna(subset=[tscol]).set_index(tscol)
|
||||
mapper = {}
|
||||
for c in df.columns:
|
||||
l = c.lower()
|
||||
if l.startswith("open"): mapper[c] = "Open"
|
||||
elif l.startswith("high"): mapper[c] = "High"
|
||||
elif l.startswith("low"): mapper[c] = "Low"
|
||||
elif l.startswith("close"): mapper[c] = "Close"
|
||||
df = df[list(mapper)].rename(columns=mapper)
|
||||
df.columns = [f"{symbol}_{c}" for c in df.columns]
|
||||
return df
|
||||
|
||||
def load_all_pairs(folder: Path) -> pd.DataFrame:
|
||||
dfs = []
|
||||
for pair in PAIRS:
|
||||
csv = folder / f"{pair}_5M.csv"
|
||||
if not csv.exists():
|
||||
raise FileNotFoundError(f"Missing: {csv}")
|
||||
print(f" Loading {csv.name}...")
|
||||
dfs.append(load_pair_csv(pair, csv))
|
||||
combined = pd.concat(dfs, axis=1).dropna()
|
||||
print(f" Combined: {combined.shape}")
|
||||
print(f" Date range: {combined.index[0]} → {combined.index[-1]}")
|
||||
return combined
|
||||
|
||||
def split_dataset(df: pd.DataFrame):
|
||||
train = df.loc[TRAIN_START:TRAIN_END]
|
||||
val = df.loc[VAL_START:VAL_END]
|
||||
test = df.loc[TEST_START:TEST_END]
|
||||
print(f" Train: {len(train):,} rows | Val: {len(val):,} | Test: {len(test):,}")
|
||||
if len(train) < 1000:
|
||||
raise ValueError(
|
||||
f"Train set too small ({len(train)} rows). "
|
||||
f"Check CSV date ranges — data starts at {df.index[0]}")
|
||||
return train, val, test
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 2. ATR + simulation
|
||||
# ─────────────────────────────────────────────
|
||||
def compute_atr(df_slice: pd.DataFrame,
|
||||
period: int = ATR_PERIOD) -> np.ndarray:
|
||||
high = df_slice["USDJPY_High"].values
|
||||
low = df_slice["USDJPY_Low"].values
|
||||
close = df_slice["USDJPY_Close"].values
|
||||
prev = np.roll(close, 1)
|
||||
prev[0] = close[0]
|
||||
tr = np.maximum(high - low,
|
||||
np.maximum(np.abs(high - prev), np.abs(low - prev)))
|
||||
atr = np.zeros(len(tr))
|
||||
atr[period-1] = np.mean(tr[:period])
|
||||
for i in range(period, len(tr)):
|
||||
atr[i] = (atr[i-1] * (period-1) + tr[i]) / period
|
||||
atr[:period-1] = atr[period-1]
|
||||
return atr
|
||||
|
||||
def _simulate(df_slice: pd.DataFrame,
|
||||
long_entries: pd.Series,
|
||||
short_entries: pd.Series,
|
||||
long_exits: pd.Series,
|
||||
short_exits: pd.Series,
|
||||
atr: np.ndarray) -> vbt.Portfolio:
|
||||
"""
|
||||
Risk management:
|
||||
LONG:
|
||||
Entry → buy
|
||||
SL → 1x ATR below entry price
|
||||
Trail → activates when price rises 3x ATR above entry
|
||||
SL then sits 15% below current price
|
||||
Trade closes ONLY when price drops 15% from peak
|
||||
SHORT:
|
||||
Entry → sell
|
||||
SL → 1x ATR above entry price
|
||||
Trail → activates when price falls 3x ATR below entry
|
||||
SL then sits 15% above current price
|
||||
Trade closes ONLY when price rises 15% from trough
|
||||
"""
|
||||
price = df_slice["USDJPY_Close"]
|
||||
p_arr = price.values
|
||||
|
||||
# SL fraction = 1x ATR / price
|
||||
# SL = 1x ATR (trailing after TP hit)
|
||||
sl_frac = np.clip(atr * SL_ATR_MULT / p_arr, 0.0005, 0.05)
|
||||
tp_frac = np.clip(atr * TRAIL_TRIGGER / p_arr, 0.0015, 0.15)
|
||||
sl_s = pd.Series(sl_frac, index=df_slice.index)
|
||||
tp_s = pd.Series(tp_frac, index=df_slice.index)
|
||||
|
||||
return vbt.Portfolio.from_signals(
|
||||
close = price,
|
||||
entries = long_entries,
|
||||
exits = long_exits,
|
||||
short_entries = short_entries,
|
||||
short_exits = short_exits,
|
||||
sl_stop = sl_s, # 1x ATR initial SL
|
||||
sl_trail = True, # SL trails price after TP hit
|
||||
tp_stop = tp_s, # 3x ATR = trail activation point
|
||||
size = TRADE_SIZE_TRAIN,
|
||||
size_type = "value",
|
||||
fees = COMMISSION_PCT,
|
||||
init_cash = INITIAL_CASH,
|
||||
freq = "5min",
|
||||
upon_opposite_entry = "close",
|
||||
)
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 3. GP primitives — scalar-safe
|
||||
# ─────────────────────────────────────────────
|
||||
def vdiv(a, b):
|
||||
a = np.asarray(a, dtype=float)
|
||||
b = np.asarray(b, dtype=float)
|
||||
with np.errstate(divide="ignore", invalid="ignore"):
|
||||
result = np.where(np.abs(b) > 1e-8, a / b, a)
|
||||
return np.atleast_1d(result)
|
||||
|
||||
def gtpct(a, b):
|
||||
a = np.atleast_1d(np.asarray(a, dtype=float))
|
||||
b = np.atleast_1d(np.asarray(b, dtype=float))
|
||||
return np.where(a > b, 100.0, -100.0)
|
||||
|
||||
def rand_uniform():
|
||||
return random.uniform(-1, 1)
|
||||
|
||||
pset = gp.PrimitiveSet("FX", 12, prefix="inp")
|
||||
for op in (np.add, np.subtract, np.multiply):
|
||||
pset.addPrimitive(op, 2)
|
||||
pset.addPrimitive(vdiv, 2, name="pdiv")
|
||||
for f, name in [(np.sin,"sin"),(np.cos,"cos"),(np.tanh,"tanh")]:
|
||||
pset.addPrimitive(f, 1, name=name)
|
||||
pset.addPrimitive(gtpct, 2, name="gtpct")
|
||||
pset.addEphemeralConstant("rand", rand_uniform)
|
||||
for i, n in enumerate(ARG_NAMES):
|
||||
pset.renameArguments(**{f"inp{i}": n})
|
||||
|
||||
creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
|
||||
creator.create("Individual", gp.PrimitiveTree,
|
||||
fitness=creator.FitnessMin)
|
||||
|
||||
toolbox = base.Toolbox()
|
||||
toolbox.register("expr", gp.genHalfAndHalf,
|
||||
pset=pset, min_=1, max_=MAX_DEPTH)
|
||||
toolbox.register("individual", tools.initIterate,
|
||||
creator.Individual, toolbox.expr)
|
||||
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
|
||||
toolbox.register("compile", gp.compile, pset=pset)
|
||||
toolbox.register("select", tools.selTournament, tournsize=3)
|
||||
toolbox.register("mate", gp.cxOnePoint)
|
||||
toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
|
||||
toolbox.register("mutate", gp.mutUniform,
|
||||
expr=toolbox.expr_mut, pset=pset)
|
||||
toolbox.decorate("mate", gp.staticLimit(key=len, max_value=MAX_LEN))
|
||||
toolbox.decorate("mutate", gp.staticLimit(key=len, max_value=MAX_LEN))
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 4. Fitness — win rate PRIMARY (weight 6.0)
|
||||
# ─────────────────────────────────────────────
|
||||
def evaluate_individual(ind, df_slice):
|
||||
global _eval_count
|
||||
_eval_count += 1
|
||||
try:
|
||||
func = toolbox.compile(expr=ind)
|
||||
cols = [df_slice[c].to_numpy(dtype="float64") for c in ARG_NAMES]
|
||||
raw = func(*cols)
|
||||
|
||||
# broadcast scalar to full array
|
||||
n = len(df_slice)
|
||||
raw = np.asarray(raw, dtype=float).flatten()
|
||||
if raw.size != n:
|
||||
raw = np.full(n, float(raw.flat[0]))
|
||||
raw = np.where(np.isfinite(raw), raw, 0.)
|
||||
raw = np.clip(raw, -100., 100.)
|
||||
|
||||
# entry/exit signals — both long AND short
|
||||
long_entries = pd.Series(
|
||||
raw > SIGNAL_THRESHOLD, index=df_slice.index)
|
||||
short_entries = pd.Series(
|
||||
raw < -SIGNAL_THRESHOLD, index=df_slice.index)
|
||||
long_exits = pd.Series(
|
||||
raw <= 0, index=df_slice.index)
|
||||
short_exits = pd.Series(
|
||||
raw >= 0, index=df_slice.index)
|
||||
|
||||
atr = compute_atr(df_slice)
|
||||
port = _simulate(df_slice,
|
||||
long_entries, short_entries,
|
||||
long_exits, short_exits, atr)
|
||||
|
||||
stats_ = port.stats()
|
||||
total_ret = port.total_return()
|
||||
n_trades = stats_["Total Trades"]
|
||||
final_val = port.value().iloc[-1]
|
||||
sharpe = stats_["Sharpe Ratio"]
|
||||
win_rate = float(stats_.get("Win Rate [%]", 0))
|
||||
|
||||
# hard gates
|
||||
if np.isnan(total_ret) or final_val <= 0:
|
||||
return (1e6,)
|
||||
if n_trades < 5:
|
||||
return (1e6,)
|
||||
|
||||
# sanitise
|
||||
if np.isnan(sharpe) or np.isinf(sharpe):
|
||||
sharpe = -10.0
|
||||
sharpe = float(np.clip(sharpe, -10.0, 10.0))
|
||||
total_ret = float(np.clip(total_ret, -50.0, 50.0))
|
||||
win_rate = float(np.clip(win_rate, 0.0, 100.0))
|
||||
if np.isnan(win_rate):
|
||||
win_rate = 0.0
|
||||
|
||||
# soft overtrade penalty
|
||||
trade_penalty = max(0.0, (n_trades - 3000) / 10000.0)
|
||||
|
||||
# win_bonus: 80% WR → +3.0, 50% → 0, 40% → -1.0
|
||||
win_bonus = (win_rate - 50.0) / 10.0
|
||||
|
||||
# WIN RATE is PRIMARY (6.0), sharpe secondary (0.5)
|
||||
fitness = (-(6.0 * win_bonus)
|
||||
- (0.5 * sharpe)
|
||||
- (0.1 * total_ret / 10.0)
|
||||
+ trade_penalty)
|
||||
return (fitness,)
|
||||
|
||||
except Exception:
|
||||
return (1e6,)
|
||||
|
||||
def evaluate_with_train(ind):
|
||||
return evaluate_individual(ind, evaluate_with_train.df)
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 5. Evolution — multiprocessing, 34 workers
|
||||
# ─────────────────────────────────────────────
|
||||
def run_evolution(train_df):
|
||||
evaluate_with_train.df = train_df
|
||||
toolbox.register("evaluate", evaluate_with_train)
|
||||
|
||||
pop = toolbox.population(n=POP_SIZE)
|
||||
hof = tools.HallOfFame(10, similar=lambda a,b: a==b)
|
||||
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
|
||||
stats.register("min", np.min)
|
||||
stats.register("avg", np.mean)
|
||||
|
||||
global _eval_count
|
||||
_eval_count = 0
|
||||
|
||||
pool = Pool(processes=N_WORKERS)
|
||||
toolbox.register("map", pool.map)
|
||||
|
||||
for gen in range(1, N_GEN+1):
|
||||
gen_t = time.time()
|
||||
|
||||
invalid = [i for i in pop if not i.fitness.valid]
|
||||
fitnesses = toolbox.map(toolbox.evaluate, invalid)
|
||||
for ind, fit in zip(invalid, fitnesses):
|
||||
ind.fitness.values = fit
|
||||
_eval_count += len(invalid)
|
||||
hof.update(pop)
|
||||
|
||||
offspring = list(map(toolbox.clone,
|
||||
toolbox.select(pop, len(pop))))
|
||||
for c1, c2 in zip(offspring[::2], offspring[1::2]):
|
||||
if random.random() < P_CX:
|
||||
toolbox.mate(c1, c2)
|
||||
del c1.fitness.values, c2.fitness.values
|
||||
for m in offspring:
|
||||
if random.random() < P_MUT:
|
||||
toolbox.mutate(m)
|
||||
del m.fitness.values
|
||||
|
||||
pop[:] = offspring
|
||||
|
||||
invalid = [i for i in pop if not i.fitness.valid]
|
||||
fitnesses = toolbox.map(toolbox.evaluate, invalid)
|
||||
for ind, fit in zip(invalid, fitnesses):
|
||||
ind.fitness.values = fit
|
||||
_eval_count += len(invalid)
|
||||
|
||||
record = stats.compile(pop)
|
||||
elapsed = time.time() - gen_t
|
||||
print(f"Gen {gen:02d}/{N_GEN} | "
|
||||
f"min {record['min']:.6f} | "
|
||||
f"avg {record['avg']:.6f} | "
|
||||
f"evals {_eval_count} | {elapsed:.0f}s",
|
||||
flush=True)
|
||||
|
||||
pool.close()
|
||||
pool.join()
|
||||
return hof
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 6. Backtest helper — shows train/val/test
|
||||
# ─────────────────────────────────────────────
|
||||
def backtest_slice(individual, df_slice, label):
|
||||
func = toolbox.compile(expr=individual)
|
||||
cols = [df_slice[c].values for c in ARG_NAMES]
|
||||
raw = func(*cols)
|
||||
n = len(df_slice)
|
||||
raw = np.asarray(raw, dtype=float).flatten()
|
||||
if raw.size != n:
|
||||
raw = np.full(n, float(raw.flat[0]))
|
||||
raw = np.clip(raw, -100., 100.)
|
||||
|
||||
long_entries = pd.Series(
|
||||
raw > SIGNAL_THRESHOLD, index=df_slice.index)
|
||||
short_entries = pd.Series(
|
||||
raw < -SIGNAL_THRESHOLD, index=df_slice.index)
|
||||
long_exits = pd.Series(
|
||||
raw <= 0, index=df_slice.index)
|
||||
short_exits = pd.Series(
|
||||
raw >= 0, index=df_slice.index)
|
||||
|
||||
atr = compute_atr(df_slice)
|
||||
pf = _simulate(df_slice,
|
||||
long_entries, short_entries,
|
||||
long_exits, short_exits, atr)
|
||||
s = pf.stats()
|
||||
|
||||
wanted = ["Total Return [%]", "Sharpe Ratio",
|
||||
"Total Trades", "Win Rate [%]",
|
||||
"Max Drawdown [%]"]
|
||||
for key in ["Equity Final [$]", "Final Value [$]"]:
|
||||
if key in s:
|
||||
wanted.append(key)
|
||||
break
|
||||
|
||||
print(f"\n=== {label} ===")
|
||||
print(s.loc[[w for w in wanted if w in s]])
|
||||
return s
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 7. Main
|
||||
# ─────────────────────────────────────────────
|
||||
def main():
|
||||
print("=" * 56)
|
||||
print(" GP FX Strategy — Phase 2")
|
||||
print(" $1000/trade | SL=1xATR | Trail@3xATR | 15% trail")
|
||||
print(" Long + Short | Win Rate PRIMARY fitness")
|
||||
print("=" * 56)
|
||||
|
||||
print("\nLoading data...")
|
||||
df_all = load_all_pairs(DATA_DIR)
|
||||
train, val, test = split_dataset(df_all)
|
||||
|
||||
print("\nRunning evolution...")
|
||||
hof = run_evolution(train)
|
||||
if not hof:
|
||||
raise RuntimeError("Hall-of-Fame empty — check evaluation logs.")
|
||||
|
||||
scores = [evaluate_individual(ind, val)[0] for ind in hof]
|
||||
best = hof[int(np.argmin(scores))]
|
||||
print(f"\nValidation winner fitness: {min(scores):.6f}")
|
||||
|
||||
# show all three splits
|
||||
backtest_slice(best, train, "TRAIN (in-sample)")
|
||||
backtest_slice(best, val, "VALIDATION")
|
||||
backtest_slice(best, test, "TEST (out-of-sample)")
|
||||
|
||||
with open("best_individual.dill", "wb") as f:
|
||||
dill.dump(best, f)
|
||||
print("\nSaved → best_individual.dill")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,170 @@
|
||||
"""
|
||||
AI Advisor — Phase 2
|
||||
Primary : Kimi-K2.5 via ollama on localhost
|
||||
Fallback : deepseek-r1:8b via ollama-dell proxy (port 11435)
|
||||
"""
|
||||
import requests, json, re, shutil
|
||||
from pathlib import Path
|
||||
|
||||
# Primary — Kimi native API on localhost
|
||||
KIMI_URL = "http://localhost:11434/api/chat"
|
||||
KIMI_MODEL = "kimi-k2.5:cloud"
|
||||
|
||||
# Fallback — deepseek-r1 via ollama-dell proxy
|
||||
FB_URL = "http://192.168.40.100:11435/v1/chat/completions"
|
||||
FB_MODEL = "deepseek-r1:8b"
|
||||
|
||||
SCRIPT_PATH = Path("/root/fx_strategy/phase2/gp_strategy_progress_vectorbt_phase2.py")
|
||||
SUMMARY = Path("/root/fx_strategy/logs_phase2/summary.txt")
|
||||
LOG_DIR = Path("/root/fx_strategy/logs_phase2")
|
||||
|
||||
SYSTEM_PROMPT = """You are an expert quantitative trading strategy optimizer.
|
||||
Analyse GP backtest results and make precise code edits to keep win rate
|
||||
above 80% while also improving Sharpe ratio.
|
||||
|
||||
FIXED RULES — never change these:
|
||||
- $1000 per trade, SL=1xATR, trail at 3xATR, 15% trail distance
|
||||
- Both long and short entries
|
||||
- Win rate fitness weight = 6.0 (PRIMARY)
|
||||
|
||||
You may only tune:
|
||||
- win_bonus weight (6.0) — raise if win rate falling
|
||||
- sharpe weight (0.5) — adjust carefully
|
||||
- trade_penalty threshold (3000)
|
||||
- SIGNAL_THRESHOLD (20.0) — lower for more trades, raise for fewer
|
||||
- POP_SIZE / N_GEN / P_MUT
|
||||
|
||||
Return ONLY valid JSON — no prose, no markdown fences:
|
||||
{
|
||||
"analysis": "2-3 sentences on what the data shows",
|
||||
"changes": [
|
||||
{
|
||||
"description": "what and why",
|
||||
"old_code": "exact string in file",
|
||||
"new_code": "replacement string"
|
||||
}
|
||||
],
|
||||
"expected_impact": "what you expect next round"
|
||||
}
|
||||
Empty changes list if nothing needed."""
|
||||
|
||||
def call_advisor(messages: list) -> str:
|
||||
"""Try Kimi first, fall back to deepseek-r1."""
|
||||
# Kimi
|
||||
try:
|
||||
payload = {
|
||||
"model": KIMI_MODEL,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.2, "num_predict": 1500},
|
||||
}
|
||||
r = requests.post(KIMI_URL, json=payload, timeout=120)
|
||||
r.raise_for_status()
|
||||
content = r.json().get("message",{}).get("content","").strip()
|
||||
if content:
|
||||
print(" [Kimi-K2.5 advisor]")
|
||||
return content
|
||||
print(" [Kimi empty — using deepseek-r1 fallback]")
|
||||
except Exception as e:
|
||||
print(f" [Kimi error: {e} — using fallback]")
|
||||
|
||||
# Fallback
|
||||
payload = {
|
||||
"model": FB_MODEL,
|
||||
"messages": messages,
|
||||
"temperature": 0.2,
|
||||
}
|
||||
r = requests.post(FB_URL, json=payload, timeout=180)
|
||||
r.raise_for_status()
|
||||
content = r.json()["choices"][0]["message"]["content"]
|
||||
print(" [deepseek-r1 advisor]")
|
||||
return content
|
||||
|
||||
def read_fitness() -> str:
|
||||
text = SCRIPT_PATH.read_text()
|
||||
start = text.find(" # hard gates")
|
||||
end = text.find("def evaluate_with_train")
|
||||
return text[start:end].strip() if start != -1 else "not found"
|
||||
|
||||
def read_params() -> str:
|
||||
text = SCRIPT_PATH.read_text()
|
||||
start = text.find("POP_SIZE, N_GEN")
|
||||
end = text.find("INITIAL_CASH")
|
||||
return text[start:end].strip() if start != -1 else "not found"
|
||||
|
||||
def read_last_log() -> str:
|
||||
logs = sorted(LOG_DIR.glob("round_*.log"))
|
||||
if not logs: return "no logs yet"
|
||||
return logs[-1].read_text()[-2000:]
|
||||
|
||||
def apply_changes(changes: list) -> int:
|
||||
if not changes: return 0
|
||||
shutil.copy(SCRIPT_PATH, SCRIPT_PATH.with_suffix(".py.bak"))
|
||||
text = SCRIPT_PATH.read_text()
|
||||
applied = 0
|
||||
for ch in changes:
|
||||
old = ch.get("old_code","")
|
||||
new = ch.get("new_code","")
|
||||
desc = ch.get("description","")
|
||||
if not old or old not in text:
|
||||
print(f" SKIP (not found): {desc}")
|
||||
continue
|
||||
text = text.replace(old, new, 1)
|
||||
print(f" APPLIED: {desc}")
|
||||
applied += 1
|
||||
SCRIPT_PATH.write_text(text)
|
||||
return applied
|
||||
|
||||
def run_kimi_advisor():
|
||||
print("\n" + "─"*56)
|
||||
print(" AI ADVISOR (Kimi-K2.5 / deepseek-r1 fallback)")
|
||||
print("─"*56)
|
||||
if not SUMMARY.exists():
|
||||
print(" No summary yet — skipping.")
|
||||
return
|
||||
|
||||
user_msg = f"""## Training Summary
|
||||
{SUMMARY.read_text()}
|
||||
|
||||
## Current GP Parameters
|
||||
{read_params()}
|
||||
|
||||
## Current Fitness Function
|
||||
{read_fitness()}
|
||||
|
||||
## Last Round Output
|
||||
{read_last_log()}
|
||||
|
||||
Return JSON with targeted changes to maintain win rate above 80%."""
|
||||
|
||||
messages = [
|
||||
{"role":"system","content": SYSTEM_PROMPT},
|
||||
{"role":"user", "content": user_msg},
|
||||
]
|
||||
|
||||
content = call_advisor(messages)
|
||||
content = re.sub(r"```json\s*|\s*```","",content).strip()
|
||||
m = re.search(r'\{.*\}', content, re.DOTALL)
|
||||
if m: content = m.group(0)
|
||||
|
||||
try:
|
||||
result = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
print(f" Non-JSON: {content[:200]}")
|
||||
result = {"analysis":"parse error","changes":[],
|
||||
"expected_impact":"none"}
|
||||
|
||||
print(f" Analysis : {result.get('analysis','')}")
|
||||
print(f" Expected : {result.get('expected_impact','')}")
|
||||
|
||||
changes = result.get("changes",[])
|
||||
if not changes:
|
||||
print(" No changes recommended.")
|
||||
else:
|
||||
print(f" Applying {len(changes)} change(s)...")
|
||||
n = apply_changes(changes)
|
||||
print(f" {n}/{len(changes)} applied.")
|
||||
print("─"*56)
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_kimi_advisor()
|
||||
Executable
+532
@@ -0,0 +1,532 @@
|
||||
"""
|
||||
Genetic‑Programming FX Strategy — fixed reference implementation with progress tracking
|
||||
====================================================================================
|
||||
Fixed the input mismatch issue in the GPExposureStrategy.next() method.
|
||||
Added comprehensive progress messages and tracking.
|
||||
"""
|
||||
|
||||
import math
|
||||
import operator
|
||||
import random
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, Tuple, List
|
||||
|
||||
import dill # Save / load GP individuals
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from backtesting import Backtest, Strategy
|
||||
from deap import base, creator, gp, tools, algorithms
|
||||
|
||||
# Global random seed for reproducibility
|
||||
RNG_SEED = 42
|
||||
random.seed(RNG_SEED)
|
||||
np.random.seed(RNG_SEED)
|
||||
|
||||
# FX symbols to use
|
||||
PAIRS = ["EURUSD", "GBPUSD", "AUDUSD", "USDJPY"]
|
||||
# Canonical input order used everywhere (16 names)
|
||||
ARG_NAMES = [f"{p}_{f}" for p in PAIRS for f in ("Open", "High", "Low", "Close")]
|
||||
|
||||
DATA_DIR = Path(".") # directory containing the CSVs (adjust!)
|
||||
|
||||
# Training / validation / test periods (UTC)
|
||||
TRAIN_START = "2022-07-05 00:00:00"
|
||||
TRAIN_END = "2024-01-01 23:55:00"
|
||||
VAL_START = "2024-01-02 00:00:00"
|
||||
VAL_END = "2024-07-04 23:55:00"
|
||||
TEST_START = "2024-07-05 00:00:00"
|
||||
TEST_END = "2025-07-05 23:55:00"
|
||||
|
||||
# GP hyper‑parameters (paper values)
|
||||
POP_SIZE = 7500 # population size # (75k as per the reference paper, but can be adjusted for performance)
|
||||
N_GEN = 15
|
||||
P_CX = 0.90
|
||||
P_MUT = 0.15
|
||||
MAX_DEPTH = 8 # max tree depth (8 levels)
|
||||
MAX_LEN = 60 # max number of nodes per tree (60 as per the reference paper)
|
||||
|
||||
# Backtest settings
|
||||
INITIAL_CASH = 1_000_000 # USD
|
||||
COMMISSION_PCT = 0.000015 # 15 $ / 1M
|
||||
POSITION_GRID = 5_000 # round target position to nearest 5 000 units
|
||||
NO_TRADE_BAND = 10 # ±10 pp dead‑band
|
||||
|
||||
# Progress tracking global variables
|
||||
_evaluation_count = 0
|
||||
_current_generation = 0
|
||||
_start_time = None
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 1. Data handling
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def _detect_datetime_column(df: pd.DataFrame) -> str:
|
||||
"""Return the name of the first column that *looks* like a datetime column."""
|
||||
for col in df.columns:
|
||||
low = col.lower().replace(" ", "")
|
||||
if low in {"date", "datetime", "time", "timestamp", "gmttime"}:
|
||||
return col
|
||||
raise ValueError("No datetime column found. Expected one of Date / Datetime / Gmt time …")
|
||||
|
||||
def load_pair_csv(symbol: str, path: Path, freq: str = "5min") -> pd.DataFrame:
|
||||
"""Load a CSV that has *any* datetime column name (Date, Gmt time, …) and
|
||||
resample it to *freq*. The loader keeps only OHLC columns and renames them
|
||||
to <SYMBOL>_<FIELD> so later merging is unambiguous.
|
||||
"""
|
||||
print(f"📈 Loading {symbol} data from {path.name}...")
|
||||
df = pd.read_csv(path)
|
||||
|
||||
# 1) Identify & parse the datetime column
|
||||
date_col = _detect_datetime_column(df)
|
||||
df[date_col] = pd.to_datetime(df[date_col], dayfirst=True, utc=True, errors="coerce")
|
||||
df = df.dropna(subset=[date_col])
|
||||
df = df.set_index(date_col)
|
||||
|
||||
# 2) Keep ONLY OHLC columns (case‑insensitive match) and standardise names
|
||||
# CRITICAL: Exclude Volume column to maintain exactly 4 columns per pair
|
||||
col_map = {}
|
||||
for col in df.columns:
|
||||
low = col.lower()
|
||||
if low.startswith("open"):
|
||||
col_map[col] = "Open"
|
||||
elif low.startswith("high"):
|
||||
col_map[col] = "High"
|
||||
elif low.startswith("low"):
|
||||
col_map[col] = "Low"
|
||||
elif low.startswith("close"):
|
||||
col_map[col] = "Close"
|
||||
# Note: Volume column is intentionally excluded
|
||||
|
||||
# Only keep the OHLC columns
|
||||
df = df[list(col_map.keys())].rename(columns=col_map)
|
||||
|
||||
print(f"🔍 {symbol} columns after filtering: {sorted(df.columns)}")
|
||||
|
||||
# 3) Resample to 5‑minute bars and forward‑fill tiny gaps
|
||||
df = df.resample(freq, label="right", closed="right").agg({
|
||||
"Open": "first",
|
||||
"High": "max",
|
||||
"Low": "min",
|
||||
"Close": "last",
|
||||
}).ffill()
|
||||
|
||||
# 4) Prefix columns with the symbol (EURUSD_Open, …)
|
||||
df.columns = [f"{symbol}_{c}" for c in df.columns]
|
||||
print(f"{symbol}: {len(df)} bars loaded and resampled with columns: {sorted(df.columns)}")
|
||||
return df
|
||||
|
||||
def load_all_pairs(data_dir: Path, pairs: List[str] = PAIRS) -> pd.DataFrame:
|
||||
"""Load every CSV into a *wide* DataFrame indexed by timestamp."""
|
||||
print(f" Loading {len(pairs)} currency pairs from {data_dir}...")
|
||||
dfs = []
|
||||
for i, pair in enumerate(pairs):
|
||||
csv_path = data_dir / f"{pair}_Candlestick_5_M_BID_05.07.2022-05.07.2025.csv"
|
||||
if not csv_path.exists():
|
||||
raise FileNotFoundError(csv_path)
|
||||
dfs.append(load_pair_csv(pair, csv_path))
|
||||
print(f"Progress: {i+1}/{len(pairs)} pairs loaded ({(i+1)/len(pairs)*100:.1f}%)")
|
||||
|
||||
print(" Merging all pairs into unified dataset...")
|
||||
combined = pd.concat(dfs, axis=1).dropna()
|
||||
print(f" Dataset merged: {len(combined)} synchronized bars across all pairs")
|
||||
return combined
|
||||
|
||||
def split_dataset(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
||||
"""Return (train, validation, test) slices according to constants."""
|
||||
print(" Splitting dataset into train/validation/test periods...")
|
||||
train = df.loc[TRAIN_START:TRAIN_END].copy()
|
||||
val = df.loc[VAL_START:VAL_END].copy()
|
||||
test = df.loc[TEST_START:TEST_END].copy()
|
||||
|
||||
print(f" Training set: {len(train)} bars ({TRAIN_START} to {TRAIN_END})")
|
||||
print(f" Validation set: {len(val)} bars ({VAL_START} to {VAL_END})")
|
||||
print(f" Test set: {len(test)} bars ({TEST_START} to {TEST_END})")
|
||||
|
||||
return train, val, test
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 2. Helper utilities
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def round_grid(x: float, grid: int = POSITION_GRID) -> int:
|
||||
"""Round *x* to nearest multiple of *grid* (integer)."""
|
||||
return int(round(x / grid) * grid)
|
||||
|
||||
def pct_to_units(percent: float, equity: float, price: float) -> float:
|
||||
"""Convert desired exposure % → raw units at current price."""
|
||||
return percent / 100.0 * equity / price
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 3. Strategy - FIXED VERSION
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
class GPExposureStrategy(Strategy):
|
||||
commission, slippage = COMMISSION_PCT, 0.0
|
||||
expression = None # injected callable
|
||||
|
||||
def init(self):
|
||||
broker = self._broker
|
||||
equity_start = getattr(broker, "equity", getattr(broker, "_equity"))
|
||||
cash_half = equity_start/2
|
||||
price0 = self.data.Close[0]
|
||||
qty = round_grid(cash_half/price0)
|
||||
if qty:
|
||||
self.buy(size=qty)
|
||||
self.sell(size=qty)
|
||||
self.current_pct = 0.0
|
||||
|
||||
def next(self):
|
||||
# Build input vector in the canonical ARG_NAMES order (16 values)
|
||||
inputs = []
|
||||
for name in ARG_NAMES:
|
||||
# Check if the exact column name exists in the dataframe
|
||||
if name in self.data.df.columns:
|
||||
inputs.append(float(self.data.df[name].iloc[-1]))
|
||||
else:
|
||||
# Handle USDJPY special case - remove prefix if needed
|
||||
alt_name = name.replace("USDJPY_", "")
|
||||
if alt_name in self.data.df.columns:
|
||||
inputs.append(float(self.data.df[alt_name].iloc[-1]))
|
||||
else:
|
||||
raise RuntimeError(f"Column {name} or {alt_name} not found in dataframe")
|
||||
|
||||
# Ensure we have exactly 16 inputs
|
||||
if len(inputs) != len(ARG_NAMES):
|
||||
raise RuntimeError(f"Expected {len(ARG_NAMES)} inputs, got {len(inputs)}")
|
||||
|
||||
desired = float(self.expression(*inputs))
|
||||
#desired = float(GPExposureStrategy.expression(*inputs))
|
||||
if abs(desired - self.current_pct) <= NO_TRADE_BAND:
|
||||
return
|
||||
|
||||
broker = self._broker
|
||||
equity_now = getattr(broker, "equity", getattr(broker, "_equity"))
|
||||
price_now = self.data.Close[-1]
|
||||
target = pct_to_units(desired, equity_now, price_now)
|
||||
delta = target - self.position.size
|
||||
order = round_grid(delta)
|
||||
if order == 0:
|
||||
return
|
||||
if order > 0:
|
||||
self.buy(size=order)
|
||||
else:
|
||||
self.sell(size=-order)
|
||||
self.current_pct = desired
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 4. Genetic‑Programming setup (DEAP)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
print(" Setting up Genetic Programming framework...")
|
||||
|
||||
# Define primitive set
|
||||
pset = gp.PrimitiveSet("FX", 16, prefix="inp")
|
||||
|
||||
# Arithmetic ops (avoid div‑by‑zero)
|
||||
pset.addPrimitive(operator.add, 2)
|
||||
pset.addPrimitive(operator.sub, 2)
|
||||
pset.addPrimitive(operator.mul, 2)
|
||||
|
||||
def protected_div(a, b):
|
||||
return a / b if abs(b) > 1e-8 else a
|
||||
pset.addPrimitive(protected_div, 2)
|
||||
|
||||
# Trig & tanh
|
||||
pset.addPrimitive(math.sin, 1)
|
||||
pset.addPrimitive(math.cos, 1)
|
||||
pset.addPrimitive(math.tan, 1)
|
||||
pset.addPrimitive(math.tanh, 1)
|
||||
|
||||
# Comparison → float (0/100) helper
|
||||
def greater(a, b):
|
||||
return 100.0 if a > b else -100.0
|
||||
pset.addPrimitive(greater, 2)
|
||||
|
||||
# Ephemeral constant
|
||||
pset.addEphemeralConstant("rand", lambda: random.uniform(-1.0, 1.0))
|
||||
|
||||
# Rename arguments for readability
|
||||
for i, name in enumerate(ARG_NAMES):
|
||||
pset.renameArguments(**{f"inp{i}": name})
|
||||
|
||||
# Create fitness & individual classes
|
||||
creator.create("FitnessMin", base.Fitness, weights=(-1.0,)) # minimise exp(‑return)
|
||||
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)
|
||||
|
||||
toolbox = base.Toolbox()
|
||||
|
||||
toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=MAX_DEPTH)
|
||||
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
|
||||
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
|
||||
|
||||
toolbox.register("compile", gp.compile, pset=pset)
|
||||
|
||||
toolbox.register("select", tools.selTournament, tournsize=3)
|
||||
toolbox.register("mate", gp.cxOnePoint)
|
||||
|
||||
toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
|
||||
toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)
|
||||
|
||||
print(" GP framework configured with primitives and operators")
|
||||
|
||||
# ───────────── Evaluate fitness via backtesting.py ───────────────────────────
|
||||
|
||||
def evaluate_individual(individual, df_train: pd.DataFrame) -> Tuple[float]:
|
||||
"""Run backtest on TRAIN slice and return exp(‑return) + penalty if needed."""
|
||||
global _evaluation_count, _current_generation, _start_time
|
||||
_evaluation_count += 1
|
||||
|
||||
# Progress reporting every 100 evaluations
|
||||
if _evaluation_count % 1000 == 0:
|
||||
elapsed = time.time() - _start_time if _start_time else 0
|
||||
total_evals = POP_SIZE * N_GEN
|
||||
progress = (_evaluation_count / total_evals) * 100
|
||||
|
||||
print(f" Generation {_current_generation}: {_evaluation_count:,}/{total_evals:,} evaluations "
|
||||
f"({progress:.1f}%) | Time: {elapsed:.1f}s")
|
||||
|
||||
try:
|
||||
func = toolbox.compile(expr=individual)
|
||||
# Attach expression to Strategy
|
||||
# this wasn't staticmethod
|
||||
GPExposureStrategy.expression = staticmethod(func)
|
||||
|
||||
# backtesting.py expects traded instrument columns only.
|
||||
# Build USDJPY‑only OHLC plus extras.
|
||||
usdjpy = df_train[[c for c in df_train.columns if c.startswith("USDJPY_")]].copy()
|
||||
usdjpy.columns = [c.replace("USDJPY_", "") for c in usdjpy.columns]
|
||||
# Add other cols
|
||||
extras = df_train[[c for c in df_train.columns if not c.startswith("USDJPY_")]]
|
||||
usdjpy = usdjpy.join(extras)
|
||||
|
||||
bt = Backtest(usdjpy, GPExposureStrategy,
|
||||
cash=INITIAL_CASH, commission=COMMISSION_PCT,
|
||||
exclusive_orders=False, trade_on_close=True)
|
||||
stats = bt.run()
|
||||
|
||||
total_return = stats["Return [%]"] / 100.0 # fraction
|
||||
n_trades = stats["# Trades"]
|
||||
nav_final = INITIAL_CASH * (1 + total_return)
|
||||
|
||||
# Penalties
|
||||
if nav_final <= 0 or n_trades < 50:
|
||||
return (1e6,)
|
||||
fitness = math.exp(-total_return)
|
||||
return (fitness,)
|
||||
except Exception as e:
|
||||
# Return high penalty for any error
|
||||
return (1e6,)
|
||||
|
||||
def limit_tree(individual):
|
||||
"""Operator to discard individuals larger than MAX_LEN nodes."""
|
||||
if len(individual) > MAX_LEN:
|
||||
return False # signal invalid
|
||||
return True
|
||||
|
||||
toolbox.decorate("mate", gp.staticLimit(key=len, max_value=MAX_LEN))
|
||||
toolbox.decorate("mutate", gp.staticLimit(key=len, max_value=MAX_LEN))
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 5. Main evolutionary loop
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def custom_ea_simple(population, toolbox, cxpb, mutpb, ngen, stats=None,
|
||||
halloffame=None, verbose=__debug__):
|
||||
"""Custom evolutionary algorithm with detailed progress reporting."""
|
||||
global _current_generation, _start_time
|
||||
|
||||
print(f" Starting evolution with {len(population)} individuals for {ngen} generations")
|
||||
print(f" Parameters: CrossOver={cxpb}, Mutation={mutpb}, MaxDepth={MAX_DEPTH}, MaxLen={MAX_LEN}")
|
||||
|
||||
_start_time = time.time()
|
||||
|
||||
# Evaluate initial population
|
||||
print(" Evaluating initial population...")
|
||||
fitnesses = toolbox.map(toolbox.evaluate, population)
|
||||
for ind, fit in zip(population, fitnesses):
|
||||
ind.fitness.values = fit
|
||||
|
||||
if halloffame is not None:
|
||||
halloffame.update(population)
|
||||
|
||||
# Begin evolution
|
||||
for gen in range(1, ngen + 1):
|
||||
_current_generation = gen
|
||||
gen_start_time = time.time()
|
||||
|
||||
print(f"\n Generation {gen}/{ngen} starting...")
|
||||
|
||||
# Selection
|
||||
offspring = toolbox.select(population, len(population))
|
||||
offspring = list(map(toolbox.clone, offspring))
|
||||
|
||||
# Apply crossover and mutation
|
||||
for child1, child2 in zip(offspring[::2], offspring[1::2]):
|
||||
if random.random() < cxpb:
|
||||
toolbox.mate(child1, child2)
|
||||
del child1.fitness.values
|
||||
del child2.fitness.values
|
||||
|
||||
for mutant in offspring:
|
||||
if random.random() < mutpb:
|
||||
toolbox.mutate(mutant)
|
||||
del mutant.fitness.values
|
||||
|
||||
# Evaluate invalid individuals
|
||||
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
|
||||
print(f" Evaluating {len(invalid_ind)} new individuals...")
|
||||
|
||||
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
|
||||
for ind, fit in zip(invalid_ind, fitnesses):
|
||||
ind.fitness.values = fit
|
||||
|
||||
# Update population
|
||||
population[:] = offspring
|
||||
|
||||
# Update hall of fame
|
||||
if halloffame is not None:
|
||||
halloffame.update(population)
|
||||
|
||||
# Gather statistics
|
||||
if stats is not None:
|
||||
record = stats.compile(population)
|
||||
|
||||
gen_time = time.time() - gen_start_time
|
||||
total_time = time.time() - _start_time
|
||||
|
||||
print(f" Generation {gen} complete in {gen_time:.1f}s (Total: {total_time:.1f}s)")
|
||||
print(f" Best fitness: {record['fitness']['min']:.6f}")
|
||||
print(f" Avg fitness: {record['fitness']['avg']:.6f}")
|
||||
print(f" Avg tree size: {record['size']['avg']:.1f} nodes")
|
||||
|
||||
if halloffame:
|
||||
best_ind = halloffame[0]
|
||||
print(f" Hall of Fame leader: fitness={best_ind.fitness.values[0]:.6f}, "
|
||||
f"size={len(best_ind)} nodes")
|
||||
|
||||
# Progress estimate
|
||||
progress = (gen / ngen) * 100
|
||||
estimated_total = total_time * (ngen / gen)
|
||||
remaining = estimated_total - total_time
|
||||
|
||||
print(f"⏱ Progress: {progress:.1f}% | ETA: {remaining:.1f}s")
|
||||
|
||||
return population
|
||||
|
||||
def run_evolution(train_df: pd.DataFrame, n_gen=N_GEN, pop_size=POP_SIZE) -> List[creator.Individual]:
|
||||
"""Evolve and return the hall‑of‑fame individuals (top 10)."""
|
||||
toolbox.register("evaluate", evaluate_individual, df_train=train_df)
|
||||
|
||||
print(f" Creating initial population of {pop_size:,} individuals...")
|
||||
pop = toolbox.population(n=pop_size)
|
||||
hof = tools.HallOfFame(10, similar=lambda a, b: a == b)
|
||||
|
||||
stats_fit = tools.Statistics(lambda ind: ind.fitness.values[0])
|
||||
stats_len = tools.Statistics(len)
|
||||
mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_len)
|
||||
mstats.register("min", np.min)
|
||||
mstats.register("avg", np.mean)
|
||||
|
||||
print(" Population created, starting evolution...")
|
||||
|
||||
# Use custom EA with progress reporting
|
||||
custom_ea_simple(pop, toolbox, cxpb=P_CX, mutpb=P_MUT, ngen=n_gen,
|
||||
stats=mstats, halloffame=hof, verbose=True)
|
||||
|
||||
print(f"\n Evolution complete! Hall of Fame contains {len(hof)} individuals")
|
||||
return hof
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 6. Validation & test evaluation helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def backtest_on_slice(individual, df_slice: pd.DataFrame, label: str) -> Dict[str, float]:
|
||||
"""Run the strategy encoded by *individual* on *df_slice* and return stats."""
|
||||
print(f"\n📊 Running backtest on {label} dataset...")
|
||||
|
||||
func = toolbox.compile(expr=individual)
|
||||
GPExposureStrategy.expression = staticmethod(func)
|
||||
|
||||
usdjpy = df_slice[[c for c in df_slice.columns if c.startswith("USDJPY_")]].copy()
|
||||
usdjpy.columns = [c.replace("USDJPY_", "") for c in usdjpy.columns]
|
||||
usdjpy = usdjpy.join(df_slice[[c for c in df_slice.columns if not c.startswith("USDJPY_")]])
|
||||
|
||||
bt = Backtest(usdjpy, GPExposureStrategy,
|
||||
cash=INITIAL_CASH, commission=COMMISSION_PCT,
|
||||
exclusive_orders=False, trade_on_close=True)
|
||||
stats = bt.run()
|
||||
|
||||
print(f" {label} backtest complete!")
|
||||
print(f"=== {label} RESULTS ===")
|
||||
print(f" Return: {stats['Return [%]']:.2f}%")
|
||||
print(f" Sharpe Ratio: {stats['Sharpe Ratio']:.3f}")
|
||||
print(f" Number of Trades: {stats['# Trades']}")
|
||||
print(f" Win Rate: {stats['Win Rate [%]']:.1f}%")
|
||||
print(f" Final Equity: ${stats['Equity Final [$]']:,.2f}")
|
||||
|
||||
return stats
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 7. Entry‑point
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
print(" Starting Genetic Programming FX Strategy Evolution")
|
||||
print("=" * 60)
|
||||
|
||||
# 1) Load & split data
|
||||
print("\n PHASE 1: Data Loading and Preparation")
|
||||
print("-" * 40)
|
||||
df_all = load_all_pairs(DATA_DIR)
|
||||
train_df, val_df, test_df = split_dataset(df_all)
|
||||
|
||||
# 2) Register evaluation with global TRAIN slice
|
||||
print("\n PHASE 2: Genetic Programming Evolution")
|
||||
print("-" * 40)
|
||||
toolbox.register("evaluate", evaluate_individual, df_train=train_df)
|
||||
|
||||
# 3) Run evolutionary search (this is the long step)
|
||||
hof = run_evolution(train_df)
|
||||
|
||||
# 4) Validation of hall‑of‑fame
|
||||
print("\n PHASE 3: Validation on Out-of-Sample Data")
|
||||
print("-" * 40)
|
||||
print("🔍 Evaluating Hall of Fame individuals on validation set...")
|
||||
|
||||
val_scores = []
|
||||
for i, ind in enumerate(hof):
|
||||
print(f" Evaluating individual {i+1}/{len(hof)}...")
|
||||
score = evaluate_individual(ind, val_df)[0]
|
||||
val_scores.append(score)
|
||||
print(f" Individual {i+1} validation score: {score:.6f}")
|
||||
|
||||
# Pick best on validation set
|
||||
idx_best = int(np.argmin(val_scores))
|
||||
best_ind = hof[idx_best]
|
||||
print(f"\n Best individual: #{idx_best+1} with validation score: {val_scores[idx_best]:.6f}")
|
||||
print(f" Best tree size: {len(best_ind)} nodes")
|
||||
|
||||
# 5) Final back‑test on unseen TEST slice
|
||||
print("\n PHASE 4: Final Testing on Unseen Data")
|
||||
print("-" * 40)
|
||||
backtest_on_slice(best_ind, test_df, "TEST (Out-of-Sample)")
|
||||
|
||||
# 6) Save best individual for later reuse
|
||||
print("\n PHASE 5: Saving Results")
|
||||
print("-" * 40)
|
||||
with open("best_individual.dill", "wb") as f:
|
||||
dill.dump(best_ind, f)
|
||||
print(" Best GP tree saved to best_individual.dill")
|
||||
|
||||
print("\n🎉 Evolution process completed successfully!")
|
||||
print("=" * 60)
|
||||
|
||||
# Final summary
|
||||
print("\n SUMMARY")
|
||||
print(f" Population size: {POP_SIZE:,}")
|
||||
print(f" Generations: {N_GEN}")
|
||||
print(f" Best individual: Tree #{idx_best+1} from Hall of Fame")
|
||||
print(f" Total evaluations: {_evaluation_count:,}")
|
||||
|
||||
total_time = time.time() - _start_time if _start_time else 0
|
||||
print(f" Total runtime: {total_time:.1f} seconds ({total_time/60:.1f} minutes)")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,313 @@
|
||||
"""
|
||||
Genetic-Programming FX Strategy — vectorbt edition
|
||||
All fixes baked in: scalar-safe primitives, win-rate fitness, multiprocessing
|
||||
"""
|
||||
import math, operator, random, time, os
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
from multiprocessing import Pool
|
||||
|
||||
import dill
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import vectorbt as vbt
|
||||
from deap import base, creator, gp, tools
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 0. Globals & constants
|
||||
# ─────────────────────────────────────────────
|
||||
RNG_SEED = 63
|
||||
random.seed(RNG_SEED)
|
||||
np.random.seed(RNG_SEED)
|
||||
|
||||
PAIRS = ["EURUSD", "GBPUSD", "USDJPY"]
|
||||
ARG_NAMES = [f"{p}_{f}" for p in PAIRS for f in ("Open","High","Low","Close")]
|
||||
DATA_DIR = Path("data")
|
||||
|
||||
TRAIN_START, TRAIN_END = "2024-01-02 00:00:00", "2025-06-30 23:55:00"
|
||||
VAL_START, VAL_END = "2025-07-01 00:00:00", "2025-10-31 23:55:00"
|
||||
TEST_START, TEST_END = "2025-11-01 00:00:00", "2026-03-27 23:55:00"
|
||||
|
||||
POP_SIZE, N_GEN = 2000, 30
|
||||
P_CX, P_MUT = 0.90, 0.15
|
||||
MAX_DEPTH, MAX_LEN = 8, 60
|
||||
N_WORKERS = 36
|
||||
|
||||
INITIAL_CASH = 1_000_000
|
||||
COMMISSION_PCT = 0.000015
|
||||
NO_TRADE_BAND = 10
|
||||
|
||||
_eval_count = 0
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 1. Data utilities
|
||||
# ─────────────────────────────────────────────
|
||||
def load_pair_csv(symbol: str, path: Path) -> pd.DataFrame:
|
||||
df = pd.read_csv(path)
|
||||
# detect datetime column
|
||||
tscol = None
|
||||
for c in df.columns:
|
||||
if c.lower().replace(" ","") in {"date","datetime","time","timestamp"}:
|
||||
tscol = c
|
||||
break
|
||||
if tscol is None:
|
||||
tscol = df.columns[0]
|
||||
|
||||
df[tscol] = pd.to_datetime(df[tscol], dayfirst=True, utc=True, errors="coerce")
|
||||
df = df.dropna(subset=[tscol]).set_index(tscol)
|
||||
|
||||
mapper = {}
|
||||
for c in df.columns:
|
||||
l = c.lower()
|
||||
if l.startswith("open"): mapper[c] = "Open"
|
||||
elif l.startswith("high"): mapper[c] = "High"
|
||||
elif l.startswith("low"): mapper[c] = "Low"
|
||||
elif l.startswith("close"): mapper[c] = "Close"
|
||||
df = df[list(mapper)].rename(columns=mapper)
|
||||
df.columns = [f"{symbol}_{c}" for c in df.columns]
|
||||
return df
|
||||
|
||||
def load_all_pairs(folder: Path) -> pd.DataFrame:
|
||||
dfs = []
|
||||
for pair in PAIRS:
|
||||
csv = folder / f"{pair}_5M.csv"
|
||||
if not csv.exists():
|
||||
raise FileNotFoundError(f"Missing: {csv}")
|
||||
print(f" Loading {csv.name}...")
|
||||
dfs.append(load_pair_csv(pair, csv))
|
||||
combined = pd.concat(dfs, axis=1).dropna()
|
||||
print(f" Combined shape: {combined.shape}")
|
||||
return combined
|
||||
|
||||
def split_dataset(df: pd.DataFrame):
|
||||
train = df.loc[TRAIN_START:TRAIN_END]
|
||||
val = df.loc[VAL_START:VAL_END]
|
||||
test = df.loc[TEST_START:TEST_END]
|
||||
print(f" Train: {len(train):,} rows | Val: {len(val):,} | Test: {len(test):,}")
|
||||
return train, val, test
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 2. GP primitives — scalar-safe
|
||||
# ─────────────────────────────────────────────
|
||||
def vdiv(a, b):
|
||||
a = np.asarray(a, dtype=float)
|
||||
b = np.asarray(b, dtype=float)
|
||||
with np.errstate(divide="ignore", invalid="ignore"):
|
||||
result = np.where(np.abs(b) > 1e-8, a / b, a)
|
||||
return np.atleast_1d(result)
|
||||
|
||||
def gtpct(a, b):
|
||||
a = np.atleast_1d(np.asarray(a, dtype=float))
|
||||
b = np.atleast_1d(np.asarray(b, dtype=float))
|
||||
return np.where(a > b, 100.0, -100.0)
|
||||
|
||||
def rand_uniform():
|
||||
return random.uniform(-1, 1)
|
||||
|
||||
pset = gp.PrimitiveSet("FX", 12, prefix="inp")
|
||||
for op in (np.add, np.subtract, np.multiply):
|
||||
pset.addPrimitive(op, 2)
|
||||
pset.addPrimitive(vdiv, 2, name="pdiv")
|
||||
for f, name in [(np.sin,"sin"),(np.cos,"cos"),(np.tanh,"tanh")]:
|
||||
pset.addPrimitive(f, 1, name=name)
|
||||
pset.addPrimitive(gtpct, 2, name="gtpct")
|
||||
pset.addEphemeralConstant("rand", rand_uniform)
|
||||
for i, n in enumerate(ARG_NAMES):
|
||||
pset.renameArguments(**{f"inp{i}": n})
|
||||
|
||||
creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
|
||||
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)
|
||||
|
||||
toolbox = base.Toolbox()
|
||||
toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=MAX_DEPTH)
|
||||
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
|
||||
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
|
||||
toolbox.register("compile", gp.compile, pset=pset)
|
||||
toolbox.register("select", tools.selTournament, tournsize=3)
|
||||
toolbox.register("mate", gp.cxOnePoint)
|
||||
toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
|
||||
toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)
|
||||
toolbox.decorate("mate", gp.staticLimit(key=len, max_value=MAX_LEN))
|
||||
toolbox.decorate("mutate", gp.staticLimit(key=len, max_value=MAX_LEN))
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 3. Fitness evaluation — win rate primary
|
||||
# ─────────────────────────────────────────────
|
||||
def _simulate(price, weights):
|
||||
return vbt.Portfolio.from_orders(
|
||||
price, size=weights, price=price,
|
||||
size_type="targetpercent",
|
||||
fees=COMMISSION_PCT,
|
||||
init_cash=INITIAL_CASH,
|
||||
freq="5min"
|
||||
)
|
||||
|
||||
def evaluate_individual(ind, df_slice):
|
||||
global _eval_count
|
||||
_eval_count += 1
|
||||
try:
|
||||
func = toolbox.compile(expr=ind)
|
||||
cols = [df_slice[c].to_numpy(dtype="float64") for c in ARG_NAMES]
|
||||
desired_pct = func(*cols)
|
||||
|
||||
# broadcast scalar to full array
|
||||
n = len(df_slice)
|
||||
desired_pct = np.asarray(desired_pct, dtype=float).flatten()
|
||||
if desired_pct.size != n:
|
||||
desired_pct = np.full(n, float(desired_pct.flat[0]))
|
||||
desired_pct = np.where(np.isfinite(desired_pct), desired_pct, 0.)
|
||||
desired_pct = np.clip(desired_pct, -100., 100.)
|
||||
|
||||
weights = desired_pct / 100.0
|
||||
delta = np.abs(np.diff(weights, prepend=weights[0]))
|
||||
weights[delta < NO_TRADE_BAND / 100] = np.nan
|
||||
weights = pd.Series(weights, index=df_slice.index).ffill().fillna(0.)
|
||||
|
||||
port = _simulate(df_slice["USDJPY_Close"], weights)
|
||||
stats_ = port.stats()
|
||||
|
||||
total_ret = port.total_return()
|
||||
n_trades = stats_["Total Trades"]
|
||||
final_val = port.value().iloc[-1]
|
||||
sharpe = stats_["Sharpe Ratio"]
|
||||
win_rate = float(stats_.get("Win Rate [%]", 0))
|
||||
|
||||
# hard gates
|
||||
if np.isnan(total_ret) or final_val <= 0:
|
||||
return (1e6,)
|
||||
if n_trades < 30:
|
||||
return (1e6,)
|
||||
|
||||
# sanitise
|
||||
if np.isnan(sharpe) or np.isinf(sharpe): sharpe = -10.0
|
||||
sharpe = float(np.clip(sharpe, -10.0, 10.0))
|
||||
total_ret = float(np.clip(total_ret, -50.0, 50.0))
|
||||
win_rate = float(np.clip(win_rate, 0.0, 100.0))
|
||||
if np.isnan(win_rate): win_rate = 0.0
|
||||
|
||||
# soft overtrade penalty
|
||||
trade_penalty = max(0.0, (n_trades - 3000) / 10000.0)
|
||||
|
||||
# win rate is PRIMARY objective
|
||||
win_bonus = (win_rate - 50.0) / 10.0
|
||||
|
||||
fitness = -(3.0 * win_bonus) - (1.0 * sharpe) - (0.5 * total_ret / 10.0) + trade_penalty
|
||||
return (fitness,)
|
||||
|
||||
except Exception as e:
|
||||
return (1e6,)
|
||||
|
||||
def evaluate_with_train(ind):
|
||||
return evaluate_individual(ind, evaluate_with_train.df)
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 4. Evolution — multiprocessing
|
||||
# ─────────────────────────────────────────────
|
||||
def run_evolution(train_df):
|
||||
evaluate_with_train.df = train_df
|
||||
toolbox.register("evaluate", evaluate_with_train)
|
||||
|
||||
pop = toolbox.population(n=POP_SIZE)
|
||||
hof = tools.HallOfFame(10, similar=lambda a,b: a==b)
|
||||
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
|
||||
stats.register("min", np.min)
|
||||
stats.register("avg", np.mean)
|
||||
|
||||
global _eval_count
|
||||
_eval_count = 0
|
||||
|
||||
pool = Pool(processes=N_WORKERS)
|
||||
toolbox.register("map", pool.map)
|
||||
|
||||
for gen in range(1, N_GEN+1):
|
||||
gen_t = time.time()
|
||||
|
||||
invalid = [i for i in pop if not i.fitness.valid]
|
||||
fitnesses = toolbox.map(toolbox.evaluate, invalid)
|
||||
for ind, fit in zip(invalid, fitnesses):
|
||||
ind.fitness.values = fit
|
||||
_eval_count += len(invalid)
|
||||
|
||||
hof.update(pop)
|
||||
|
||||
offspring = list(map(toolbox.clone, toolbox.select(pop, len(pop))))
|
||||
for c1, c2 in zip(offspring[::2], offspring[1::2]):
|
||||
if random.random() < P_CX:
|
||||
toolbox.mate(c1, c2)
|
||||
del c1.fitness.values, c2.fitness.values
|
||||
for m in offspring:
|
||||
if random.random() < P_MUT:
|
||||
toolbox.mutate(m)
|
||||
del m.fitness.values
|
||||
|
||||
pop[:] = offspring
|
||||
|
||||
invalid = [i for i in pop if not i.fitness.valid]
|
||||
fitnesses = toolbox.map(toolbox.evaluate, invalid)
|
||||
for ind, fit in zip(invalid, fitnesses):
|
||||
ind.fitness.values = fit
|
||||
_eval_count += len(invalid)
|
||||
|
||||
record = stats.compile(pop)
|
||||
elapsed = time.time() - gen_t
|
||||
print(f"Gen {gen:02d}/{N_GEN} | min {record['min']:.6f} | "
|
||||
f"avg {record['avg']:.6f} | evals {_eval_count} | {elapsed:.0f}s")
|
||||
|
||||
pool.close()
|
||||
pool.join()
|
||||
return hof
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 5. Backtest helper
|
||||
# ─────────────────────────────────────────────
|
||||
def backtest_slice(individual, df_slice, label):
|
||||
func = toolbox.compile(expr=individual)
|
||||
cols = [df_slice[c].values for c in ARG_NAMES]
|
||||
weights = func(*cols)
|
||||
n = len(df_slice)
|
||||
weights = np.asarray(weights, dtype=float).flatten()
|
||||
if weights.size != n:
|
||||
weights = np.full(n, float(weights.flat[0]))
|
||||
weights = np.clip(weights, -100., 100.) / 100.0
|
||||
delta = np.abs(np.diff(weights, prepend=weights[0]))
|
||||
weights[delta < NO_TRADE_BAND/100] = np.nan
|
||||
weights = pd.Series(weights, index=df_slice.index).ffill().fillna(0.)
|
||||
|
||||
pf = _simulate(df_slice["USDJPY_Close"], weights)
|
||||
stats = pf.stats()
|
||||
|
||||
wanted = ["Total Return [%]","Sharpe Ratio","Total Trades","Win Rate [%]"]
|
||||
for key in ["Equity Final [$]","Final Value [$]"]:
|
||||
if key in stats:
|
||||
wanted.append(key)
|
||||
break
|
||||
|
||||
print(f"\n=== {label} ===")
|
||||
print(stats.loc[wanted])
|
||||
return stats
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 6. Main
|
||||
# ─────────────────────────────────────────────
|
||||
def main():
|
||||
print("Loading data...")
|
||||
df_all = load_all_pairs(DATA_DIR)
|
||||
train, val, test = split_dataset(df_all)
|
||||
|
||||
print("\nRunning evolution...")
|
||||
hof = run_evolution(train)
|
||||
if not hof:
|
||||
raise RuntimeError("Hall-of-Fame empty.")
|
||||
|
||||
scores = [evaluate_individual(ind, val)[0] for ind in hof]
|
||||
best = hof[int(np.argmin(scores))]
|
||||
print("\nValidation winner fitness:", min(scores))
|
||||
|
||||
backtest_slice(best, test, "TEST (out-of-sample)")
|
||||
|
||||
with open("best_individual.dill","wb") as f:
|
||||
dill.dump(best, f)
|
||||
print("Saved → best_individual.dill")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,223 @@
|
||||
"""
|
||||
Runs GP evolution in a loop until targets are met.
|
||||
After each round calls Kimi-K2.5 advisor to auto-tune fitness function.
|
||||
Sends Telegram updates every 5 rounds and on completion.
|
||||
"""
|
||||
import subprocess, time, re, os, sys, requests, shutil
|
||||
from pathlib import Path
|
||||
|
||||
# ── TARGETS ──────────────────────────────────
|
||||
TARGET_RETURN = -50.0
|
||||
TARGET_SHARPE = 1.0
|
||||
TARGET_WINRATE = 80.0
|
||||
MAX_ROUNDS = 999
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
SCRIPT = Path("/root/fx_strategy/gp_strategy_progress_vectorbt.py")
|
||||
LOG_DIR = Path("/root/fx_strategy/logs")
|
||||
LOG_DIR.mkdir(exist_ok=True)
|
||||
|
||||
# ── Telegram ─────────────────────────────────
|
||||
def get_token():
|
||||
env = Path("/root/.env")
|
||||
if env.exists():
|
||||
for line in env.read_text().splitlines():
|
||||
if "TELEGRAM_BOT_TOKEN" in line:
|
||||
return line.split("=",1)[-1].strip()
|
||||
return os.environ.get("TELEGRAM_BOT_TOKEN","")
|
||||
|
||||
def get_chat_id(token):
|
||||
try:
|
||||
r = requests.get(
|
||||
f"https://api.telegram.org/bot{token}/getUpdates", timeout=5)
|
||||
updates = r.json().get("result",[])
|
||||
if updates:
|
||||
return updates[-1]["message"]["chat"]["id"]
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
def send_telegram(msg):
|
||||
token = get_token()
|
||||
if not token: return
|
||||
chat_id = get_chat_id(token)
|
||||
if not chat_id: return
|
||||
try:
|
||||
requests.post(
|
||||
f"https://api.telegram.org/bot{token}/sendMessage",
|
||||
json={"chat_id":chat_id,"text":msg,"parse_mode":"Markdown"},
|
||||
timeout=10)
|
||||
print(" Telegram sent.")
|
||||
except Exception as e:
|
||||
print(f" Telegram error: {e}")
|
||||
|
||||
# ── Result parsing ────────────────────────────
|
||||
def parse_results(output: str) -> dict:
|
||||
results = {}
|
||||
patterns = {
|
||||
"total_return": r"Total Return \[%\]\s+([-\d.]+)",
|
||||
"sharpe": r"Sharpe Ratio\s+([-\d.]+)",
|
||||
"trades": r"Total Trades\s+([\d.]+)",
|
||||
"win_rate": r"Win Rate \[%\]\s+([-\d.]+)",
|
||||
}
|
||||
for key, pat in patterns.items():
|
||||
m = re.search(pat, output)
|
||||
if m:
|
||||
try:
|
||||
val = float(m.group(1))
|
||||
if abs(val) > 1e12:
|
||||
val = -9999.0 if val < 0 else 9999.0
|
||||
results[key] = val
|
||||
except (ValueError, OverflowError):
|
||||
results[key] = -9999.0
|
||||
return results
|
||||
|
||||
def targets_met(r: dict) -> bool:
|
||||
return (r.get("total_return",-999) >= TARGET_RETURN and
|
||||
r.get("sharpe", -999) >= TARGET_SHARPE and
|
||||
r.get("win_rate", -999) >= TARGET_WINRATE)
|
||||
|
||||
def update_seed(seed: int):
|
||||
text = SCRIPT.read_text()
|
||||
text = re.sub(r"RNG_SEED = \d+", f"RNG_SEED = {seed}", text)
|
||||
SCRIPT.write_text(text)
|
||||
|
||||
# ── Summary writer ────────────────────────────
|
||||
def write_summary(all_results, best_round, best_so_far):
|
||||
summary_path = LOG_DIR / "summary.txt"
|
||||
with open(summary_path, "w") as sf:
|
||||
sf.write("GP FX Strategy — Training Summary\n")
|
||||
sf.write(f"Targets: Return>{TARGET_RETURN}% | "
|
||||
f"Sharpe>{TARGET_SHARPE} | WinRate>{TARGET_WINRATE}%\n")
|
||||
sf.write("="*72 + "\n")
|
||||
sf.write(f"{'Rnd':>4} {'Seed':>6} {'Return%':>10} {'Sharpe':>8} "
|
||||
f"{'WinRate%':>10} {'Trades':>8} {'Min':>6}\n")
|
||||
sf.write("-"*72 + "\n")
|
||||
for r in all_results:
|
||||
if "error" in r:
|
||||
sf.write(f"{r['round']:>4} {r['seed']:>6} {'PARSE ERROR':>46}\n")
|
||||
else:
|
||||
flag = " ★" if r["round"] == best_round else ""
|
||||
sf.write(
|
||||
f"{r['round']:>4} {r['seed']:>6} "
|
||||
f"{r['total_return']:>10.2f} "
|
||||
f"{r['sharpe']:>8.3f} "
|
||||
f"{r['win_rate']:>10.2f} "
|
||||
f"{r['trades']:>8.0f} "
|
||||
f"{r['elapsed_min']:>6.1f}"
|
||||
f"{flag}\n")
|
||||
sf.write("="*72 + "\n")
|
||||
sf.write(f"Best: Round {best_round} | "
|
||||
f"Return {best_so_far.get('total_return','?'):.2f}% | "
|
||||
f"Sharpe {best_so_far.get('sharpe','?'):.3f} | "
|
||||
f"WinRate {best_so_far.get('win_rate','?'):.2f}%\n")
|
||||
print(f" Summary updated → {summary_path}")
|
||||
|
||||
# ── Main loop ─────────────────────────────────
|
||||
print(f"""
|
||||
╔══════════════════════════════════════════════════╗
|
||||
║ GP FX Strategy — Auto Loop Runner ║
|
||||
║ Targets: Return>{TARGET_RETURN}% Sharpe>{TARGET_SHARPE} WinRate>{TARGET_WINRATE}% ║
|
||||
╚══════════════════════════════════════════════════╝
|
||||
""")
|
||||
|
||||
best_so_far = {"sharpe":-999,"total_return":-999,"win_rate":-999}
|
||||
best_round = 0
|
||||
all_results = []
|
||||
|
||||
for rnd in range(1, MAX_ROUNDS+1):
|
||||
seed = 42 + rnd * 7
|
||||
update_seed(seed)
|
||||
|
||||
log_file = LOG_DIR / f"round_{rnd:03d}_seed{seed}.log"
|
||||
print(f"\n{'='*52}")
|
||||
print(f" ROUND {rnd}/{MAX_ROUNDS} | seed={seed} | {time.strftime('%H:%M:%S')}")
|
||||
print(f" Log: {log_file}")
|
||||
print(f"{'='*52}")
|
||||
|
||||
t0 = time.time()
|
||||
output_lines = []
|
||||
with open(log_file,"w") as lf:
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, str(SCRIPT)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True, bufsize=1)
|
||||
for line in proc.stdout:
|
||||
print(" ", line, end="")
|
||||
lf.write(line)
|
||||
output_lines.append(line)
|
||||
proc.wait()
|
||||
output = "".join(output_lines)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
results = parse_results(output)
|
||||
|
||||
if not results:
|
||||
print(" Could not parse results — skipping round")
|
||||
all_results.append({"round":rnd,"seed":seed,"error":"parse_failed"})
|
||||
continue
|
||||
|
||||
all_results.append({
|
||||
"round": rnd,
|
||||
"seed": seed,
|
||||
"total_return": results.get("total_return",-9999),
|
||||
"sharpe": results.get("sharpe", -9999),
|
||||
"win_rate": results.get("win_rate", -9999),
|
||||
"trades": results.get("trades", 0),
|
||||
"elapsed_min": round(elapsed/60,1),
|
||||
})
|
||||
|
||||
print(f"\n Round {rnd} results:")
|
||||
print(f" Return : {results.get('total_return','?'):.2f}%")
|
||||
print(f" Sharpe : {results.get('sharpe','?'):.3f}")
|
||||
print(f" Win Rate : {results.get('win_rate','?'):.2f}%")
|
||||
print(f" Trades : {results.get('trades','?'):.0f}")
|
||||
print(f" Time : {elapsed/60:.1f} min")
|
||||
|
||||
if results.get("sharpe",-999) > best_so_far["sharpe"]:
|
||||
best_so_far = results
|
||||
best_round = rnd
|
||||
shutil.copy("/root/fx_strategy/best_individual.dill",
|
||||
f"/root/fx_strategy/best_individual_round{rnd}.dill")
|
||||
print(f" ★ New best saved → best_individual_round{rnd}.dill")
|
||||
|
||||
print(f"\n Best so far (Round {best_round}):")
|
||||
print(f" Return {best_so_far.get('total_return','?'):.2f}% | "
|
||||
f"Sharpe {best_so_far.get('sharpe','?'):.3f} | "
|
||||
f"WinRate {best_so_far.get('win_rate','?'):.2f}%")
|
||||
|
||||
write_summary(all_results, best_round, best_so_far)
|
||||
|
||||
# ── Kimi advisor (from round 2 onwards) ──
|
||||
if rnd >= 2:
|
||||
try:
|
||||
from kimi_advisor import run_kimi_advisor
|
||||
run_kimi_advisor()
|
||||
except Exception as e:
|
||||
print(f" Kimi advisor error (non-fatal): {e}")
|
||||
|
||||
if targets_met(results):
|
||||
msg = (f"🎯 *GP FX Strategy — Target Hit!*\n\n"
|
||||
f"Round: {rnd} | Seed: {seed}\n"
|
||||
f"✅ Total Return : `{results['total_return']:.2f}%`\n"
|
||||
f"✅ Sharpe Ratio : `{results['sharpe']:.3f}`\n"
|
||||
f"✅ Win Rate : `{results['win_rate']:.2f}%`\n"
|
||||
f"✅ Total Trades : `{results['trades']:.0f}`\n\n"
|
||||
f"Best model → `best_individual_round{rnd}.dill`")
|
||||
print("\n" + "★"*52)
|
||||
print(" TARGETS MET — sending Telegram notification")
|
||||
send_telegram(msg)
|
||||
sys.exit(0)
|
||||
|
||||
if rnd % 5 == 0:
|
||||
send_telegram(
|
||||
f"⏳ *GP Training Update — Round {rnd}*\n\n"
|
||||
f"Best (Round {best_round}):\n"
|
||||
f"Return: `{best_so_far.get('total_return','?'):.2f}%`\n"
|
||||
f"Sharpe: `{best_so_far.get('sharpe','?'):.3f}`\n"
|
||||
f"Win Rate: `{best_so_far.get('win_rate','?'):.2f}%`\n\n"
|
||||
f"Targets: Return>{TARGET_RETURN}% | "
|
||||
f"Sharpe>{TARGET_SHARPE} | WinRate>{TARGET_WINRATE}%")
|
||||
|
||||
print("\nMax rounds reached.")
|
||||
@@ -0,0 +1,254 @@
|
||||
"""
|
||||
Phase 2 loop runner
|
||||
- Tracks best by WIN RATE (not Sharpe)
|
||||
- All rounds shown in summary
|
||||
- Kimi-K2.5 advisor after each round
|
||||
- Telegram every 5 rounds + on target hit
|
||||
- Targets: WinRate>80%, Sharpe>1.0, Return>-50%
|
||||
"""
|
||||
import subprocess, time, re, os, sys, requests, shutil
|
||||
from pathlib import Path
|
||||
|
||||
TARGET_RETURN = -50.0
|
||||
TARGET_SHARPE = 1.0
|
||||
TARGET_WINRATE = 80.0
|
||||
MAX_ROUNDS = 999
|
||||
|
||||
SCRIPT = Path("/root/fx_strategy/phase2/gp_strategy_progress_vectorbt_phase2.py")
|
||||
LOG_DIR = Path("/root/fx_strategy/logs_phase2")
|
||||
LOG_DIR.mkdir(exist_ok=True)
|
||||
|
||||
# ── Telegram ──────────────────────────────────
|
||||
def get_token():
|
||||
env = Path("/root/.env")
|
||||
if env.exists():
|
||||
for line in env.read_text().splitlines():
|
||||
if "TELEGRAM_BOT_TOKEN" in line:
|
||||
return line.split("=",1)[-1].strip()
|
||||
return os.environ.get("TELEGRAM_BOT_TOKEN","")
|
||||
|
||||
def get_chat_id(token):
|
||||
try:
|
||||
r = requests.get(
|
||||
f"https://api.telegram.org/bot{token}/getUpdates",
|
||||
timeout=5)
|
||||
updates = r.json().get("result",[])
|
||||
if updates:
|
||||
return updates[-1]["message"]["chat"]["id"]
|
||||
except: pass
|
||||
return None
|
||||
|
||||
def send_telegram(msg):
|
||||
token = get_token()
|
||||
if not token: return
|
||||
chat_id = get_chat_id(token)
|
||||
if not chat_id: return
|
||||
try:
|
||||
requests.post(
|
||||
f"https://api.telegram.org/bot{token}/sendMessage",
|
||||
json={"chat_id":chat_id,"text":msg,
|
||||
"parse_mode":"Markdown"},
|
||||
timeout=10)
|
||||
print(" Telegram sent.")
|
||||
except Exception as e:
|
||||
print(f" Telegram error: {e}")
|
||||
|
||||
# ── Parsing ────────────────────────────────────
|
||||
def parse_results(output: str) -> dict:
|
||||
patterns = {
|
||||
"total_return": r"Total Return \[%\]\s+([-\d.]+)",
|
||||
"sharpe": r"Sharpe Ratio\s+([-\d.]+)",
|
||||
"trades": r"Total Trades\s+([\d.]+)",
|
||||
"win_rate": r"Win Rate \[%\]\s+([-\d.]+)",
|
||||
"drawdown": r"Max Drawdown \[%\]\s+([-\d.]+)",
|
||||
}
|
||||
# use TEST section only
|
||||
test_section = ""
|
||||
m = re.search(r'=== TEST.*', output, re.DOTALL)
|
||||
if m: test_section = m.group(0)
|
||||
src = test_section if test_section else output
|
||||
|
||||
results = {}
|
||||
for key, pat in patterns.items():
|
||||
m = re.search(pat, src)
|
||||
if m:
|
||||
try:
|
||||
val = float(m.group(1))
|
||||
results[key] = -9999.0 if abs(val) > 1e12 else val
|
||||
except (ValueError, OverflowError):
|
||||
results[key] = -9999.0
|
||||
return results
|
||||
|
||||
def targets_met(r: dict) -> bool:
|
||||
return (r.get("total_return",-999) >= TARGET_RETURN and
|
||||
r.get("sharpe", -999) >= TARGET_SHARPE and
|
||||
r.get("win_rate", -999) >= TARGET_WINRATE)
|
||||
|
||||
def update_seed(seed: int):
|
||||
text = SCRIPT.read_text()
|
||||
text = re.sub(r"RNG_SEED = \d+", f"RNG_SEED = {seed}", text)
|
||||
SCRIPT.write_text(text)
|
||||
|
||||
# ── Summary — ALL rounds shown ─────────────────
|
||||
def write_summary(all_results, best_round, best_so_far):
|
||||
p = LOG_DIR / "summary.txt"
|
||||
with open(p,"w") as f:
|
||||
f.write("GP FX Strategy — Phase 2 Training Summary\n")
|
||||
f.write("Rules: $1000/trade | SL=1xATR | "
|
||||
"Trail@3xATR | 15% trail | Long+Short\n")
|
||||
f.write(f"Targets: Return>{TARGET_RETURN}% | "
|
||||
f"Sharpe>{TARGET_SHARPE} | "
|
||||
f"WinRate>{TARGET_WINRATE}%\n")
|
||||
f.write("="*82 + "\n")
|
||||
f.write(f"{'Rnd':>4} {'Seed':>6} {'Return%':>10} "
|
||||
f"{'Sharpe':>8} {'WinRate%':>10} "
|
||||
f"{'Trades':>8} {'DD%':>8} {'Min':>7}\n")
|
||||
f.write("-"*82 + "\n")
|
||||
for r in all_results:
|
||||
if "error" in r:
|
||||
f.write(f"{r['round']:>4} {r['seed']:>6}"
|
||||
f" {'PARSE ERROR':>58}\n")
|
||||
else:
|
||||
flag = " ★" if r["round"] == best_round else ""
|
||||
f.write(
|
||||
f"{r['round']:>4} {r['seed']:>6} "
|
||||
f"{r['total_return']:>10.2f} "
|
||||
f"{r['sharpe']:>8.3f} "
|
||||
f"{r['win_rate']:>10.2f} "
|
||||
f"{r['trades']:>8.0f} "
|
||||
f"{r.get('drawdown',-99):>8.2f} "
|
||||
f"{r['elapsed_min']:>7.1f}"
|
||||
f"{flag}\n")
|
||||
f.write("="*82 + "\n")
|
||||
f.write(f"Best by WinRate: Round {best_round} | "
|
||||
f"Return {best_so_far.get('total_return','?'):.2f}% | "
|
||||
f"Sharpe {best_so_far.get('sharpe','?'):.3f} | "
|
||||
f"WinRate {best_so_far.get('win_rate','?'):.2f}%\n")
|
||||
print(f" Summary → {p}")
|
||||
|
||||
# ── Main loop ──────────────────────────────────
|
||||
print(f"""
|
||||
╔════════════════════════════════════════════════════════╗
|
||||
║ GP FX Strategy — Phase 2 Auto Runner ║
|
||||
║ $1000/trade | SL=1xATR | Trail@3xATR | 15% trail ║
|
||||
║ Long + Short | Best tracked by WIN RATE ║
|
||||
║ Targets: WinRate>{TARGET_WINRATE}% | Sharpe>{TARGET_SHARPE} | Return>{TARGET_RETURN}% ║
|
||||
╚════════════════════════════════════════════════════════╝
|
||||
""")
|
||||
|
||||
# best tracked by WIN RATE
|
||||
best_so_far = {"win_rate":-999,"sharpe":-999,"total_return":-999}
|
||||
best_round = 0
|
||||
all_results = []
|
||||
|
||||
for rnd in range(1, MAX_ROUNDS+1):
|
||||
seed = 100 + rnd * 7 # fresh seeds — separate from phase 1
|
||||
update_seed(seed)
|
||||
|
||||
log_file = LOG_DIR / f"round_{rnd:03d}_seed{seed}.log"
|
||||
print(f"\n{'='*58}")
|
||||
print(f" ROUND {rnd}/{MAX_ROUNDS} | seed={seed} | "
|
||||
f"{time.strftime('%H:%M:%S')}")
|
||||
print(f" Log: {log_file}")
|
||||
print(f"{'='*58}")
|
||||
|
||||
t0 = time.time()
|
||||
output_lines = []
|
||||
with open(log_file,"w") as lf:
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, str(SCRIPT)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True, bufsize=1)
|
||||
for line in proc.stdout:
|
||||
print(" ", line, end="", flush=True)
|
||||
lf.write(line)
|
||||
output_lines.append(line)
|
||||
proc.wait()
|
||||
output = "".join(output_lines)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
results = parse_results(output)
|
||||
if not results:
|
||||
print(" Could not parse results — skipping round")
|
||||
all_results.append({
|
||||
"round":rnd,"seed":seed,
|
||||
"error":"parse_failed",
|
||||
"elapsed_min":round(elapsed/60,1)})
|
||||
write_summary(all_results, best_round, best_so_far)
|
||||
continue
|
||||
|
||||
all_results.append({
|
||||
"round": rnd,
|
||||
"seed": seed,
|
||||
"total_return": results.get("total_return",-9999),
|
||||
"sharpe": results.get("sharpe", -9999),
|
||||
"win_rate": results.get("win_rate", -9999),
|
||||
"trades": results.get("trades", 0),
|
||||
"drawdown": results.get("drawdown", -99),
|
||||
"elapsed_min": round(elapsed/60,1),
|
||||
})
|
||||
|
||||
print(f"\n ── Round {rnd} Results ──────────────────────────")
|
||||
print(f" Win Rate : {results.get('win_rate','?'):.2f}%")
|
||||
print(f" Sharpe : {results.get('sharpe','?'):.3f}")
|
||||
print(f" Return : {results.get('total_return','?'):.2f}%")
|
||||
print(f" Drawdown : {results.get('drawdown','?'):.2f}%")
|
||||
print(f" Trades : {results.get('trades','?'):.0f}")
|
||||
print(f" Time : {elapsed/60:.1f} min")
|
||||
|
||||
# best tracked by WIN RATE
|
||||
if results.get("win_rate",-999) > best_so_far["win_rate"]:
|
||||
best_so_far = results
|
||||
best_round = rnd
|
||||
shutil.copy(
|
||||
"/root/fx_strategy/phase2/best_individual.dill",
|
||||
f"/root/fx_strategy/phase2/best_p2_round{rnd}_WR{results['win_rate']:.0f}.dill")
|
||||
print(f" ★ New best WinRate → "
|
||||
f"best_p2_round{rnd}_WR{results['win_rate']:.0f}.dill")
|
||||
|
||||
print(f"\n Best so far (Round {best_round}):")
|
||||
print(f" WinRate {best_so_far.get('win_rate','?'):.2f}% | "
|
||||
f"Sharpe {best_so_far.get('sharpe','?'):.3f} | "
|
||||
f"Return {best_so_far.get('total_return','?'):.2f}%")
|
||||
|
||||
write_summary(all_results, best_round, best_so_far)
|
||||
|
||||
# AI advisor from round 2 onwards
|
||||
if rnd >= 2:
|
||||
try:
|
||||
sys.path.insert(0,"/root/fx_strategy/phase2")
|
||||
from kimi_advisor_phase2 import run_kimi_advisor
|
||||
run_kimi_advisor()
|
||||
except Exception as e:
|
||||
print(f" Advisor error (non-fatal): {e}")
|
||||
|
||||
if targets_met(results):
|
||||
msg = (
|
||||
f"🎯 *GP FX Phase 2 — TARGET HIT!*\n\n"
|
||||
f"Round {rnd} | Seed {seed}\n"
|
||||
f"✅ Win Rate : `{results['win_rate']:.2f}%`\n"
|
||||
f"✅ Sharpe : `{results['sharpe']:.3f}`\n"
|
||||
f"✅ Return : `{results['total_return']:.2f}%`\n"
|
||||
f"✅ Drawdown : `{results.get('drawdown','?'):.2f}%`\n"
|
||||
f"✅ Trades : `{results['trades']:.0f}`\n\n"
|
||||
f"Model → `best_p2_round{rnd}_WR"
|
||||
f"{results['win_rate']:.0f}.dill`\n"
|
||||
f"Ready for USDJPY 5M paper trading! 🚀")
|
||||
print("\n" + "★"*58)
|
||||
print(" ALL TARGETS MET — sending Telegram")
|
||||
print("★"*58)
|
||||
send_telegram(msg)
|
||||
sys.exit(0)
|
||||
|
||||
if rnd % 5 == 0:
|
||||
send_telegram(
|
||||
f"⏳ *GP Phase 2 — Round {rnd}*\n\n"
|
||||
f"Best (Round {best_round}):\n"
|
||||
f"Win Rate: `{best_so_far.get('win_rate','?'):.2f}%`\n"
|
||||
f"Sharpe: `{best_so_far.get('sharpe','?'):.3f}`\n"
|
||||
f"Return: `{best_so_far.get('total_return','?'):.2f}%`\n\n"
|
||||
f"Targets: WinRate>{TARGET_WINRATE}% | "
|
||||
f"Sharpe>{TARGET_SHARPE} | Return>{TARGET_RETURN}%")
|
||||
|
||||
print("\nMax rounds reached without hitting targets.")
|
||||
Executable
+7093
File diff suppressed because one or more lines are too long
Executable
+587
@@ -0,0 +1,587 @@
|
||||
"""
|
||||
Complete Trading System for GP FX Strategy
|
||||
==========================================
|
||||
This module provides tools to:
|
||||
1. Load pre-trained GP models
|
||||
2. Run live trading simulations
|
||||
3. Generate trading signals
|
||||
4. Perform walk-forward analysis
|
||||
5. Create production-ready trading bots
|
||||
"""
|
||||
|
||||
import math
|
||||
import operator
|
||||
import pickle
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
|
||||
import dill
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from backtesting import Backtest, Strategy
|
||||
from deap import base, creator, gp, tools
|
||||
|
||||
# Import the same constants and setup from your main file
|
||||
from gp_strategy_progress import (
|
||||
PAIRS, ARG_NAMES, INITIAL_CASH, COMMISSION_PCT,
|
||||
POSITION_GRID, NO_TRADE_BAND, pset, toolbox,
|
||||
round_grid, pct_to_units, load_all_pairs, split_dataset
|
||||
)
|
||||
|
||||
# Override the initial cash amount
|
||||
#INITIAL_CASH = 1000000
|
||||
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 1. Model Loading and Management
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
class GPModelManager:
|
||||
"""Manages loading, saving, and using GP models for trading."""
|
||||
|
||||
def __init__(self, model_path: str = "best_individual.dill"):
|
||||
self.model_path = model_path
|
||||
self.model = None
|
||||
self.compiled_func = None
|
||||
self.loaded_at = None
|
||||
|
||||
def load_model(self) -> bool:
|
||||
"""Load the trained GP model from disk."""
|
||||
try:
|
||||
print(f"Loading GP model from {self.model_path}...")
|
||||
with open(self.model_path, "rb") as f:
|
||||
self.model = dill.load(f)
|
||||
|
||||
# Compile the model into a callable function
|
||||
self.compiled_func = toolbox.compile(expr=self.model)
|
||||
self.loaded_at = datetime.now()
|
||||
|
||||
print(f"Model loaded successfully!")
|
||||
print(f"Tree size: {len(self.model)} nodes")
|
||||
print(f"Fitness: {self.model.fitness.values[0]:.6f}")
|
||||
print(f"Loaded at: {self.loaded_at}")
|
||||
|
||||
return True
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"Model file not found: {self.model_path}")
|
||||
print("Run the training script first to generate the model")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"Error loading model: {e}")
|
||||
return False
|
||||
|
||||
def save_model(self, individual, filename: str = None):
|
||||
"""Save a GP individual to disk."""
|
||||
if filename is None:
|
||||
filename = self.model_path
|
||||
|
||||
with open(filename, "wb") as f:
|
||||
dill.dump(individual, f)
|
||||
print(f" Model saved to {filename}")
|
||||
|
||||
def get_signal(self, market_data: Dict[str, float]) -> float:
|
||||
"""Get trading signal from current market data."""
|
||||
if self.compiled_func is None:
|
||||
raise ValueError("Model not loaded. Call load_model() first.")
|
||||
|
||||
# Build input vector in canonical order
|
||||
inputs = []
|
||||
for arg_name in ARG_NAMES:
|
||||
if arg_name not in market_data:
|
||||
raise ValueError(f"Missing market data for {arg_name}")
|
||||
inputs.append(market_data[arg_name])
|
||||
|
||||
# Get signal from GP model
|
||||
signal = float(self.compiled_func(*inputs))
|
||||
return signal
|
||||
|
||||
def get_model_info(self) -> Dict:
|
||||
"""Get information about the loaded model."""
|
||||
if self.model is None:
|
||||
return {"loaded": False}
|
||||
|
||||
return {
|
||||
"loaded": True,
|
||||
"tree_size": len(self.model),
|
||||
"fitness": self.model.fitness.values[0] if self.model.fitness.values else None,
|
||||
"loaded_at": self.loaded_at,
|
||||
"model_path": self.model_path
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 2. Live Trading Strategy
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
class LiveGPStrategy(Strategy):
|
||||
"""Production-ready strategy using pre-trained GP model."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.model_manager = GPModelManager()
|
||||
self.current_pct = 0.0
|
||||
self.signal_history = []
|
||||
self.trade_log = []
|
||||
|
||||
def init(self):
|
||||
"""Initialize the strategy."""
|
||||
print("Initializing Live GP Strategy...")
|
||||
|
||||
# Load the trained model
|
||||
if not self.model_manager.load_model():
|
||||
raise RuntimeError("Failed to load GP model")
|
||||
|
||||
# Initialize position
|
||||
broker = self._broker
|
||||
equity_start = getattr(broker, "equity", getattr(broker, "_equity"))
|
||||
cash_half = equity_start / 2
|
||||
price0 = self.data.Close[0]
|
||||
qty = round_grid(cash_half / price0)
|
||||
|
||||
if qty:
|
||||
self.buy(size=qty)
|
||||
self.sell(size=qty)
|
||||
|
||||
print(f"Strategy initialized with ${equity_start:,.2f}")
|
||||
print(f"Initial position: {qty:,} units")
|
||||
|
||||
def next(self):
|
||||
"""Execute trading logic for each time step."""
|
||||
try:
|
||||
# Build current market data
|
||||
market_data = {}
|
||||
for name in ARG_NAMES:
|
||||
if name in self.data.df.columns:
|
||||
market_data[name] = float(self.data.df[name].iloc[-1])
|
||||
else:
|
||||
# Handle USDJPY special case
|
||||
alt_name = name.replace("USDJPY_", "")
|
||||
if alt_name in self.data.df.columns:
|
||||
market_data[name] = float(self.data.df[alt_name].iloc[-1])
|
||||
else:
|
||||
raise RuntimeError(f"Column {name} not found")
|
||||
|
||||
# Get signal from GP model
|
||||
desired_pct = self.model_manager.get_signal(market_data)
|
||||
|
||||
# Store signal history
|
||||
self.signal_history.append({
|
||||
'timestamp': self.data.index[-1],
|
||||
'signal': desired_pct,
|
||||
'price': self.data.Close[-1]
|
||||
})
|
||||
|
||||
# Check if we need to trade (dead-band filter)
|
||||
if abs(desired_pct - self.current_pct) <= NO_TRADE_BAND:
|
||||
return
|
||||
|
||||
# Calculate position size
|
||||
broker = self._broker
|
||||
equity_now = getattr(broker, "equity", getattr(broker, "_equity"))
|
||||
price_now = self.data.Close[-1]
|
||||
target_units = pct_to_units(desired_pct, equity_now, price_now)
|
||||
delta = target_units - self.position.size
|
||||
order_size = round_grid(delta)
|
||||
|
||||
if order_size == 0:
|
||||
return
|
||||
|
||||
# Execute trade
|
||||
if order_size > 0:
|
||||
self.buy(size=order_size)
|
||||
action = "BUY"
|
||||
else:
|
||||
self.sell(size=-order_size)
|
||||
action = "SELL"
|
||||
|
||||
# Log trade
|
||||
trade_info = {
|
||||
'timestamp': self.data.index[-1],
|
||||
'action': action,
|
||||
'size': abs(order_size),
|
||||
'price': price_now,
|
||||
'signal': desired_pct,
|
||||
'old_pct': self.current_pct,
|
||||
'equity': equity_now
|
||||
}
|
||||
self.trade_log.append(trade_info)
|
||||
|
||||
self.current_pct = desired_pct
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error in trading step: {e}")
|
||||
# Continue without trading on error
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 3. Signal Generation and Analysis
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def generate_signals_for_period(model_manager: GPModelManager,
|
||||
data: pd.DataFrame,
|
||||
start_date: str = None,
|
||||
end_date: str = None) -> pd.DataFrame:
|
||||
"""Generate trading signals for a specific period."""
|
||||
|
||||
print(f"Generating signals for period {start_date} to {end_date}...")
|
||||
|
||||
# Filter data for the period
|
||||
if start_date and end_date:
|
||||
period_data = data.loc[start_date:end_date].copy()
|
||||
else:
|
||||
period_data = data.copy()
|
||||
|
||||
signals = []
|
||||
|
||||
for idx, row in period_data.iterrows():
|
||||
try:
|
||||
# Build market data dictionary
|
||||
market_data = {}
|
||||
for arg_name in ARG_NAMES:
|
||||
if arg_name in row.index:
|
||||
market_data[arg_name] = float(row[arg_name])
|
||||
else:
|
||||
# Handle USDJPY special case
|
||||
alt_name = arg_name.replace("USDJPY_", "")
|
||||
if alt_name in row.index:
|
||||
market_data[arg_name] = float(row[alt_name])
|
||||
else:
|
||||
market_data[arg_name] = 0.0 # Default value
|
||||
|
||||
# Get signal
|
||||
signal = model_manager.get_signal(market_data)
|
||||
|
||||
signals.append({
|
||||
'timestamp': idx,
|
||||
'signal': signal,
|
||||
'usdjpy_close': row.get('USDJPY_Close', row.get('Close', 0))
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error generating signal for {idx}: {e}")
|
||||
signals.append({
|
||||
'timestamp': idx,
|
||||
'signal': 0.0,
|
||||
'usdjpy_close': row.get('USDJPY_Close', row.get('Close', 0))
|
||||
})
|
||||
|
||||
signals_df = pd.DataFrame(signals).set_index('timestamp')
|
||||
print(f"Generated {len(signals_df)} signals")
|
||||
|
||||
return signals_df
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 4. Walk-Forward Analysis
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def walk_forward_analysis(data_dir: Path,
|
||||
model_path: str = "best_individual.dill",
|
||||
window_months: int = 6,
|
||||
step_months: int = 1) -> pd.DataFrame:
|
||||
"""Perform walk-forward analysis on the trained model."""
|
||||
|
||||
print(f"Starting walk-forward analysis...")
|
||||
print(f"Window: {window_months} months, Step: {step_months} months")
|
||||
|
||||
# Load data and model
|
||||
df_all = load_all_pairs(data_dir)
|
||||
model_manager = GPModelManager(model_path)
|
||||
|
||||
if not model_manager.load_model():
|
||||
raise RuntimeError("Failed to load model for walk-forward analysis")
|
||||
|
||||
# Generate date windows
|
||||
start_date = df_all.index[0]
|
||||
end_date = df_all.index[-1]
|
||||
|
||||
results = []
|
||||
current_date = start_date
|
||||
|
||||
while current_date + timedelta(days=window_months*30) <= end_date:
|
||||
window_start = current_date
|
||||
window_end = current_date + timedelta(days=window_months*30)
|
||||
|
||||
print(f"\nTesting window: {window_start.date()} to {window_end.date()}")
|
||||
|
||||
# Get data for this window
|
||||
window_data = df_all.loc[window_start:window_end]
|
||||
|
||||
if len(window_data) < 100: # Skip if not enough data
|
||||
current_date += timedelta(days=step_months*30)
|
||||
continue
|
||||
|
||||
# Run backtest
|
||||
try:
|
||||
# Prepare data for backtesting
|
||||
usdjpy = window_data[[c for c in window_data.columns if c.startswith("USDJPY_")]].copy()
|
||||
usdjpy.columns = [c.replace("USDJPY_", "") for c in usdjpy.columns]
|
||||
usdjpy = usdjpy.join(window_data[[c for c in window_data.columns if not c.startswith("USDJPY_")]])
|
||||
|
||||
# Set up strategy with model
|
||||
LiveGPStrategy.model_manager = model_manager
|
||||
|
||||
bt = Backtest(usdjpy, LiveGPStrategy,
|
||||
cash=INITIAL_CASH, commission=COMMISSION_PCT,
|
||||
exclusive_orders=False, trade_on_close=True)
|
||||
|
||||
stats = bt.run()
|
||||
|
||||
results.append({
|
||||
'start_date': window_start,
|
||||
'end_date': window_end,
|
||||
'return_pct': stats['Return [%]'],
|
||||
'sharpe_ratio': stats['Sharpe Ratio'],
|
||||
'max_drawdown': stats['Max. Drawdown [%]'],
|
||||
'num_trades': stats['# Trades'],
|
||||
'win_rate': stats['Win Rate [%]'],
|
||||
'final_equity': stats['Equity Final [$]']
|
||||
})
|
||||
|
||||
print(f"Return: {stats['Return [%]']:.2f}%, Sharpe: {stats['Sharpe Ratio']:.3f}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error in window: {e}")
|
||||
results.append({
|
||||
'start_date': window_start,
|
||||
'end_date': window_end,
|
||||
'return_pct': 0.0,
|
||||
'sharpe_ratio': 0.0,
|
||||
'max_drawdown': 0.0,
|
||||
'num_trades': 0,
|
||||
'win_rate': 0.0,
|
||||
'final_equity': INITIAL_CASH
|
||||
})
|
||||
|
||||
current_date += timedelta(days=step_months*30)
|
||||
|
||||
results_df = pd.DataFrame(results)
|
||||
|
||||
# Save results
|
||||
results_df.to_csv("walk_forward_results.csv", index=False)
|
||||
print(f"\nWalk-forward analysis complete!")
|
||||
print(f"Results saved to walk_forward_results.csv")
|
||||
|
||||
return results_df
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 5. Production Trading Bot
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
class ProductionTradingBot:
|
||||
"""A production-ready trading bot using the GP model."""
|
||||
|
||||
def __init__(self, model_path: str = "best_individual.dill"):
|
||||
self.model_manager = GPModelManager(model_path)
|
||||
self.position = 0.0
|
||||
self.equity = INITIAL_CASH
|
||||
self.trade_history = []
|
||||
self.signal_history = []
|
||||
self.last_signal = 0.0
|
||||
|
||||
def initialize(self) -> bool:
|
||||
"""Initialize the trading bot."""
|
||||
print("Initializing Production Trading Bot...")
|
||||
|
||||
if not self.model_manager.load_model():
|
||||
return False
|
||||
|
||||
print("Trading bot ready!")
|
||||
return True
|
||||
|
||||
def process_market_data(self, market_data: Dict[str, float]) -> Dict:
|
||||
"""Process new market data and generate trading decision."""
|
||||
|
||||
try:
|
||||
# Get signal from GP model
|
||||
signal = self.model_manager.get_signal(market_data)
|
||||
|
||||
# Store signal
|
||||
self.signal_history.append({
|
||||
'timestamp': datetime.now(),
|
||||
'signal': signal,
|
||||
'price': market_data.get('USDJPY_Close', 0)
|
||||
})
|
||||
|
||||
# Calculate position change
|
||||
position_change = 0.0
|
||||
if abs(signal - self.last_signal) > NO_TRADE_BAND:
|
||||
# Calculate target position in units
|
||||
current_price = market_data.get('USDJPY_Close', 1.0)
|
||||
target_units = pct_to_units(signal, self.equity, current_price)
|
||||
position_change = target_units - self.position
|
||||
|
||||
if abs(position_change) > POSITION_GRID:
|
||||
position_change = round_grid(position_change)
|
||||
self.position += position_change
|
||||
self.last_signal = signal
|
||||
|
||||
# Log trade
|
||||
trade = {
|
||||
'timestamp': datetime.now(),
|
||||
'action': 'BUY' if position_change > 0 else 'SELL',
|
||||
'size': abs(position_change),
|
||||
'price': current_price,
|
||||
'signal': signal,
|
||||
'new_position': self.position
|
||||
}
|
||||
self.trade_history.append(trade)
|
||||
|
||||
print(f"{trade['action']} {trade['size']:,} units at {trade['price']:.5f}")
|
||||
|
||||
return {
|
||||
'signal': signal,
|
||||
'position_change': position_change,
|
||||
'current_position': self.position,
|
||||
'should_trade': abs(position_change) > 0,
|
||||
'timestamp': datetime.now()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing market data: {e}")
|
||||
return {
|
||||
'signal': 0.0,
|
||||
'position_change': 0.0,
|
||||
'current_position': self.position,
|
||||
'should_trade': False,
|
||||
'error': str(e),
|
||||
'timestamp': datetime.now()
|
||||
}
|
||||
|
||||
def get_status(self) -> Dict:
|
||||
"""Get current bot status."""
|
||||
return {
|
||||
'position': self.position,
|
||||
'equity': self.equity,
|
||||
'last_signal': self.last_signal,
|
||||
'total_trades': len(self.trade_history),
|
||||
'model_info': self.model_manager.get_model_info(),
|
||||
'last_update': datetime.now()
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# 6. Example Usage Functions
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
import matplotlib.pyplot as plt
|
||||
def backtest_saved_model(
|
||||
data_dir: Path,
|
||||
model_path: str = "best_individual.dill",
|
||||
start_date: str = None,
|
||||
end_date: str = None,
|
||||
plot: bool = False, # <-- add this
|
||||
):
|
||||
print(f"Backtesting saved model: {model_path}", flush=True)
|
||||
|
||||
# Load data once
|
||||
print("Loading data once...", flush=True)
|
||||
df_all = load_all_pairs(data_dir)
|
||||
|
||||
if start_date and end_date:
|
||||
df_test = df_all.loc[start_date:end_date]
|
||||
print(f"Testing period: {start_date} to {end_date} | bars={len(df_test)}", flush=True)
|
||||
else:
|
||||
df_test = df_all
|
||||
print(f"Testing on full dataset | bars={len(df_test)}", flush=True)
|
||||
|
||||
# Load model once
|
||||
model_manager = GPModelManager(model_path)
|
||||
if not model_manager.load_model():
|
||||
return None
|
||||
|
||||
# Inject model into strategy to avoid double loading
|
||||
LiveGPStrategy.model_manager = model_manager
|
||||
|
||||
# Prepare USDJPY base + extras
|
||||
usdjpy = df_test[[c for c in df_test.columns if c.startswith("USDJPY_")]].copy()
|
||||
usdjpy.columns = [c.replace("USDJPY_", "") for c in usdjpy.columns]
|
||||
extras = df_test[[c for c in df_test.columns if not c.startswith("USDJPY_")]]
|
||||
usdjpy = usdjpy.join(extras)
|
||||
|
||||
bt = Backtest(
|
||||
usdjpy,
|
||||
LiveGPStrategy,
|
||||
margin=1/5,
|
||||
cash=INITIAL_CASH,
|
||||
commission=COMMISSION_PCT,
|
||||
exclusive_orders=False,
|
||||
trade_on_close=True,
|
||||
)
|
||||
|
||||
print("Running backtest (this can take time)...", flush=True)
|
||||
stats = bt.run()
|
||||
|
||||
print("\nBACKTEST RESULTS")
|
||||
print(f"Return: {stats['Return [%]']:.2f}%")
|
||||
print(f"Sharpe Ratio: {stats['Sharpe Ratio']:.3f}")
|
||||
print(f"Max Drawdown: {stats['Max. Drawdown [%]']:.2f}%")
|
||||
print(f"Number of Trades: {stats['# Trades']}")
|
||||
print(f"Win Rate: {stats['Win Rate [%]']:.1f}%")
|
||||
print(f"Final Equity: ${stats['Equity Final [$]']:,.2f}")
|
||||
|
||||
if plot:
|
||||
try:
|
||||
print("\nPlotting…", flush=True)
|
||||
bt.plot()
|
||||
except Exception as e:
|
||||
print(f"Could not plot: {e}")
|
||||
|
||||
return stats
|
||||
|
||||
def demo_live_trading():
|
||||
"""Demonstrate live trading with the production bot."""
|
||||
|
||||
print("Live Trading Demo")
|
||||
print("=" * 50)
|
||||
|
||||
# Initialize bot
|
||||
bot = ProductionTradingBot()
|
||||
if not bot.initialize():
|
||||
print("Failed to initialize trading bot")
|
||||
return
|
||||
|
||||
# Simulate some market data updates
|
||||
print("\nSimulating market data updates...")
|
||||
|
||||
for i in range(10):
|
||||
# Generate random market data (in practice, this comes from your data feed)
|
||||
market_data = {}
|
||||
for arg_name in ARG_NAMES:
|
||||
market_data[arg_name] = np.random.uniform(1.0, 1.5) # Random prices
|
||||
|
||||
# Process market data
|
||||
decision = bot.process_market_data(market_data)
|
||||
|
||||
print(f"Step {i+1}: Signal={decision['signal']:.2f}, "
|
||||
f"Position={decision['current_position']:.0f}, "
|
||||
f"Trade={decision['should_trade']}")
|
||||
|
||||
time.sleep(0.1) # Simulate time between updates
|
||||
|
||||
# Show final status
|
||||
status = bot.get_status()
|
||||
print(f"\nFinal Status:")
|
||||
print(f"Position: {status['position']:.0f}")
|
||||
print(f"Total Trades: {status['total_trades']}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage
|
||||
print("GP Trading System - Model Usage Examples")
|
||||
print("=" * 60)
|
||||
|
||||
# Example 1: Backtest a saved model
|
||||
print("\n1. Backtesting saved model...")
|
||||
backtest_saved_model(Path("."), start_date="2024-07-05", end_date="2025-07-05")
|
||||
|
||||
# Example 2: Generate signals for a period
|
||||
#print("\n2. Generating signals...")
|
||||
# model = GPModelManager()
|
||||
# if model.load_model():
|
||||
# data = load_all_pairs(Path("."))
|
||||
# signals = generate_signals_for_period(model, data, "2025-01-01", "2025-02-01")
|
||||
# print(f"Generated {len(signals)} signals")
|
||||
|
||||
# Example 3: Live trading demo
|
||||
#print("\n3. Live trading demo...")
|
||||
#demo_live_trading()
|
||||
Reference in New Issue
Block a user