feat: Organize notebooks and add stationarity checks
This commit is contained in:
Mohammad Aghdam
+134
@@ -0,0 +1,134 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# VSCode
|
||||
.vscode/
|
||||
*.code-workspace
|
||||
|
||||
# Custom
|
||||
*.csv
|
||||
*.db
|
||||
*.log
|
||||
*.xlsx
|
||||
*.pkl
|
||||
|
||||
|
||||
|
||||
|
||||
# Developer notes
|
||||
NOTES.md
|
||||
@@ -1,5 +1,4 @@
|
||||
# AlphaFlow-MT5-ML-DL-Trading-Lab
|
||||
## Multi-strategy MT5 research lab for ML/DL/time-series trading: data → modeling → backtests → tuning → prototype execution.
|
||||
# AlphaFlow ML & DL Trading Bot Project
|
||||
|
||||
A comprehensive **machine learning and deep learning trading framework** that covers the entire workflow:
|
||||
|
||||
@@ -42,49 +41,61 @@ This project provides a flexible **template** for you to **create and add your o
|
||||
|
||||
## Repository Structure
|
||||
```bash
|
||||
# ML Bot Trading Repository Structure
|
||||
# AlphaFlow ML & DL Trading Bot Repository Structure
|
||||
|
||||
ml_bot_trading/
|
||||
AlphaFlow-MT5-ML-DL-Trading-Lab/
|
||||
├── data/
|
||||
│ ├── data_loader.py # MetaTrader 5 data retrieval
|
||||
│ ├── data_loader.py # MetaTrader 5 data retrieval
|
||||
│
|
||||
├── features/
|
||||
│ ├── feature_engineering.py # Technical indicators, custom features
|
||||
│ ├── labeling.py # Labeling methods: next-bar, multi-bar, double-barrier, regime detection
|
||||
│ ├── feature_engineering.py # Technical indicators, stationarity checks, custom features
|
||||
│ ├── labeling_schemes.py # Labeling methods: next-bar, multi-bar, double-barrier, regime detection
|
||||
│
|
||||
├── models/
|
||||
│ ├── model_training.py # Model selection, hyperparam tuning
|
||||
│ ├── saved_models/ # Folder for .pkl pipelines (best_rf_pipeline.pkl, etc.)
|
||||
│ ├── model_training.py # Model selection, hyperparameter tuning (Optuna, GridSearchCV)
|
||||
│ ├── saved_models/ # Folder for saved model pipelines (.pkl, .joblib)
|
||||
│
|
||||
├── backtests/
|
||||
│ ├── simple_backtest.py # Simple Pythonic backtest logic
|
||||
│ ├── vectorbt_backtest.py # VectorBT-based backtesting template
|
||||
│ ├── simple_backtest.py # Simple event-driven backtest logic
|
||||
│ ├── vectorbt_backtest.py # VectorBT-based backtesting template
|
||||
│
|
||||
├── live_trading/
|
||||
│ ├── regression_returns.py # Live trading script for regression returns
|
||||
│ ├── multi_bar.py # Live trading script for multi-bar classification
|
||||
│ ├── double_barrier.py # Live trading script for double-barrier labeling
|
||||
│ ├── regime_detection.py # Live trading script for regime detection
|
||||
│ ├── ... (live trading scripts)
|
||||
│
|
||||
├── notebooks/
|
||||
│ ├── dl_notebooks/
|
||||
│ │ ├── 00_eda_visualization.ipynb
|
||||
│ │ ├── 01_backtests_regression_returns_dl.ipynb
|
||||
│ │ ├── 01_live_trading_regression_returns_dl.ipynb
|
||||
│ │ ├── 02_time_series_arima_sarima_var_lstmprice.ipynb
|
||||
│ │
|
||||
│ ├── eda_notebooks/
|
||||
│ │ ├── 00_eda_visualization.ipynb
|
||||
│ │
|
||||
│ ├── ml_notebooks/
|
||||
│ │ ├── 01_backtests_regression_returns.ipynb
|
||||
│ │ ├── 01_live_trading_regression_returns.ipynb
|
||||
│ │ ├── 02_backtests_multi_bar_classification.ipynb
|
||||
│ │ ├── 02_live_trading_multi_bar_classification.ipynb
|
||||
│ │ ├── 03_backtests_double_barrier_labeling.ipynb
|
||||
│ │ ├── 03_live_trading_double_barrier_labeling.ipynb
|
||||
│ │ ├── 04_backtests_regime_detection.ipynb
|
||||
│ │ ├── 04_live_trading_regime_detection.ipynb
|
||||
│ ├── exploratory/
|
||||
│ │ ├── eda_visualization.ipynb
|
||||
│ │ ├── integrated_pipeline_pair_trading.ipynb
|
||||
│ │ └── kalman_filters_solution.ipynb
|
||||
│ ├── strategies/
|
||||
│ │ ├── ml/
|
||||
│ │ │ ├── backtesting/
|
||||
│ │ │ │ ├── regression_returns.ipynb
|
||||
│ │ │ │ ├── double_barrier_labeling.ipynb
|
||||
│ │ │ │ ├── multi_bar_classification.ipynb
|
||||
│ │ │ │ ├── multi_bar_classification_multisymbol.ipynb
|
||||
│ │ │ │ ├── multi_bar_classification_multisymbol_core_features.ipynb
|
||||
│ │ │ │ ├── multi_bar_classification_multisymbol_core_features_stocks.ipynb
|
||||
│ │ │ │ ├── regime_detection.ipynb
|
||||
│ │ │ │ ├── momentum_strategy.ipynb
|
||||
│ │ │ │ ├── pairs_trading_cointegration.ipynb
|
||||
│ │ │ │ └── pairs_trading_clustering.ipynb
|
||||
│ │ │ └── live_trading/
|
||||
│ │ │ ├── regression_returns.ipynb
|
||||
│ │ │ ├── double_barrier_labeling.ipynb
|
||||
│ │ │ ├── multi_bar_classification.ipynb
|
||||
│ │ │ ├── multi_bar_classification_multisymbol_core_features.ipynb
|
||||
│ │ │ ├── regime_detection.ipynb
|
||||
│ │ │ ├── pairs_trading_cointegration.ipynb
|
||||
│ │ │ └── pairs_trading_clustering.ipynb
|
||||
│ │ └── dl/
|
||||
│ │ ├── backtesting/
|
||||
│ │ │ ├── regression_returns.ipynb
|
||||
│ │ │ └── multi_bar_classification_multisymbol.ipynb
|
||||
│ │ └── live_trading/
|
||||
│ │ └── regression_returns.ipynb
|
||||
│ └── time_series/
|
||||
│ └── arima_sarima_var_lstm.ipynb
|
||||
│
|
||||
├── requirements.txt
|
||||
├── README.md
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# vectorbt_backtest.py
|
||||
# backtests/vectorbt_backtest.py
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
@@ -15,66 +15,29 @@ def run_vectorbt_backtest(
|
||||
threshold=0.0
|
||||
):
|
||||
"""
|
||||
Runs a vectorbt backtest for a given pre-trained model.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model : fitted scikit-learn model
|
||||
Already fitted model (e.g. RandomForestRegressor).
|
||||
X : pd.DataFrame
|
||||
The full feature DataFrame (or the portion you want to backtest).
|
||||
selected_features : list
|
||||
List of feature names used by the model.
|
||||
data : pd.DataFrame
|
||||
Original DataFrame containing at least a 'close' column.
|
||||
scaler : fitted scaler
|
||||
The StandardScaler (or other) used to scale features.
|
||||
init_cash : float
|
||||
Starting capital for the backtest.
|
||||
freq : str
|
||||
Frequency for vectorbt (e.g. '4H', '1D').
|
||||
threshold : float
|
||||
Minimum absolute predicted return to place a trade (optional).
|
||||
|
||||
Returns
|
||||
-------
|
||||
pf : vbt.Portfolio
|
||||
The resulting vectorbt portfolio object.
|
||||
Simple, short-enabled backtest using target exposure (-1, 0, +1).
|
||||
"""
|
||||
# 1) Subset X to the selected features
|
||||
# features -> scale -> predict
|
||||
X_sel = X[selected_features]
|
||||
preds = model.predict(scaler.transform(X_sel))
|
||||
|
||||
# 2) Scale
|
||||
X_scaled = scaler.transform(X_sel)
|
||||
|
||||
# 3) Generate predictions
|
||||
preds = model.predict(X_scaled)
|
||||
|
||||
# 4) Convert predictions to signals
|
||||
# Optionally use threshold to reduce whipsaws
|
||||
# map preds -> {-1, 0, 1}
|
||||
if threshold > 0.0:
|
||||
signals = np.where(preds > threshold, 1, np.where(preds < -threshold, -1, 0))
|
||||
exposure = np.where(preds > threshold, 1.0,
|
||||
np.where(preds < -threshold, -1.0, 0.0))
|
||||
else:
|
||||
signals = np.sign(preds)
|
||||
exposure = np.sign(preds).astype(float)
|
||||
|
||||
# 5) Align signals with close prices
|
||||
close_prices = data.loc[X_sel.index, "close"]
|
||||
# If signals is shorter or the same length
|
||||
if len(signals) < len(close_prices):
|
||||
# Pad signals with 0 if needed
|
||||
signals = np.append(signals, [0]*(len(close_prices)-len(signals)))
|
||||
# align to prices
|
||||
close = data.loc[X_sel.index, "close"]
|
||||
target = pd.Series(exposure, index=close.index)
|
||||
|
||||
signals_s = pd.Series(signals, index=close_prices.index)
|
||||
# Align if any missing indexes
|
||||
close_prices, signals_s = close_prices.align(signals_s, join="inner", axis=0)
|
||||
|
||||
# 6) Run vectorbt Portfolio
|
||||
pf = vbt.Portfolio.from_signals(
|
||||
close_prices,
|
||||
entries=signals_s > 0,
|
||||
exits=signals_s < 0,
|
||||
# build portfolio: -1 short, 0 flat, +1 long
|
||||
pf = vbt.Portfolio.from_orders(
|
||||
close=close,
|
||||
size=target,
|
||||
size_type='targetpercent',
|
||||
init_cash=init_cash,
|
||||
freq=freq
|
||||
)
|
||||
|
||||
return pf
|
||||
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
import streamlit as st
|
||||
import sqlite3
|
||||
import pandas as pd
|
||||
from streamlit_autorefresh import st_autorefresh
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
# --- Constants ---
|
||||
N_FORWARD = 3 # Set this to match your trading bot's N_FORWARD
|
||||
DB_PATH = Path("live_signals.db")
|
||||
TABLE_NAME = "signals"
|
||||
COL_TIMESTAMP = "timestamp"
|
||||
COL_SYMBOL = "symbol"
|
||||
COL_PREDICTION = "prediction"
|
||||
|
||||
# --- Page Config ---
|
||||
st.set_page_config(page_title="AlphaFlow Live Signals", layout="wide")
|
||||
st.title("📈 AlphaFlow Trading Bot - Live Signals")
|
||||
|
||||
# Auto-refresh every 60 seconds
|
||||
st_autorefresh(interval=60_000, key="refresh")
|
||||
|
||||
# --- Data Loading ---
|
||||
@st.cache_data(ttl=30)
|
||||
def load_data() -> Optional[pd.DataFrame]:
|
||||
"""Load signal data from the SQLite database."""
|
||||
if not DB_PATH.exists():
|
||||
return None
|
||||
try:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
df = pd.read_sql(f'SELECT * FROM {TABLE_NAME} ORDER BY {COL_TIMESTAMP} DESC', conn)
|
||||
conn.close()
|
||||
return df
|
||||
except (sqlite3.Error, pd.errors.DatabaseError) as e:
|
||||
st.error(f"Database error: {e}")
|
||||
return None
|
||||
|
||||
def display_latest_signals(df: pd.DataFrame):
|
||||
st.subheader("Latest Signal Per Symbol")
|
||||
latest = (
|
||||
df.sort_values(COL_TIMESTAMP)
|
||||
.groupby(COL_SYMBOL)
|
||||
.tail(1)
|
||||
.sort_values(COL_SYMBOL)
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
st.dataframe(latest[[COL_SYMBOL, COL_PREDICTION, COL_TIMESTAMP]], use_container_width=True)
|
||||
|
||||
def display_recent_signals(df: pd.DataFrame):
|
||||
st.subheader(f"Latest {N_FORWARD} Signals Per Symbol")
|
||||
recent = (
|
||||
df.sort_values([COL_SYMBOL, COL_TIMESTAMP])
|
||||
.groupby(COL_SYMBOL)
|
||||
.tail(N_FORWARD)
|
||||
.sort_values([COL_SYMBOL, COL_TIMESTAMP])
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
st.dataframe(recent[[COL_SYMBOL, COL_PREDICTION, COL_TIMESTAMP]], use_container_width=True)
|
||||
|
||||
# Optional: Show a quick mini-forecast chart per symbol
|
||||
st.write("---")
|
||||
st.subheader("Mini Signal Forecasts (per symbol)")
|
||||
for symbol in sorted(df[COL_SYMBOL].unique()):
|
||||
mini_df = (
|
||||
df[df[COL_SYMBOL] == symbol]
|
||||
.sort_values(COL_TIMESTAMP)
|
||||
.tail(N_FORWARD)
|
||||
)
|
||||
# Only show if there is more than one unique value
|
||||
if mini_df[COL_PREDICTION].nunique() > 1:
|
||||
st.write(f"**{symbol}**")
|
||||
st.line_chart(mini_df.set_index(COL_TIMESTAMP)[[COL_PREDICTION]], height=100, use_container_width=True)
|
||||
|
||||
def display_signal_history(df: pd.DataFrame):
|
||||
with st.expander("Show Full Signal History"):
|
||||
st.dataframe(df[[COL_SYMBOL, COL_PREDICTION, COL_TIMESTAMP]], use_container_width=True)
|
||||
|
||||
def display_signal_distribution(df: pd.DataFrame):
|
||||
st.subheader("Signal Distribution (All Time)")
|
||||
signal_counts = df.groupby([COL_SYMBOL, COL_PREDICTION]).size().unstack(fill_value=0)
|
||||
st.bar_chart(signal_counts)
|
||||
|
||||
def main():
|
||||
"""Main function to run the Streamlit dashboard."""
|
||||
# Show signal legend and refresh time
|
||||
st.markdown("""
|
||||
| Signal | Meaning |
|
||||
|--------|---------|
|
||||
| -1 | **Sell**|
|
||||
| 0 | **Flat**|
|
||||
| 1 | **Buy** |
|
||||
""")
|
||||
st.caption(f"Last refreshed: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')} (server time, probably UTC)")
|
||||
|
||||
df = load_data()
|
||||
|
||||
if df is None or df.empty:
|
||||
st.warning("No signals found in the database yet. Please wait for signals to be generated.")
|
||||
return
|
||||
|
||||
view_mode = st.radio(
|
||||
"View Mode",
|
||||
["Latest Signal Per Symbol", f"Latest {N_FORWARD} Signals Per Symbol"],
|
||||
horizontal=True,
|
||||
)
|
||||
|
||||
if view_mode == "Latest Signal Per Symbol":
|
||||
display_latest_signals(df)
|
||||
else:
|
||||
display_recent_signals(df)
|
||||
|
||||
display_signal_history(df)
|
||||
display_signal_distribution(df)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+474
-245
@@ -1,166 +1,188 @@
|
||||
# feature_engineering.py
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import warnings
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import math
|
||||
import ta
|
||||
from statsmodels.tsa.stattools import adfuller
|
||||
from scipy.fftpack import fft
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
from scipy.fftpack import fft # simple global FFT (optional)
|
||||
from statsmodels.tsa.stattools import adfuller, kpss
|
||||
from sklearn.preprocessing import StandardScaler # keep for downstream pipelines
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Stationarity configuration (can be overridden at call time)
|
||||
# =============================================================================
|
||||
STATIONARITY_CFG: Dict = {
|
||||
"enabled": True,
|
||||
"adf_alpha": 0.05, # want ADF p < alpha
|
||||
"kpss_alpha": 0.05, # want KPSS p > alpha
|
||||
"keep_original": False, # keep both original and stationary variant
|
||||
"max_diff": 2, # maximum extra differencing attempts
|
||||
"seasonal_period": None, # e.g., 6 for 4H bars ~ daily; None to skip
|
||||
"transform_order": ["pct_change", "diff1", "log_diff1", "seasonal_diff"],
|
||||
# columns we never transform (raw OHLCV by default)
|
||||
"exclude_cols": {"open", "high", "low", "close", "tick_volume", "volume",
|
||||
"rolling_adf_stat", "rolling_adf_pval", "stationary_flag"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# 1) TA-LIB FEATURES (add_all_ta_features)
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
# 1) TA-LIB FEATURES (ta library)
|
||||
# =============================================================================
|
||||
def add_all_ta_features(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Adds a wide range of technical analysis indicators to the DataFrame
|
||||
using the 'ta' library. Modifies the DataFrame in place.
|
||||
Adds a wide range of technical analysis indicators using the 'ta' library.
|
||||
Modifies the DataFrame in place and returns it for chaining.
|
||||
"""
|
||||
df = ta.add_all_ta_features(
|
||||
df, open="open", high="high", low="low", close="close", volume="tick_volume", fillna=True
|
||||
df,
|
||||
open="open",
|
||||
high="high",
|
||||
low="low",
|
||||
close="close",
|
||||
volume="tick_volume",
|
||||
fillna=True,
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def create_custom_feature(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Example custom feature. For instance, a rolling mean of the close price.
|
||||
"""
|
||||
"""Example custom feature: a rolling mean of the close price."""
|
||||
df["rolling_mean_10"] = df["close"].rolling(window=10).mean()
|
||||
return df
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# 2) MISCELLANEOUS FEATURES
|
||||
# --------------------------------------------------------------------
|
||||
def spread(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Calculates the spread between 'high' and 'low' columns.
|
||||
"""
|
||||
df_copy = df.copy()
|
||||
df_copy["spread"] = df_copy["high"] - df_copy["low"]
|
||||
return df_copy
|
||||
|
||||
def auto_corr_multi(df: pd.DataFrame, col: str, n: int = 50, lags: list = [1, 3, 5, 10]) -> pd.DataFrame:
|
||||
"""
|
||||
Computes rolling autocorrelation for multiple lags.
|
||||
"""
|
||||
df_copy = df.copy()
|
||||
# =============================================================================
|
||||
# 2) MISCELLANEOUS FEATURES
|
||||
# =============================================================================
|
||||
def spread(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Spread between high and low."""
|
||||
dfc = df.copy()
|
||||
dfc["spread"] = dfc["high"] - dfc["low"]
|
||||
return dfc
|
||||
|
||||
|
||||
def auto_corr_multi(
|
||||
df: pd.DataFrame, col: str, n: int = 50, lags: List[int] = [1, 3, 5, 10]
|
||||
) -> pd.DataFrame:
|
||||
"""Rolling autocorrelation for multiple lags."""
|
||||
dfc = df.copy()
|
||||
for lag in lags:
|
||||
df_copy[f"autocorr_{lag}"] = (
|
||||
df_copy[col]
|
||||
dfc[f"autocorr_{lag}"] = (
|
||||
dfc[col]
|
||||
.rolling(window=n, min_periods=n)
|
||||
.apply(lambda x: x.autocorr(lag=lag), raw=False)
|
||||
)
|
||||
return df_copy
|
||||
return dfc
|
||||
|
||||
|
||||
def candle_information(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Adds candle-specific features:
|
||||
- candle_way
|
||||
- fill
|
||||
- amplitude
|
||||
- candle_way (1 if close > open else 0)
|
||||
- fill (real body / range)
|
||||
- amplitude (abs(close - open) / open)
|
||||
"""
|
||||
df_copy = df.copy()
|
||||
df_copy["candle_way"] = 0
|
||||
df_copy.loc[df_copy["close"] > df_copy["open"], "candle_way"] = 1
|
||||
|
||||
df_copy["fill"] = (
|
||||
np.abs(df_copy["close"] - df_copy["open"])
|
||||
/ (df_copy["high"] - df_copy["low"] + 1e-5)
|
||||
)
|
||||
df_copy["amplitude"] = (
|
||||
np.abs(df_copy["close"] - df_copy["open"])
|
||||
/ (df_copy["open"] + 1e-5)
|
||||
)
|
||||
return df_copy
|
||||
dfc = df.copy()
|
||||
dfc["candle_way"] = (dfc["close"] > dfc["open"]).astype(int)
|
||||
rng = (dfc["high"] - dfc["low"]).replace(0, np.nan)
|
||||
dfc["fill"] = (dfc["close"] - dfc["open"]).abs() / (rng + 1e-5)
|
||||
dfc["amplitude"] = (dfc["close"] - dfc["open"]).abs() / (dfc["open"].abs() + 1e-5)
|
||||
return dfc
|
||||
|
||||
|
||||
def log_transform(df: pd.DataFrame, col: str, n: int) -> pd.DataFrame:
|
||||
"""
|
||||
Log-transform a column + compute % change over 'n' bars.
|
||||
Create log(price) and n-period log-return: log_ret_n = log(col).diff(n).
|
||||
"""
|
||||
df_copy = df.copy()
|
||||
df_copy[f"log_{col}"] = np.log(df_copy[col])
|
||||
df_copy[f"ret_log_{n}"] = df_copy[f"log_{col}"].pct_change(periods=n)
|
||||
return df_copy
|
||||
dfc = df.copy()
|
||||
# clip to avoid log(0); if strictly positive, you can drop clip
|
||||
dfc[f"log_{col}"] = np.log(dfc[col].clip(lower=1e-12))
|
||||
dfc[f"log_ret_{n}"] = dfc[f"log_{col}"].diff(n)
|
||||
return dfc
|
||||
|
||||
|
||||
def mathematical_derivatives(df: pd.DataFrame, col: str) -> pd.DataFrame:
|
||||
"""
|
||||
Adds 'velocity' and 'acceleration' for a given column.
|
||||
"""
|
||||
df_copy = df.copy()
|
||||
df_copy["velocity"] = df_copy[col].diff()
|
||||
df_copy["acceleration"] = df_copy["velocity"].diff()
|
||||
return df_copy
|
||||
"""Velocity and acceleration for a given column."""
|
||||
dfc = df.copy()
|
||||
dfc["velocity"] = dfc[col].diff()
|
||||
dfc["acceleration"] = dfc["velocity"].diff()
|
||||
return dfc
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
# =============================================================================
|
||||
# 3) VOLATILITY ESTIMATORS
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
def parkinson_estimator(window: pd.DataFrame) -> float:
|
||||
n = len(window)
|
||||
if n < 1:
|
||||
return np.nan
|
||||
sum_sq = np.sum(np.log(window['high'] / window['low']) ** 2)
|
||||
sum_sq = np.sum(np.log(window["high"] / window["low"]) ** 2)
|
||||
return math.sqrt(sum_sq / (4 * math.log(2) * n))
|
||||
|
||||
|
||||
def moving_parkinson_estimator(df: pd.DataFrame, window_size: int = 30) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
rolling_vol = pd.Series(dtype="float64", index=df_copy.index)
|
||||
for i in range(window_size, len(df_copy)):
|
||||
w = df_copy.iloc[i - window_size : i]
|
||||
dfc = df.copy()
|
||||
rolling_vol = pd.Series(dtype="float64", index=dfc.index)
|
||||
for i in range(window_size, len(dfc)):
|
||||
w = dfc.iloc[i - window_size : i]
|
||||
rolling_vol.iloc[i] = parkinson_estimator(w)
|
||||
df_copy["rolling_volatility_parkinson"] = rolling_vol
|
||||
return df_copy
|
||||
dfc["rolling_volatility_parkinson"] = rolling_vol
|
||||
return dfc
|
||||
|
||||
|
||||
def yang_zhang_estimator(window: pd.DataFrame) -> float:
|
||||
n = len(window)
|
||||
if n < 1:
|
||||
return np.nan
|
||||
term1 = np.log(window['high'] / window['low']) ** 2
|
||||
term2 = np.log(window['close'] / window['open']) ** 2
|
||||
term1 = np.log(window["high"] / window["low"]) ** 2
|
||||
term2 = np.log(window["close"] / window["open"]) ** 2
|
||||
return math.sqrt(np.mean(term1 + term2))
|
||||
|
||||
def moving_yang_zhang_estimator(df: pd.DataFrame, window_size: int = 30) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
rolling_vol = pd.Series(dtype="float64", index=df_copy.index)
|
||||
for i in range(window_size, len(df_copy)):
|
||||
w = df_copy.iloc[i - window_size : i]
|
||||
rolling_vol.iloc[i] = yang_zhang_estimator(w)
|
||||
df_copy["rolling_volatility_yang_zhang"] = rolling_vol
|
||||
return df_copy
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
def moving_yang_zhang_estimator(df: pd.DataFrame, window_size: int = 30) -> pd.DataFrame:
|
||||
dfc = df.copy()
|
||||
rolling_vol = pd.Series(dtype="float64", index=dfc.index)
|
||||
for i in range(window_size, len(dfc)):
|
||||
w = dfc.iloc[i - window_size : i]
|
||||
rolling_vol.iloc[i] = yang_zhang_estimator(w)
|
||||
dfc["rolling_volatility_yang_zhang"] = rolling_vol
|
||||
return dfc
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 4) MARKET REGIME / DC EVENTS
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
def dc_event(P: float, Pext: float, threshold: float) -> int:
|
||||
dc = 0
|
||||
var = (P - Pext) / Pext
|
||||
if var >= threshold:
|
||||
dc = 1
|
||||
elif var <= -threshold:
|
||||
dc = -1
|
||||
return dc
|
||||
return 1
|
||||
if var <= -threshold:
|
||||
return -1
|
||||
return 0
|
||||
|
||||
def calculate_dc(df: pd.DataFrame, threshold: float = 0.01) -> tuple:
|
||||
df_copy = df.copy()
|
||||
prices = df_copy['close'].values
|
||||
dc_events_up, dc_events_down = [], []
|
||||
|
||||
def calculate_dc(df: pd.DataFrame, threshold: float = 0.01) -> Tuple[List[int], List[int]]:
|
||||
dfc = df.copy()
|
||||
prices = dfc["close"].values
|
||||
dc_up, dc_down = [], []
|
||||
Pext = prices[0]
|
||||
direction = 0
|
||||
for i in range(1, len(prices)):
|
||||
P = prices[i]
|
||||
dc_flag = dc_event(P, Pext, threshold)
|
||||
if dc_flag == 1:
|
||||
dc_events_up.append(i)
|
||||
flag = dc_event(P, Pext, threshold)
|
||||
if flag == 1:
|
||||
dc_up.append(i)
|
||||
direction = 1
|
||||
Pext = P
|
||||
elif dc_flag == -1:
|
||||
dc_events_down.append(i)
|
||||
elif flag == -1:
|
||||
dc_down.append(i)
|
||||
direction = -1
|
||||
Pext = P
|
||||
else:
|
||||
@@ -168,214 +190,421 @@ def calculate_dc(df: pd.DataFrame, threshold: float = 0.01) -> tuple:
|
||||
Pext = P
|
||||
elif direction == -1 and P < Pext:
|
||||
Pext = P
|
||||
return dc_events_up, dc_events_down
|
||||
return dc_up, dc_down
|
||||
|
||||
def calculate_trend(dc_events_up: list, dc_events_down: list, df: pd.DataFrame):
|
||||
trend_events_down = []
|
||||
trend_events_up = []
|
||||
trend_events_down.extend(sorted(dc_events_down))
|
||||
trend_events_up.extend(sorted(dc_events_up))
|
||||
|
||||
def calculate_trend(dc_events_up: List[int], dc_events_down: List[int], df: pd.DataFrame):
|
||||
trend_events_down = list(sorted(dc_events_down))
|
||||
trend_events_up = list(sorted(dc_events_up))
|
||||
return trend_events_down, trend_events_up
|
||||
|
||||
|
||||
def market_regime_dc(df: pd.DataFrame, threshold: float = 0.01) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
dc_up, dc_down = calculate_dc(df_copy, threshold=threshold)
|
||||
t_down, t_up = calculate_trend(dc_up, dc_down, df_copy)
|
||||
df_copy['market_regime'] = np.nan
|
||||
df_copy.loc[t_up, 'market_regime'] = 1
|
||||
df_copy.loc[t_down, 'market_regime'] = 0
|
||||
df_copy['market_regime'] = df_copy['market_regime'].ffill().bfill()
|
||||
return df_copy
|
||||
dfc = df.copy()
|
||||
dc_up, dc_down = calculate_dc(dfc, threshold=threshold)
|
||||
t_down, t_up = calculate_trend(dc_up, dc_down, dfc)
|
||||
dfc["market_regime"] = np.nan
|
||||
dfc.loc[t_up, "market_regime"] = 1
|
||||
dfc.loc[t_down, "market_regime"] = 0
|
||||
dfc["market_regime"] = dfc["market_regime"].ffill().bfill()
|
||||
return dfc
|
||||
|
||||
def kama_market_regime(df: pd.DataFrame, col: str = 'close', n1: int = 10, n2: int = 30) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
short_kama = df_copy[col].ewm(span=n1, adjust=False).mean()
|
||||
long_kama = df_copy[col].ewm(span=n2, adjust=False).mean()
|
||||
df_copy['kama_diff'] = short_kama - long_kama
|
||||
df_copy['kama_trend'] = (df_copy['kama_diff'] >= 0).astype(int)
|
||||
return df_copy
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
def kama_market_regime(df: pd.DataFrame, col: str = "close", n1: int = 10, n2: int = 30) -> pd.DataFrame:
|
||||
dfc = df.copy()
|
||||
short_kama = dfc[col].ewm(span=n1, adjust=False).mean()
|
||||
long_kama = dfc[col].ewm(span=n2, adjust=False).mean()
|
||||
dfc["kama_diff"] = short_kama - long_kama
|
||||
dfc["kama_trend"] = (dfc["kama_diff"] >= 0).astype(int)
|
||||
return dfc
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 5) GAP & DISPLACEMENT
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
def gap_detection(df: pd.DataFrame, lookback: int = 1) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
df_copy['Bullish_gap_inf'] = np.nan
|
||||
df_copy['Bullish_gap_sup'] = np.nan
|
||||
df_copy['Bullish_gap_size'] = np.nan
|
||||
df_copy['Bearish_gap_inf'] = np.nan
|
||||
df_copy['Bearish_gap_sup'] = np.nan
|
||||
df_copy['Bearish_gap_size'] = np.nan
|
||||
for i in range(lookback, len(df_copy)):
|
||||
prev_high = df_copy['high'].iloc[i - lookback]
|
||||
prev_low = df_copy['low'].iloc[i - lookback]
|
||||
curr_high = df_copy['high'].iloc[i]
|
||||
curr_low = df_copy['low'].iloc[i]
|
||||
dfc = df.copy()
|
||||
cols = [
|
||||
"Bullish_gap_inf",
|
||||
"Bullish_gap_sup",
|
||||
"Bullish_gap_size",
|
||||
"Bearish_gap_inf",
|
||||
"Bearish_gap_sup",
|
||||
"Bearish_gap_size",
|
||||
]
|
||||
for c in cols:
|
||||
dfc[c] = np.nan
|
||||
for i in range(lookback, len(dfc)):
|
||||
prev_high = dfc["high"].iloc[i - lookback]
|
||||
prev_low = dfc["low"].iloc[i - lookback]
|
||||
curr_high = dfc["high"].iloc[i]
|
||||
curr_low = dfc["low"].iloc[i]
|
||||
if curr_low > prev_high:
|
||||
df_copy.at[df_copy.index[i], 'Bullish_gap_inf'] = prev_high
|
||||
df_copy.at[df_copy.index[i], 'Bullish_gap_sup'] = curr_low
|
||||
df_copy.at[df_copy.index[i], 'Bullish_gap_size'] = curr_low - prev_high
|
||||
dfc.at[dfc.index[i], "Bullish_gap_inf"] = prev_high
|
||||
dfc.at[dfc.index[i], "Bullish_gap_sup"] = curr_low
|
||||
dfc.at[dfc.index[i], "Bullish_gap_size"] = curr_low - prev_high
|
||||
if curr_high < prev_low:
|
||||
df_copy.at[df_copy.index[i], 'Bearish_gap_inf'] = curr_high
|
||||
df_copy.at[df_copy.index[i], 'Bearish_gap_sup'] = prev_low
|
||||
df_copy.at[df_copy.index[i], 'Bearish_gap_size'] = prev_low - curr_high
|
||||
return df_copy
|
||||
dfc.at[dfc.index[i], "Bearish_gap_inf"] = curr_high
|
||||
dfc.at[dfc.index[i], "Bearish_gap_sup"] = prev_low
|
||||
dfc.at[dfc.index[i], "Bearish_gap_size"] = prev_low - curr_high
|
||||
return dfc
|
||||
|
||||
|
||||
def displacement_detection(
|
||||
df: pd.DataFrame,
|
||||
type_range: str = 'standard',
|
||||
strenght: float = 3.0,
|
||||
period: int = 20
|
||||
df: pd.DataFrame, type_range: str = "standard", strenght: float = 3.0, period: int = 20
|
||||
) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
if type_range == 'standard':
|
||||
df_copy['candle_range'] = np.abs(df_copy['close'] - df_copy['open'])
|
||||
elif type_range == 'extrem':
|
||||
df_copy['candle_range'] = np.abs(df_copy['high'] - df_copy['low'])
|
||||
dfc = df.copy()
|
||||
if type_range == "standard":
|
||||
dfc["candle_range"] = (dfc["close"] - dfc["open"]).abs()
|
||||
elif type_range == "extrem":
|
||||
dfc["candle_range"] = (dfc["high"] - dfc["low"]).abs()
|
||||
else:
|
||||
raise ValueError("Invalid 'type_range'. Use 'standard' or 'extrem'.")
|
||||
|
||||
df_copy['Variation'] = np.abs(df_copy['close'] / df_copy['open'] - 1)
|
||||
df_copy['STD'] = df_copy['candle_range'].rolling(period).std()
|
||||
df_copy['displacement'] = 0
|
||||
mask = df_copy['candle_range'] > strenght * df_copy['STD']
|
||||
df_copy.loc[mask, 'displacement'] = 1
|
||||
df_copy['red_displacement'] = (
|
||||
df_copy['displacement'] & df_copy['displacement'].shift(1).fillna(0)
|
||||
).astype(int)
|
||||
return df_copy
|
||||
dfc["Variation"] = (dfc["close"] / dfc["open"] - 1).abs()
|
||||
dfc["STD"] = dfc["candle_range"].rolling(period).std()
|
||||
dfc["displacement"] = 0
|
||||
mask = dfc["candle_range"] > strenght * dfc["STD"]
|
||||
dfc.loc[mask, "displacement"] = 1
|
||||
dfc["red_displacement"] = (dfc["displacement"] & dfc["displacement"].shift(1).fillna(0)).astype(int)
|
||||
return dfc
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# 6) ROLLING ADF (Stationarity)
|
||||
# --------------------------------------------------------------------
|
||||
def rolling_adf_with_flag(df: pd.DataFrame, col: str = 'close', window_size: int = 50, p_value_threshold=0.05) -> pd.DataFrame:
|
||||
"""
|
||||
Computes rolling ADF test and adds a stationarity flag (1=stationary, 0=non-stationary).
|
||||
"""
|
||||
df_copy = df.copy()
|
||||
adf_stat = pd.Series(dtype="float64", index=df_copy.index)
|
||||
adf_pval = pd.Series(dtype="float64", index=df_copy.index)
|
||||
stationarity_flag = pd.Series(dtype="int", index=df_copy.index)
|
||||
|
||||
for i in range(window_size, len(df_copy)):
|
||||
slice_data = df_copy[col].iloc[i - window_size : i].values
|
||||
# =============================================================================
|
||||
# 6) ROLLING ADF DIAGNOSTIC (optional; not used for gating)
|
||||
# =============================================================================
|
||||
def rolling_adf_with_flag(
|
||||
df: pd.DataFrame, col: str = "close", window_size: int = 50, p_value_threshold=0.05
|
||||
) -> pd.DataFrame:
|
||||
"""Compute rolling ADF p-values and a stationarity flag (diagnostic)."""
|
||||
dfc = df.copy()
|
||||
adf_stat = pd.Series(dtype="float64", index=dfc.index)
|
||||
adf_pval = pd.Series(dtype="float64", index=dfc.index)
|
||||
flag = pd.Series(dtype="float64", index=dfc.index)
|
||||
|
||||
for i in range(window_size, len(dfc)):
|
||||
slice_data = dfc[col].iloc[i - window_size : i].values
|
||||
try:
|
||||
result = adfuller(slice_data, autolag='AIC')
|
||||
result = adfuller(slice_data, autolag="AIC")
|
||||
adf_stat.iloc[i] = result[0]
|
||||
adf_pval.iloc[i] = result[1]
|
||||
stationarity_flag.iloc[i] = 1 if result[1] < p_value_threshold else 0
|
||||
except:
|
||||
flag.iloc[i] = 1 if result[1] < p_value_threshold else 0
|
||||
except Exception:
|
||||
adf_stat.iloc[i] = np.nan
|
||||
adf_pval.iloc[i] = np.nan
|
||||
stationarity_flag.iloc[i] = np.nan
|
||||
flag.iloc[i] = np.nan
|
||||
|
||||
df_copy['rolling_adf_stat'] = adf_stat
|
||||
df_copy['rolling_adf_pval'] = adf_pval
|
||||
df_copy['stationary_flag'] = stationarity_flag # 1 = stationary, 0 = non-stationary
|
||||
|
||||
return df_copy
|
||||
dfc["rolling_adf_stat"] = adf_stat
|
||||
dfc["rolling_adf_pval"] = adf_pval
|
||||
dfc["stationary_flag"] = flag
|
||||
return dfc
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
# 7) DOUBLE-BARRIER LABEL
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
def set_double_barrier_label(
|
||||
df: pd.DataFrame,
|
||||
up: float = 0.005,
|
||||
down: float = 0.005,
|
||||
horizon: int = 50
|
||||
df: pd.DataFrame, up: float = 0.005, down: float = 0.005, horizon: int = 50
|
||||
) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
closes = df_copy["close"].values
|
||||
dfc = df.copy()
|
||||
closes = dfc["close"].values
|
||||
labels = np.full(len(closes), np.nan)
|
||||
|
||||
for i in range(len(closes)):
|
||||
current_price = closes[i]
|
||||
upper_barrier = current_price * (1 + up)
|
||||
lower_barrier = current_price * (1 - down)
|
||||
current = closes[i]
|
||||
upper = current * (1 + up)
|
||||
lower = current * (1 - down)
|
||||
end = min(i + horizon, len(closes))
|
||||
for forward_i in range(i + 1, end):
|
||||
if closes[forward_i] >= upper_barrier:
|
||||
for j in range(i + 1, end):
|
||||
if closes[j] >= upper:
|
||||
labels[i] = 1
|
||||
break
|
||||
elif closes[forward_i] <= lower_barrier:
|
||||
if closes[j] <= lower:
|
||||
labels[i] = 0
|
||||
break
|
||||
df_copy["barrier_label"] = labels
|
||||
df_copy.dropna(subset=["barrier_label"], inplace=True)
|
||||
return df_copy
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
dfc["barrier_label"] = labels
|
||||
dfc.dropna(subset=["barrier_label"], inplace=True)
|
||||
return dfc
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 8) FUTURE MARKET REGIME (Directional-Change Example)
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
def future_DC_market_regime(df: pd.DataFrame, threshold: float = 0.03, horizon: int = 10) -> pd.DataFrame:
|
||||
df_copy = df.copy()
|
||||
df_copy['future_return'] = df_copy['close'].shift(-horizon) / df_copy['close'] - 1.0
|
||||
df_copy['future_market_regime'] = np.nan
|
||||
df_copy.loc[df_copy['future_return'] >= threshold, 'future_market_regime'] = 1
|
||||
df_copy.loc[df_copy['future_return'] <= -threshold, 'future_market_regime'] = 0
|
||||
df_copy.dropna(subset=['future_market_regime'], inplace=True)
|
||||
return df_copy
|
||||
dfc = df.copy()
|
||||
dfc["future_return"] = dfc["close"].shift(-horizon) / dfc["close"] - 1.0
|
||||
dfc["future_market_regime"] = np.nan
|
||||
dfc.loc[dfc["future_return"] >= threshold, "future_market_regime"] = 1
|
||||
dfc.loc[dfc["future_return"] <= -threshold, "future_market_regime"] = 0
|
||||
dfc.dropna(subset=["future_market_regime"], inplace=True)
|
||||
return dfc
|
||||
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# 9) Introduce Fourier & Wavelet Features for Cyclical Pattern Recognition
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
# =============================================================================
|
||||
# 9) Fourier features (global)
|
||||
# =============================================================================
|
||||
def add_fourier_features(df: pd.DataFrame, col: str = "close", n_components: int = 5) -> pd.DataFrame:
|
||||
"""
|
||||
Extracts the top 'n_components' Fourier coefficients from price data.
|
||||
Global FFT magnitudes (same values on all rows). For truly time-local
|
||||
frequency content, implement a rolling FFT (heavier) instead.
|
||||
"""
|
||||
fft_vals = np.abs(fft(df[col].values))
|
||||
dfc = df.copy()
|
||||
fft_vals = np.abs(fft(dfc[col].values))
|
||||
for i in range(1, n_components + 1):
|
||||
df[f'fft_comp_{i}'] = fft_vals[i]
|
||||
return df
|
||||
dfc[f"fft_comp_{i}"] = fft_vals[i] if i < len(fft_vals) else np.nan
|
||||
return dfc
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# 10) Optimize ADF Test for Model Selection
|
||||
# --------------------------------------------------------------------
|
||||
# =============================================================================
|
||||
# 10) Stationarity: ADF+KPSS with safe transforms
|
||||
# =============================================================================
|
||||
def _is_stationary(series: pd.Series, adf_alpha: float, kpss_alpha: float) -> Dict:
|
||||
s = series.dropna().astype(float)
|
||||
if len(s) < 30:
|
||||
return {"adf_p": np.nan, "kpss_p": np.nan, "stationary": False}
|
||||
try:
|
||||
adf_p = adfuller(s, autolag="AIC")[1]
|
||||
except Exception:
|
||||
adf_p = np.nan
|
||||
try:
|
||||
kpss_p = kpss(s, regression="c", nlags="auto")[1]
|
||||
except Exception:
|
||||
kpss_p = np.nan
|
||||
ok_adf = (not np.isnan(adf_p)) and (adf_p < adf_alpha)
|
||||
ok_kpss = (not np.isnan(kpss_p)) and (kpss_p > kpss_alpha)
|
||||
return {"adf_p": adf_p, "kpss_p": kpss_p, "stationary": (ok_adf and ok_kpss)}
|
||||
|
||||
def apply_differencing_if_needed(df: pd.DataFrame, col: str = "close", threshold: float = 0.05) -> pd.DataFrame:
|
||||
|
||||
def _apply_transform(s: pd.Series, kind: str, seasonal_period: Optional[int]) -> pd.Series:
|
||||
if kind == "pct_change":
|
||||
return s.pct_change()
|
||||
if kind == "diff1":
|
||||
return s.diff(1)
|
||||
if kind == "log_diff1":
|
||||
return np.log1p(s.clip(lower=0)).diff(1)
|
||||
if kind == "seasonal_diff" and seasonal_period and seasonal_period > 1:
|
||||
return s.diff(seasonal_period)
|
||||
return s # fallback
|
||||
|
||||
|
||||
def ensure_stationary_features(
|
||||
df: pd.DataFrame,
|
||||
cols: Optional[List[str]] = None,
|
||||
cfg: Dict = STATIONARITY_CFG,
|
||||
exclude: Optional[List[str]] = None,
|
||||
) -> Tuple[pd.DataFrame, Dict]:
|
||||
"""
|
||||
If ADF p-value > threshold (non-stationary), apply first differencing.
|
||||
For each numeric feature, test ADF+KPSS. If non-stationary,
|
||||
apply transforms in cfg['transform_order'] (no look-ahead), re-test,
|
||||
and keep only features that pass. Returns (df_out, report).
|
||||
"""
|
||||
if df['rolling_adf_pval'].iloc[-1] > threshold: # Check last rolling p-value
|
||||
df[f"{col}_diff"] = df[col] - df[col].shift(1) # First differencing
|
||||
return df.dropna()
|
||||
if not cfg.get("enabled", True):
|
||||
return df, {"enabled": False}
|
||||
|
||||
df_out = df.copy()
|
||||
report: Dict = {"config": cfg, "features": {}}
|
||||
|
||||
# choose candidate columns
|
||||
if cols is None:
|
||||
cols = df_out.select_dtypes(include=[np.number]).columns.tolist()
|
||||
|
||||
exclude_set = set(cfg.get("exclude_cols", set()))
|
||||
if exclude:
|
||||
exclude_set.update(exclude)
|
||||
cols = [c for c in cols if c not in exclude_set]
|
||||
|
||||
drop_cols, added_cols = [], []
|
||||
|
||||
for col in cols:
|
||||
s = df_out[col]
|
||||
base = _is_stationary(s, cfg["adf_alpha"], cfg["kpss_alpha"])
|
||||
entry = {"original": base, "applied": None, "final_col": col}
|
||||
|
||||
if base["stationary"]:
|
||||
report["features"][col] = entry
|
||||
continue
|
||||
|
||||
# try configured transforms
|
||||
applied = False
|
||||
for kind in cfg["transform_order"]:
|
||||
s_t = _apply_transform(s, kind, cfg.get("seasonal_period"))
|
||||
test_t = _is_stationary(s_t, cfg["adf_alpha"], cfg["kpss_alpha"])
|
||||
if test_t["stationary"]:
|
||||
new_col = f"{col}__{kind}"
|
||||
df_out[new_col] = s_t
|
||||
entry["applied"] = {"transform": kind, **test_t}
|
||||
entry["final_col"] = new_col
|
||||
added_cols.append(new_col)
|
||||
if not cfg["keep_original"]:
|
||||
drop_cols.append(col)
|
||||
applied = True
|
||||
break
|
||||
|
||||
# fallback: deeper differencing up to max_diff
|
||||
if not applied:
|
||||
s_f = s.copy()
|
||||
for d in range(1, int(cfg.get("max_diff", 2)) + 1):
|
||||
s_f = s_f.diff(1)
|
||||
test_f = _is_stationary(s_f, cfg["adf_alpha"], cfg["kpss_alpha"])
|
||||
if test_f["stationary"]:
|
||||
new_col = f"{col}__diff{d}"
|
||||
df_out[new_col] = s_f
|
||||
entry["applied"] = {"transform": f"diff{d}", **test_f}
|
||||
entry["final_col"] = new_col
|
||||
added_cols.append(new_col)
|
||||
if not cfg["keep_original"]:
|
||||
drop_cols.append(col)
|
||||
applied = True
|
||||
break
|
||||
|
||||
report["features"][col] = entry
|
||||
|
||||
if drop_cols:
|
||||
df_out = df_out.drop(columns=list(set(drop_cols)))
|
||||
|
||||
# clean NaNs introduced by differencing
|
||||
df_out = df_out.dropna()
|
||||
|
||||
return df_out, report
|
||||
|
||||
|
||||
# Backward-compatible simple wrapper (kept for API parity with earlier drafts)
|
||||
def apply_stationarity_test(
|
||||
df: pd.DataFrame, threshold: float = 0.05
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Legacy simple ADF-only differencing (kept for backward compatibility).
|
||||
Prefer `ensure_stationary_features` for robust ADF+KPSS handling.
|
||||
"""
|
||||
cfg = STATIONARITY_CFG.copy()
|
||||
cfg["adf_alpha"] = threshold
|
||||
df_out, _ = ensure_stationary_features(df, cfg=cfg)
|
||||
return df_out
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# 11) Normalize Feature Distributions (Scaling)
|
||||
# --------------------------------------------------------------------
|
||||
def scale_features(df: pd.DataFrame, cols_to_scale: list) -> pd.DataFrame:
|
||||
|
||||
# =============================================================================
|
||||
# 11) Scaling
|
||||
# =============================================================================
|
||||
def scale_features(df: pd.DataFrame, cols_to_scale: List[str]) -> pd.DataFrame:
|
||||
"""
|
||||
Fit-transform scaling on the entire frame (risk of leakage).
|
||||
Prefer putting scalers INSIDE your ML pipeline (fit on train only),
|
||||
or use rolling_zscore below for backtests.
|
||||
"""
|
||||
dfc = df.copy()
|
||||
scaler = StandardScaler()
|
||||
df[cols_to_scale] = scaler.fit_transform(df[cols_to_scale])
|
||||
return df
|
||||
dfc[cols_to_scale] = scaler.fit_transform(dfc[cols_to_scale])
|
||||
return dfc
|
||||
|
||||
|
||||
def rolling_zscore(
|
||||
df: pd.DataFrame, cols: List[str], window: int = 200, min_periods: int = 50
|
||||
) -> pd.DataFrame:
|
||||
"""Rolling standardization to avoid look-ahead leakage."""
|
||||
dfc = df.copy()
|
||||
mu = dfc[cols].rolling(window, min_periods=min_periods).mean()
|
||||
sd = dfc[cols].rolling(window, min_periods=min_periods).std().replace(0, np.nan)
|
||||
dfc[cols] = (dfc[cols] - mu) / sd
|
||||
return dfc
|
||||
|
||||
# 12) SINGLE PIPELINE EXAMPLE
|
||||
# --------------------------------------------------------------------
|
||||
def create_features(df: pd.DataFrame, col: str = "close", window_size: int = 30) -> pd.DataFrame:
|
||||
|
||||
# =============================================================================
|
||||
# 12) PIPELINES
|
||||
# =============================================================================
|
||||
def create_features(
|
||||
df: pd.DataFrame,
|
||||
col: str = "close",
|
||||
window_size: int = 30,
|
||||
enforce_stationarity: bool = True,
|
||||
stationarity_cfg: Optional[Dict] = None,
|
||||
use_rolling_zscore: bool = True,
|
||||
zscore_window: int = 200,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Optimized pipeline integrating TA, autocorrelation, stationarity, Fourier transform, and normalization.
|
||||
Integrated feature pipeline (TA, autocorr, volatility, Fourier, stationarity).
|
||||
Uses ADF+KPSS gating (ensure_stationary_features) and optional rolling z-score.
|
||||
"""
|
||||
df = add_all_ta_features(df) # Adds TA indicators
|
||||
df = spread(df) # Adds 'spread'
|
||||
df = auto_corr_multi(df, col='close') # Multi-lag autocorrelation
|
||||
df = rolling_adf_with_flag(df) # ADF with stationarity flag
|
||||
|
||||
df = log_transform(df, col, 5) # Log transform
|
||||
df = moving_yang_zhang_estimator(df, window_size)
|
||||
df = moving_parkinson_estimator(df, window_size)
|
||||
|
||||
df = add_fourier_features(df, col="close") # Fourier Transform for cyclic detection
|
||||
df = apply_differencing_if_needed(df, col="close") # Ensure stationarity
|
||||
dfc = df.copy()
|
||||
|
||||
# Normalize all numeric features
|
||||
df = scale_features(df, df.select_dtypes(include=[np.number]).columns.tolist())
|
||||
# TA & misc
|
||||
dfc = add_all_ta_features(dfc)
|
||||
dfc = spread(dfc)
|
||||
dfc = auto_corr_multi(dfc, col="close")
|
||||
dfc = rolling_adf_with_flag(dfc, col="close") # diagnostic
|
||||
|
||||
return df
|
||||
# transforms & volatility
|
||||
dfc = log_transform(dfc, col, 5)
|
||||
dfc = moving_yang_zhang_estimator(dfc, window_size)
|
||||
dfc = moving_parkinson_estimator(dfc, window_size)
|
||||
|
||||
# frequency features (global simple FFT)
|
||||
dfc = add_fourier_features(dfc, col="close")
|
||||
|
||||
# Stationarity enforcement
|
||||
if enforce_stationarity:
|
||||
cfg = STATIONARITY_CFG.copy()
|
||||
if stationarity_cfg:
|
||||
cfg.update(stationarity_cfg)
|
||||
dfc, _ = ensure_stationary_features(dfc, cfg=cfg)
|
||||
|
||||
# Scaling (choose one: rolling z-score here OR scaling inside ML pipeline)
|
||||
numeric_cols = dfc.select_dtypes(include=[np.number]).columns.tolist()
|
||||
if use_rolling_zscore and len(numeric_cols) > 0:
|
||||
dfc = rolling_zscore(dfc, numeric_cols, window=zscore_window)
|
||||
# else: keep raw; or scale later in your sklearn/Keras pipeline
|
||||
|
||||
dfc = dfc.dropna()
|
||||
return dfc
|
||||
|
||||
|
||||
def add_core_features(
|
||||
df: pd.DataFrame,
|
||||
enforce_stationarity: bool = True,
|
||||
stationarity_cfg: Optional[Dict] = None,
|
||||
) -> pd.DataFrame:
|
||||
"""Lightweight core feature set + optional stationarity enforcement."""
|
||||
dfc = df.copy()
|
||||
|
||||
# Trend & momentum
|
||||
dfc["sma_20"] = dfc["close"].rolling(20).mean()
|
||||
dfc["ema_20"] = dfc["close"].ewm(span=20, adjust=False).mean()
|
||||
dfc["kama_10"] = dfc["close"].ewm(span=10, adjust=False).mean() # placeholder
|
||||
dfc["rsi_14"] = ta.momentum.rsi(dfc["close"], window=14)
|
||||
macd = ta.trend.macd(dfc["close"])
|
||||
macd_signal = ta.trend.macd_signal(dfc["close"])
|
||||
dfc["macd_diff"] = macd - macd_signal
|
||||
|
||||
# Volatility & volume
|
||||
dfc["atr_14"] = ta.volatility.average_true_range(dfc["high"], dfc["low"], dfc["close"], window=14)
|
||||
dfc["obv"] = ta.volume.on_balance_volume(dfc["close"], dfc["tick_volume"])
|
||||
dfc["rolling_std_20"] = dfc["close"].rolling(20).std()
|
||||
|
||||
# Structure & candle
|
||||
dfc = spread(dfc)
|
||||
dfc = candle_information(dfc)
|
||||
|
||||
# Autocorrelation
|
||||
dfc = auto_corr_multi(dfc, col="close", n=50, lags=[1, 5, 10])
|
||||
|
||||
# Regime
|
||||
dfc = kama_market_regime(dfc, col="close", n1=10, n2=30)
|
||||
dfc["ma_short"] = dfc["close"].rolling(20).mean()
|
||||
dfc["ma_long"] = dfc["close"].rolling(50).mean()
|
||||
dfc["market_regime"] = 0
|
||||
dfc.loc[dfc["ma_short"] > dfc["ma_long"], "market_regime"] = 1
|
||||
dfc.loc[dfc["ma_short"] < dfc["ma_long"], "market_regime"] = -1
|
||||
|
||||
# Diagnostic rolling ADF on close
|
||||
dfc = rolling_adf_with_flag(dfc, col="close", window_size=50)
|
||||
|
||||
dfc = dfc.dropna().reset_index(drop=True)
|
||||
|
||||
# Stationarity gating on derived features
|
||||
if enforce_stationarity:
|
||||
cfg = STATIONARITY_CFG.copy()
|
||||
if stationarity_cfg:
|
||||
cfg.update(stationarity_cfg)
|
||||
dfc, _ = ensure_stationary_features(dfc, cfg=cfg)
|
||||
|
||||
return dfc
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,517 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Kalman Filters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In this lab you will:\n",
|
||||
"\n",
|
||||
"- Estimate Moving Average\n",
|
||||
"- Use Kalman Filters to calculate the mean and covariance of our time series\n",
|
||||
"- Modify a Pairs trading function to make use of Kalman Filters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## What is a Kalman Filter?\n",
|
||||
"\n",
|
||||
"The Kalman filter is an algorithm that uses noisy observations of a system over time to estimate the parameters of the system (some of which are unobservable) and predict future observations. At each time step, it makes a prediction, takes in a measurement, and updates itself based on how the prediction and measurement compare.\n",
|
||||
"\n",
|
||||
"The algorithm is as follows:\n",
|
||||
"1. Take as input a mathematical model of the system, i.e.\n",
|
||||
" * the transition matrix, which tells us how the system evolves from one state to another. For instance, if we are modeling the movement of a car, then the next values of position and velocity can be computed from the previous ones using kinematic equations. Alternatively, if we have a system which is fairly stable, we might model its evolution as a random walk. If you want to read up on Kalman filters, note that this matrix is usually called $A$.\n",
|
||||
" * the observation matrix, which tells us the next measurement we should expect given the predicted next state. If we are measuring the position of the car, we just extract the position values stored in the state. For a more complex example, consider estimating a linear regression model for the data. Then our state is the coefficients of the model, and we can predict the next measurement from the linear equation. This is denoted $H$.\n",
|
||||
" * any control factors that affect the state transitions but are not part of the measurements. For instance, if our car were falling, gravity would be a control factor. If the noise does not have mean 0, it should be shifted over and the offset put into the control factors. The control factors are summarized in a matrix $B$ with time-varying control vector $u_t$, which give the offset $Bu_t$.\n",
|
||||
" * covariance matrices of the transition noise (i.e. noise in the evolution of the system) and measurement noise, denoted $Q$ and $R$, respectively.\n",
|
||||
"2. Take as input an initial estimate of the state of the system and the error of the estimate, $\\mu_0$ and $\\sigma_0$.\n",
|
||||
"3. At each timestep:\n",
|
||||
" * estimate the current state of the system $x_t$ using the transition matrix\n",
|
||||
" * take as input new measurements $z_t$\n",
|
||||
" * use the conditional probability of the measurements given the state, taking into account the uncertainties of the measurement and the state estimate, to update the estimated current state of the system $x_t$ and the covariance matrix of the estimate $P_t$\n",
|
||||
"\n",
|
||||
"[This graphic](https://upload.wikimedia.org/wikipedia/commons/a/a5/Basic_concept_of_Kalman_filtering.svg) illustrates the procedure followed by the algorithm. \n",
|
||||
"\n",
|
||||
"It's very important for the algorithm to keep track of the covariances of its estimates. This way, it can give us a more nuanced result than simply a point value when we ask for it, and it can use its confidence to decide how much to be influenced by new measurements during the update process. The more certain it is of its estimate of the state, the more skeptical it will be of measurements that disagree with the state.\n",
|
||||
"\n",
|
||||
"By default, the errors are assumed to be normally distributed, and this assumption allows the algorithm to calculate precise confidence intervals. It can, however, be implemented for non-normal errors."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Install dependencies"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install pykalman"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install qq-training-wheels auquan_toolbox --upgrade"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import a Kalman filter and other useful libraries\n",
|
||||
"from pykalman import KalmanFilter\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from scipy import poly1d\n",
|
||||
"\n",
|
||||
"from backtester.dataSource.yahoo_data_source import YahooStockDataSource\n",
|
||||
"from datetime import datetime\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Toy example: falling ball\n",
|
||||
"\n",
|
||||
"Imagine we have a falling ball whose motion we are tracking with a camera. The state of the ball consists of its position and velocity. We know that we have the relationship $x_t = x_{t-1} + v_{t-1}\\tau - \\frac{1}{2} g \\tau^2$, where $\\tau$ is the time (in seconds) elapsed between $t-1$ and $t$ and $g$ is gravitational acceleration. Meanwhile, our camera can tell us the position of the ball every second, but we know from the manufacturer that the camera accuracy, translated into the position of the ball, implies variance in the position estimate of about 3 meters.\n",
|
||||
"\n",
|
||||
"In order to use a Kalman filter, we need to give it transition and observation matrices, transition and observation covariance matrices, and the initial state. The state of the system is (position, velocity), so it follows the transition matrix\n",
|
||||
"$$ \\left( \\begin{array}{cc}\n",
|
||||
"1 & \\tau \\\\\n",
|
||||
"0 & 1 \\end{array} \\right) $$\n",
|
||||
"\n",
|
||||
"with offset $(-\\tau^2 \\cdot g/2, -\\tau\\cdot g)$. The observation matrix just extracts the position coordinate, (1 0), since we are measuring position. We know that the observation variance is 1, and transition covariance is 0 since we will be simulating the data the same way we specified our model. For the initial state, let's feed our model something bogus like (30, 10) and see how our system evolves."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tau = 0.1\n",
|
||||
"\n",
|
||||
"# Set up the filter\n",
|
||||
"kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, # position is 1-dimensional, (x,v) is 2-dimensional\n",
|
||||
" initial_state_mean=[30,10],\n",
|
||||
" initial_state_covariance=np.eye(2),\n",
|
||||
" transition_matrices=[[1,tau], [0,1]],\n",
|
||||
" observation_matrices=[[1,0]],\n",
|
||||
" observation_covariance=3,\n",
|
||||
" transition_covariance=np.zeros((2,2)),\n",
|
||||
" transition_offsets=[-4.9*tau**2, -9.8*tau])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create a simulation of a ball falling for 40 units of time (each of length tau)\n",
|
||||
"times = np.arange(40)\n",
|
||||
"actual = -4.9*tau**2*times**2\n",
|
||||
"\n",
|
||||
"# Simulate the noisy camera data\n",
|
||||
"sim = actual + 3*np.random.randn(40)\n",
|
||||
"\n",
|
||||
"# Run filter on camera data\n",
|
||||
"state_means, state_covs = kf.filter(sim)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"plt.figure(figsize=(15,7))\n",
|
||||
"plt.plot(times, state_means[:,0])\n",
|
||||
"plt.plot(times, sim)\n",
|
||||
"plt.plot(times, actual)\n",
|
||||
"plt.legend(['Filter estimate', 'Camera data', 'Actual'])\n",
|
||||
"plt.xlabel('Time')\n",
|
||||
"plt.ylabel('Height');"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"At each point in time we plot the state estimate <i>after</i> accounting for the most recent measurement, which is why we are not at position 30 at time 0. The filter's attentiveness to the measurements allows it to correct for the initial bogus state we gave it. Then, by weighing its model and knowledge of the physical laws against new measurements, it is able to filter out much of the noise in the camera data. Meanwhile the confidence in the estimate increases with time, as shown by the graph below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Plot variances of x and v, extracting the appropriate values from the covariance matrix\n",
|
||||
"plt.figure(figsize=(15,7))\n",
|
||||
"plt.plot(times, state_covs[:,0,0])\n",
|
||||
"plt.plot(times, state_covs[:,1,1])\n",
|
||||
"plt.legend(['Var(x)', 'Var(v)'])\n",
|
||||
"plt.ylabel('Variance')\n",
|
||||
"plt.xlabel('Time');"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The Kalman filter can also do <i>smoothing</i>, which takes in all of the input data at once and then constructs its best guess for the state of the system in each period post factum. That is, it does not provide online, running estimates, but instead uses all of the data to estimate the historical state, which is useful if we only want to use the data after we have collected all of it."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Use smoothing to estimate what the state of the system has been\n",
|
||||
"smoothed_state_means, _ = kf.smooth(sim)\n",
|
||||
"\n",
|
||||
"# Plot results\n",
|
||||
"plt.figure(figsize=(15,7))\n",
|
||||
"plt.plot(times, smoothed_state_means[:,0])\n",
|
||||
"plt.plot(times, sim)\n",
|
||||
"plt.plot(times, actual)\n",
|
||||
"plt.legend(['Smoothed estimate', 'Camera data', 'Actual'])\n",
|
||||
"plt.xlabel('Time')\n",
|
||||
"plt.ylabel('Height');"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Example: Estimating Moving Average\n",
|
||||
"\n",
|
||||
"Because the Kalman filter updates its estimates at every time step and tends to weigh recent observations more than older ones, it can be used to estimate rolling parameters of the data. When using a Kalman filter, there's no window length that we need to specify. This is useful for computing the moving average or for smoothing out estimates of other quantities.\n",
|
||||
"\n",
|
||||
"Below, we'll use both a Kalman filter and an n-day moving average to estimate the rolling mean of a dataset. We construct the inputs to the Kalman filter as follows:\n",
|
||||
"\n",
|
||||
"* The mean is the model's guess for the mean of the distribution from which measurements are drawn. This means our prediction of the next value is equal to our estimate of the mean. \n",
|
||||
"* Hopefully the mean describes our observations well, hence it shouldn't change significantly when we add an observation. This implies we can assume that it evolves as a random walk with a small error term. We set the transition matrix to 1 and transition covariance matrix is a small number.\n",
|
||||
"* We assume that the observations have variance 1 around the rolling mean (1 is chosen randomly). \n",
|
||||
"* Our initial guess for the mean is 0, but the filter realizes that that is incorrect and adjusts."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pykalman import KalmanFilter\n",
|
||||
"from backtester.dataSource.yahoo_data_source import YahooStockDataSource\n",
|
||||
"\n",
|
||||
"# Load pricing data for a security\n",
|
||||
"startDateStr = '2012/12/31'\n",
|
||||
"endDateStr = '2017/12/31'\n",
|
||||
"cachedFolderName = './yahooData/'\n",
|
||||
"dataSetId = 'testPairsTrading'\n",
|
||||
"instrumentIds = ['SPY','MSFT','ADBE']\n",
|
||||
"ds = YahooStockDataSource(cachedFolderName=cachedFolderName,\n",
|
||||
" dataSetId=dataSetId,\n",
|
||||
" instrumentIds=instrumentIds,\n",
|
||||
" startDateStr=startDateStr,\n",
|
||||
" endDateStr=endDateStr,\n",
|
||||
" event='history')\n",
|
||||
"\n",
|
||||
"# Get adjusted closing price\n",
|
||||
"data = ds.getBookDataByFeature()['adjClose']\n",
|
||||
"\n",
|
||||
"# Data for Adobe\n",
|
||||
"S1 = data['ADBE']\n",
|
||||
"# Data for Microsoft\n",
|
||||
"S2 = data['MSFT']\n",
|
||||
"\n",
|
||||
"# Take ratio of the adjusted closing prices\n",
|
||||
"x = S1/S2\n",
|
||||
"\n",
|
||||
"# Construct a Kalman filter\n",
|
||||
"kf = KalmanFilter(transition_matrices = [1],\n",
|
||||
" observation_matrices = [1],\n",
|
||||
" initial_state_mean = 0,\n",
|
||||
" initial_state_covariance = 1,\n",
|
||||
" observation_covariance=1,\n",
|
||||
" transition_covariance=.01)\n",
|
||||
"\n",
|
||||
"# Use the observed values of the price to get a rolling mean\n",
|
||||
"state_means, _ = kf.filter(x.values)\n",
|
||||
"state_means = pd.Series(state_means.flatten(), index=x.index)\n",
|
||||
"\n",
|
||||
"# Compute the rolling mean with various lookback windows\n",
|
||||
"mean30 = x.rolling(window = 10).mean()\n",
|
||||
"mean60 = x.rolling(window = 30).mean()\n",
|
||||
"mean90 = x.rolling(window = 60).mean()\n",
|
||||
"\n",
|
||||
"# Plot original data and estimated mean\n",
|
||||
"plt.figure(figsize=(15,7))\n",
|
||||
"plt.plot(state_means[60:], '-b', lw=2, )\n",
|
||||
"plt.plot(x[60:],'-g',lw=1.5)\n",
|
||||
"plt.plot(mean30[60:], 'm', lw=1)\n",
|
||||
"plt.plot(mean60[60:], 'y', lw=1)\n",
|
||||
"plt.plot(mean90[60:], 'c', lw=1)\n",
|
||||
"plt.title('Kalman filter estimate of average')\n",
|
||||
"plt.legend(['Kalman Estimate', 'X', '30-day Moving Average', '60-day Moving Average','90-day Moving Average'])\n",
|
||||
"plt.xlabel('Day')\n",
|
||||
"plt.ylabel('Price');"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Observations\n",
|
||||
"\n",
|
||||
"As you can see, the estimate from Kalman Filter is usually somewhere between day 30 and day 60 moving average. This could be because the Filter updates its knowledge of the world based on the most recent data. The advantage of the Kalman filter is that we don't need to select a window length. It makes predictions based on the underlying model (that we set parameters for) and the data itself. We do open ourselves up to overfitting with some of the initialization parameters for the filter, but those are slightly easier to objectively define. There's no free lunch and we can't eliminate overfitting, but a Kalman Filter is more rigorous than a moving average and generally better."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Another interesting application of Kalman Filters, Beta Estimation for Linear Regression can be found here [Dr. Aidan O'Mahony's blog.](http://www.thealgoengineer.com/2014/online_linear_regression_kalman_filter/)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We'll be using Kalman filters for Pairs trading the subsequent notebook. Make sure you try to run the examples given here with various hyperparameters for the underlying Kalman filter model to get comfortable with the same and developing a better understanding in the process. For example you can try out the following:\n",
|
||||
"1. Use multi dimensional transition matrices so as to use more of past information for making predictions at each point\n",
|
||||
"2. Try different values of observation and transition covariance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Example: Pairs Trading\n",
|
||||
"\n",
|
||||
"In the previous notebook we made use of 60 day window for calculating mean and standard deviation of our time series. Now we'll be replacing that with Kalman filters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Let's get the same data that we used in the previous notebook"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"startDateStr = '2007/12/01'\n",
|
||||
"endDateStr = '2017/12/01'\n",
|
||||
"cachedFolderName = 'yahooData/'\n",
|
||||
"dataSetId = 'testPairsTrading2'\n",
|
||||
"instrumentIds = ['ADBE','MSFT']\n",
|
||||
"ds = YahooStockDataSource(cachedFolderName=cachedFolderName,\n",
|
||||
" dataSetId=dataSetId,\n",
|
||||
" instrumentIds=instrumentIds,\n",
|
||||
" startDateStr=startDateStr,\n",
|
||||
" endDateStr=endDateStr,\n",
|
||||
" event='history')\n",
|
||||
"data = ds.getBookDataByFeature()['adjClose']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### A quick visualization of error and standard deviations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"S1, S2 = data['ADBE'].iloc[:1762], data['MSFT'].iloc[:1762]\n",
|
||||
"ratios = S1/S2\n",
|
||||
"\n",
|
||||
"kf = KalmanFilter(transition_matrices = [1],\n",
|
||||
" observation_matrices = [1],\n",
|
||||
" initial_state_mean = 0,\n",
|
||||
" initial_state_covariance = 1,\n",
|
||||
" observation_covariance=1,\n",
|
||||
" transition_covariance=.0001)\n",
|
||||
"\n",
|
||||
"state_means, state_cov = kf.filter(ratios.values)\n",
|
||||
"state_means, state_std = state_means.squeeze(), np.std(state_cov.squeeze())\n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(15,7))\n",
|
||||
"plt.plot(ratios.values - state_means, 'm', lw=1)\n",
|
||||
"plt.plot(np.sqrt(state_cov.squeeze()), 'y', lw=1)\n",
|
||||
"plt.plot(-np.sqrt(state_cov.squeeze()), 'c', lw=1)\n",
|
||||
"plt.title('Kalman filter estimate')\n",
|
||||
"plt.legend(['Error: real_value - mean', 'std', '-std'])\n",
|
||||
"plt.xlabel('Day')\n",
|
||||
"plt.ylabel('Value');\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We'll be using the z score in the same way as before. Our strategy is to go long or short only in the areas where the |error| is greater than one standard deviation. Since 1 day price could be noisy, we'll be using 5 day average for a particular day's price"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Let's modify our trading function to make use of Kalman Filter while keeping the same logic for carrying out trades"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def trade(S1, S2):\n",
|
||||
" # Compute rolling mean and rolling standard deviation\n",
|
||||
" ratios = S1/S2\n",
|
||||
" \n",
|
||||
" kf = KalmanFilter(transition_matrices = [1],\n",
|
||||
" observation_matrices = [1],\n",
|
||||
" initial_state_mean = 0,\n",
|
||||
" initial_state_covariance = 1,\n",
|
||||
" observation_covariance=1,\n",
|
||||
" transition_covariance=.001)\n",
|
||||
" \n",
|
||||
" state_means, state_cov = kf.filter(ratios.values)\n",
|
||||
" state_means, state_std = state_means.squeeze(), np.std(state_cov.squeeze())\n",
|
||||
" \n",
|
||||
" window = 5\n",
|
||||
" ma = ratios.rolling(window=window,\n",
|
||||
" center=False).mean()\n",
|
||||
" zscore = (ma - state_means)/state_std\n",
|
||||
" \n",
|
||||
" # Simulate trading\n",
|
||||
" # Start with no money and no positions\n",
|
||||
" money = 0\n",
|
||||
" countS1 = 0\n",
|
||||
" countS2 = 0\n",
|
||||
" for i in range(len(ratios)):\n",
|
||||
" # Sell short if the z-score is > 1\n",
|
||||
" if zscore[i] > 1:\n",
|
||||
" money += S1[i] - S2[i] * ratios[i]\n",
|
||||
" countS1 -= 1\n",
|
||||
" countS2 += ratios[i]\n",
|
||||
" # Buy long if the z-score is < 1\n",
|
||||
" elif zscore[i] < -1:\n",
|
||||
" money -= S1[i] - S2[i] * ratios[i]\n",
|
||||
" countS1 += 1\n",
|
||||
" countS2 -= ratios[i]\n",
|
||||
" # Clear positions if the z-score between -.5 and .5\n",
|
||||
" elif abs(zscore[i]) < 0.5:\n",
|
||||
" money += countS1*S1[i] + S2[i] * countS2\n",
|
||||
" countS1 = 0\n",
|
||||
" countS2 = 0\n",
|
||||
"# print('Z-score: '+ str(zscore[i]), countS1, countS2, S1[i] , S2[i])\n",
|
||||
" return money\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trade(data['ADBE'].iloc[:1762], data['MSFT'].iloc[:1762])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The strategy is still profitable! You can try changing the hyperparameters of the Kalman Filter and see how it affects the PnL. The results might not be always better than the mean over moving window. You can try this with other instruments as well."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"environment": {
|
||||
"name": "tf2-2-2-gpu.2-2.m50",
|
||||
"type": "gcloud",
|
||||
"uri": "gcr.io/deeplearning-platform-release/tf2-2-2-gpu.2-2:m50"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.6"
|
||||
},
|
||||
"varInspector": {
|
||||
"cols": {
|
||||
"lenName": 16,
|
||||
"lenType": 16,
|
||||
"lenVar": 40
|
||||
},
|
||||
"kernels_config": {
|
||||
"python": {
|
||||
"delete_cmd_postfix": "",
|
||||
"delete_cmd_prefix": "del ",
|
||||
"library": "var_list.py",
|
||||
"varRefreshCmd": "print(var_dic_list())"
|
||||
},
|
||||
"r": {
|
||||
"delete_cmd_postfix": ") ",
|
||||
"delete_cmd_prefix": "rm(",
|
||||
"library": "var_list.r",
|
||||
"varRefreshCmd": "cat(var_dic_list()) "
|
||||
}
|
||||
},
|
||||
"types_to_exclude": [
|
||||
"module",
|
||||
"function",
|
||||
"builtin_function_or_method",
|
||||
"instance",
|
||||
"_Feature"
|
||||
],
|
||||
"window_display": false
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,461 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loaded pipeline from models/saved_models/best_rf_mb_pipeline.pkl\n",
|
||||
"Checking market status...\n",
|
||||
"Market is closed. No actions performed.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# LIVE TRADING CODE FOR MULTI-BAR CLASSIFICATION\n",
|
||||
"\n",
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"import warnings\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"# 1) SET PROJECT ROOT AND UPDATE PATH/WORKING DIRECTORY\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"project_root = Path.cwd().parent.parent # Adjust if your notebook is in notebooks/time_series\n",
|
||||
"sys.path.append(str(project_root))\n",
|
||||
"os.chdir(str(project_root))\n",
|
||||
"warnings.filterwarnings(\"ignore\")\n",
|
||||
"\n",
|
||||
"import warnings\n",
|
||||
"warnings.filterwarnings(\"ignore\")\n",
|
||||
"import MetaTrader5 as mt5\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import ta\n",
|
||||
"from datetime import datetime, timedelta\n",
|
||||
"import time\n",
|
||||
"import logging\n",
|
||||
"import joblib\n",
|
||||
"\n",
|
||||
"# Setup logging\n",
|
||||
"logging.basicConfig(\n",
|
||||
" filename='models/saved_models/trading_app1.log',\n",
|
||||
" level=logging.INFO,\n",
|
||||
" format='%(asctime)s %(levelname)s:%(message)s',\n",
|
||||
" datefmt='%Y-%m-%d %H:%M:%S'\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"def log_and_print(message, is_error=False):\n",
|
||||
" \"\"\"\n",
|
||||
" Logs and prints a message.\n",
|
||||
" If is_error=True, logs at the ERROR level; otherwise logs at INFO level.\n",
|
||||
" \"\"\"\n",
|
||||
" if is_error:\n",
|
||||
" logging.error(message)\n",
|
||||
" else:\n",
|
||||
" logging.info(message)\n",
|
||||
" print(message)\n",
|
||||
"\n",
|
||||
"# Update the login credentials and server information accordingly\n",
|
||||
"name = 66677507\n",
|
||||
"key = 'ST746$nG38'\n",
|
||||
"serv = 'ICMarketsSC-Demo'\n",
|
||||
"\n",
|
||||
"# Global variables\n",
|
||||
"SYMBOL = \"EURUSD\"\n",
|
||||
"LOT_SIZE = 0.01\n",
|
||||
"TIMEFRAME = mt5.TIMEFRAME_D1\n",
|
||||
"N_BARS = 50000\n",
|
||||
"MAGIC_NUMBER = 234003\n",
|
||||
"SLEEP_TIME = 86400 # 24 hours in seconds\n",
|
||||
"COMMENT_ML = \"RFFV-D\"\n",
|
||||
"\n",
|
||||
"# If you still need feature selection, you can keep this helper function:\n",
|
||||
"def select_features_rf_reg(X, y, estimator, max_features=20):\n",
|
||||
" \"\"\"\n",
|
||||
" Example helper function for feature selection using RandomForest.\n",
|
||||
" \"\"\"\n",
|
||||
" from sklearn.feature_selection import SelectFromModel\n",
|
||||
" selector = SelectFromModel(estimator=estimator, threshold=-np.inf, max_features=max_features).fit(X, y)\n",
|
||||
" X_transformed = selector.transform(X)\n",
|
||||
" selected_features_mask = selector.get_support()\n",
|
||||
" return X_transformed, selected_features_mask\n",
|
||||
"\n",
|
||||
"class TradingApp:\n",
|
||||
" def __init__(self, symbol, lot_size, magic_number):\n",
|
||||
" self.symbol = symbol\n",
|
||||
" self.lot_size = lot_size\n",
|
||||
" self.magic_number = magic_number\n",
|
||||
" self.pipeline = None # We'll store the loaded classification pipeline here\n",
|
||||
" self.last_retrain_time = None\n",
|
||||
"\n",
|
||||
" def get_data(self, symbol, n, timeframe):\n",
|
||||
" \"\"\"\n",
|
||||
" Fetch 'n' bars of historical data for the given symbol and timeframe.\n",
|
||||
" \"\"\"\n",
|
||||
" rates = mt5.copy_rates_from_pos(symbol, timeframe, 0, n)\n",
|
||||
" rates_frame = pd.DataFrame(rates)\n",
|
||||
" rates_frame['time'] = pd.to_datetime(rates_frame['time'], unit='s')\n",
|
||||
" rates_frame.set_index('time', inplace=True)\n",
|
||||
" return rates_frame\n",
|
||||
"\n",
|
||||
" def add_all_ta_features(self, df):\n",
|
||||
" \"\"\"\n",
|
||||
" Add technical analysis features to the DataFrame using the 'ta' library.\n",
|
||||
" \"\"\"\n",
|
||||
" df = ta.add_all_ta_features(\n",
|
||||
" df, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"tick_volume\", fillna=True\n",
|
||||
" )\n",
|
||||
" return df\n",
|
||||
"\n",
|
||||
" def load_pipeline(self, pipeline_path):\n",
|
||||
" \"\"\"\n",
|
||||
" Loads a pre-trained classification pipeline (e.g., 'best_rf_pipeline.pkl').\n",
|
||||
" This pipeline is expected to produce SHIFTED labels [0,1,2].\n",
|
||||
" \"\"\"\n",
|
||||
" self.pipeline = joblib.load(pipeline_path)\n",
|
||||
" logging.info(f\"Loaded pipeline from {pipeline_path}\")\n",
|
||||
" log_and_print(f\"Loaded pipeline from {pipeline_path}\")\n",
|
||||
"\n",
|
||||
" def ml_signal_generation(self, symbol, n_bars, timeframe):\n",
|
||||
" \"\"\"\n",
|
||||
" Generate buy/sell signals using the loaded classification pipeline.\n",
|
||||
" The pipeline outputs SHIFTED labels in {0,1,2} => we SHIFT them back to {-1,0,+1}.\n",
|
||||
" We'll interpret +1 => buy, -1 => sell, 0 => no trade.\n",
|
||||
" \"\"\"\n",
|
||||
" if self.pipeline is None:\n",
|
||||
" logging.error(\"No pipeline loaded. Call load_pipeline(...) first.\")\n",
|
||||
" return False, False, True, True\n",
|
||||
"\n",
|
||||
" # 1) Fetch new data\n",
|
||||
" df = self.get_data(symbol, n_bars, timeframe)\n",
|
||||
"\n",
|
||||
" # 2) Add TA features\n",
|
||||
" df = self.add_all_ta_features(df)\n",
|
||||
" df.fillna(method='ffill', inplace=True)\n",
|
||||
"\n",
|
||||
" # 3) Prepare the features\n",
|
||||
" X_new = df # The pipeline must handle columns in the correct order.\n",
|
||||
"\n",
|
||||
" # 4) Predict SHIFTED classes\n",
|
||||
" preds_shifted = self.pipeline.predict(X_new)\n",
|
||||
" # SHIFT them back: 0->-1, 1->0, 2->+1\n",
|
||||
" preds = preds_shifted - 1\n",
|
||||
"\n",
|
||||
" # Get the latest predicted class\n",
|
||||
" latest_pred = preds[-1]\n",
|
||||
" # If latest_pred == +1 => buy signal\n",
|
||||
" # If latest_pred == -1 => sell signal\n",
|
||||
" # If 0 => do nothing\n",
|
||||
" buy_signal = (latest_pred == 1)\n",
|
||||
" sell_signal = (latest_pred == -1)\n",
|
||||
"\n",
|
||||
" return buy_signal, sell_signal, not buy_signal, not sell_signal\n",
|
||||
"\n",
|
||||
" def orders(self, symbol, lot, is_buy=True, id_position=None, sl=None, tp=None):\n",
|
||||
" \"\"\"\n",
|
||||
" Place an order (BUY or SELL) for the specified symbol and lot size.\n",
|
||||
" \"\"\"\n",
|
||||
" symbol_info = mt5.symbol_info(symbol)\n",
|
||||
" if symbol_info is None:\n",
|
||||
" log_and_print(f\"Symbol {symbol} not found, can't place order.\", is_error=True)\n",
|
||||
" return \"Symbol not found\"\n",
|
||||
"\n",
|
||||
" # Make sure symbol is visible\n",
|
||||
" if not symbol_info.visible:\n",
|
||||
" if not mt5.symbol_select(symbol, True):\n",
|
||||
" log_and_print(f\"Failed to select symbol {symbol}\", is_error=True)\n",
|
||||
" return \"Symbol not visible or could not be selected.\"\n",
|
||||
"\n",
|
||||
" tick_info = mt5.symbol_info_tick(symbol)\n",
|
||||
" if tick_info is None:\n",
|
||||
" log_and_print(f\"Could not get tick info for {symbol}.\", is_error=True)\n",
|
||||
" return \"Tick info unavailable\"\n",
|
||||
"\n",
|
||||
" # Check for valid bid/ask\n",
|
||||
" if tick_info.bid <= 0 or tick_info.ask <= 0:\n",
|
||||
" log_and_print(\n",
|
||||
" f\"Zero or invalid bid/ask for {symbol}: bid={tick_info.bid}, ask={tick_info.ask}\",\n",
|
||||
" is_error=True\n",
|
||||
" )\n",
|
||||
" return \"Invalid prices\"\n",
|
||||
"\n",
|
||||
" # LOT SIZE VALIDATION\n",
|
||||
" lot = max(lot, symbol_info.volume_min)\n",
|
||||
" step = symbol_info.volume_step\n",
|
||||
" if step > 0:\n",
|
||||
" remainder = lot % step\n",
|
||||
" if remainder != 0:\n",
|
||||
" lot = lot - remainder + step\n",
|
||||
" if lot > symbol_info.volume_max:\n",
|
||||
" lot = symbol_info.volume_max\n",
|
||||
"\n",
|
||||
" log_and_print(\n",
|
||||
" f\"Adjusted lot size to {lot} (min={symbol_info.volume_min}, \"\n",
|
||||
" f\"step={symbol_info.volume_step}, max={symbol_info.volume_max})\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Force ORDER_FILLING_IOC\n",
|
||||
" filling_mode = 1 # ORDER_FILLING_IOC\n",
|
||||
"\n",
|
||||
" order_type = mt5.ORDER_TYPE_BUY if is_buy else mt5.ORDER_TYPE_SELL\n",
|
||||
" order_price = tick_info.ask if is_buy else tick_info.bid\n",
|
||||
" deviation = 20\n",
|
||||
"\n",
|
||||
" request = {\n",
|
||||
" \"action\": mt5.TRADE_ACTION_DEAL,\n",
|
||||
" \"symbol\": symbol,\n",
|
||||
" \"volume\": lot,\n",
|
||||
" \"type\": order_type,\n",
|
||||
" \"deviation\": deviation,\n",
|
||||
" \"magic\": self.magic_number,\n",
|
||||
" \"comment\": COMMENT_ML,\n",
|
||||
" \"type_time\": mt5.ORDER_TIME_GTC,\n",
|
||||
" \"type_filling\": filling_mode,\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" if sl is not None:\n",
|
||||
" request[\"sl\"] = sl\n",
|
||||
" if tp is not None:\n",
|
||||
" request[\"tp\"] = tp\n",
|
||||
" if id_position is not None:\n",
|
||||
" request[\"position\"] = id_position\n",
|
||||
"\n",
|
||||
" log_and_print(f\"Sending order request: {request}\")\n",
|
||||
" result = mt5.order_send(request)\n",
|
||||
"\n",
|
||||
" order_type_str = \"BUY\" if is_buy else \"SELL\"\n",
|
||||
" if result is None or result.retcode != mt5.TRADE_RETCODE_DONE:\n",
|
||||
" error_message = f\"Order failed for {symbol}\"\n",
|
||||
" if result:\n",
|
||||
" error_message += f\", retcode={result.retcode}, comment={result.comment}\"\n",
|
||||
" additional_info = (\n",
|
||||
" f\"Date/Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\\n\"\n",
|
||||
" f\"Order Type: {order_type_str}\\n\"\n",
|
||||
" f\"Lot Size: {lot}\\n\"\n",
|
||||
" f\"SL: {sl if sl else 'None'}\\n\"\n",
|
||||
" f\"TP: {tp if tp else 'None'}\\n\"\n",
|
||||
" f\"Comment: {COMMENT_ML}\\n\"\n",
|
||||
" f\"Request: {request}\\n\"\n",
|
||||
" f\"Result: {result}\"\n",
|
||||
" )\n",
|
||||
" # If you want notifications, you could log or handle them differently here.\n",
|
||||
" log_and_print(f\"Order failed details: {additional_info}\", is_error=True)\n",
|
||||
" else:\n",
|
||||
" success_message = f\"Order successful for {symbol}, comment={result.comment}\"\n",
|
||||
" additional_info = (\n",
|
||||
" f\"Date/Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\\n\"\n",
|
||||
" f\"Order Type: {order_type_str}\\n\"\n",
|
||||
" f\"Lot Size: {lot}\\n\"\n",
|
||||
" f\"SL: {sl if sl else 'None'}\\n\"\n",
|
||||
" f\"TP: {tp if tp else 'None'}\\n\"\n",
|
||||
" f\"Comment: {COMMENT_ML}\"\n",
|
||||
" )\n",
|
||||
" # If you want notifications, you could log or handle them differently here.\n",
|
||||
" log_and_print(success_message)\n",
|
||||
"\n",
|
||||
" def get_positions_by_magic(self, symbol, magic_number):\n",
|
||||
" \"\"\"\n",
|
||||
" Retrieve positions for a specific symbol and magic number.\n",
|
||||
" \"\"\"\n",
|
||||
" all_positions = mt5.positions_get(symbol=symbol)\n",
|
||||
" if not all_positions:\n",
|
||||
" log_and_print(\"No positions found.\", is_error=False)\n",
|
||||
" return []\n",
|
||||
" return [pos for pos in all_positions if pos.magic == magic_number]\n",
|
||||
"\n",
|
||||
" def run_strategy(self, symbol, lot, buy_signal, sell_signal):\n",
|
||||
" \"\"\"\n",
|
||||
" Run the trading strategy logic based on buy/sell signals.\n",
|
||||
" \"\"\"\n",
|
||||
" log_and_print(\"------------------------------------------------------------------\")\n",
|
||||
" log_and_print(\n",
|
||||
" f\"Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}, \"\n",
|
||||
" f\"SYMBOL: {symbol}, BUY SIGNAL: {buy_signal}, SELL SIGNAL: {sell_signal}\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" positions = self.get_positions_by_magic(symbol, self.magic_number)\n",
|
||||
" has_buy = any(pos.type == mt5.POSITION_TYPE_BUY for pos in positions)\n",
|
||||
" has_sell = any(pos.type == mt5.POSITION_TYPE_SELL for pos in positions)\n",
|
||||
"\n",
|
||||
" if buy_signal and not has_buy:\n",
|
||||
" if has_sell:\n",
|
||||
" log_and_print(\"Existing sell positions found. Attempting to close...\")\n",
|
||||
" if self.close_position(symbol, is_buy=True):\n",
|
||||
" log_and_print(\"Sell positions closed. Placing new buy order.\")\n",
|
||||
" self.orders(symbol, lot, is_buy=True)\n",
|
||||
" else:\n",
|
||||
" log_and_print(\"Failed to close sell positions.\")\n",
|
||||
" else:\n",
|
||||
" self.orders(symbol, lot, is_buy=True)\n",
|
||||
" elif sell_signal and not has_sell:\n",
|
||||
" if has_buy:\n",
|
||||
" log_and_print(\"Existing buy positions found. Attempting to close...\")\n",
|
||||
" if self.close_position(symbol, is_buy=False):\n",
|
||||
" log_and_print(\"Buy positions closed. Placing new sell order.\")\n",
|
||||
" self.orders(symbol, lot, is_buy=False)\n",
|
||||
" else:\n",
|
||||
" log_and_print(\"Failed to close buy positions.\")\n",
|
||||
" else:\n",
|
||||
" self.orders(symbol, lot, is_buy=False)\n",
|
||||
" else:\n",
|
||||
" log_and_print(\"Appropriate position already exists or no signal to act on.\")\n",
|
||||
"\n",
|
||||
" def close_position(self, symbol, is_buy):\n",
|
||||
" \"\"\"\n",
|
||||
" Closes positions of the opposite type (BUY/SELL) for this app's magic number.\n",
|
||||
" \"\"\"\n",
|
||||
" positions = mt5.positions_get(symbol=symbol)\n",
|
||||
" if not positions:\n",
|
||||
" log_and_print(f\"No positions to close for symbol: {symbol}\")\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
" initial_balance = mt5.account_info().balance\n",
|
||||
" closed_any = False\n",
|
||||
"\n",
|
||||
" for position in positions:\n",
|
||||
" # Close positions of the opposite type with the same magic number\n",
|
||||
" if position.magic == self.magic_number and (\n",
|
||||
" (is_buy and position.type == mt5.POSITION_TYPE_SELL) or\n",
|
||||
" (not is_buy and position.type == mt5.POSITION_TYPE_BUY)\n",
|
||||
" ):\n",
|
||||
" close_request = {\n",
|
||||
" \"action\": mt5.TRADE_ACTION_DEAL,\n",
|
||||
" \"symbol\": symbol,\n",
|
||||
" \"volume\": position.volume,\n",
|
||||
" \"type\": mt5.ORDER_TYPE_BUY if position.type == mt5.POSITION_TYPE_SELL else mt5.ORDER_TYPE_SELL,\n",
|
||||
" \"position\": position.ticket,\n",
|
||||
" \"deviation\": 20,\n",
|
||||
" \"magic\": self.magic_number,\n",
|
||||
" \"comment\": COMMENT_ML,\n",
|
||||
" \"type_time\": mt5.ORDER_TIME_GTC,\n",
|
||||
" \"type_filling\": mt5.ORDER_FILLING_RETURN,\n",
|
||||
" }\n",
|
||||
" result = mt5.order_send(close_request)\n",
|
||||
" if result.retcode != mt5.TRADE_RETCODE_DONE:\n",
|
||||
" error_message = f\"Failed to close position {position.ticket} for {symbol}: {result.retcode}\"\n",
|
||||
" log_and_print(error_message, is_error=True)\n",
|
||||
" # If you want notifications, you could log or handle them differently here.\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"Successfully closed position {position.ticket} for {symbol}\")\n",
|
||||
" closed_any = True\n",
|
||||
"\n",
|
||||
" if closed_any:\n",
|
||||
" final_balance = mt5.account_info().balance\n",
|
||||
" profit = final_balance - initial_balance\n",
|
||||
" success_message = f\"Closed positions successfully, Profit: {profit}\"\n",
|
||||
" log_and_print(success_message)\n",
|
||||
" return True\n",
|
||||
"\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
" def check_and_execute_trades(self):\n",
|
||||
" \"\"\"\n",
|
||||
" Convenience method to perform the entire flow:\n",
|
||||
" generate signals, run strategy, and deselect symbol.\n",
|
||||
" \"\"\"\n",
|
||||
" mt5.symbol_select(self.symbol, True)\n",
|
||||
" buy, sell, _, _ = self.ml_signal_generation(self.symbol, N_BARS, TIMEFRAME)\n",
|
||||
" self.run_strategy(self.symbol, self.lot_size, buy, sell)\n",
|
||||
" mt5.symbol_select(self.symbol, False)\n",
|
||||
" log_and_print(\"Waiting for new signals...\")\n",
|
||||
"\n",
|
||||
"def is_market_open():\n",
|
||||
" \"\"\"\n",
|
||||
" Check if the current time is within the typical Forex trading session, adjusted for CET/CEST.\n",
|
||||
" Market closes at Friday 10:00 PM CET and opens at Sunday 11:00 PM CET. \n",
|
||||
" It is closed all day Saturday.\n",
|
||||
" \"\"\"\n",
|
||||
" current_time_utc = datetime.utcnow()\n",
|
||||
" # Adjust for Central European Time (UTC+1) or Central European Summer Time (UTC+2)\n",
|
||||
" current_time_cet = (\n",
|
||||
" current_time_utc + timedelta(hours=2) \n",
|
||||
" if time.localtime().tm_isdst \n",
|
||||
" else current_time_utc + timedelta(hours=1)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Friday after 10 PM CET\n",
|
||||
" if current_time_cet.weekday() == 4 and current_time_cet.hour >= 22:\n",
|
||||
" return False\n",
|
||||
" # Sunday before 11 PM CET\n",
|
||||
" elif current_time_cet.weekday() == 6 and current_time_cet.hour < 23:\n",
|
||||
" return False\n",
|
||||
" # All day Saturday\n",
|
||||
" elif current_time_cet.weekday() == 5:\n",
|
||||
" return False\n",
|
||||
" return True\n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" try:\n",
|
||||
" if not mt5.initialize(login=name, server=serv, password=key):\n",
|
||||
" log_and_print(\"Failed to initialize MetaTrader 5\", is_error=True)\n",
|
||||
" exit()\n",
|
||||
"\n",
|
||||
" app = TradingApp(symbol=SYMBOL, lot_size=LOT_SIZE, magic_number=MAGIC_NUMBER)\n",
|
||||
"\n",
|
||||
" # 1) Load the classification pipeline\n",
|
||||
" pipeline_path = \"models/saved_models/best_rf_mb_pipeline.pkl\"\n",
|
||||
" app.load_pipeline(pipeline_path)\n",
|
||||
"\n",
|
||||
" while True:\n",
|
||||
" log_and_print(\"Checking market status...\")\n",
|
||||
" if is_market_open():\n",
|
||||
" log_and_print(\"Market is open. Executing trades...\")\n",
|
||||
"\n",
|
||||
" # 2) Generate signals using the loaded pipeline\n",
|
||||
" # This pipeline is classification-based => SHIFTED labels [0,1,2]\n",
|
||||
" # ml_signal_generation() SHIFTs them back to [-1,0,+1] for signals\n",
|
||||
" buy_signal, sell_signal, _, _ = app.ml_signal_generation(\n",
|
||||
" symbol=app.symbol,\n",
|
||||
" n_bars=N_BARS,\n",
|
||||
" timeframe=TIMEFRAME\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # 3) Run strategy\n",
|
||||
" app.run_strategy(app.symbol, app.lot_size, buy_signal, sell_signal)\n",
|
||||
" else:\n",
|
||||
" log_and_print(\"Market is closed. No actions performed.\")\n",
|
||||
"\n",
|
||||
" time.sleep(SLEEP_TIME)\n",
|
||||
"\n",
|
||||
" except KeyboardInterrupt:\n",
|
||||
" log_and_print(\"Shutdown signal received.\")\n",
|
||||
" # If you need a notification here, handle it (e.g., log, email, etc.).\n",
|
||||
" except Exception as e:\n",
|
||||
" error_message = f\"An error occurred: {e}\"\n",
|
||||
" log_and_print(error_message, is_error=True)\n",
|
||||
" # If you need a notification here, handle it (e.g., log, email, etc.).\n",
|
||||
" finally:\n",
|
||||
" mt5.shutdown()\n",
|
||||
" log_and_print(\"MetaTrader 5 shutdown completed.\")\n",
|
||||
" # If you need a notification here, handle it (e.g., log, email, etc.).\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "ml",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+17
-18
@@ -6200,7 +6200,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -41440,33 +41440,32 @@
|
||||
"# 4) MAKE PREDICTIONS (NO RETRAINING)\n",
|
||||
"###########################################################\n",
|
||||
"print(\"\\nPredicting on out-of-sample data (No folds, no retraining)...\")\n",
|
||||
"preds = best_model.predict(X_seq)\n",
|
||||
"preds = best_model.predict(X_seq, verbose=0).reshape(-1) # flatten to (n,)\n",
|
||||
"mse = mean_squared_error(y_seq, preds)\n",
|
||||
"\n",
|
||||
"###########################################################\n",
|
||||
"# 5) CONVERT PREDICTIONS TO TRADING SIGNALS\n",
|
||||
"# 5) BACKTEST via target exposure (-1, 0, +1) << UPDATED\n",
|
||||
"###########################################################\n",
|
||||
"signals = np.where(preds > threshold, 1, np.where(preds < -threshold, -1, 0)).flatten()\n",
|
||||
"# Map regression predictions -> exposure {-1, 0, +1}\n",
|
||||
"exposure = np.where(preds > threshold, 1.0,\n",
|
||||
" np.where(preds < -threshold, -1.0, 0.0)).astype(float)\n",
|
||||
"\n",
|
||||
"# Align signals with dataset index\n",
|
||||
"# Align prices exactly to the prediction rows (no padding)\n",
|
||||
"df_test = df.iloc[lookback:].copy()\n",
|
||||
"close_prices = df_test[\"close\"]\n",
|
||||
"close = df_test[\"close\"]\n",
|
||||
"exposure = pd.Series(exposure, index=close.index)\n",
|
||||
"\n",
|
||||
"# Pad signals if needed\n",
|
||||
"if len(signals) < len(close_prices):\n",
|
||||
" signals = np.append(signals, [0] * (len(close_prices) - len(signals)))\n",
|
||||
"# Optional: trade on the next bar to avoid look-ahead (set to 0 for same-bar)\n",
|
||||
"execution_lag = 1\n",
|
||||
"if execution_lag > 0:\n",
|
||||
" exposure = exposure.shift(execution_lag).fillna(0.0)\n",
|
||||
"\n",
|
||||
"signals_s = pd.Series(signals, index=close_prices.index)\n",
|
||||
"\n",
|
||||
"###########################################################\n",
|
||||
"# 6) RUN FULL BACKTEST USING VECTORBT\n",
|
||||
"###########################################################\n",
|
||||
"print(\"\\nRunning Full Backtest on the Last 2000 Bars...\")\n",
|
||||
"\n",
|
||||
"pf = vbt.Portfolio.from_signals(\n",
|
||||
" close_prices,\n",
|
||||
" entries=signals_s > 0,\n",
|
||||
" exits=signals_s < 0,\n",
|
||||
"pf = vbt.Portfolio.from_orders(\n",
|
||||
" close=close,\n",
|
||||
" size=exposure, # -1 short, 0 flat, +1 long\n",
|
||||
" size_type='targetpercent',\n",
|
||||
" init_cash=10000,\n",
|
||||
" freq='4H',\n",
|
||||
" fees=fees\n",
|
||||
+26
-32
@@ -1109,7 +1109,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -41322,15 +41322,13 @@
|
||||
"df = add_all_ta_features(data)\n",
|
||||
"\n",
|
||||
"# Create double-barrier classification labels\n",
|
||||
"# e.g., up=0.005, down=0.005 => ±0.5% barrier, horizon=20 bars\n",
|
||||
"# Example: up=0.005, down=0.005 => ±0.5% barrier, horizon=20 bars\n",
|
||||
"df_lbl = create_labels_double_barrier(df, up=0.005, down=0.005, horizon=20)\n",
|
||||
"\n",
|
||||
"# Separate features and labels\n",
|
||||
"X = df_lbl.drop(columns=[\"barrier_label\"])\n",
|
||||
"y = df_lbl[\"barrier_label\"]\n",
|
||||
"\n",
|
||||
"# Shift labels from [-1,0,+1] => [0,1,2] for the classifier\n",
|
||||
"y_shifted = y + 1 # -1 → 0, 0 → 1, +1 → 2\n",
|
||||
"y = df_lbl[\"barrier_label\"] # in {-1, 0, +1}\n",
|
||||
"y_shifted = y + 1 # in {0, 1, 2} for the classifier if needed\n",
|
||||
"\n",
|
||||
"###########################################################\n",
|
||||
"# 2) LOAD PRE-TRAINED CLASSIFICATION MODEL (NO RETRAINING)\n",
|
||||
@@ -41344,54 +41342,50 @@
|
||||
"# Subset X to match these columns\n",
|
||||
"X_test = X[trained_columns]\n",
|
||||
"\n",
|
||||
"# Predict on the new dataset\n",
|
||||
"# Predict on the new dataset (model expects labels in {0,1,2})\n",
|
||||
"preds_shifted = best_pipeline.predict(X_test)\n",
|
||||
"\n",
|
||||
"# Convert predictions back to [-1,0,+1]\n",
|
||||
"# Convert predictions back to {-1, 0, +1}\n",
|
||||
"preds = preds_shifted - 1\n",
|
||||
"\n",
|
||||
"# Compute accuracy score against true labels (also shifted back)\n",
|
||||
"y_true_unshifted = y_shifted - 1\n",
|
||||
"accuracy = accuracy_score(y_true_unshifted, preds)\n",
|
||||
"# Accuracy on the exact prediction rows\n",
|
||||
"accuracy = accuracy_score(y.loc[X_test.index], preds)\n",
|
||||
"print(f\"\\nOut-of-Sample Accuracy: {accuracy:.4f}\")\n",
|
||||
"\n",
|
||||
"###########################################################\n",
|
||||
"# 3) CONVERT PREDICTIONS TO SIGNALS & BACKTEST\n",
|
||||
"# 3) BACKTEST via target exposure (-1, 0, +1) << UPDATED\n",
|
||||
"###########################################################\n",
|
||||
"# +1 => buy, -1 => sell, 0 => no position\n",
|
||||
"signals = preds\n",
|
||||
"# predictions already in {-1, 0, +1}\n",
|
||||
"exposure = pd.Series(preds.astype(float), index=X_test.index)\n",
|
||||
"\n",
|
||||
"print(\"\\nRunning Full Backtest on the Last 2000 Bars...\")\n",
|
||||
"# Align prices exactly to prediction rows (no padding)\n",
|
||||
"close = df_lbl.loc[X_test.index, \"close\"]\n",
|
||||
"\n",
|
||||
"df_full = df_lbl.copy()\n",
|
||||
"close_prices_full = df_full[\"close\"]\n",
|
||||
"\n",
|
||||
"# Pad signals if needed\n",
|
||||
"if len(signals) < len(close_prices_full):\n",
|
||||
" signals = np.append(signals, [0] * (len(close_prices_full) - len(signals)))\n",
|
||||
"\n",
|
||||
"signals_s = pd.Series(signals, index=close_prices_full.index)\n",
|
||||
"# Optional: trade on next bar to avoid look-ahead (set to 0 for same-bar)\n",
|
||||
"execution_lag = 1\n",
|
||||
"if execution_lag > 0:\n",
|
||||
" exposure = exposure.shift(execution_lag).fillna(0.0)\n",
|
||||
"\n",
|
||||
"fees = 0.0002 # 0.02% transaction cost per trade\n",
|
||||
"\n",
|
||||
"pf_full = vbt.Portfolio.from_signals(\n",
|
||||
" close_prices_full,\n",
|
||||
" entries=signals_s > 0,\n",
|
||||
" exits=signals_s < 0,\n",
|
||||
"pf = vbt.Portfolio.from_orders(\n",
|
||||
" close=close,\n",
|
||||
" size=exposure, # -1 short, 0 flat, +1 long\n",
|
||||
" size_type='targetpercent',\n",
|
||||
" init_cash=10000,\n",
|
||||
" freq='4H',\n",
|
||||
" fees=fees\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"total_return = pf_full.total_return()\n",
|
||||
"sharpe_ratio = pf_full.sharpe_ratio()\n",
|
||||
"total_return = pf.total_return()\n",
|
||||
"sharpe_ratio = pf.sharpe_ratio()\n",
|
||||
"\n",
|
||||
"print(\"\\nFull Backtest Results:\")\n",
|
||||
"print(f\"Accuracy={accuracy:.2f}, Return={total_return:.2f}%, Sharpe={sharpe_ratio:.2f}\")\n",
|
||||
"print(pf_full.stats())\n",
|
||||
"print(f\"Accuracy={accuracy:.4f}, Return={total_return:.2f}%, Sharpe={sharpe_ratio:.2f}\")\n",
|
||||
"print(pf.stats())\n",
|
||||
"\n",
|
||||
"# Optional: Plot the backtest results\n",
|
||||
"fig = pf_full.plot()\n",
|
||||
"fig = pf.plot()\n",
|
||||
"fig.show()\n"
|
||||
]
|
||||
}
|
||||
+642164
File diff suppressed because one or more lines are too long
+3139431
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+24
-30
@@ -732,7 +732,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -33856,10 +33856,8 @@
|
||||
"\n",
|
||||
"# Separate features and labels\n",
|
||||
"X = df_lbl.drop(columns=[\"regime_label\", \"ma_short\", \"ma_long\"])\n",
|
||||
"y = df_lbl[\"regime_label\"]\n",
|
||||
"\n",
|
||||
"# Shift labels from [-1,0,+1] → [0,1,2] for classifier\n",
|
||||
"y_shifted = y + 1\n",
|
||||
"y = df_lbl[\"regime_label\"] # in {-1, 0, +1}\n",
|
||||
"y_shifted = y + 1 # in {0,1,2} if your classifier used shifted labels\n",
|
||||
"\n",
|
||||
"###########################################################\n",
|
||||
"# 2) LOAD PRE-TRAINED CLASSIFICATION MODEL (NO RETRAINING)\n",
|
||||
@@ -33876,51 +33874,47 @@
|
||||
"# Predict on the new dataset\n",
|
||||
"preds_shifted = best_pipeline.predict(X_test)\n",
|
||||
"\n",
|
||||
"# Shift predictions back to [-1,0,+1]\n",
|
||||
"# Shift predictions back to {-1, 0, +1}\n",
|
||||
"preds = preds_shifted - 1\n",
|
||||
"\n",
|
||||
"# Compute accuracy score against true labels (also shifted back)\n",
|
||||
"y_true_unshifted = y_shifted - 1\n",
|
||||
"accuracy = accuracy_score(y_true_unshifted, preds)\n",
|
||||
"# Accuracy on the exact prediction rows\n",
|
||||
"accuracy = accuracy_score(y.loc[X_test.index], preds)\n",
|
||||
"print(f\"\\nOut-of-Sample Accuracy: {accuracy:.4f}\")\n",
|
||||
"\n",
|
||||
"###########################################################\n",
|
||||
"# 3) CONVERT PREDICTIONS TO SIGNALS & BACKTEST\n",
|
||||
"# 3) BACKTEST via target exposure (-1, 0, +1) << UPDATED\n",
|
||||
"###########################################################\n",
|
||||
"# +1 => buy, -1 => sell, 0 => no position\n",
|
||||
"signals = preds\n",
|
||||
"# predictions already in {-1, 0, +1}\n",
|
||||
"exposure = pd.Series(preds.astype(float), index=X_test.index)\n",
|
||||
"\n",
|
||||
"print(\"\\nRunning Full Backtest on the Last 2000 Bars...\")\n",
|
||||
"# Align prices exactly to prediction rows (no padding)\n",
|
||||
"close = df_lbl.loc[X_test.index, \"close\"]\n",
|
||||
"\n",
|
||||
"df_full = df_lbl.copy()\n",
|
||||
"close_prices_full = df_full[\"close\"]\n",
|
||||
"\n",
|
||||
"# Pad signals if needed\n",
|
||||
"if len(signals) < len(close_prices_full):\n",
|
||||
" signals = np.append(signals, [0] * (len(close_prices_full) - len(signals)))\n",
|
||||
"\n",
|
||||
"signals_s = pd.Series(signals, index=close_prices_full.index)\n",
|
||||
"# Optional: trade on next bar to avoid look-ahead (set to 0 for same-bar)\n",
|
||||
"execution_lag = 1\n",
|
||||
"if execution_lag > 0:\n",
|
||||
" exposure = exposure.shift(execution_lag).fillna(0.0)\n",
|
||||
"\n",
|
||||
"fees = 0.0002 # 0.02% transaction cost per trade\n",
|
||||
"\n",
|
||||
"pf_full = vbt.Portfolio.from_signals(\n",
|
||||
" close_prices_full,\n",
|
||||
" entries=signals_s > 0,\n",
|
||||
" exits=signals_s < 0,\n",
|
||||
"pf = vbt.Portfolio.from_orders(\n",
|
||||
" close=close,\n",
|
||||
" size=exposure, # -1 short, 0 flat, +1 long\n",
|
||||
" size_type='targetpercent',\n",
|
||||
" init_cash=10000,\n",
|
||||
" freq='4H',\n",
|
||||
" fees=fees\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"total_return = pf_full.total_return()\n",
|
||||
"sharpe_ratio = pf_full.sharpe_ratio()\n",
|
||||
"total_return = pf.total_return()\n",
|
||||
"sharpe_ratio = pf.sharpe_ratio()\n",
|
||||
"\n",
|
||||
"print(\"\\nFull Backtest Results:\")\n",
|
||||
"print(f\"Accuracy={accuracy:.2f}, Return={total_return:.2f}%, Sharpe={sharpe_ratio:.2f}\")\n",
|
||||
"print(pf_full.stats())\n",
|
||||
"print(f\"Accuracy={accuracy:.4f}, Return={total_return:.2f}%, Sharpe={sharpe_ratio:.2f}\")\n",
|
||||
"print(pf.stats())\n",
|
||||
"\n",
|
||||
"# Optional: Plot the backtest\n",
|
||||
"fig = pf_full.plot()\n",
|
||||
"fig = pf.plot()\n",
|
||||
"fig.show()\n"
|
||||
]
|
||||
}
|
||||
+22
-16
@@ -4386,7 +4386,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -39127,31 +39127,37 @@
|
||||
"# 2) Subset X to match the columns the pipeline was trained on\n",
|
||||
"X_test = X[trained_columns]\n",
|
||||
"\n",
|
||||
"# 3) Predict on the new dataset\n",
|
||||
"# 3) Predict on the new dataset (pipeline handles scaling + model)\n",
|
||||
"preds = best_pipeline.predict(X_test)\n",
|
||||
"mse = mean_squared_error(y, preds)\n",
|
||||
"\n",
|
||||
"# Align target to prediction index for a fair MSE\n",
|
||||
"y_test = y.loc[X_test.index]\n",
|
||||
"mse = mean_squared_error(y_test, preds)\n",
|
||||
"\n",
|
||||
"###########################################################\n",
|
||||
"# 3) GENERATE TRADING SIGNALS & BACKTEST\n",
|
||||
"# 3) BACKTEST via target exposure (-1, 0, +1) << UPDATED\n",
|
||||
"###########################################################\n",
|
||||
"threshold = 0.0005 # Min predicted return for a trade\n",
|
||||
"signals = np.where(preds > threshold, 1, np.where(preds < -threshold, -1, 0))\n",
|
||||
"threshold = 0.0005 # Min predicted return magnitude to take a position\n",
|
||||
"\n",
|
||||
"print(\"\\nRunning Full Backtest on the Last 2000 Bars...\")\n",
|
||||
"# Map regression predictions -> exposure {-1, 0, +1}\n",
|
||||
"exposure = np.where(preds > threshold, 1.0,\n",
|
||||
" np.where(preds < -threshold, -1.0, 0.0)).astype(float)\n",
|
||||
"\n",
|
||||
"# Align signals with the dataset\n",
|
||||
"close_prices = df[\"close\"]\n",
|
||||
"if len(signals) < len(close_prices):\n",
|
||||
" signals = np.append(signals, [0] * (len(close_prices) - len(signals)))\n",
|
||||
"# Align prices exactly to prediction rows (no padding)\n",
|
||||
"close = df.loc[X_test.index, \"close\"]\n",
|
||||
"\n",
|
||||
"signals_s = pd.Series(signals, index=close_prices.index)\n",
|
||||
"# Optional: trade on next bar to avoid look-ahead (set to 0 for same-bar)\n",
|
||||
"execution_lag = 1\n",
|
||||
"exposure = pd.Series(exposure, index=close.index)\n",
|
||||
"if execution_lag > 0:\n",
|
||||
" exposure = exposure.shift(execution_lag).fillna(0.0)\n",
|
||||
"\n",
|
||||
"fees = 0.0002 # 0.02% transaction cost per trade\n",
|
||||
"\n",
|
||||
"pf = vbt.Portfolio.from_signals(\n",
|
||||
" close_prices,\n",
|
||||
" entries=signals_s > 0,\n",
|
||||
" exits=signals_s < 0,\n",
|
||||
"pf = vbt.Portfolio.from_orders(\n",
|
||||
" close=close,\n",
|
||||
" size=exposure, # -1 short, 0 flat, +1 long\n",
|
||||
" size_type='targetpercent',\n",
|
||||
" init_cash=10000,\n",
|
||||
" freq='4H',\n",
|
||||
" fees=fees\n",
|
||||
File diff suppressed because it is too large
Load Diff
+778
@@ -0,0 +1,778 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Multi-Symbol Version"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loaded pipeline from models/h1_models/US500_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/US2000_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/UK100_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/JP225_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/XAUUSD_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/DE40_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/US30_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/USTEC_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/BTCUSD_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/AAPL.NAS_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/MSFT.NAS_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/GOOG.NAS_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/AMZN.NAS_H1_best_model.pkl\n",
|
||||
"Loaded pipeline from models/h1_models/TSLA.NAS_H1_best_model.pkl\n",
|
||||
"Checking market status...\n",
|
||||
"Market is open. Executing trades...\n",
|
||||
"🟠 Skipping AAPL.NAS: US market not open yet.\n",
|
||||
"🔵 Checking Symbol: US30\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:19:56, SYMBOL: US30, BUY SIGNAL: False, SELL SIGNAL: False\n",
|
||||
"No positions found.\n",
|
||||
"🟤 US30: Flat signal. No position open.\n",
|
||||
"🔵 Checking Symbol: US2000\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:03, SYMBOL: US2000, BUY SIGNAL: False, SELL SIGNAL: False\n",
|
||||
"No positions found.\n",
|
||||
"🟤 US2000: Flat signal. No position open.\n",
|
||||
"🔵 Checking Symbol: DE40\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:09, SYMBOL: DE40, BUY SIGNAL: False, SELL SIGNAL: True\n",
|
||||
"No positions found.\n",
|
||||
"🔴 DE40: Placing new SELL order.\n",
|
||||
"Adjusted lot size to 0.1 (min=0.1, step=0.1, max=250.0)\n",
|
||||
"Sending order request: {'action': 1, 'symbol': 'DE40', 'volume': 0.1, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Order failed details: Date/Time: 2025-05-26 23:20:10\n",
|
||||
"Order Type: SELL\n",
|
||||
"Lot Size: 0.1\n",
|
||||
"SL: None\n",
|
||||
"TP: None\n",
|
||||
"Comment: RFFV-D\n",
|
||||
"Request: {'action': 1, 'symbol': 'DE40', 'volume': 0.1, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Result: OrderSendResult(retcode=10018, deal=0, order=0, volume=0.0, price=0.0, bid=0.0, ask=0.0, comment='Market closed', request_id=1768314453, retcode_external=0, request=TradeRequest(action=1, magic=234003, order=0, symbol='DE40', volume=0.1, price=0.0, stoplimit=0.0, sl=0.0, tp=0.0, deviation=20, type=1, type_filling=1, type_time=0, expiration=0, comment='RFFV-D', position=0, position_by=0))\n",
|
||||
"Symbol DE40 added to retry queue for later attempt.\n",
|
||||
"🔵 Checking Symbol: USTEC\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:17, SYMBOL: USTEC, BUY SIGNAL: False, SELL SIGNAL: False\n",
|
||||
"No positions found.\n",
|
||||
"🟤 USTEC: Flat signal. No position open.\n",
|
||||
"🔵 Checking Symbol: XAUUSD\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:23, SYMBOL: XAUUSD, BUY SIGNAL: False, SELL SIGNAL: True\n",
|
||||
"No positions found.\n",
|
||||
"🔴 XAUUSD: Placing new SELL order.\n",
|
||||
"Adjusted lot size to 0.01 (min=0.01, step=0.01, max=100.0)\n",
|
||||
"Sending order request: {'action': 1, 'symbol': 'XAUUSD', 'volume': 0.01, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Order failed details: Date/Time: 2025-05-26 23:20:26\n",
|
||||
"Order Type: SELL\n",
|
||||
"Lot Size: 0.01\n",
|
||||
"SL: None\n",
|
||||
"TP: None\n",
|
||||
"Comment: RFFV-D\n",
|
||||
"Request: {'action': 1, 'symbol': 'XAUUSD', 'volume': 0.01, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Result: OrderSendResult(retcode=10018, deal=0, order=0, volume=0.0, price=0.0, bid=0.0, ask=0.0, comment='Market closed', request_id=1768314454, retcode_external=0, request=TradeRequest(action=1, magic=234003, order=0, symbol='XAUUSD', volume=0.01, price=0.0, stoplimit=0.0, sl=0.0, tp=0.0, deviation=20, type=1, type_filling=1, type_time=0, expiration=0, comment='RFFV-D', position=0, position_by=0))\n",
|
||||
"Symbol XAUUSD added to retry queue for later attempt.\n",
|
||||
"🟠 Skipping AMZN.NAS: US market not open yet.\n",
|
||||
"🔵 Checking Symbol: BTCUSD\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:32, SYMBOL: BTCUSD, BUY SIGNAL: True, SELL SIGNAL: False\n",
|
||||
"No positions found.\n",
|
||||
"🟢 BTCUSD: Placing new BUY order.\n",
|
||||
"Adjusted lot size to 0.01 (min=0.01, step=0.01, max=10.0)\n",
|
||||
"Sending order request: {'action': 1, 'symbol': 'BTCUSD', 'volume': 0.01, 'type': 0, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Order successful for BTCUSD, comment=Request executed\n",
|
||||
"🟠 Skipping TSLA.NAS: US market not open yet.\n",
|
||||
"🟠 Skipping GOOG.NAS: US market not open yet.\n",
|
||||
"🔵 Checking Symbol: UK100\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:42, SYMBOL: UK100, BUY SIGNAL: False, SELL SIGNAL: False\n",
|
||||
"No positions found.\n",
|
||||
"🟤 UK100: Flat signal. No position open.\n",
|
||||
"🟠 Skipping MSFT.NAS: US market not open yet.\n",
|
||||
"🔵 Checking Symbol: US500\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:48, SYMBOL: US500, BUY SIGNAL: False, SELL SIGNAL: True\n",
|
||||
"No positions found.\n",
|
||||
"🔴 US500: Placing new SELL order.\n",
|
||||
"Adjusted lot size to 0.1 (min=0.1, step=0.1, max=250.0)\n",
|
||||
"Sending order request: {'action': 1, 'symbol': 'US500', 'volume': 0.1, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Order failed details: Date/Time: 2025-05-26 23:20:48\n",
|
||||
"Order Type: SELL\n",
|
||||
"Lot Size: 0.1\n",
|
||||
"SL: None\n",
|
||||
"TP: None\n",
|
||||
"Comment: RFFV-D\n",
|
||||
"Request: {'action': 1, 'symbol': 'US500', 'volume': 0.1, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Result: OrderSendResult(retcode=10018, deal=0, order=0, volume=0.0, price=0.0, bid=0.0, ask=0.0, comment='Market closed', request_id=1768314456, retcode_external=0, request=TradeRequest(action=1, magic=234003, order=0, symbol='US500', volume=0.1, price=0.0, stoplimit=0.0, sl=0.0, tp=0.0, deviation=20, type=1, type_filling=1, type_time=0, expiration=0, comment='RFFV-D', position=0, position_by=0))\n",
|
||||
"Symbol US500 added to retry queue for later attempt.\n",
|
||||
"🔵 Checking Symbol: JP225\n",
|
||||
"------------------------------------------------------------------\n",
|
||||
"🟡 Date: 2025-05-26 23:20:53, SYMBOL: JP225, BUY SIGNAL: False, SELL SIGNAL: True\n",
|
||||
"No positions found.\n",
|
||||
"🔴 JP225: Placing new SELL order.\n",
|
||||
"Adjusted lot size to 1.0 (min=1.0, step=1.0, max=250.0)\n",
|
||||
"Sending order request: {'action': 1, 'symbol': 'JP225', 'volume': 1.0, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Order failed details: Date/Time: 2025-05-26 23:20:55\n",
|
||||
"Order Type: SELL\n",
|
||||
"Lot Size: 1.0\n",
|
||||
"SL: None\n",
|
||||
"TP: None\n",
|
||||
"Comment: RFFV-D\n",
|
||||
"Request: {'action': 1, 'symbol': 'JP225', 'volume': 1.0, 'type': 1, 'deviation': 20, 'magic': 234003, 'comment': 'RFFV-D', 'type_time': 0, 'type_filling': 1}\n",
|
||||
"Result: OrderSendResult(retcode=10018, deal=0, order=0, volume=0.0, price=0.0, bid=0.0, ask=0.0, comment='Market closed', request_id=1768314457, retcode_external=0, request=TradeRequest(action=1, magic=234003, order=0, symbol='JP225', volume=1.0, price=0.0, stoplimit=0.0, sl=0.0, tp=0.0, deviation=20, type=1, type_filling=1, type_time=0, expiration=0, comment='RFFV-D', position=0, position_by=0))\n",
|
||||
"Symbol JP225 added to retry queue for later attempt.\n",
|
||||
"✅ Successful Orders: 1\n",
|
||||
"❌ Failed Orders: 4\n",
|
||||
"⏳ Retry Queue: ['DE40', 'XAUUSD', 'US500', 'JP225']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# LIVE TRADING CODE FOR MULTI-BAR CLASSIFICATION\n",
|
||||
"\n",
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"import warnings\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"# 1) SET PROJECT ROOT AND UPDATE PATH/WORKING DIRECTORY\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"project_root = Path.cwd().parent.parent # Adjust if your notebook is in notebooks/time_series\n",
|
||||
"sys.path.append(str(project_root))\n",
|
||||
"os.chdir(str(project_root))\n",
|
||||
"warnings.filterwarnings(\"ignore\")\n",
|
||||
"\n",
|
||||
"import warnings\n",
|
||||
"warnings.filterwarnings(\"ignore\")\n",
|
||||
"import MetaTrader5 as mt5\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import ta\n",
|
||||
"from datetime import datetime, timedelta\n",
|
||||
"import time\n",
|
||||
"import logging\n",
|
||||
"import joblib\n",
|
||||
"from features.feature_engineering import add_core_features\n",
|
||||
"\n",
|
||||
"import sqlite3\n",
|
||||
"import pandas as pd\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Setup logging\n",
|
||||
"logging.basicConfig(\n",
|
||||
" filename='models/saved_models/trading_app1.log',\n",
|
||||
" level=logging.INFO,\n",
|
||||
" format='%(asctime)s %(levelname)s:%(message)s',\n",
|
||||
" datefmt='%Y-%m-%d %H:%M:%S'\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"def log_and_print(message, is_error=False):\n",
|
||||
" \"\"\"\n",
|
||||
" Logs and prints a message.\n",
|
||||
" If is_error=True, logs at the ERROR level; otherwise logs at INFO level.\n",
|
||||
" \"\"\"\n",
|
||||
" if is_error:\n",
|
||||
" logging.error(message)\n",
|
||||
" else:\n",
|
||||
" logging.info(message)\n",
|
||||
" print(message)\n",
|
||||
"\n",
|
||||
"# Update the login credentials and server information accordingly\n",
|
||||
"#name = 66677507\n",
|
||||
"#key = 'ST746$nG38'\n",
|
||||
"#serv = 'ICMarketsSC-Demo'\n",
|
||||
"\n",
|
||||
"# Global variables\n",
|
||||
"symbols = [\n",
|
||||
" \"US500\", \"US2000\", \"UK100\", \"JP225\", \"XAUUSD\", \"DE40\", \"US30\",\n",
|
||||
" \"USTEC\", \"BTCUSD\", \"AAPL.NAS\", \"MSFT.NAS\", \"GOOG.NAS\", \"AMZN.NAS\", \"TSLA.NAS\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"lot_sizes = {\n",
|
||||
" \n",
|
||||
" \"US500\": 0.1,\n",
|
||||
" \"US2000\": 0.1,\n",
|
||||
" \"UK100\": 0.1,\n",
|
||||
" \"DE40\": 0.1,\n",
|
||||
" \"USTEC\": 0.1,\n",
|
||||
" \"JP225\": 1.00,\n",
|
||||
"\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Add Forex and Stocks with 0.01 lot size\n",
|
||||
"for symbol in symbols:\n",
|
||||
" if symbol not in lot_sizes:\n",
|
||||
" lot_sizes[symbol] = 0.01\n",
|
||||
"\n",
|
||||
"model_paths = {\n",
|
||||
" symbol: f\"models/h1_models/{symbol}_H1_best_model.pkl\" for symbol in symbols\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"TIMEFRAME = mt5.TIMEFRAME_H1\n",
|
||||
"N_BARS = 1000\n",
|
||||
"MAGIC_NUMBER = 234003\n",
|
||||
"SLEEP_TIME = 3600 # 1 hour\n",
|
||||
"COMMENT_ML = \"RFFV-D\"\n",
|
||||
"\n",
|
||||
"success_count = 0\n",
|
||||
"fail_count = 0\n",
|
||||
"retry_queue = []\n",
|
||||
"# Global setting for whether to close open positions on flat (neutral) signal\n",
|
||||
"CLOSE_ON_FLAT = True\n",
|
||||
"N_FORWARD = 3 # Number of bars ahead to predict and save\n",
|
||||
"\n",
|
||||
"# If you still need feature selection, you can keep this helper function:\n",
|
||||
"def select_features_rf_reg(X, y, estimator, max_features=20):\n",
|
||||
" \"\"\"\n",
|
||||
" Example helper function for feature selection using RandomForest.\n",
|
||||
" \"\"\"\n",
|
||||
" from sklearn.feature_selection import SelectFromModel\n",
|
||||
" selector = SelectFromModel(estimator=estimator, threshold=-np.inf, max_features=max_features).fit(X, y)\n",
|
||||
" X_transformed = selector.transform(X)\n",
|
||||
" selected_features_mask = selector.get_support()\n",
|
||||
" return X_transformed, selected_features_mask\n",
|
||||
"\n",
|
||||
"def is_us_stock(symbol):\n",
|
||||
" return symbol.endswith(\".NAS\") or symbol.endswith(\".NYSE\")\n",
|
||||
"\n",
|
||||
"def is_us_market_open():\n",
|
||||
" \"\"\"\n",
|
||||
" Check if US stock market is open based on Switzerland time (CET/CEST).\n",
|
||||
" \"\"\"\n",
|
||||
" now = datetime.now()\n",
|
||||
"\n",
|
||||
" if now.weekday() >= 5: # Saturday or Sunday\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
" market_open = now.replace(hour=15, minute=30, second=0, microsecond=0)\n",
|
||||
" market_close = now.replace(hour=22, minute=0, second=0, microsecond=0)\n",
|
||||
"\n",
|
||||
" return market_open <= now <= market_close\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TradingApp:\n",
|
||||
"\n",
|
||||
" def __init__(self, symbol, lot_size, magic_number):\n",
|
||||
" self.symbol = symbol\n",
|
||||
" self.lot_size = lot_size\n",
|
||||
" self.magic_number = magic_number\n",
|
||||
" self.model = None # previously self.pipeline\n",
|
||||
" self.scaler = None\n",
|
||||
" self.features = None\n",
|
||||
" self.last_retrain_time = None\n",
|
||||
"\n",
|
||||
" def get_data(self, symbol, n, timeframe):\n",
|
||||
" \"\"\"\n",
|
||||
" Fetch 'n' bars of historical data for the given symbol and timeframe.\n",
|
||||
" \"\"\"\n",
|
||||
" rates = mt5.copy_rates_from_pos(symbol, timeframe, 0, n)\n",
|
||||
" rates_frame = pd.DataFrame(rates)\n",
|
||||
" rates_frame['time'] = pd.to_datetime(rates_frame['time'], unit='s')\n",
|
||||
" rates_frame.set_index('time', inplace=True)\n",
|
||||
" return rates_frame\n",
|
||||
"\n",
|
||||
" def add_core_features(self, df):\n",
|
||||
" \"\"\"\n",
|
||||
" Add only the core features to the DataFrame (the same ones used in model training).\n",
|
||||
" \"\"\"\n",
|
||||
" df = add_core_features(df)\n",
|
||||
" return df\n",
|
||||
"\n",
|
||||
" def load_pipeline(self, pipeline_path):\n",
|
||||
" pipeline_loaded = joblib.load(pipeline_path)\n",
|
||||
"\n",
|
||||
" if isinstance(pipeline_loaded, dict):\n",
|
||||
" self.model = pipeline_loaded.get(\"model\")\n",
|
||||
" self.scaler = pipeline_loaded.get(\"scaler\", None)\n",
|
||||
" self.features = pipeline_loaded.get(\"features\", None)\n",
|
||||
" else:\n",
|
||||
" self.model = pipeline_loaded\n",
|
||||
" self.scaler = None\n",
|
||||
" self.features = None\n",
|
||||
"\n",
|
||||
" logging.info(f\"Loaded model from {pipeline_path}\")\n",
|
||||
" log_and_print(f\"✅ Loaded model from {pipeline_path}\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" def ml_signal_generation(self, symbol, n_bars, timeframe):\n",
|
||||
" if self.model is None:\n",
|
||||
" logging.error(\"❌ No model loaded.\")\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
" df = self.get_data(symbol, n_bars, timeframe)\n",
|
||||
" df = self.add_core_features(df)\n",
|
||||
" df.fillna(method='ffill', inplace=True)\n",
|
||||
"\n",
|
||||
" features = self.features if self.features else [\n",
|
||||
" \"sma_20\", \"ema_20\", \"kama_10\", \"rsi_14\", \"macd_diff\",\n",
|
||||
" \"atr_14\", \"obv\", \"rolling_std_20\", \"spread\", \"fill\", \"amplitude\",\n",
|
||||
" \"autocorr_1\", \"autocorr_5\", \"autocorr_10\", \"market_regime\", \"stationary_flag\"\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" X_new = df[features].dropna()\n",
|
||||
"\n",
|
||||
" if X_new.empty:\n",
|
||||
" logging.error(f\"❌ No valid feature rows for prediction on {symbol}.\")\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
" if self.scaler:\n",
|
||||
" X_scaled = self.scaler.transform(X_new)\n",
|
||||
" else:\n",
|
||||
" X_scaled = X_new\n",
|
||||
"\n",
|
||||
" preds_shifted = self.model.predict(X_scaled)\n",
|
||||
" preds = preds_shifted - 1\n",
|
||||
"\n",
|
||||
" if all(p == 0 for p in preds[-N_FORWARD:]):\n",
|
||||
" log_and_print(f\"⚪ {symbol}: All {N_FORWARD} predictions are flat (0).\")\n",
|
||||
"\n",
|
||||
" return preds[-N_FORWARD:]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" def orders(self, symbol, lot, is_buy=True, id_position=None, sl=None, tp=None):\n",
|
||||
" \"\"\"\n",
|
||||
" Place an order (BUY or SELL) for the specified symbol and lot size.\n",
|
||||
" \"\"\"\n",
|
||||
" global success_count, fail_count, retry_queue\n",
|
||||
"\n",
|
||||
" symbol_info = mt5.symbol_info(symbol)\n",
|
||||
" if symbol_info is None:\n",
|
||||
" log_and_print(f\"Symbol {symbol} not found, can't place order.\", is_error=True)\n",
|
||||
" fail_count += 1\n",
|
||||
" return \"Symbol not found\"\n",
|
||||
"\n",
|
||||
" # Make sure symbol is visible\n",
|
||||
" if not symbol_info.visible:\n",
|
||||
" if not mt5.symbol_select(symbol, True):\n",
|
||||
" log_and_print(f\"Failed to select symbol {symbol}\", is_error=True)\n",
|
||||
" fail_count += 1\n",
|
||||
" return \"Symbol not visible or could not be selected.\"\n",
|
||||
"\n",
|
||||
" tick_info = mt5.symbol_info_tick(symbol)\n",
|
||||
" if tick_info is None:\n",
|
||||
" log_and_print(f\"Could not get tick info for {symbol}.\", is_error=True)\n",
|
||||
" fail_count += 1\n",
|
||||
" return \"Tick info unavailable\"\n",
|
||||
"\n",
|
||||
" # Check for valid bid/ask\n",
|
||||
" if tick_info.bid <= 0 or tick_info.ask <= 0:\n",
|
||||
" log_and_print(\n",
|
||||
" f\"Zero or invalid bid/ask for {symbol}: bid={tick_info.bid}, ask={tick_info.ask}\",\n",
|
||||
" is_error=True\n",
|
||||
" )\n",
|
||||
" fail_count += 1\n",
|
||||
" return \"Invalid prices\"\n",
|
||||
"\n",
|
||||
" # LOT SIZE VALIDATION\n",
|
||||
" lot = max(lot, symbol_info.volume_min)\n",
|
||||
" step = symbol_info.volume_step\n",
|
||||
" if step > 0:\n",
|
||||
" remainder = lot % step\n",
|
||||
" if remainder != 0:\n",
|
||||
" lot = lot - remainder + step\n",
|
||||
" if lot > symbol_info.volume_max:\n",
|
||||
" lot = symbol_info.volume_max\n",
|
||||
"\n",
|
||||
" log_and_print(\n",
|
||||
" f\"Adjusted lot size to {lot} (min={symbol_info.volume_min}, \"\n",
|
||||
" f\"step={symbol_info.volume_step}, max={symbol_info.volume_max})\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Force ORDER_FILLING_IOC\n",
|
||||
" filling_mode = 1 # ORDER_FILLING_IOC\n",
|
||||
"\n",
|
||||
" order_type = mt5.ORDER_TYPE_BUY if is_buy else mt5.ORDER_TYPE_SELL\n",
|
||||
" order_price = tick_info.ask if is_buy else tick_info.bid\n",
|
||||
" deviation = 20\n",
|
||||
"\n",
|
||||
" request = {\n",
|
||||
" \"action\": mt5.TRADE_ACTION_DEAL,\n",
|
||||
" \"symbol\": symbol,\n",
|
||||
" \"volume\": lot,\n",
|
||||
" \"type\": order_type,\n",
|
||||
" \"deviation\": deviation,\n",
|
||||
" \"magic\": self.magic_number,\n",
|
||||
" \"comment\": COMMENT_ML,\n",
|
||||
" \"type_time\": mt5.ORDER_TIME_GTC,\n",
|
||||
" \"type_filling\": filling_mode,\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" if sl is not None:\n",
|
||||
" request[\"sl\"] = sl\n",
|
||||
" if tp is not None:\n",
|
||||
" request[\"tp\"] = tp\n",
|
||||
" if id_position is not None:\n",
|
||||
" request[\"position\"] = id_position\n",
|
||||
"\n",
|
||||
" log_and_print(f\"Sending order request: {request}\")\n",
|
||||
" result = mt5.order_send(request)\n",
|
||||
"\n",
|
||||
" order_type_str = \"BUY\" if is_buy else \"SELL\"\n",
|
||||
"\n",
|
||||
" if result is None or result.retcode != mt5.TRADE_RETCODE_DONE:\n",
|
||||
" fail_count += 1\n",
|
||||
" error_message = f\"Order failed for {symbol}\"\n",
|
||||
" if result:\n",
|
||||
" error_message += f\", retcode={result.retcode}, comment={result.comment}\"\n",
|
||||
" additional_info = (\n",
|
||||
" f\"Date/Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\\n\"\n",
|
||||
" f\"Order Type: {order_type_str}\\n\"\n",
|
||||
" f\"Lot Size: {lot}\\n\"\n",
|
||||
" f\"SL: {sl if sl else 'None'}\\n\"\n",
|
||||
" f\"TP: {tp if tp else 'None'}\\n\"\n",
|
||||
" f\"Comment: {COMMENT_ML}\\n\"\n",
|
||||
" f\"Request: {request}\\n\"\n",
|
||||
" f\"Result: {result}\"\n",
|
||||
" )\n",
|
||||
" log_and_print(f\"Order failed details: {additional_info}\", is_error=True)\n",
|
||||
"\n",
|
||||
" # If market closed or only closing allowed => add to retry queue\n",
|
||||
" if result is not None and result.retcode in [10018, 10044]:\n",
|
||||
" retry_queue.append((symbol, datetime.now() + timedelta(minutes=15)))\n",
|
||||
" log_and_print(f\"Symbol {symbol} added to retry queue for later attempt.\", is_error=False)\n",
|
||||
"\n",
|
||||
" else:\n",
|
||||
" success_count += 1\n",
|
||||
" success_message = f\"Order successful for {symbol}, comment={result.comment}\"\n",
|
||||
" additional_info = (\n",
|
||||
" f\"Date/Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\\n\"\n",
|
||||
" f\"Order Type: {order_type_str}\\n\"\n",
|
||||
" f\"Lot Size: {lot}\\n\"\n",
|
||||
" f\"SL: {sl if sl else 'None'}\\n\"\n",
|
||||
" f\"TP: {tp if tp else 'None'}\\n\"\n",
|
||||
" f\"Comment: {COMMENT_ML}\"\n",
|
||||
" )\n",
|
||||
" log_and_print(success_message)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" def get_positions_by_magic(self, symbol, magic_number):\n",
|
||||
" \"\"\"\n",
|
||||
" Retrieve positions for a specific symbol and magic number.\n",
|
||||
" \"\"\"\n",
|
||||
" all_positions = mt5.positions_get(symbol=symbol)\n",
|
||||
" if not all_positions:\n",
|
||||
" log_and_print(\"No positions found.\", is_error=False)\n",
|
||||
" return []\n",
|
||||
" return [pos for pos in all_positions if pos.magic == magic_number]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" def run_strategy(self, symbol, lot, buy_signal, sell_signal):\n",
|
||||
" \"\"\"\n",
|
||||
" Run the trading strategy logic based on buy/sell signals, including flat signals.\n",
|
||||
" \"\"\"\n",
|
||||
" log_and_print(\"------------------------------------------------------------------\")\n",
|
||||
" log_and_print(\n",
|
||||
" f\"🟡 Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}, \"\n",
|
||||
" f\"SYMBOL: {symbol}, BUY SIGNAL: {buy_signal}, SELL SIGNAL: {sell_signal}\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" positions = self.get_positions_by_magic(symbol, self.magic_number)\n",
|
||||
" has_buy = any(pos.type == mt5.POSITION_TYPE_BUY for pos in positions)\n",
|
||||
" has_sell = any(pos.type == mt5.POSITION_TYPE_SELL for pos in positions)\n",
|
||||
"\n",
|
||||
" if buy_signal and not has_buy:\n",
|
||||
" if has_sell:\n",
|
||||
" log_and_print(f\"🔄 {symbol}: Existing SELL position found. Attempting to close it...\")\n",
|
||||
" if self.close_position(symbol, is_buy=True):\n",
|
||||
" log_and_print(f\"✅ {symbol}: Closed SELL. Placing new BUY order.\")\n",
|
||||
" self.orders(symbol, lot, is_buy=True)\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"❌ {symbol}: Failed to close SELL position.\")\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"🟢 {symbol}: Placing new BUY order.\")\n",
|
||||
" self.orders(symbol, lot, is_buy=True)\n",
|
||||
"\n",
|
||||
" elif sell_signal and not has_sell:\n",
|
||||
" if has_buy:\n",
|
||||
" log_and_print(f\"🔄 {symbol}: Existing BUY position found. Attempting to close it...\")\n",
|
||||
" if self.close_position(symbol, is_buy=False):\n",
|
||||
" log_and_print(f\"✅ {symbol}: Closed BUY. Placing new SELL order.\")\n",
|
||||
" self.orders(symbol, lot, is_buy=False)\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"❌ {symbol}: Failed to close BUY position.\")\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"🔴 {symbol}: Placing new SELL order.\")\n",
|
||||
" self.orders(symbol, lot, is_buy=False)\n",
|
||||
"\n",
|
||||
" elif not buy_signal and not sell_signal:\n",
|
||||
" if has_buy or has_sell:\n",
|
||||
" if CLOSE_ON_FLAT:\n",
|
||||
" log_and_print(f\"⚪ {symbol}: Flat signal. CLOSE_ON_FLAT is True, attempting to close open position.\")\n",
|
||||
" success = self.close_position(symbol, is_buy=has_sell)\n",
|
||||
" if success:\n",
|
||||
" log_and_print(f\"✅ {symbol}: Flat signal - Position closed.\")\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"❌ {symbol}: Flat signal - Failed to close position.\")\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"⚪ {symbol}: Flat signal but CLOSE_ON_FLAT is False. Holding current position.\")\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"🟤 {symbol}: Flat signal. No position open.\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" def close_position(self, symbol, is_buy):\n",
|
||||
" \"\"\"\n",
|
||||
" Closes positions of the opposite type (BUY/SELL) for this app's magic number.\n",
|
||||
" \"\"\"\n",
|
||||
" positions = mt5.positions_get(symbol=symbol)\n",
|
||||
" if not positions:\n",
|
||||
" log_and_print(f\"No positions to close for symbol: {symbol}\")\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
" initial_balance = mt5.account_info().balance\n",
|
||||
" closed_any = False\n",
|
||||
"\n",
|
||||
" for position in positions:\n",
|
||||
" if position.magic == self.magic_number and (\n",
|
||||
" (is_buy and position.type == mt5.POSITION_TYPE_SELL) or\n",
|
||||
" (not is_buy and position.type == mt5.POSITION_TYPE_BUY)\n",
|
||||
" ):\n",
|
||||
" # First try ORDER_FILLING_RETURN\n",
|
||||
" close_request = {\n",
|
||||
" \"action\": mt5.TRADE_ACTION_DEAL,\n",
|
||||
" \"symbol\": symbol,\n",
|
||||
" \"volume\": position.volume,\n",
|
||||
" \"type\": mt5.ORDER_TYPE_BUY if position.type == mt5.POSITION_TYPE_SELL else mt5.ORDER_TYPE_SELL,\n",
|
||||
" \"position\": position.ticket,\n",
|
||||
" \"deviation\": 20,\n",
|
||||
" \"magic\": self.magic_number,\n",
|
||||
" \"comment\": COMMENT_ML,\n",
|
||||
" \"type_time\": mt5.ORDER_TIME_GTC,\n",
|
||||
" \"type_filling\": mt5.ORDER_FILLING_RETURN, # Try RETURN first\n",
|
||||
" }\n",
|
||||
" result = mt5.order_send(close_request)\n",
|
||||
"\n",
|
||||
" if result is None or result.retcode != mt5.TRADE_RETCODE_DONE:\n",
|
||||
" # Retry WITH ORDER_FILLING_IOC instead of removing filling type\n",
|
||||
" log_and_print(f\"⚠️ Filling mode error for {symbol} position {position.ticket}, retrying with ORDER_FILLING_IOC...\")\n",
|
||||
" \n",
|
||||
" close_request[\"type_filling\"] = mt5.ORDER_FILLING_IOC # Force IOC\n",
|
||||
" result_retry = mt5.order_send(close_request)\n",
|
||||
"\n",
|
||||
" if result_retry is None or result_retry.retcode != mt5.TRADE_RETCODE_DONE:\n",
|
||||
" log_and_print(f\"❌ Retry failed to close {symbol} — retcode: {result_retry.retcode}, comment: {result_retry.comment}\", is_error=True)\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"✅ Retry succeeded in closing {symbol} position {position.ticket}\")\n",
|
||||
" closed_any = True\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"✅ Successfully closed position {position.ticket} for {symbol}\")\n",
|
||||
" closed_any = True\n",
|
||||
"\n",
|
||||
" if closed_any:\n",
|
||||
" final_balance = mt5.account_info().balance\n",
|
||||
" profit = final_balance - initial_balance\n",
|
||||
" log_and_print(f\"✅ Closed positions successfully, Profit: {profit}\")\n",
|
||||
" return True\n",
|
||||
" else:\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" def check_and_execute_trades(self):\n",
|
||||
" \"\"\"\n",
|
||||
" Convenience method to perform the entire flow:\n",
|
||||
" generate signals, run strategy, and deselect symbol.\n",
|
||||
" \"\"\"\n",
|
||||
" mt5.symbol_select(self.symbol, True)\n",
|
||||
" buy, sell, _, _ = self.ml_signal_generation(self.symbol, N_BARS, TIMEFRAME)\n",
|
||||
" self.run_strategy(self.symbol, self.lot_size, buy, sell)\n",
|
||||
" mt5.symbol_select(self.symbol, False)\n",
|
||||
" log_and_print(\"Waiting for new signals...\")\n",
|
||||
"\n",
|
||||
"def is_market_open():\n",
|
||||
" \"\"\"\n",
|
||||
" Check if the current time is within the typical Forex trading session, adjusted for CET/CEST.\n",
|
||||
" Market closes at Friday 10:00 PM CET and opens at Sunday 11:00 PM CET. \n",
|
||||
" It is closed all day Saturday.\n",
|
||||
" \"\"\"\n",
|
||||
" current_time_utc = datetime.utcnow()\n",
|
||||
" # Adjust for Central European Time (UTC+1) or Central European Summer Time (UTC+2)\n",
|
||||
" current_time_cet = (\n",
|
||||
" current_time_utc + timedelta(hours=2) \n",
|
||||
" if time.localtime().tm_isdst \n",
|
||||
" else current_time_utc + timedelta(hours=1)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Friday after 10 PM CET\n",
|
||||
" if current_time_cet.weekday() == 4 and current_time_cet.hour >= 22:\n",
|
||||
" return False\n",
|
||||
" # Sunday before 11 PM CET\n",
|
||||
" elif current_time_cet.weekday() == 6 and current_time_cet.hour < 23:\n",
|
||||
" return False\n",
|
||||
" # All day Saturday\n",
|
||||
" elif current_time_cet.weekday() == 5:\n",
|
||||
" return False\n",
|
||||
" return True\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def save_signal_to_db(symbol, prediction, timestamp=None):\n",
|
||||
" conn = sqlite3.connect('live_signals.db')\n",
|
||||
" c = conn.cursor()\n",
|
||||
" c.execute('''\n",
|
||||
" CREATE TABLE IF NOT EXISTS signals (\n",
|
||||
" id INTEGER PRIMARY KEY AUTOINCREMENT,\n",
|
||||
" symbol TEXT,\n",
|
||||
" prediction INTEGER,\n",
|
||||
" timestamp TEXT,\n",
|
||||
" UNIQUE(symbol, prediction, timestamp)\n",
|
||||
" )\n",
|
||||
" ''')\n",
|
||||
" if timestamp is None:\n",
|
||||
" timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')\n",
|
||||
" try:\n",
|
||||
" c.execute(\n",
|
||||
" 'INSERT OR IGNORE INTO signals (symbol, prediction, timestamp) VALUES (?, ?, ?)',\n",
|
||||
" (symbol, prediction, timestamp)\n",
|
||||
" )\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Failed to save to db: {e}\")\n",
|
||||
" conn.commit()\n",
|
||||
" conn.close()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" try:\n",
|
||||
" if not mt5.initialize():\n",
|
||||
" log_and_print(\"Failed to initialize MetaTrader 5\", is_error=True)\n",
|
||||
" exit()\n",
|
||||
"\n",
|
||||
" # 1) Create a TradingApp per symbol\n",
|
||||
" apps = {}\n",
|
||||
" for symbol in symbols:\n",
|
||||
" app = TradingApp(symbol=symbol, lot_size=lot_sizes[symbol], magic_number=MAGIC_NUMBER)\n",
|
||||
" app.load_pipeline(model_paths[symbol])\n",
|
||||
" apps[symbol] = app\n",
|
||||
"\n",
|
||||
" # 2) Initialize tracking variables\n",
|
||||
" success_count = 0\n",
|
||||
" fail_count = 0\n",
|
||||
" retry_queue = []\n",
|
||||
"\n",
|
||||
" while True:\n",
|
||||
" log_and_print(\"Checking market status...\")\n",
|
||||
" if is_market_open():\n",
|
||||
" log_and_print(\"Market is open. Executing trades...\")\n",
|
||||
"\n",
|
||||
" # Handle retry queue\n",
|
||||
" now = datetime.now()\n",
|
||||
" retry_symbols_ready = [item for item in retry_queue if item[1] <= now]\n",
|
||||
" retry_queue = [item for item in retry_queue if item[1] > now]\n",
|
||||
"\n",
|
||||
" symbols_to_process = set(apps.keys())\n",
|
||||
"\n",
|
||||
" # Add retry symbols first\n",
|
||||
" symbols_ready_to_retry = [item[0] for item in retry_symbols_ready]\n",
|
||||
" if symbols_ready_to_retry:\n",
|
||||
" log_and_print(f\"🔁 Retrying symbols: {symbols_ready_to_retry}\")\n",
|
||||
" symbols_to_process = set(symbols_ready_to_retry) | symbols_to_process\n",
|
||||
"\n",
|
||||
" for symbol in symbols_to_process:\n",
|
||||
" app = apps[symbol]\n",
|
||||
" try:\n",
|
||||
" if is_us_stock(symbol) and not is_us_market_open():\n",
|
||||
" log_and_print(f\"🟠 Skipping {symbol}: US market not open yet.\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" log_and_print(f\"🔵 Checking Symbol: {symbol}\")\n",
|
||||
"\n",
|
||||
" # --- Get N-step-ahead predictions ---\n",
|
||||
" multi_preds = app.ml_signal_generation(\n",
|
||||
" symbol=app.symbol,\n",
|
||||
" n_bars=N_BARS,\n",
|
||||
" timeframe=TIMEFRAME\n",
|
||||
" )\n",
|
||||
" if multi_preds is None:\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" # --- Get last bar time for timestamping ---\n",
|
||||
" df_data = app.get_data(app.symbol, N_BARS, TIMEFRAME)\n",
|
||||
" last_bar_time = df_data.index[-1]\n",
|
||||
"\n",
|
||||
" # --- Save all N_FORWARD predictions ---\n",
|
||||
" for i, pred in enumerate(multi_preds):\n",
|
||||
" future_time = (last_bar_time + pd.Timedelta(hours=i+1)).strftime('%Y-%m-%d %H:%M:%S')\n",
|
||||
" save_signal_to_db(app.symbol, int(pred), timestamp=future_time)\n",
|
||||
"\n",
|
||||
" # --- Use first prediction for live trading ---\n",
|
||||
" buy_signal = (multi_preds[0] == 1)\n",
|
||||
" sell_signal = (multi_preds[0] == -1)\n",
|
||||
" app.run_strategy(app.symbol, app.lot_size, buy_signal, sell_signal)\n",
|
||||
"\n",
|
||||
" except Exception as e:\n",
|
||||
" log_and_print(f\"⚠️ Error processing {symbol}: {e}\", is_error=True)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # After the trading round, log results\n",
|
||||
" log_and_print(f\"✅ Successful Orders: {success_count}\")\n",
|
||||
" log_and_print(f\"❌ Failed Orders: {fail_count}\")\n",
|
||||
" log_and_print(f\"⏳ Retry Queue: {[item[0] for item in retry_queue]}\")\n",
|
||||
"\n",
|
||||
" else:\n",
|
||||
" log_and_print(\"Market is closed. Waiting...\")\n",
|
||||
"\n",
|
||||
" time.sleep(SLEEP_TIME)\n",
|
||||
"\n",
|
||||
" except KeyboardInterrupt:\n",
|
||||
" log_and_print(\"Shutdown signal received.\")\n",
|
||||
"\n",
|
||||
" except Exception as e:\n",
|
||||
" error_message = f\"An error occurred: {e}\"\n",
|
||||
" log_and_print(error_message, is_error=True)\n",
|
||||
"\n",
|
||||
" finally:\n",
|
||||
" mt5.shutdown()\n",
|
||||
" log_and_print(\"MetaTrader 5 shutdown completed.\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "ml",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.16"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"import warnings\n",
|
||||
"from pathlib import Path\n",
|
||||
"import MetaTrader5 as mt5\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"from statsmodels.tsa.stattools import coint\n",
|
||||
"import time\n",
|
||||
"import logging\n",
|
||||
"\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"# 1) SETUP LOGGING & ENVIRONMENT\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"warnings.filterwarnings(\"ignore\")\n",
|
||||
"\n",
|
||||
"# Logging setup\n",
|
||||
"logging.basicConfig(\n",
|
||||
" filename=\"models/saved_models/pair_trading.log\",\n",
|
||||
" level=logging.INFO,\n",
|
||||
" format=\"%(asctime)s %(levelname)s:%(message)s\",\n",
|
||||
" datefmt=\"%Y-%m-%d %H:%M:%S\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"def log_and_print(message, is_error=False):\n",
|
||||
" if is_error:\n",
|
||||
" logging.error(message)\n",
|
||||
" else:\n",
|
||||
" logging.info(message)\n",
|
||||
" print(message)\n",
|
||||
"\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"# 2) ACCOUNT CONFIGURATION\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"name = 7889999\n",
|
||||
"key = \"hdgdggFxEG38\"\n",
|
||||
"serv = \"ICMarketsSC-Demo\"\n",
|
||||
"\n",
|
||||
"# Global parameters\n",
|
||||
"PAIR1 = \"AUDUSD\"\n",
|
||||
"PAIR2 = \"NZDUSD\"\n",
|
||||
"LOT_SIZE = 0.01\n",
|
||||
"TIMEFRAME = mt5.TIMEFRAME_H1 # Adjust as needed\n",
|
||||
"N_BARS = 5000 # More data for stable pair trading\n",
|
||||
"MAGIC_NUMBER = 77777\n",
|
||||
"SLEEP_TIME = 3600 # 1 Hour sleep cycle\n",
|
||||
"\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"# 3) MT5 FUNCTIONS\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"def get_data(symbol, n, timeframe):\n",
|
||||
" \"\"\"Fetches historical price data from MT5.\"\"\"\n",
|
||||
" rates = mt5.copy_rates_from_pos(symbol, timeframe, 0, n)\n",
|
||||
" if rates is None:\n",
|
||||
" raise ValueError(f\"Could not retrieve data for {symbol}\")\n",
|
||||
" df = pd.DataFrame(rates)\n",
|
||||
" df[\"time\"] = pd.to_datetime(df[\"time\"], unit=\"s\")\n",
|
||||
" df.set_index(\"time\", inplace=True)\n",
|
||||
" return df[[\"close\"]]\n",
|
||||
"\n",
|
||||
"def check_cointegration(symbol1, symbol2, n_bars):\n",
|
||||
" \"\"\"Performs cointegration test between two assets.\"\"\"\n",
|
||||
" df1 = get_data(symbol1, n_bars, TIMEFRAME)\n",
|
||||
" df2 = get_data(symbol2, n_bars, TIMEFRAME)\n",
|
||||
"\n",
|
||||
" score, p_value, _ = coint(df1[\"close\"], df2[\"close\"])\n",
|
||||
" log_and_print(f\"⚖️ Cointegration Test p-value ({symbol1} & {symbol2}): {p_value:.4f}\")\n",
|
||||
" \n",
|
||||
" return p_value < 0.05 # True if cointegrated\n",
|
||||
"\n",
|
||||
"def compute_z_score(spread, lookback=60):\n",
|
||||
" \"\"\"Calculates the rolling Z-score of the spread.\"\"\"\n",
|
||||
" mean = spread.rolling(lookback).mean()\n",
|
||||
" std = spread.rolling(lookback).std()\n",
|
||||
" return (spread - mean) / std\n",
|
||||
"\n",
|
||||
"def generate_pair_signals(pair1, pair2, lookback=60, entry_threshold=1.5, exit_threshold=0.5):\n",
|
||||
" \"\"\"Computes Z-score signals for pair trading.\"\"\"\n",
|
||||
" df1 = get_data(pair1, N_BARS, TIMEFRAME)\n",
|
||||
" df2 = get_data(pair2, N_BARS, TIMEFRAME)\n",
|
||||
"\n",
|
||||
" # Compute spread\n",
|
||||
" spread = df1[\"close\"] - df2[\"close\"]\n",
|
||||
" z_score = compute_z_score(spread, lookback)\n",
|
||||
"\n",
|
||||
" df = pd.DataFrame({\"spread\": spread, \"z_score\": z_score})\n",
|
||||
" \n",
|
||||
" # Trading signals\n",
|
||||
" df[\"long_signal\"] = df[\"z_score\"] < -entry_threshold # Buy Pair1, Sell Pair2\n",
|
||||
" df[\"short_signal\"] = df[\"z_score\"] > entry_threshold # Sell Pair1, Buy Pair2\n",
|
||||
" df[\"exit_signal\"] = df[\"z_score\"].abs() < exit_threshold # Exit trade\n",
|
||||
"\n",
|
||||
" return df\n",
|
||||
"\n",
|
||||
"def place_order(symbol, lot, is_buy, magic):\n",
|
||||
" \"\"\"Places an order on MT5.\"\"\"\n",
|
||||
" order_type = mt5.ORDER_TYPE_BUY if is_buy else mt5.ORDER_TYPE_SELL\n",
|
||||
" deviation = 20\n",
|
||||
"\n",
|
||||
" request = {\n",
|
||||
" \"action\": mt5.TRADE_ACTION_DEAL,\n",
|
||||
" \"symbol\": symbol,\n",
|
||||
" \"volume\": lot,\n",
|
||||
" \"type\": order_type,\n",
|
||||
" \"deviation\": deviation,\n",
|
||||
" \"magic\": magic,\n",
|
||||
" \"comment\": \"Pair Trading Bot\",\n",
|
||||
" \"type_time\": mt5.ORDER_TIME_GTC,\n",
|
||||
" \"type_filling\": mt5.ORDER_FILLING_IOC,\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" result = mt5.order_send(request)\n",
|
||||
" if result is None or result.retcode != mt5.TRADE_RETCODE_DONE:\n",
|
||||
" log_and_print(f\"❌ Order failed for {symbol}: {result.retcode}\", is_error=True)\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"✅ Order placed for {symbol}\")\n",
|
||||
"\n",
|
||||
"def manage_trades(pair1, pair2, lot_size, signals):\n",
|
||||
" \"\"\"Executes pair trading strategy based on signals.\"\"\"\n",
|
||||
" log_and_print(f\"📈 Running Pair Trading Strategy for {pair1} & {pair2}\")\n",
|
||||
"\n",
|
||||
" latest_signal = signals.iloc[-1]\n",
|
||||
"\n",
|
||||
" if latest_signal[\"long_signal\"]:\n",
|
||||
" log_and_print(f\"📉 Enter SHORT {pair2} & LONG {pair1}\")\n",
|
||||
" place_order(pair1, lot_size, is_buy=True, magic=MAGIC_NUMBER)\n",
|
||||
" place_order(pair2, lot_size, is_buy=False, magic=MAGIC_NUMBER)\n",
|
||||
"\n",
|
||||
" elif latest_signal[\"short_signal\"]:\n",
|
||||
" log_and_print(f\"📈 Enter LONG {pair2} & SHORT {pair1}\")\n",
|
||||
" place_order(pair1, lot_size, is_buy=False, magic=MAGIC_NUMBER)\n",
|
||||
" place_order(pair2, lot_size, is_buy=True, magic=MAGIC_NUMBER)\n",
|
||||
"\n",
|
||||
" elif latest_signal[\"exit_signal\"]:\n",
|
||||
" log_and_print(f\"❌ Exiting positions for {pair1} & {pair2}\")\n",
|
||||
" close_positions(pair1, MAGIC_NUMBER)\n",
|
||||
" close_positions(pair2, MAGIC_NUMBER)\n",
|
||||
"\n",
|
||||
"def close_positions(symbol, magic):\n",
|
||||
" \"\"\"Closes all open positions for a given symbol and magic number.\"\"\"\n",
|
||||
" positions = mt5.positions_get(symbol=symbol)\n",
|
||||
" if positions:\n",
|
||||
" for pos in positions:\n",
|
||||
" if pos.magic == magic:\n",
|
||||
" close_request = {\n",
|
||||
" \"action\": mt5.TRADE_ACTION_DEAL,\n",
|
||||
" \"symbol\": symbol,\n",
|
||||
" \"volume\": pos.volume,\n",
|
||||
" \"type\": mt5.ORDER_TYPE_BUY if pos.type == mt5.ORDER_TYPE_SELL else mt5.ORDER_TYPE_SELL,\n",
|
||||
" \"position\": pos.ticket,\n",
|
||||
" \"magic\": magic,\n",
|
||||
" \"comment\": \"Closing Pair Trade\",\n",
|
||||
" }\n",
|
||||
" result = mt5.order_send(close_request)\n",
|
||||
" if result.retcode == mt5.TRADE_RETCODE_DONE:\n",
|
||||
" log_and_print(f\"✅ Closed position for {symbol}\")\n",
|
||||
" else:\n",
|
||||
" log_and_print(f\"❌ Failed to close position for {symbol}\", is_error=True)\n",
|
||||
"\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"# 4) MAIN TRADING LOOP\n",
|
||||
"# ---------------------------------------------------------------------------\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" try:\n",
|
||||
" if not mt5.initialize(login=name, server=serv, password=key):\n",
|
||||
" log_and_print(\"Failed to initialize MetaTrader 5\", is_error=True)\n",
|
||||
" exit()\n",
|
||||
"\n",
|
||||
" # Validate Cointegration\n",
|
||||
" if not check_cointegration(PAIR1, PAIR2, N_BARS):\n",
|
||||
" log_and_print(f\"⚠️ {PAIR1} & {PAIR2} are NOT cointegrated. Exiting.\", is_error=True)\n",
|
||||
" mt5.shutdown()\n",
|
||||
" exit()\n",
|
||||
"\n",
|
||||
" while True:\n",
|
||||
" log_and_print(\"🔄 Fetching New Data & Running Strategy...\")\n",
|
||||
"\n",
|
||||
" # Generate signals\n",
|
||||
" signals = generate_pair_signals(PAIR1, PAIR2)\n",
|
||||
"\n",
|
||||
" # Execute trades\n",
|
||||
" manage_trades(PAIR1, PAIR2, LOT_SIZE, signals)\n",
|
||||
"\n",
|
||||
" # Sleep before next check\n",
|
||||
" time.sleep(SLEEP_TIME)\n",
|
||||
"\n",
|
||||
" except KeyboardInterrupt:\n",
|
||||
" log_and_print(\"Shutdown signal received.\")\n",
|
||||
" except Exception as e:\n",
|
||||
" log_and_print(f\"An error occurred: {e}\", is_error=True)\n",
|
||||
" finally:\n",
|
||||
" mt5.shutdown()\n",
|
||||
" log_and_print(\"MetaTrader 5 shutdown completed.\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "ml",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.11.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+2
-11
@@ -44,6 +44,7 @@
|
||||
"import time\n",
|
||||
"import logging\n",
|
||||
"import joblib\n",
|
||||
"from data.data_loader import get_data_mt5\n",
|
||||
"\n",
|
||||
"# Setup logging\n",
|
||||
"logging.basicConfig(\n",
|
||||
@@ -95,16 +96,6 @@
|
||||
" self.pipeline = None # We'll store the loaded pipeline here\n",
|
||||
" self.last_retrain_time = None\n",
|
||||
"\n",
|
||||
" def get_data(self, symbol, n, timeframe):\n",
|
||||
" \"\"\"\n",
|
||||
" Fetch 'n' bars of historical data for the given symbol and timeframe.\n",
|
||||
" \"\"\"\n",
|
||||
" rates = mt5.copy_rates_from_pos(symbol, timeframe, 0, n)\n",
|
||||
" rates_frame = pd.DataFrame(rates)\n",
|
||||
" rates_frame['time'] = pd.to_datetime(rates_frame['time'], unit='s')\n",
|
||||
" rates_frame.set_index('time', inplace=True)\n",
|
||||
" return rates_frame\n",
|
||||
"\n",
|
||||
" def add_all_ta_features(self, df):\n",
|
||||
" \"\"\"\n",
|
||||
" Add technical analysis features to the DataFrame using the 'ta' library.\n",
|
||||
@@ -138,7 +129,7 @@
|
||||
" return False, False, True, True\n",
|
||||
"\n",
|
||||
" # 1) Fetch new data\n",
|
||||
" df = self.get_data(symbol, n_bars, timeframe)\n",
|
||||
" df = get_data_mt5(symbol, n_bars, timeframe)\n",
|
||||
" # 2) Add TA features (if your pipeline doesn't handle feature eng, do it here)\n",
|
||||
" df = self.add_all_ta_features(df)\n",
|
||||
" df.fillna(method='ffill', inplace=True)\n",
|
||||
+30
-28
@@ -56096,7 +56096,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"id": "b0ab0198",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@@ -91474,6 +91474,7 @@
|
||||
"lookback = 10 # Must match what was used during training\n",
|
||||
"threshold = 0.002 # e.g. 0.2% implied return threshold\n",
|
||||
"fees = 0.0002 # e.g. 0.02% transaction cost per trade\n",
|
||||
"execution_lag = 1 # trade on the NEXT bar to avoid look-ahead (set 0 for same-bar)\n",
|
||||
"\n",
|
||||
"###############################################################################\n",
|
||||
"# 5) HELPER: CREATE SEQUENCES FOR INFERENCE\n",
|
||||
@@ -91495,40 +91496,47 @@
|
||||
"X_full_seq = create_sequences(df_scaled, lookback)\n",
|
||||
"\n",
|
||||
"# Predict using the entire dataset\n",
|
||||
"preds_scaled = best_model.predict(X_full_seq)\n",
|
||||
"preds_scaled = best_model.predict(X_full_seq, verbose=0)\n",
|
||||
"\n",
|
||||
"# Convert scaled predictions back to original price domain\n",
|
||||
"preds_inverted = scaler.inverse_transform(preds_scaled)\n",
|
||||
"\n",
|
||||
"# Align predictions with the actual index\n",
|
||||
"# Index that corresponds to the predictions\n",
|
||||
"test_index = df_selected.index[lookback:]\n",
|
||||
"if len(preds_inverted) > len(test_index):\n",
|
||||
" preds_inverted = preds_inverted[:len(test_index)]\n",
|
||||
"\n",
|
||||
"# Convert predicted price → implied return\n",
|
||||
"actual_prices = df_selected['close'].values[lookback:]\n",
|
||||
"implied_returns = (preds_inverted.flatten() - actual_prices) / actual_prices\n",
|
||||
"# Ensure alignment (trim if any slight length mismatch)\n",
|
||||
"n = min(len(test_index), len(preds_inverted))\n",
|
||||
"test_index = test_index[:n]\n",
|
||||
"preds_inverted = preds_inverted[:n].reshape(-1)\n",
|
||||
"\n",
|
||||
"# Convert returns to signals: Buy (1), Sell (-1), Hold (0)\n",
|
||||
"signals = np.where(implied_returns > threshold, 1,\n",
|
||||
" np.where(implied_returns < -threshold, -1, 0))\n",
|
||||
"# Actual prices aligned to prediction rows\n",
|
||||
"actual_prices = df_selected['close'].reindex(test_index).values\n",
|
||||
"\n",
|
||||
"# Compute implied returns from predicted price vs actual price\n",
|
||||
"implied_returns = (preds_inverted - actual_prices) / actual_prices\n",
|
||||
"\n",
|
||||
"# Compute RMSE of price forecasts\n",
|
||||
"rmse = np.sqrt(mean_squared_error(actual_prices, preds_inverted))\n",
|
||||
"\n",
|
||||
"###############################################################################\n",
|
||||
"# 7) BACKTEST WITH VECTORBT\n",
|
||||
"# 7) BACKTEST WITH VECTORBT (TARGET EXPOSURE: -1, 0, +1)\n",
|
||||
"###############################################################################\n",
|
||||
"print(\"\\n=== Running Full Backtest on Entire Dataset ===\")\n",
|
||||
"\n",
|
||||
"# Ensure signals align with price data\n",
|
||||
"close_prices = df_selected['close'].iloc[lookback:]\n",
|
||||
"if len(signals) < len(close_prices):\n",
|
||||
" signals = np.append(signals, [0] * (len(close_prices) - len(signals)))\n",
|
||||
"# Convert returns to exposure: Long (+1), Short (-1), Flat (0)\n",
|
||||
"exposure = np.where(implied_returns > threshold, 1.0,\n",
|
||||
" np.where(implied_returns < -threshold, -1.0, 0.0)).astype(float)\n",
|
||||
"\n",
|
||||
"signals_s = pd.Series(signals, index=close_prices.index)\n",
|
||||
"# Align exposure with price series; optional next-bar execution to avoid look-ahead\n",
|
||||
"close = df_selected['close'].reindex(test_index)\n",
|
||||
"exposure_s = pd.Series(exposure, index=test_index)\n",
|
||||
"if execution_lag > 0:\n",
|
||||
" exposure_s = exposure_s.shift(execution_lag).fillna(0.0)\n",
|
||||
"\n",
|
||||
"pf = vbt.Portfolio.from_signals(\n",
|
||||
" close_prices,\n",
|
||||
" entries=signals_s > 0,\n",
|
||||
" exits=signals_s < 0,\n",
|
||||
"pf = vbt.Portfolio.from_orders(\n",
|
||||
" close=close,\n",
|
||||
" size=exposure_s, # -1 short, 0 flat, +1 long\n",
|
||||
" size_type='targetpercent',\n",
|
||||
" init_cash=10000,\n",
|
||||
" freq='4H',\n",
|
||||
" fees=fees\n",
|
||||
@@ -91540,13 +91548,7 @@
|
||||
"total_return = pf.total_return()\n",
|
||||
"sharpe_ratio = pf.sharpe_ratio()\n",
|
||||
"\n",
|
||||
"# Compute RMSE (optional)\n",
|
||||
"mse = mean_squared_error(actual_prices[:len(preds_inverted)], preds_inverted)\n",
|
||||
"rmse = np.sqrt(mse)\n",
|
||||
"\n",
|
||||
"print(f\"\\nFull Backtest RMSE={rmse:.2f}, Return={total_return:.2%}, Sharpe={sharpe_ratio:.2f}\")\n",
|
||||
"\n",
|
||||
"# Print portfolio statistics\n",
|
||||
"print(f\"\\nFull Backtest RMSE={rmse:.4f}, Return={total_return:.2%}, Sharpe={sharpe_ratio:.2f}\")\n",
|
||||
"print(\"\\nVectorbt Full Dataset Stats:\")\n",
|
||||
"print(pf.stats())\n",
|
||||
"\n",
|
||||
Reference in New Issue
Block a user