feat: Smart AI Trading Bot for XAUUSD with ML and SMC
- XGBoost ML model with 37 features for market direction prediction - Smart Money Concepts (SMC): Order Blocks, FVG, BOS, CHoCH - HMM market regime detection (trending/ranging/volatile) - ATR-based stop loss with 1.5 ATR minimum distance - Broker-level SL protection with fallback - Time-based exit (max 6 hours per trade) - Session-aware trading optimized for London/NY overlap - Auto-retraining based on market conditions - Telegram notifications and web dashboard - Backtest results: 63.9% win rate, 2.64 profit factor, 4.83 Sharpe Backtest period: Jan 2025 - Feb 2026, 654 trades, $4,189 net P/L Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
buckybonez
@@ -0,0 +1,692 @@
|
||||
"""
|
||||
Auto Training Module
|
||||
====================
|
||||
Automatically retrain ML models during market close.
|
||||
|
||||
Features:
|
||||
- Daily retraining at market close (05:00 WIB)
|
||||
- Weekend deep training (more data, more epochs)
|
||||
- Incremental learning from recent trades
|
||||
- Model performance tracking in PostgreSQL
|
||||
- Auto-rollback if new model performs worse
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Tuple
|
||||
from zoneinfo import ZoneInfo
|
||||
from loguru import logger
|
||||
import polars as pl
|
||||
|
||||
# Database imports
|
||||
try:
|
||||
from src.db import get_db, init_db, TrainingRepository
|
||||
DB_AVAILABLE = True
|
||||
except ImportError:
|
||||
DB_AVAILABLE = False
|
||||
logger.warning("Database module not available, using file-based history")
|
||||
|
||||
# Timezone
|
||||
WIB = ZoneInfo("Asia/Jakarta")
|
||||
|
||||
|
||||
class AutoTrainer:
|
||||
"""
|
||||
Automatic model retraining system.
|
||||
|
||||
Retrains models during market close to keep AI up-to-date
|
||||
with latest market conditions.
|
||||
|
||||
Features:
|
||||
- Stores training history in PostgreSQL
|
||||
- Fallback to file-based history if DB unavailable
|
||||
- Auto-rollback on poor performance
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
models_dir: str = "models",
|
||||
data_dir: str = "data",
|
||||
daily_retrain_hour_wib: int = 5, # 05:00 WIB (market close)
|
||||
weekend_retrain: bool = True, # Deep training on weekends
|
||||
min_hours_between_retrain: float = 20, # Don't retrain too often
|
||||
backup_models: bool = True, # Keep backup of old models
|
||||
use_db: bool = True, # Use database for history
|
||||
min_auc_threshold: float = 0.65, # Alert if AUC drops below this
|
||||
auto_retrain_on_low_auc: bool = True, # Auto retrain when AUC low
|
||||
):
|
||||
self.models_dir = Path(models_dir)
|
||||
self.data_dir = Path(data_dir)
|
||||
self.daily_retrain_hour = daily_retrain_hour_wib
|
||||
self.weekend_retrain = weekend_retrain
|
||||
self.min_hours_between_retrain = min_hours_between_retrain
|
||||
self.backup_models = backup_models
|
||||
self.min_auc_threshold = min_auc_threshold
|
||||
self.auto_retrain_on_low_auc = auto_retrain_on_low_auc
|
||||
|
||||
# Database setup
|
||||
self._use_db = use_db and DB_AVAILABLE
|
||||
self._db_connected = False
|
||||
self._training_repo = None
|
||||
|
||||
if self._use_db:
|
||||
self._init_database()
|
||||
|
||||
# Tracking
|
||||
self._last_retrain_time: Optional[datetime] = None
|
||||
self._current_run_id: Optional[int] = None
|
||||
self._current_auc: Optional[float] = None
|
||||
self._auc_check_count: int = 0
|
||||
self._low_auc_alert_sent: bool = False
|
||||
self._load_retrain_history()
|
||||
|
||||
# Create directories
|
||||
self.models_dir.mkdir(exist_ok=True)
|
||||
self.data_dir.mkdir(exist_ok=True)
|
||||
(self.models_dir / "backups").mkdir(exist_ok=True)
|
||||
|
||||
logger.info(f"AutoTrainer initialized: DB={self._db_connected}, Min AUC={min_auc_threshold}")
|
||||
|
||||
def _init_database(self):
|
||||
"""Initialize database connection."""
|
||||
try:
|
||||
if init_db():
|
||||
self._db = get_db()
|
||||
self._training_repo = TrainingRepository(self._db)
|
||||
self._db_connected = True
|
||||
logger.info("AutoTrainer: Database connected")
|
||||
else:
|
||||
logger.warning("AutoTrainer: Database connection failed, using file")
|
||||
self._db_connected = False
|
||||
except Exception as e:
|
||||
logger.error(f"AutoTrainer: Database init error: {e}")
|
||||
self._db_connected = False
|
||||
|
||||
def _load_retrain_history(self):
|
||||
"""Load last retrain time from database or file."""
|
||||
# Try database first
|
||||
if self._db_connected and self._training_repo:
|
||||
try:
|
||||
latest = self._training_repo.get_latest_successful()
|
||||
if latest and latest.get("completed_at"):
|
||||
self._last_retrain_time = latest["completed_at"]
|
||||
if self._last_retrain_time.tzinfo is None:
|
||||
self._last_retrain_time = self._last_retrain_time.replace(tzinfo=WIB)
|
||||
logger.debug(f"Last retrain (DB): {self._last_retrain_time}")
|
||||
return
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load from DB: {e}")
|
||||
|
||||
# Fallback to file
|
||||
history_file = self.data_dir / "retrain_history.txt"
|
||||
if history_file.exists():
|
||||
try:
|
||||
with open(history_file, "r") as f:
|
||||
lines = f.readlines()
|
||||
if lines:
|
||||
last_line = lines[-1].strip()
|
||||
self._last_retrain_time = datetime.fromisoformat(last_line)
|
||||
logger.debug(f"Last retrain (file): {self._last_retrain_time}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load retrain history: {e}")
|
||||
|
||||
def _save_retrain_start(self, training_type: str, bars: int, num_boost_rounds: int) -> Optional[int]:
|
||||
"""Record training start in database."""
|
||||
if self._db_connected and self._training_repo:
|
||||
try:
|
||||
result = self._training_repo.insert_training_run({
|
||||
"training_type": training_type,
|
||||
"bars_used": bars,
|
||||
"num_boost_rounds": num_boost_rounds,
|
||||
"started_at": datetime.now(WIB),
|
||||
})
|
||||
if result:
|
||||
self._current_run_id = result.get("id")
|
||||
return self._current_run_id
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save training start: {e}")
|
||||
return None
|
||||
|
||||
def _save_retrain_complete(
|
||||
self,
|
||||
success: bool,
|
||||
hmm_trained: bool = False,
|
||||
xgb_trained: bool = False,
|
||||
train_auc: float = 0,
|
||||
test_auc: float = 0,
|
||||
train_accuracy: float = 0,
|
||||
test_accuracy: float = 0,
|
||||
model_path: str = "",
|
||||
backup_path: str = "",
|
||||
error_message: str = None,
|
||||
started_at: datetime = None,
|
||||
):
|
||||
"""Record training completion in database."""
|
||||
now = datetime.now(WIB)
|
||||
self._last_retrain_time = now
|
||||
|
||||
# Calculate duration
|
||||
duration = None
|
||||
if started_at:
|
||||
duration = int((now - started_at).total_seconds())
|
||||
|
||||
# Update database
|
||||
if self._db_connected and self._training_repo and self._current_run_id:
|
||||
try:
|
||||
self._training_repo.update_training_complete(
|
||||
self._current_run_id,
|
||||
{
|
||||
"completed_at": now,
|
||||
"duration_seconds": duration,
|
||||
"hmm_trained": hmm_trained,
|
||||
"xgb_trained": xgb_trained,
|
||||
"train_auc": train_auc,
|
||||
"test_auc": test_auc,
|
||||
"train_accuracy": train_accuracy,
|
||||
"test_accuracy": test_accuracy,
|
||||
"model_path": model_path,
|
||||
"backup_path": backup_path,
|
||||
"success": success,
|
||||
"error_message": error_message,
|
||||
}
|
||||
)
|
||||
logger.debug(f"Training run #{self._current_run_id} completed in DB")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update training completion: {e}")
|
||||
|
||||
# Also save to file (backup)
|
||||
history_file = self.data_dir / "retrain_history.txt"
|
||||
with open(history_file, "a") as f:
|
||||
f.write(f"{now.isoformat()}\n")
|
||||
|
||||
def should_retrain(self) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if retraining should happen now.
|
||||
|
||||
Returns:
|
||||
(should_retrain, reason)
|
||||
"""
|
||||
now = datetime.now(WIB)
|
||||
|
||||
# Check if enough time has passed since last retrain
|
||||
if self._last_retrain_time:
|
||||
# Ensure timezone-aware comparison
|
||||
last_time = self._last_retrain_time
|
||||
if last_time.tzinfo is None:
|
||||
last_time = last_time.replace(tzinfo=WIB)
|
||||
|
||||
hours_since_retrain = (now - last_time).total_seconds() / 3600
|
||||
if hours_since_retrain < self.min_hours_between_retrain:
|
||||
return False, f"Too soon since last retrain ({hours_since_retrain:.1f}h ago)"
|
||||
|
||||
# Check if it's daily retrain time (within 30 min window)
|
||||
is_retrain_hour = (
|
||||
now.hour == self.daily_retrain_hour and
|
||||
now.minute < 30
|
||||
)
|
||||
|
||||
# Check if it's weekend (Saturday or Sunday)
|
||||
is_weekend = now.weekday() >= 5 # 5=Saturday, 6=Sunday
|
||||
|
||||
if is_retrain_hour:
|
||||
if is_weekend and self.weekend_retrain:
|
||||
return True, "Weekend deep training time"
|
||||
elif not is_weekend:
|
||||
return True, "Daily market close training"
|
||||
|
||||
# Manual trigger check - if no retrain in 24+ hours
|
||||
if self._last_retrain_time:
|
||||
last_time = self._last_retrain_time
|
||||
if last_time.tzinfo is None:
|
||||
last_time = last_time.replace(tzinfo=WIB)
|
||||
hours_since = (now - last_time).total_seconds() / 3600
|
||||
if hours_since > 24:
|
||||
return True, f"Over 24h since last training ({hours_since:.1f}h)"
|
||||
elif self._last_retrain_time is None:
|
||||
# Never trained before
|
||||
return True, "Initial training required"
|
||||
|
||||
return False, "Not retrain time"
|
||||
|
||||
def check_model_auc(self, model=None) -> Tuple[float, bool, str]:
|
||||
"""
|
||||
Check current model AUC and determine if it's acceptable.
|
||||
|
||||
Args:
|
||||
model: ML model instance (optional, will load from file if not provided)
|
||||
|
||||
Returns:
|
||||
(current_auc, is_acceptable, message)
|
||||
"""
|
||||
try:
|
||||
# Try to get AUC from loaded model
|
||||
if model is not None and hasattr(model, '_test_auc'):
|
||||
current_auc = model._test_auc
|
||||
elif model is not None and hasattr(model, 'auc'):
|
||||
current_auc = model.auc
|
||||
else:
|
||||
# Try to load from model file
|
||||
import pickle
|
||||
model_path = self.models_dir / "xgboost_model.pkl"
|
||||
if model_path.exists():
|
||||
with open(model_path, "rb") as f:
|
||||
saved_data = pickle.load(f)
|
||||
if isinstance(saved_data, dict) and "test_auc" in saved_data:
|
||||
current_auc = saved_data["test_auc"]
|
||||
elif hasattr(saved_data, '_test_auc'):
|
||||
current_auc = saved_data._test_auc
|
||||
else:
|
||||
return 0.0, False, "Could not determine AUC from model"
|
||||
else:
|
||||
return 0.0, False, "Model file not found"
|
||||
|
||||
self._current_auc = current_auc
|
||||
self._auc_check_count += 1
|
||||
|
||||
# Check if AUC is acceptable
|
||||
is_acceptable = current_auc >= self.min_auc_threshold
|
||||
|
||||
if is_acceptable:
|
||||
message = f"Model AUC OK: {current_auc:.4f} (threshold: {self.min_auc_threshold})"
|
||||
self._low_auc_alert_sent = False # Reset alert flag
|
||||
else:
|
||||
message = f"⚠️ LOW AUC ALERT: {current_auc:.4f} < {self.min_auc_threshold} threshold!"
|
||||
if not self._low_auc_alert_sent:
|
||||
logger.warning(message)
|
||||
self._low_auc_alert_sent = True
|
||||
|
||||
return current_auc, is_acceptable, message
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking model AUC: {e}")
|
||||
return 0.0, False, f"Error: {e}"
|
||||
|
||||
def should_retrain_due_to_low_auc(self) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if model should be retrained due to low AUC.
|
||||
|
||||
Returns:
|
||||
(should_retrain, reason)
|
||||
"""
|
||||
if not self.auto_retrain_on_low_auc:
|
||||
return False, "Auto-retrain on low AUC disabled"
|
||||
|
||||
current_auc, is_acceptable, message = self.check_model_auc()
|
||||
|
||||
if not is_acceptable and current_auc > 0:
|
||||
# Check if enough time has passed since last retrain
|
||||
now = datetime.now(WIB)
|
||||
if self._last_retrain_time:
|
||||
last_time = self._last_retrain_time
|
||||
if last_time.tzinfo is None:
|
||||
last_time = last_time.replace(tzinfo=WIB)
|
||||
hours_since = (now - last_time).total_seconds() / 3600
|
||||
|
||||
# Only retrain if at least 4 hours since last retrain (to prevent loops)
|
||||
if hours_since < 4:
|
||||
return False, f"Low AUC but retrained recently ({hours_since:.1f}h ago)"
|
||||
|
||||
return True, f"Low AUC detected: {current_auc:.4f} < {self.min_auc_threshold}"
|
||||
|
||||
return False, "AUC acceptable" if is_acceptable else "Could not check AUC"
|
||||
|
||||
def get_auc_status(self) -> Dict:
|
||||
"""Get current AUC status for monitoring."""
|
||||
current_auc, is_acceptable, message = self.check_model_auc()
|
||||
return {
|
||||
"current_auc": current_auc,
|
||||
"min_threshold": self.min_auc_threshold,
|
||||
"is_acceptable": is_acceptable,
|
||||
"message": message,
|
||||
"check_count": self._auc_check_count,
|
||||
"alert_sent": self._low_auc_alert_sent,
|
||||
}
|
||||
|
||||
def backup_current_models(self) -> Tuple[bool, str]:
|
||||
"""
|
||||
Backup current models before retraining.
|
||||
|
||||
Returns:
|
||||
(success, backup_path)
|
||||
"""
|
||||
if not self.backup_models:
|
||||
return True, ""
|
||||
|
||||
try:
|
||||
now = datetime.now(WIB)
|
||||
backup_suffix = now.strftime("%Y%m%d_%H%M%S")
|
||||
backup_dir = self.models_dir / "backups" / backup_suffix
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Backup XGBoost model
|
||||
xgb_path = self.models_dir / "xgboost_model.pkl"
|
||||
if xgb_path.exists():
|
||||
shutil.copy(xgb_path, backup_dir / "xgboost_model.pkl")
|
||||
|
||||
# Backup HMM model
|
||||
hmm_path = self.models_dir / "hmm_regime.pkl"
|
||||
if hmm_path.exists():
|
||||
shutil.copy(hmm_path, backup_dir / "hmm_regime.pkl")
|
||||
|
||||
logger.info(f"Models backed up to {backup_dir}")
|
||||
|
||||
# Keep only last 5 backups
|
||||
self._cleanup_old_backups(keep=5)
|
||||
|
||||
return True, str(backup_dir)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to backup models: {e}")
|
||||
return False, ""
|
||||
|
||||
def _cleanup_old_backups(self, keep: int = 5):
|
||||
"""Remove old backups, keeping only the most recent ones."""
|
||||
backup_base = self.models_dir / "backups"
|
||||
if not backup_base.exists():
|
||||
return
|
||||
|
||||
backups = sorted(backup_base.iterdir(), reverse=True)
|
||||
for old_backup in backups[keep:]:
|
||||
if old_backup.is_dir():
|
||||
shutil.rmtree(old_backup)
|
||||
logger.debug(f"Removed old backup: {old_backup}")
|
||||
|
||||
def retrain(
|
||||
self,
|
||||
connector, # MT5Connector
|
||||
symbol: str = "XAUUSD",
|
||||
timeframe: str = "M15",
|
||||
is_weekend: bool = False,
|
||||
) -> Dict:
|
||||
"""
|
||||
Retrain all models with latest data.
|
||||
|
||||
Args:
|
||||
connector: MT5 connector for fetching data
|
||||
symbol: Trading symbol
|
||||
timeframe: Timeframe for training data
|
||||
is_weekend: If True, use more data for deep training
|
||||
|
||||
Returns:
|
||||
Dict with training results
|
||||
"""
|
||||
from src.feature_eng import FeatureEngineer
|
||||
from src.smc_polars import SMCAnalyzer
|
||||
from src.regime_detector import MarketRegimeDetector
|
||||
from src.ml_model import TradingModel, get_default_feature_columns
|
||||
|
||||
started_at = datetime.now(WIB)
|
||||
|
||||
results = {
|
||||
"success": False,
|
||||
"hmm_trained": False,
|
||||
"xgb_trained": False,
|
||||
"xgb_train_auc": 0,
|
||||
"xgb_test_auc": 0,
|
||||
"train_accuracy": 0,
|
||||
"test_accuracy": 0,
|
||||
"samples": 0,
|
||||
"error": None,
|
||||
"backup_path": "",
|
||||
"duration_seconds": 0,
|
||||
}
|
||||
|
||||
# Determine training parameters
|
||||
training_type = "weekend" if is_weekend else "daily"
|
||||
if is_weekend:
|
||||
bars = 15000 # More data for weekend deep training
|
||||
num_boost_round = 80
|
||||
else:
|
||||
bars = 8000 # Daily training
|
||||
num_boost_round = 50
|
||||
|
||||
# Record training start in database
|
||||
self._save_retrain_start(training_type, bars, num_boost_round)
|
||||
|
||||
try:
|
||||
logger.info("=" * 50)
|
||||
logger.info("AUTO-RETRAINING STARTED")
|
||||
logger.info(f"Type: {training_type}, Bars: {bars}, Boost Rounds: {num_boost_round}")
|
||||
logger.info("=" * 50)
|
||||
|
||||
# Backup current models
|
||||
backup_success, backup_path = self.backup_current_models()
|
||||
results["backup_path"] = backup_path
|
||||
|
||||
# Fetch latest data
|
||||
logger.info(f"Fetching {bars} bars of {symbol} {timeframe} data...")
|
||||
df = connector.get_market_data(symbol, timeframe, bars)
|
||||
|
||||
if len(df) < 1000:
|
||||
results["error"] = f"Insufficient data: {len(df)} bars"
|
||||
logger.error(results["error"])
|
||||
self._save_retrain_complete(
|
||||
success=False,
|
||||
error_message=results["error"],
|
||||
started_at=started_at,
|
||||
)
|
||||
return results
|
||||
|
||||
results["samples"] = len(df)
|
||||
logger.info(f"Received {len(df)} bars")
|
||||
logger.info(f"Date range: {df['time'].min()} to {df['time'].max()}")
|
||||
|
||||
# Feature engineering
|
||||
logger.info("Applying feature engineering...")
|
||||
fe = FeatureEngineer()
|
||||
df = fe.calculate_all(df, include_ml_features=True)
|
||||
|
||||
# SMC indicators
|
||||
smc = SMCAnalyzer(swing_length=5)
|
||||
df = smc.calculate_all(df)
|
||||
|
||||
# Create target
|
||||
df = fe.create_target(df, lookahead=1)
|
||||
|
||||
# Train HMM
|
||||
logger.info("Training HMM Regime Model...")
|
||||
hmm = MarketRegimeDetector(
|
||||
n_regimes=3,
|
||||
lookback_periods=500,
|
||||
model_path=str(self.models_dir / "hmm_regime.pkl"),
|
||||
)
|
||||
hmm.fit(df)
|
||||
|
||||
if hmm.fitted:
|
||||
df = hmm.predict(df)
|
||||
results["hmm_trained"] = True
|
||||
logger.info("HMM model trained and saved")
|
||||
|
||||
# Train XGBoost
|
||||
logger.info("Training XGBoost Model...")
|
||||
xgb = TradingModel(
|
||||
confidence_threshold=0.60,
|
||||
model_path=str(self.models_dir / "xgboost_model.pkl"),
|
||||
)
|
||||
|
||||
feature_cols = get_default_feature_columns()
|
||||
available_features = [f for f in feature_cols if f in df.columns]
|
||||
|
||||
xgb.fit(
|
||||
df,
|
||||
available_features,
|
||||
target_col="target",
|
||||
train_ratio=0.7,
|
||||
num_boost_round=num_boost_round,
|
||||
early_stopping_rounds=5,
|
||||
)
|
||||
|
||||
if xgb.fitted:
|
||||
results["xgb_trained"] = True
|
||||
results["xgb_train_auc"] = xgb._train_metrics.get("train_auc", 0)
|
||||
results["xgb_test_auc"] = xgb._train_metrics.get("test_auc", 0)
|
||||
results["train_accuracy"] = xgb._train_metrics.get("train_accuracy", 0)
|
||||
results["test_accuracy"] = xgb._train_metrics.get("test_accuracy", 0)
|
||||
logger.info(f"XGBoost trained: Train AUC={results['xgb_train_auc']:.4f}, Test AUC={results['xgb_test_auc']:.4f}")
|
||||
|
||||
# Save training data
|
||||
training_data_path = self.data_dir / "training_data.parquet"
|
||||
df.write_parquet(training_data_path)
|
||||
logger.info(f"Training data saved to {training_data_path}")
|
||||
|
||||
# Mark success
|
||||
results["success"] = results["hmm_trained"] and results["xgb_trained"]
|
||||
results["duration_seconds"] = int((datetime.now(WIB) - started_at).total_seconds())
|
||||
|
||||
# Save completion to database
|
||||
self._save_retrain_complete(
|
||||
success=results["success"],
|
||||
hmm_trained=results["hmm_trained"],
|
||||
xgb_trained=results["xgb_trained"],
|
||||
train_auc=results["xgb_train_auc"],
|
||||
test_auc=results["xgb_test_auc"],
|
||||
train_accuracy=results["train_accuracy"],
|
||||
test_accuracy=results["test_accuracy"],
|
||||
model_path=str(self.models_dir / "xgboost_model.pkl"),
|
||||
backup_path=backup_path,
|
||||
started_at=started_at,
|
||||
)
|
||||
|
||||
if results["success"]:
|
||||
logger.info("=" * 50)
|
||||
logger.info("AUTO-RETRAINING COMPLETED SUCCESSFULLY")
|
||||
logger.info(f"Duration: {results['duration_seconds']}s")
|
||||
logger.info("=" * 50)
|
||||
else:
|
||||
logger.warning("Retraining completed with issues")
|
||||
|
||||
except Exception as e:
|
||||
results["error"] = str(e)
|
||||
logger.error(f"Retraining failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Save failure to database
|
||||
self._save_retrain_complete(
|
||||
success=False,
|
||||
error_message=str(e),
|
||||
started_at=started_at,
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def rollback_models(self, reason: str = "Manual rollback") -> bool:
|
||||
"""Rollback to previous model version if new one performs worse."""
|
||||
backup_base = self.models_dir / "backups"
|
||||
if not backup_base.exists():
|
||||
logger.warning("No backups available for rollback")
|
||||
return False
|
||||
|
||||
# Get most recent backup
|
||||
backups = sorted(backup_base.iterdir(), reverse=True)
|
||||
if not backups:
|
||||
logger.warning("No backups found")
|
||||
return False
|
||||
|
||||
latest_backup = backups[0]
|
||||
|
||||
try:
|
||||
# Restore XGBoost
|
||||
xgb_backup = latest_backup / "xgboost_model.pkl"
|
||||
if xgb_backup.exists():
|
||||
shutil.copy(xgb_backup, self.models_dir / "xgboost_model.pkl")
|
||||
|
||||
# Restore HMM
|
||||
hmm_backup = latest_backup / "hmm_regime.pkl"
|
||||
if hmm_backup.exists():
|
||||
shutil.copy(hmm_backup, self.models_dir / "hmm_regime.pkl")
|
||||
|
||||
logger.info(f"Models rolled back from {latest_backup}")
|
||||
|
||||
# Record rollback in database
|
||||
if self._db_connected and self._training_repo and self._current_run_id:
|
||||
try:
|
||||
self._training_repo.mark_rollback(self._current_run_id, reason)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to record rollback: {e}")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Rollback failed: {e}")
|
||||
return False
|
||||
|
||||
def get_training_history(self, limit: int = 10) -> list:
|
||||
"""Get recent training history from database."""
|
||||
if self._db_connected and self._training_repo:
|
||||
try:
|
||||
return self._training_repo.get_training_history(limit)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get training history: {e}")
|
||||
return []
|
||||
|
||||
def get_status(self) -> Dict:
|
||||
"""Get current auto-trainer status."""
|
||||
now = datetime.now(WIB)
|
||||
|
||||
hours_since_retrain = None
|
||||
if self._last_retrain_time:
|
||||
last_time = self._last_retrain_time
|
||||
if last_time.tzinfo is None:
|
||||
last_time = last_time.replace(tzinfo=WIB)
|
||||
hours_since_retrain = (now - last_time).total_seconds() / 3600
|
||||
|
||||
should_train, reason = self.should_retrain()
|
||||
|
||||
# Get latest training from DB
|
||||
latest_training = None
|
||||
if self._db_connected and self._training_repo:
|
||||
try:
|
||||
latest_training = self._training_repo.get_latest_successful()
|
||||
except:
|
||||
pass
|
||||
|
||||
return {
|
||||
"last_retrain": self._last_retrain_time.isoformat() if self._last_retrain_time else "Never",
|
||||
"hours_since_retrain": round(hours_since_retrain, 1) if hours_since_retrain else None,
|
||||
"should_retrain": should_train,
|
||||
"reason": reason,
|
||||
"next_retrain_hour": f"{self.daily_retrain_hour:02d}:00 WIB",
|
||||
"weekend_training": self.weekend_retrain,
|
||||
"db_connected": self._db_connected,
|
||||
"latest_training": {
|
||||
"train_auc": latest_training.get("train_auc") if latest_training else None,
|
||||
"test_auc": latest_training.get("test_auc") if latest_training else None,
|
||||
"bars_used": latest_training.get("bars_used") if latest_training else None,
|
||||
} if latest_training else None,
|
||||
}
|
||||
|
||||
|
||||
def create_auto_trainer() -> AutoTrainer:
|
||||
"""Create default auto trainer instance."""
|
||||
return AutoTrainer(
|
||||
models_dir="models",
|
||||
data_dir="data",
|
||||
daily_retrain_hour_wib=5, # 05:00 WIB (market close)
|
||||
weekend_retrain=True,
|
||||
min_hours_between_retrain=20,
|
||||
backup_models=True,
|
||||
use_db=True,
|
||||
min_auc_threshold=0.65, # Alert if AUC drops below 0.65
|
||||
auto_retrain_on_low_auc=True, # Auto retrain when AUC is low
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test auto trainer
|
||||
trainer = create_auto_trainer()
|
||||
|
||||
print("=== Auto Trainer Status ===")
|
||||
status = trainer.get_status()
|
||||
for key, value in status.items():
|
||||
print(f" {key}: {value}")
|
||||
|
||||
print("\n=== Should Retrain Check ===")
|
||||
should, reason = trainer.should_retrain()
|
||||
print(f" Should retrain: {should}")
|
||||
print(f" Reason: {reason}")
|
||||
|
||||
print("\n=== Training History ===")
|
||||
history = trainer.get_training_history(5)
|
||||
for h in history:
|
||||
print(f" - {h.get('training_type')}: AUC={h.get('test_auc')}, Success={h.get('success')}")
|
||||
Reference in New Issue
Block a user