693 lines
26 KiB
Python
693 lines
26 KiB
Python
|
"""
|
||
|
|
Auto Training Module
|
||
|
|
====================
|
||
|
|
Automatically retrain ML models during market close.
|
||
|
|
|
||
|
|
Features:
|
||
|
|
- Daily retraining at market close (05:00 WIB)
|
||
|
|
- Weekend deep training (more data, more epochs)
|
||
|
|
- Incremental learning from recent trades
|
||
|
|
- Model performance tracking in PostgreSQL
|
||
|
|
- Auto-rollback if new model performs worse
|
||
|
|
"""
|
||
|
|
|
||
|
|
import os
|
||
|
|
import shutil
|
||
|
|
from datetime import datetime, timedelta
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Optional, Dict, Tuple
|
||
|
|
from zoneinfo import ZoneInfo
|
||
|
|
from loguru import logger
|
||
|
|
import polars as pl
|
||
|
|
|
||
|
|
# Database imports
|
||
|
|
try:
|
||
|
|
from src.db import get_db, init_db, TrainingRepository
|
||
|
|
DB_AVAILABLE = True
|
||
|
|
except ImportError:
|
||
|
|
DB_AVAILABLE = False
|
||
|
|
logger.warning("Database module not available, using file-based history")
|
||
|
|
|
||
|
|
# Timezone
|
||
|
|
WIB = ZoneInfo("Asia/Jakarta")
|
||
|
|
|
||
|
|
|
||
|
|
class AutoTrainer:
|
||
|
|
"""
|
||
|
|
Automatic model retraining system.
|
||
|
|
|
||
|
|
Retrains models during market close to keep AI up-to-date
|
||
|
|
with latest market conditions.
|
||
|
|
|
||
|
|
Features:
|
||
|
|
- Stores training history in PostgreSQL
|
||
|
|
- Fallback to file-based history if DB unavailable
|
||
|
|
- Auto-rollback on poor performance
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
models_dir: str = "models",
|
||
|
|
data_dir: str = "data",
|
||
|
|
daily_retrain_hour_wib: int = 5, # 05:00 WIB (market close)
|
||
|
|
weekend_retrain: bool = True, # Deep training on weekends
|
||
|
|
min_hours_between_retrain: float = 20, # Don't retrain too often
|
||
|
|
backup_models: bool = True, # Keep backup of old models
|
||
|
|
use_db: bool = True, # Use database for history
|
||
|
|
min_auc_threshold: float = 0.65, # Alert if AUC drops below this
|
||
|
|
auto_retrain_on_low_auc: bool = True, # Auto retrain when AUC low
|
||
|
|
):
|
||
|
|
self.models_dir = Path(models_dir)
|
||
|
|
self.data_dir = Path(data_dir)
|
||
|
|
self.daily_retrain_hour = daily_retrain_hour_wib
|
||
|
|
self.weekend_retrain = weekend_retrain
|
||
|
|
self.min_hours_between_retrain = min_hours_between_retrain
|
||
|
|
self.backup_models = backup_models
|
||
|
|
self.min_auc_threshold = min_auc_threshold
|
||
|
|
self.auto_retrain_on_low_auc = auto_retrain_on_low_auc
|
||
|
|
|
||
|
|
# Database setup
|
||
|
|
self._use_db = use_db and DB_AVAILABLE
|
||
|
|
self._db_connected = False
|
||
|
|
self._training_repo = None
|
||
|
|
|
||
|
|
if self._use_db:
|
||
|
|
self._init_database()
|
||
|
|
|
||
|
|
# Tracking
|
||
|
|
self._last_retrain_time: Optional[datetime] = None
|
||
|
|
self._current_run_id: Optional[int] = None
|
||
|
|
self._current_auc: Optional[float] = None
|
||
|
|
self._auc_check_count: int = 0
|
||
|
|
self._low_auc_alert_sent: bool = False
|
||
|
|
self._load_retrain_history()
|
||
|
|
|
||
|
|
# Create directories
|
||
|
|
self.models_dir.mkdir(exist_ok=True)
|
||
|
|
self.data_dir.mkdir(exist_ok=True)
|
||
|
|
(self.models_dir / "backups").mkdir(exist_ok=True)
|
||
|
|
|
||
|
|
logger.info(f"AutoTrainer initialized: DB={self._db_connected}, Min AUC={min_auc_threshold}")
|
||
|
|
|
||
|
|
def _init_database(self):
|
||
|
|
"""Initialize database connection."""
|
||
|
|
try:
|
||
|
|
if init_db():
|
||
|
|
self._db = get_db()
|
||
|
|
self._training_repo = TrainingRepository(self._db)
|
||
|
|
self._db_connected = True
|
||
|
|
logger.info("AutoTrainer: Database connected")
|
||
|
|
else:
|
||
|
|
logger.warning("AutoTrainer: Database connection failed, using file")
|
||
|
|
self._db_connected = False
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"AutoTrainer: Database init error: {e}")
|
||
|
|
self._db_connected = False
|
||
|
|
|
||
|
|
def _load_retrain_history(self):
|
||
|
|
"""Load last retrain time from database or file."""
|
||
|
|
# Try database first
|
||
|
|
if self._db_connected and self._training_repo:
|
||
|
|
try:
|
||
|
|
latest = self._training_repo.get_latest_successful()
|
||
|
|
if latest and latest.get("completed_at"):
|
||
|
|
self._last_retrain_time = latest["completed_at"]
|
||
|
|
if self._last_retrain_time.tzinfo is None:
|
||
|
|
self._last_retrain_time = self._last_retrain_time.replace(tzinfo=WIB)
|
||
|
|
logger.debug(f"Last retrain (DB): {self._last_retrain_time}")
|
||
|
|
return
|
||
|
|
except Exception as e:
|
||
|
|
logger.warning(f"Could not load from DB: {e}")
|
||
|
|
|
||
|
|
# Fallback to file
|
||
|
|
history_file = self.data_dir / "retrain_history.txt"
|
||
|
|
if history_file.exists():
|
||
|
|
try:
|
||
|
|
with open(history_file, "r") as f:
|
||
|
|
lines = f.readlines()
|
||
|
|
if lines:
|
||
|
|
last_line = lines[-1].strip()
|
||
|
|
self._last_retrain_time = datetime.fromisoformat(last_line)
|
||
|
|
logger.debug(f"Last retrain (file): {self._last_retrain_time}")
|
||
|
|
except Exception as e:
|
||
|
|
logger.warning(f"Could not load retrain history: {e}")
|
||
|
|
|
||
|
|
def _save_retrain_start(self, training_type: str, bars: int, num_boost_rounds: int) -> Optional[int]:
|
||
|
|
"""Record training start in database."""
|
||
|
|
if self._db_connected and self._training_repo:
|
||
|
|
try:
|
||
|
|
result = self._training_repo.insert_training_run({
|
||
|
|
"training_type": training_type,
|
||
|
|
"bars_used": bars,
|
||
|
|
"num_boost_rounds": num_boost_rounds,
|
||
|
|
"started_at": datetime.now(WIB),
|
||
|
|
})
|
||
|
|
if result:
|
||
|
|
self._current_run_id = result.get("id")
|
||
|
|
return self._current_run_id
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Failed to save training start: {e}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def _save_retrain_complete(
|
||
|
|
self,
|
||
|
|
success: bool,
|
||
|
|
hmm_trained: bool = False,
|
||
|
|
xgb_trained: bool = False,
|
||
|
|
train_auc: float = 0,
|
||
|
|
test_auc: float = 0,
|
||
|
|
train_accuracy: float = 0,
|
||
|
|
test_accuracy: float = 0,
|
||
|
|
model_path: str = "",
|
||
|
|
backup_path: str = "",
|
||
|
|
error_message: str = None,
|
||
|
|
started_at: datetime = None,
|
||
|
|
):
|
||
|
|
"""Record training completion in database."""
|
||
|
|
now = datetime.now(WIB)
|
||
|
|
self._last_retrain_time = now
|
||
|
|
|
||
|
|
# Calculate duration
|
||
|
|
duration = None
|
||
|
|
if started_at:
|
||
|
|
duration = int((now - started_at).total_seconds())
|
||
|
|
|
||
|
|
# Update database
|
||
|
|
if self._db_connected and self._training_repo and self._current_run_id:
|
||
|
|
try:
|
||
|
|
self._training_repo.update_training_complete(
|
||
|
|
self._current_run_id,
|
||
|
|
{
|
||
|
|
"completed_at": now,
|
||
|
|
"duration_seconds": duration,
|
||
|
|
"hmm_trained": hmm_trained,
|
||
|
|
"xgb_trained": xgb_trained,
|
||
|
|
"train_auc": train_auc,
|
||
|
|
"test_auc": test_auc,
|
||
|
|
"train_accuracy": train_accuracy,
|
||
|
|
"test_accuracy": test_accuracy,
|
||
|
|
"model_path": model_path,
|
||
|
|
"backup_path": backup_path,
|
||
|
|
"success": success,
|
||
|
|
"error_message": error_message,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
logger.debug(f"Training run #{self._current_run_id} completed in DB")
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Failed to update training completion: {e}")
|
||
|
|
|
||
|
|
# Also save to file (backup)
|
||
|
|
history_file = self.data_dir / "retrain_history.txt"
|
||
|
|
with open(history_file, "a") as f:
|
||
|
|
f.write(f"{now.isoformat()}\n")
|
||
|
|
|
||
|
|
def should_retrain(self) -> Tuple[bool, str]:
|
||
|
|
"""
|
||
|
|
Check if retraining should happen now.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
(should_retrain, reason)
|
||
|
|
"""
|
||
|
|
now = datetime.now(WIB)
|
||
|
|
|
||
|
|
# Check if enough time has passed since last retrain
|
||
|
|
if self._last_retrain_time:
|
||
|
|
# Ensure timezone-aware comparison
|
||
|
|
last_time = self._last_retrain_time
|
||
|
|
if last_time.tzinfo is None:
|
||
|
|
last_time = last_time.replace(tzinfo=WIB)
|
||
|
|
|
||
|
|
hours_since_retrain = (now - last_time).total_seconds() / 3600
|
||
|
|
if hours_since_retrain < self.min_hours_between_retrain:
|
||
|
|
return False, f"Too soon since last retrain ({hours_since_retrain:.1f}h ago)"
|
||
|
|
|
||
|
|
# Check if it's daily retrain time (within 30 min window)
|
||
|
|
is_retrain_hour = (
|
||
|
|
now.hour == self.daily_retrain_hour and
|
||
|
|
now.minute < 30
|
||
|
|
)
|
||
|
|
|
||
|
|
# Check if it's weekend (Saturday or Sunday)
|
||
|
|
is_weekend = now.weekday() >= 5 # 5=Saturday, 6=Sunday
|
||
|
|
|
||
|
|
if is_retrain_hour:
|
||
|
|
if is_weekend and self.weekend_retrain:
|
||
|
|
return True, "Weekend deep training time"
|
||
|
|
elif not is_weekend:
|
||
|
|
return True, "Daily market close training"
|
||
|
|
|
||
|
|
# Manual trigger check - if no retrain in 24+ hours
|
||
|
|
if self._last_retrain_time:
|
||
|
|
last_time = self._last_retrain_time
|
||
|
|
if last_time.tzinfo is None:
|
||
|
|
last_time = last_time.replace(tzinfo=WIB)
|
||
|
|
hours_since = (now - last_time).total_seconds() / 3600
|
||
|
|
if hours_since > 24:
|
||
|
|
return True, f"Over 24h since last training ({hours_since:.1f}h)"
|
||
|
|
elif self._last_retrain_time is None:
|
||
|
|
# Never trained before
|
||
|
|
return True, "Initial training required"
|
||
|
|
|
||
|
|
return False, "Not retrain time"
|
||
|
|
|
||
|
|
def check_model_auc(self, model=None) -> Tuple[float, bool, str]:
|
||
|
|
"""
|
||
|
|
Check current model AUC and determine if it's acceptable.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
model: ML model instance (optional, will load from file if not provided)
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
(current_auc, is_acceptable, message)
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
# Try to get AUC from loaded model
|
||
|
|
if model is not None and hasattr(model, '_test_auc'):
|
||
|
|
current_auc = model._test_auc
|
||
|
|
elif model is not None and hasattr(model, 'auc'):
|
||
|
|
current_auc = model.auc
|
||
|
|
else:
|
||
|
|
# Try to load from model file
|
||
|
|
import pickle
|
||
|
|
model_path = self.models_dir / "xgboost_model.pkl"
|
||
|
|
if model_path.exists():
|
||
|
|
with open(model_path, "rb") as f:
|
||
|
|
saved_data = pickle.load(f)
|
||
|
|
if isinstance(saved_data, dict) and "test_auc" in saved_data:
|
||
|
|
current_auc = saved_data["test_auc"]
|
||
|
|
elif hasattr(saved_data, '_test_auc'):
|
||
|
|
current_auc = saved_data._test_auc
|
||
|
|
else:
|
||
|
|
return 0.0, False, "Could not determine AUC from model"
|
||
|
|
else:
|
||
|
|
return 0.0, False, "Model file not found"
|
||
|
|
|
||
|
|
self._current_auc = current_auc
|
||
|
|
self._auc_check_count += 1
|
||
|
|
|
||
|
|
# Check if AUC is acceptable
|
||
|
|
is_acceptable = current_auc >= self.min_auc_threshold
|
||
|
|
|
||
|
|
if is_acceptable:
|
||
|
|
message = f"Model AUC OK: {current_auc:.4f} (threshold: {self.min_auc_threshold})"
|
||
|
|
self._low_auc_alert_sent = False # Reset alert flag
|
||
|
|
else:
|
||
|
|
message = f"⚠️ LOW AUC ALERT: {current_auc:.4f} < {self.min_auc_threshold} threshold!"
|
||
|
|
if not self._low_auc_alert_sent:
|
||
|
|
logger.warning(message)
|
||
|
|
self._low_auc_alert_sent = True
|
||
|
|
|
||
|
|
return current_auc, is_acceptable, message
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Error checking model AUC: {e}")
|
||
|
|
return 0.0, False, f"Error: {e}"
|
||
|
|
|
||
|
|
def should_retrain_due_to_low_auc(self) -> Tuple[bool, str]:
|
||
|
|
"""
|
||
|
|
Check if model should be retrained due to low AUC.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
(should_retrain, reason)
|
||
|
|
"""
|
||
|
|
if not self.auto_retrain_on_low_auc:
|
||
|
|
return False, "Auto-retrain on low AUC disabled"
|
||
|
|
|
||
|
|
current_auc, is_acceptable, message = self.check_model_auc()
|
||
|
|
|
||
|
|
if not is_acceptable and current_auc > 0:
|
||
|
|
# Check if enough time has passed since last retrain
|
||
|
|
now = datetime.now(WIB)
|
||
|
|
if self._last_retrain_time:
|
||
|
|
last_time = self._last_retrain_time
|
||
|
|
if last_time.tzinfo is None:
|
||
|
|
last_time = last_time.replace(tzinfo=WIB)
|
||
|
|
hours_since = (now - last_time).total_seconds() / 3600
|
||
|
|
|
||
|
|
# Only retrain if at least 4 hours since last retrain (to prevent loops)
|
||
|
|
if hours_since < 4:
|
||
|
|
return False, f"Low AUC but retrained recently ({hours_since:.1f}h ago)"
|
||
|
|
|
||
|
|
return True, f"Low AUC detected: {current_auc:.4f} < {self.min_auc_threshold}"
|
||
|
|
|
||
|
|
return False, "AUC acceptable" if is_acceptable else "Could not check AUC"
|
||
|
|
|
||
|
|
def get_auc_status(self) -> Dict:
|
||
|
|
"""Get current AUC status for monitoring."""
|
||
|
|
current_auc, is_acceptable, message = self.check_model_auc()
|
||
|
|
return {
|
||
|
|
"current_auc": current_auc,
|
||
|
|
"min_threshold": self.min_auc_threshold,
|
||
|
|
"is_acceptable": is_acceptable,
|
||
|
|
"message": message,
|
||
|
|
"check_count": self._auc_check_count,
|
||
|
|
"alert_sent": self._low_auc_alert_sent,
|
||
|
|
}
|
||
|
|
|
||
|
|
def backup_current_models(self) -> Tuple[bool, str]:
|
||
|
|
"""
|
||
|
|
Backup current models before retraining.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
(success, backup_path)
|
||
|
|
"""
|
||
|
|
if not self.backup_models:
|
||
|
|
return True, ""
|
||
|
|
|
||
|
|
try:
|
||
|
|
now = datetime.now(WIB)
|
||
|
|
backup_suffix = now.strftime("%Y%m%d_%H%M%S")
|
||
|
|
backup_dir = self.models_dir / "backups" / backup_suffix
|
||
|
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
||
|
|
|
||
|
|
# Backup XGBoost model
|
||
|
|
xgb_path = self.models_dir / "xgboost_model.pkl"
|
||
|
|
if xgb_path.exists():
|
||
|
|
shutil.copy(xgb_path, backup_dir / "xgboost_model.pkl")
|
||
|
|
|
||
|
|
# Backup HMM model
|
||
|
|
hmm_path = self.models_dir / "hmm_regime.pkl"
|
||
|
|
if hmm_path.exists():
|
||
|
|
shutil.copy(hmm_path, backup_dir / "hmm_regime.pkl")
|
||
|
|
|
||
|
|
logger.info(f"Models backed up to {backup_dir}")
|
||
|
|
|
||
|
|
# Keep only last 5 backups
|
||
|
|
self._cleanup_old_backups(keep=5)
|
||
|
|
|
||
|
|
return True, str(backup_dir)
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Failed to backup models: {e}")
|
||
|
|
return False, ""
|
||
|
|
|
||
|
|
def _cleanup_old_backups(self, keep: int = 5):
|
||
|
|
"""Remove old backups, keeping only the most recent ones."""
|
||
|
|
backup_base = self.models_dir / "backups"
|
||
|
|
if not backup_base.exists():
|
||
|
|
return
|
||
|
|
|
||
|
|
backups = sorted(backup_base.iterdir(), reverse=True)
|
||
|
|
for old_backup in backups[keep:]:
|
||
|
|
if old_backup.is_dir():
|
||
|
|
shutil.rmtree(old_backup)
|
||
|
|
logger.debug(f"Removed old backup: {old_backup}")
|
||
|
|
|
||
|
|
def retrain(
|
||
|
|
self,
|
||
|
|
connector, # MT5Connector
|
||
|
|
symbol: str = "XAUUSD",
|
||
|
|
timeframe: str = "M15",
|
||
|
|
is_weekend: bool = False,
|
||
|
|
) -> Dict:
|
||
|
|
"""
|
||
|
|
Retrain all models with latest data.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
connector: MT5 connector for fetching data
|
||
|
|
symbol: Trading symbol
|
||
|
|
timeframe: Timeframe for training data
|
||
|
|
is_weekend: If True, use more data for deep training
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
Dict with training results
|
||
|
|
"""
|
||
|
|
from src.feature_eng import FeatureEngineer
|
||
|
|
from src.smc_polars import SMCAnalyzer
|
||
|
|
from src.regime_detector import MarketRegimeDetector
|
||
|
|
from src.ml_model import TradingModel, get_default_feature_columns
|
||
|
|
|
||
|
|
started_at = datetime.now(WIB)
|
||
|
|
|
||
|
|
results = {
|
||
|
|
"success": False,
|
||
|
|
"hmm_trained": False,
|
||
|
|
"xgb_trained": False,
|
||
|
|
"xgb_train_auc": 0,
|
||
|
|
"xgb_test_auc": 0,
|
||
|
|
"train_accuracy": 0,
|
||
|
|
"test_accuracy": 0,
|
||
|
|
"samples": 0,
|
||
|
|
"error": None,
|
||
|
|
"backup_path": "",
|
||
|
|
"duration_seconds": 0,
|
||
|
|
}
|
||
|
|
|
||
|
|
# Determine training parameters
|
||
|
|
training_type = "weekend" if is_weekend else "daily"
|
||
|
|
if is_weekend:
|
||
|
|
bars = 15000 # More data for weekend deep training
|
||
|
|
num_boost_round = 80
|
||
|
|
else:
|
||
|
|
bars = 8000 # Daily training
|
||
|
|
num_boost_round = 50
|
||
|
|
|
||
|
|
# Record training start in database
|
||
|
|
self._save_retrain_start(training_type, bars, num_boost_round)
|
||
|
|
|
||
|
|
try:
|
||
|
|
logger.info("=" * 50)
|
||
|
|
logger.info("AUTO-RETRAINING STARTED")
|
||
|
|
logger.info(f"Type: {training_type}, Bars: {bars}, Boost Rounds: {num_boost_round}")
|
||
|
|
logger.info("=" * 50)
|
||
|
|
|
||
|
|
# Backup current models
|
||
|
|
backup_success, backup_path = self.backup_current_models()
|
||
|
|
results["backup_path"] = backup_path
|
||
|
|
|
||
|
|
# Fetch latest data
|
||
|
|
logger.info(f"Fetching {bars} bars of {symbol} {timeframe} data...")
|
||
|
|
df = connector.get_market_data(symbol, timeframe, bars)
|
||
|
|
|
||
|
|
if len(df) < 1000:
|
||
|
|
results["error"] = f"Insufficient data: {len(df)} bars"
|
||
|
|
logger.error(results["error"])
|
||
|
|
self._save_retrain_complete(
|
||
|
|
success=False,
|
||
|
|
error_message=results["error"],
|
||
|
|
started_at=started_at,
|
||
|
|
)
|
||
|
|
return results
|
||
|
|
|
||
|
|
results["samples"] = len(df)
|
||
|
|
logger.info(f"Received {len(df)} bars")
|
||
|
|
logger.info(f"Date range: {df['time'].min()} to {df['time'].max()}")
|
||
|
|
|
||
|
|
# Feature engineering
|
||
|
|
logger.info("Applying feature engineering...")
|
||
|
|
fe = FeatureEngineer()
|
||
|
|
df = fe.calculate_all(df, include_ml_features=True)
|
||
|
|
|
||
|
|
# SMC indicators
|
||
|
|
smc = SMCAnalyzer(swing_length=5)
|
||
|
|
df = smc.calculate_all(df)
|
||
|
|
|
||
|
|
# Create target
|
||
|
|
df = fe.create_target(df, lookahead=1)
|
||
|
|
|
||
|
|
# Train HMM
|
||
|
|
logger.info("Training HMM Regime Model...")
|
||
|
|
hmm = MarketRegimeDetector(
|
||
|
|
n_regimes=3,
|
||
|
|
lookback_periods=500,
|
||
|
|
model_path=str(self.models_dir / "hmm_regime.pkl"),
|
||
|
|
)
|
||
|
|
hmm.fit(df)
|
||
|
|
|
||
|
|
if hmm.fitted:
|
||
|
|
df = hmm.predict(df)
|
||
|
|
results["hmm_trained"] = True
|
||
|
|
logger.info("HMM model trained and saved")
|
||
|
|
|
||
|
|
# Train XGBoost
|
||
|
|
logger.info("Training XGBoost Model...")
|
||
|
|
xgb = TradingModel(
|
||
|
|
confidence_threshold=0.60,
|
||
|
|
model_path=str(self.models_dir / "xgboost_model.pkl"),
|
||
|
|
)
|
||
|
|
|
||
|
|
feature_cols = get_default_feature_columns()
|
||
|
|
available_features = [f for f in feature_cols if f in df.columns]
|
||
|
|
|
||
|
|
xgb.fit(
|
||
|
|
df,
|
||
|
|
available_features,
|
||
|
|
target_col="target",
|
||
|
|
train_ratio=0.7,
|
||
|
|
num_boost_round=num_boost_round,
|
||
|
|
early_stopping_rounds=5,
|
||
|
|
)
|
||
|
|
|
||
|
|
if xgb.fitted:
|
||
|
|
results["xgb_trained"] = True
|
||
|
|
results["xgb_train_auc"] = xgb._train_metrics.get("train_auc", 0)
|
||
|
|
results["xgb_test_auc"] = xgb._train_metrics.get("test_auc", 0)
|
||
|
|
results["train_accuracy"] = xgb._train_metrics.get("train_accuracy", 0)
|
||
|
|
results["test_accuracy"] = xgb._train_metrics.get("test_accuracy", 0)
|
||
|
|
logger.info(f"XGBoost trained: Train AUC={results['xgb_train_auc']:.4f}, Test AUC={results['xgb_test_auc']:.4f}")
|
||
|
|
|
||
|
|
# Save training data
|
||
|
|
training_data_path = self.data_dir / "training_data.parquet"
|
||
|
|
df.write_parquet(training_data_path)
|
||
|
|
logger.info(f"Training data saved to {training_data_path}")
|
||
|
|
|
||
|
|
# Mark success
|
||
|
|
results["success"] = results["hmm_trained"] and results["xgb_trained"]
|
||
|
|
results["duration_seconds"] = int((datetime.now(WIB) - started_at).total_seconds())
|
||
|
|
|
||
|
|
# Save completion to database
|
||
|
|
self._save_retrain_complete(
|
||
|
|
success=results["success"],
|
||
|
|
hmm_trained=results["hmm_trained"],
|
||
|
|
xgb_trained=results["xgb_trained"],
|
||
|
|
train_auc=results["xgb_train_auc"],
|
||
|
|
test_auc=results["xgb_test_auc"],
|
||
|
|
train_accuracy=results["train_accuracy"],
|
||
|
|
test_accuracy=results["test_accuracy"],
|
||
|
|
model_path=str(self.models_dir / "xgboost_model.pkl"),
|
||
|
|
backup_path=backup_path,
|
||
|
|
started_at=started_at,
|
||
|
|
)
|
||
|
|
|
||
|
|
if results["success"]:
|
||
|
|
logger.info("=" * 50)
|
||
|
|
logger.info("AUTO-RETRAINING COMPLETED SUCCESSFULLY")
|
||
|
|
logger.info(f"Duration: {results['duration_seconds']}s")
|
||
|
|
logger.info("=" * 50)
|
||
|
|
else:
|
||
|
|
logger.warning("Retraining completed with issues")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
results["error"] = str(e)
|
||
|
|
logger.error(f"Retraining failed: {e}")
|
||
|
|
import traceback
|
||
|
|
traceback.print_exc()
|
||
|
|
|
||
|
|
# Save failure to database
|
||
|
|
self._save_retrain_complete(
|
||
|
|
success=False,
|
||
|
|
error_message=str(e),
|
||
|
|
started_at=started_at,
|
||
|
|
)
|
||
|
|
|
||
|
|
return results
|
||
|
|
|
||
|
|
def rollback_models(self, reason: str = "Manual rollback") -> bool:
|
||
|
|
"""Rollback to previous model version if new one performs worse."""
|
||
|
|
backup_base = self.models_dir / "backups"
|
||
|
|
if not backup_base.exists():
|
||
|
|
logger.warning("No backups available for rollback")
|
||
|
|
return False
|
||
|
|
|
||
|
|
# Get most recent backup
|
||
|
|
backups = sorted(backup_base.iterdir(), reverse=True)
|
||
|
|
if not backups:
|
||
|
|
logger.warning("No backups found")
|
||
|
|
return False
|
||
|
|
|
||
|
|
latest_backup = backups[0]
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Restore XGBoost
|
||
|
|
xgb_backup = latest_backup / "xgboost_model.pkl"
|
||
|
|
if xgb_backup.exists():
|
||
|
|
shutil.copy(xgb_backup, self.models_dir / "xgboost_model.pkl")
|
||
|
|
|
||
|
|
# Restore HMM
|
||
|
|
hmm_backup = latest_backup / "hmm_regime.pkl"
|
||
|
|
if hmm_backup.exists():
|
||
|
|
shutil.copy(hmm_backup, self.models_dir / "hmm_regime.pkl")
|
||
|
|
|
||
|
|
logger.info(f"Models rolled back from {latest_backup}")
|
||
|
|
|
||
|
|
# Record rollback in database
|
||
|
|
if self._db_connected and self._training_repo and self._current_run_id:
|
||
|
|
try:
|
||
|
|
self._training_repo.mark_rollback(self._current_run_id, reason)
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Failed to record rollback: {e}")
|
||
|
|
|
||
|
|
return True
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Rollback failed: {e}")
|
||
|
|
return False
|
||
|
|
|
||
|
|
def get_training_history(self, limit: int = 10) -> list:
|
||
|
|
"""Get recent training history from database."""
|
||
|
|
if self._db_connected and self._training_repo:
|
||
|
|
try:
|
||
|
|
return self._training_repo.get_training_history(limit)
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Failed to get training history: {e}")
|
||
|
|
return []
|
||
|
|
|
||
|
|
def get_status(self) -> Dict:
|
||
|
|
"""Get current auto-trainer status."""
|
||
|
|
now = datetime.now(WIB)
|
||
|
|
|
||
|
|
hours_since_retrain = None
|
||
|
|
if self._last_retrain_time:
|
||
|
|
last_time = self._last_retrain_time
|
||
|
|
if last_time.tzinfo is None:
|
||
|
|
last_time = last_time.replace(tzinfo=WIB)
|
||
|
|
hours_since_retrain = (now - last_time).total_seconds() / 3600
|
||
|
|
|
||
|
|
should_train, reason = self.should_retrain()
|
||
|
|
|
||
|
|
# Get latest training from DB
|
||
|
|
latest_training = None
|
||
|
|
if self._db_connected and self._training_repo:
|
||
|
|
try:
|
||
|
|
latest_training = self._training_repo.get_latest_successful()
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
return {
|
||
|
|
"last_retrain": self._last_retrain_time.isoformat() if self._last_retrain_time else "Never",
|
||
|
|
"hours_since_retrain": round(hours_since_retrain, 1) if hours_since_retrain else None,
|
||
|
|
"should_retrain": should_train,
|
||
|
|
"reason": reason,
|
||
|
|
"next_retrain_hour": f"{self.daily_retrain_hour:02d}:00 WIB",
|
||
|
|
"weekend_training": self.weekend_retrain,
|
||
|
|
"db_connected": self._db_connected,
|
||
|
|
"latest_training": {
|
||
|
|
"train_auc": latest_training.get("train_auc") if latest_training else None,
|
||
|
|
"test_auc": latest_training.get("test_auc") if latest_training else None,
|
||
|
|
"bars_used": latest_training.get("bars_used") if latest_training else None,
|
||
|
|
} if latest_training else None,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def create_auto_trainer() -> AutoTrainer:
|
||
|
|
"""Create default auto trainer instance."""
|
||
|
|
return AutoTrainer(
|
||
|
|
models_dir="models",
|
||
|
|
data_dir="data",
|
||
|
|
daily_retrain_hour_wib=5, # 05:00 WIB (market close)
|
||
|
|
weekend_retrain=True,
|
||
|
|
min_hours_between_retrain=20,
|
||
|
|
backup_models=True,
|
||
|
|
use_db=True,
|
||
|
|
min_auc_threshold=0.65, # Alert if AUC drops below 0.65
|
||
|
|
auto_retrain_on_low_auc=True, # Auto retrain when AUC is low
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
# Test auto trainer
|
||
|
|
trainer = create_auto_trainer()
|
||
|
|
|
||
|
|
print("=== Auto Trainer Status ===")
|
||
|
|
status = trainer.get_status()
|
||
|
|
for key, value in status.items():
|
||
|
|
print(f" {key}: {value}")
|
||
|
|
|
||
|
|
print("\n=== Should Retrain Check ===")
|
||
|
|
should, reason = trainer.should_retrain()
|
||
|
|
print(f" Should retrain: {should}")
|
||
|
|
print(f" Reason: {reason}")
|
||
|
|
|
||
|
|
print("\n=== Training History ===")
|
||
|
|
history = trainer.get_training_history(5)
|
||
|
|
for h in history:
|
||
|
|
print(f" - {h.get('training_type')}: AUC={h.get('test_auc')}, Success={h.get('success')}")
|