commit 71a885dfe383bcea9bb493b901a95ed222c6fd96 Author: Abdoul Ahad Binizi Date: Thu Jun 25 14:00:20 2026 +0300 Initial commit - AHAD QUANT v1 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..84c82db --- /dev/null +++ b/.env.example @@ -0,0 +1,112 @@ +# ══════════════════════════════════════════════════════════════════ +# AHAD QUANT — Configuration (.env) +# Copier ce fichier : cp .env.example .env +# Puis remplir vos valeurs — NE JAMAIS commiter .env sur GitHub +# ══════════════════════════════════════════════════════════════════ + +# ─── Broker principal ───────────────────────────────────────────── +# Valeurs : oanda | mt5 | alpaca | ccxt | ib | paper +EXCHANGE=oanda + +# ─── Source de données historiques ──────────────────────────────── +# auto | oanda | yfinance | twelvedata | alphavantage | mt5 | ccxt +DATA_SOURCE=auto + +# ─── Mode paper (simulation sans argent réel) ───────────────────── +PAPER_MODE=true +PAPER_INITIAL_BALANCE=10000.0 + +# ─── OANDA (broker Forex principal) ─────────────────────────────── +# Inscription : https://www.oanda.com/ +# Clé API : Mon Compte → Gestion des clés API v20 +OANDA_API_KEY= +OANDA_ACCOUNT_ID= +OANDA_PRACTICE=true # true = compte démo | false = compte réel + +# ─── MetaTrader 5 (optionnel — Windows uniquement) ──────────────── +MT5_LOGIN=0 +MT5_PASSWORD= +MT5_SERVER= +MT5_BRIDGE_ENABLED=false +MT5_FILES_PATH= # Ex: C:\Users\Vous\AppData\Roaming\MetaQuotes\Terminal\\MQL5\Files +MT5_SYMBOL_SUFFIX= # Ex: .m pour ICMarkets + +# ─── Alpaca Markets (optionnel) ─────────────────────────────────── +# Inscription : https://alpaca.markets/ +ALPACA_API_KEY= +ALPACA_SECRET= +ALPACA_PAPER=true + +# ─── CCXT — broker générique (optionnel) ────────────────────────── +CCXT_BROKER= +CCXT_API_KEY= +CCXT_API_SECRET= +CCXT_PASSPHRASE= +CCXT_SANDBOX=false + +# ─── Twelve Data (données historiques — gratuit 800 req/j) ──────── +# Inscription : https://twelvedata.com/ +TWELVE_DATA_API_KEY= + +# ─── Alpha Vantage (données historiques — gratuit 25 req/j) ─────── +# Inscription : https://www.alphavantage.co/support/#api-key +ALPHA_VANTAGE_API_KEY= + +# ─── Interactive Brokers (optionnel) ────────────────────────────── +IB_HOST=127.0.0.1 +IB_PORT=7497 +IB_CLIENT_ID=1 + +# ─── Interface Web ───────────────────────────────────────────────── +# ⚠️ OBLIGATOIRE sur VPS exposé à internet — choisir un token fort +# Exemple : openssl rand -hex 32 +WEB_UI_TOKEN= +# Origines CORS autorisées (par défaut : localhost seulement) +WEB_UI_CORS_ORIGINS=http://localhost:8080,http://127.0.0.1:8080 + +# ─── Paramètres de trading ──────────────────────────────────────── +LEVERAGE=30 +MAX_POSITIONS=5 +RISK_PER_TRADE=0.01 # 1% du capital par trade +MAX_DAILY_LOSS_PCT=0.03 # Stop trading si -3% dans la journée + +# ─── Stop Loss / Take Profit ────────────────────────────────────── +STOP_LOSS_PCT=0.0050 # 50 pips sur EUR/USD +TAKE_PROFIT_PCT=0.0075 # 75 pips — R:R 1:1.5 + +# ─── Modèle IA ──────────────────────────────────────────────────── +MIN_CONFIDENCE=0.72 # Seuil minimum pour prendre un trade +USE_ENSEMBLE=true +USE_REGIME_FILTER=false +MODEL_PATH=model.pkl +ENSEMBLE_MODEL_PATH=model_ensemble.pkl + +# ─── Lot Forex ──────────────────────────────────────────────────── +MIN_LOT_SIZE=0.01 # 1 micro lot = 1 000 unités +MAX_LOT_SIZE=10.0 # 10 lots standard = 1 000 000 unités + +# ─── Circuit Breaker ────────────────────────────────────────────── +CIRCUIT_BREAKER_LOSSES=3 # Pause après 3 pertes consécutives +CIRCUIT_BREAKER_COOLDOWN=3600 # Pause de 1 heure + +# ─── Données ────────────────────────────────────────────────────── +CANDLE_INTERVAL=1h +DATA_DIR=data + +# ─── Agent RL (optionnel) ───────────────────────────────────────── +USE_RL_AGENT=false +RL_MODEL_PATH=rl_agent +RL_SCALER_PATH=rl_scaler.pkl +RL_MODE=filter # filter | override + +# ─── Apprentissage continu ──────────────────────────────────────── +CONTINUOUS_LEARNING_ENABLED=true +AUTO_RETRAIN_ENABLED=false +AUTO_RETRAIN_INTERVAL_HOURS=24 + +# ─── Session filter (optionnel) ─────────────────────────────────── +SESSION_FILTER_ENABLED=false +SESSION_LONDON_START=7 +SESSION_LONDON_END=16 +SESSION_NY_START=12 +SESSION_NY_END=21 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4286ee7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# ── Secrets — ne JAMAIS committer ────────────────────────────────────────── +.env +*.env +!.env.example + +# ── Python ────────────────────────────────────────────────────────────────── +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +.eggs/ +venv/ +.venv/ +env/ + +# ── État runtime du bot (régénéré à chaque exécution) ─────────────────────── +bot_state.json +paper_state.json +last_retrain.json + +# ── Modèles entraînés / données (trop volumineux pour git) ────────────────── +*.pkl +*.pt +*.pth +*.h5 +*.onnx +data/* +!data/.gitkeep + +# ── Logs ───────────────────────────────────────────────────────────────────── +*.log +logs/ + +# ── OS / éditeurs ──────────────────────────────────────────────────────────── +.DS_Store +Thumbs.db +.vscode/ +.idea/ + +# ── Jupyter ────────────────────────────────────────────────────────────────── +.ipynb_checkpoints/ diff --git a/AHAD_QUANT_Colab_Training.ipynb b/AHAD_QUANT_Colab_Training.ipynb new file mode 100644 index 0000000..d8063c3 --- /dev/null +++ b/AHAD_QUANT_Colab_Training.ipynb @@ -0,0 +1,1242 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "BsjhisC9CtK8" + }, + "source": [ + "# 🧠 AHAD QUANT V4 — Entraînement Complet sur Google Colab\n", + "\n", + "> **GPU recommandé** : Runtime → Modifier le type de Runtime → T4 GPU (gratuit)\n", + "\n", + "## 📋 Étapes\n", + "| # | Étape | Durée estimée |\n", + "|---|-------|---------------|\n", + "| 1 | Upload du projet + installation | ~5 min |\n", + "| 2 | Téléchargement données Forex | ~5 min |\n", + "| 3 | Entraînement ML (LGB + XGB + RF + TFT + TGRU) | ~45 min |\n", + "| 4 | Entraînement RL (PPO) | ~60 min |\n", + "| 5 | Export modèle unifié + téléchargement | ~2 min |\n", + "\n", + "---\n", + "⚠️ **Garde cet onglet actif** — Colab déconnecte après 90 min d'inactivité.\n", + "En cas de déconnexion : relancer depuis la **cellule 4.2** avec `--resume`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wyUAORUuCtK_" + }, + "source": [ + "## 📦 ÉTAPE 1 — Upload & Installation" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 160 + }, + "id": "rTjwFHDUCtLA", + "outputId": "532d5b73-5d63-4fdb-dd91-1b68c7903853" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "📁 Sélectionne le fichier ahad_quant_v32_FINAL_updated_v4.zip ...\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " \n", + " Upload widget is only available when the cell has been executed in the\n", + " current browser session. Please rerun this cell to enable.\n", + " \n", + " " + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Saving ahad_quant_v32_CLEAN.zip to ahad_quant_v32_CLEAN.zip\n", + "\n", + "✅ Reçu : ahad_quant_v32_CLEAN.zip (0.2 MB)\n", + "📂 Projet extrait dans : /content/ahad_quant/ahad_quant_v32_fixed\n", + "✅ Répertoire de travail configuré\n" + ] + } + ], + "source": [ + "# ── 1.1 Upload du ZIP AHAD QUANT ──────────────────────────────────────────────\n", + "from google.colab import files\n", + "import os, zipfile, shutil\n", + "\n", + "print('📁 Sélectionne le fichier ahad_quant_v32_FINAL_updated_v4.zip ...')\n", + "uploaded = files.upload()\n", + "\n", + "zip_name = list(uploaded.keys())[0]\n", + "print(f'\\n✅ Reçu : {zip_name} ({os.path.getsize(zip_name)/1024/1024:.1f} MB)')\n", + "\n", + "# Extraire dans /content/ahad_quant\n", + "os.makedirs('/content/ahad_quant', exist_ok=True)\n", + "with zipfile.ZipFile(zip_name, 'r') as z:\n", + " z.extractall('/content/ahad_quant')\n", + "\n", + "# Trouver le dossier extrait\n", + "subdirs = [d for d in os.listdir('/content/ahad_quant') if os.path.isdir(f'/content/ahad_quant/{d}')]\n", + "PROJECT_DIR = f'/content/ahad_quant/{subdirs[0]}'\n", + "print(f'📂 Projet extrait dans : {PROJECT_DIR}')\n", + "os.chdir(PROJECT_DIR)\n", + "print('✅ Répertoire de travail configuré')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HAPFJvJkCtLC", + "outputId": "2356f3dd-ff8b-4d15-c261-6d2e2437889d" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "📦 Installation des packages...\n", + "\n", + "✅ Packages installés\n" + ] + } + ], + "source": [ + "# ── 1.2 Installation des dépendances ─────────────────────────────────────────\n", + "print('📦 Installation des packages...')\n", + "\n", + "# Packages standards depuis PyPI\n", + "!pip install -q lightgbm xgboost scikit-learn optuna yfinance stable-baselines3[extra] gymnasium shimmy python-dotenv\n", + "\n", + "# Torch GPU (CUDA 11.8) depuis l'index PyTorch dédié\n", + "!pip install -q torch torchvision --index-url https://download.pytorch.org/whl/cu118\n", + "\n", + "print('\\n✅ Packages installés')" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ScKxJIYmCtLD", + "outputId": "3896a169-43f0-4d7d-8566-2dd1f3c4afc0" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "✅ GPU détecté : Tesla T4 (14.6 GB)\n", + "\n", + "📋 Config active :\n", + " DATA_SOURCE = yfinance\n", + " PAPER_MODE = true\n", + " MAX_SEQ_PER_PAIR = 4000\n", + " AHAD_QUANT_BATCH_SIZE = 512\n", + " AHAD_QUANT_MAX_EPOCHS = 20\n" + ] + } + ], + "source": [ + "# ── 1.3 Configuration GPU + variables d'environnement ────────────────────────\n", + "import torch, os\n", + "\n", + "GPU_OK = torch.cuda.is_available()\n", + "if GPU_OK:\n", + " gpu_name = torch.cuda.get_device_name(0)\n", + " gpu_mem = torch.cuda.get_device_properties(0).total_memory / 1024**3\n", + " print(f'✅ GPU détecté : {gpu_name} ({gpu_mem:.1f} GB)')\n", + "else:\n", + " print('⚠️ Pas de GPU — entraînement plus lent (CPU). Active le GPU dans Runtime → Modifier le type de Runtime.')\n", + "\n", + "# Variables d'env pour AHAD QUANT\n", + "os.environ.update({\n", + " 'DATA_SOURCE' : 'yfinance', # source données (gratuit)\n", + " 'EXCHANGE' : 'paper', # pas de broker réel\n", + " 'PAPER_MODE' : 'true',\n", + " 'PAPER_INITIAL_BALANCE' : '10000',\n", + " # Mémoire : RAM SYSTÈME Colab (~12 Gi), indépendante de la VRAM du GPU\n", + " 'MAX_SEQ_PER_PAIR' : '4000',\n", + " # Batch size TFT/TGRU — GPU peut prendre 512, CPU limité\n", + " 'AHAD_QUANT_BATCH_SIZE' : '512' if GPU_OK else '128',\n", + " 'AHAD_QUANT_MAX_EPOCHS' : '20' if GPU_OK else '10',\n", + "})\n", + "\n", + "print('\\n📋 Config active :')\n", + "for k in ['DATA_SOURCE','PAPER_MODE','MAX_SEQ_PER_PAIR','AHAD_QUANT_BATCH_SIZE','AHAD_QUANT_MAX_EPOCHS']:\n", + " print(f' {k} = {os.environ[k]}')" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "52yD66NnCtLE", + "outputId": "b0b873ed-ca6b-4642-b2e1-2566a2ec43ac" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n", + "✅ Drive monté — sauvegarde dans : /content/drive/MyDrive/AhadQuant_Models\n" + ] + } + ], + "source": [ + "# ── 1.4 (Optionnel) Montage Google Drive pour sauvegarde auto ────────────────\n", + "USE_DRIVE = True # ← Mettre False si tu ne veux pas utiliser Drive\n", + "\n", + "DRIVE_SAVE_DIR = None\n", + "if USE_DRIVE:\n", + " from google.colab import drive\n", + " drive.mount('/content/drive')\n", + " DRIVE_SAVE_DIR = '/content/drive/MyDrive/AhadQuant_Models'\n", + " os.makedirs(DRIVE_SAVE_DIR, exist_ok=True)\n", + " print(f'✅ Drive monté — sauvegarde dans : {DRIVE_SAVE_DIR}')\n", + "else:\n", + " print('ℹ️ Drive désactivé — les modèles seront téléchargés en fin de notebook')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KnX1i32NCtLE" + }, + "source": [ + "## 📊 ÉTAPE 2 — Téléchargement des données Forex" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Gvan90BJCtLE", + "outputId": "745ad356-e0cb-43c6-fa02-dc8c14abc111" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "📡 Téléchargement des données historiques Forex (Yahoo Finance)...\n", + " 25 paires × 730 jours × 1h ≈ 17 000 bougies/paire\n", + "\n", + "\n", + "✅ Données téléchargées : 25 fichiers dans data/\n" + ] + } + ], + "source": [ + "# ── 2.1 Download 730 jours de données 1h pour les 25 paires ─────────────────\n", + "import subprocess, sys\n", + "\n", + "print('📡 Téléchargement des données historiques Forex (Yahoo Finance)...')\n", + "print(' 25 paires × 730 jours × 1h ≈ 17 000 bougies/paire\\n')\n", + "\n", + "result = subprocess.run(\n", + " [sys.executable, 'download_data.py'],\n", + " capture_output=False,\n", + " text=True\n", + ")\n", + "\n", + "if result.returncode == 0:\n", + " data_files = [f for f in os.listdir('data') if f.endswith('.json')] if os.path.exists('data') else []\n", + " print(f'\\n✅ Données téléchargées : {len(data_files)} fichiers dans data/')\n", + "else:\n", + " print(f'\\n❌ Erreur download (code {result.returncode})')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fs-zpnwoCtLF" + }, + "source": [ + "## 🤖 ÉTAPE 3 — Entraînement ML (Ensemble + Deep Learning)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "otoBfmYuCtLF", + "outputId": "a0cfa69b-d997-4f1b-df85-50f107e94998" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "🧠 Lancement entraînement ML...\n", + " LightGBM + XGBoost + RF + TFT + TransformerGRU + Meta-learner\n", + "\n", + "[DL] PyTorch 2.11.0+cu128 detected — TFT + TGRU enabled\n", + "=================================================================\n", + " AHAD QUANT — Ensemble Training Pipeline (V5)\n", + " LightGBM + XGBoost + RF + TFT + TransformerGRU + Meta-learner\n", + "=================================================================\n", + "\n", + "[0/7] Données disponibles : 25 fichiers dans data/\n", + "\n", + "[1/7] Loading data and building tabular features...\n", + " ✅ EURUSD — 17,222 samples (719 jours)\n", + " ✅ GBPUSD — 17,223 samples (719 jours)\n", + " ✅ USDJPY — 17,109 samples (714 jours)\n", + " ✅ USDCHF — 17,163 samples (716 jours)\n", + " ✅ AUDUSD — 17,321 samples (723 jours)\n", + " ✅ NZDUSD — 17,314 samples (723 jours)\n", + " ✅ USDCAD — 17,326 samples (723 jours)\n", + " ✅ EURGBP — 17,248 samples (720 jours)\n", + " ✅ EURJPY — 17,234 samples (719 jours)\n", + " ✅ EURCAD — 17,249 samples (720 jours)\n", + " ✅ EURCHF — 17,233 samples (719 jours)\n", + " ✅ EURAUD — 17,247 samples (720 jours)\n", + " ✅ EURNZD — 17,240 samples (720 jours)\n", + " ✅ GBPJPY — 17,233 samples (719 jours)\n", + " ✅ GBPCAD — 17,313 samples (723 jours)\n", + " ✅ GBPCHF — 17,230 samples (719 jours)\n", + " ✅ GBPAUD — 17,309 samples (723 jours)\n", + " ✅ AUDCAD — 17,318 samples (723 jours)\n", + " ✅ AUDNZD — 17,315 samples (723 jours)\n", + " ✅ AUDJPY — 17,234 samples (719 jours)\n", + " ✅ AUDCHF — 17,283 samples (722 jours)\n", + " ✅ CADJPY — 17,237 samples (720 jours)\n", + " ✅ CHFJPY — 17,221 samples (719 jours)\n", + " ✅ NZDJPY — 17,230 samples (719 jours)\n", + " ✅ NZDCAD — 17,309 samples (723 jours)\n", + "\n", + " Total: 431,361 samples | 62 features | 50.40% long labels\n", + "\n", + "[1b/7] Building sequence dataset (SEQ_LEN=168)...\n", + " → sampled 4,000 / 17,051 sequences (RAM cap)\n", + " [EURUSD ] 4,000 sequences\n", + " → sampled 4,000 / 17,052 sequences (RAM cap)\n", + " [GBPUSD ] 4,000 sequences\n", + " → sampled 4,000 / 16,938 sequences (RAM cap)\n", + " [USDJPY ] 4,000 sequences\n", + " → sampled 4,000 / 16,992 sequences (RAM cap)\n", + " [USDCHF ] 4,000 sequences\n", + " → sampled 4,000 / 17,150 sequences (RAM cap)\n", + " [AUDUSD ] 4,000 sequences\n", + " → sampled 4,000 / 17,143 sequences (RAM cap)\n", + " [NZDUSD ] 4,000 sequences\n", + " → sampled 4,000 / 17,155 sequences (RAM cap)\n", + " [USDCAD ] 4,000 sequences\n", + " → sampled 4,000 / 17,077 sequences (RAM cap)\n", + " [EURGBP ] 4,000 sequences\n", + " → sampled 4,000 / 17,063 sequences (RAM cap)\n", + " [EURJPY ] 4,000 sequences\n", + " → sampled 4,000 / 17,078 sequences (RAM cap)\n", + " [EURCAD ] 4,000 sequences\n", + " → sampled 4,000 / 17,062 sequences (RAM cap)\n", + " [EURCHF ] 4,000 sequences\n", + " → sampled 4,000 / 17,076 sequences (RAM cap)\n", + " [EURAUD ] 4,000 sequences\n", + " → sampled 4,000 / 17,069 sequences (RAM cap)\n", + " [EURNZD ] 4,000 sequences\n", + " → sampled 4,000 / 17,062 sequences (RAM cap)\n", + " [GBPJPY ] 4,000 sequences\n", + " → sampled 4,000 / 17,142 sequences (RAM cap)\n", + " [GBPCAD ] 4,000 sequences\n", + " → sampled 4,000 / 17,059 sequences (RAM cap)\n", + " [GBPCHF ] 4,000 sequences\n", + " → sampled 4,000 / 17,138 sequences (RAM cap)\n", + " [GBPAUD ] 4,000 sequences\n", + " → sampled 4,000 / 17,147 sequences (RAM cap)\n", + " [AUDCAD ] 4,000 sequences\n", + " → sampled 4,000 / 17,144 sequences (RAM cap)\n", + " [AUDNZD ] 4,000 sequences\n", + " → sampled 4,000 / 17,063 sequences (RAM cap)\n", + " [AUDJPY ] 4,000 sequences\n", + " → sampled 4,000 / 17,112 sequences (RAM cap)\n", + " [AUDCHF ] 4,000 sequences\n", + " → sampled 4,000 / 17,066 sequences (RAM cap)\n", + " [CADJPY ] 4,000 sequences\n", + " → sampled 4,000 / 17,050 sequences (RAM cap)\n", + " [CHFJPY ] 4,000 sequences\n", + " → sampled 4,000 / 17,059 sequences (RAM cap)\n", + " [NZDJPY ] 4,000 sequences\n", + " → sampled 4,000 / 17,138 sequences (RAM cap)\n", + " [NZDCAD ] 4,000 sequences\n", + "\n", + " Total : 100,000 sequences | shape X_seq=(100000, 168, 62) | 50.00% long labels\n", + " Sequence samples : 100,000 | shape: (100000, 168, 62)\n", + " Seq splits: train=70,000 | val=15,000 | test=15,000\n", + "\n", + "[2/7] Walk-forward cross-validation (tabular)...\n", + "\n", + " Walk-forward CV (4 windows):\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[500]\tvalid_0's binary_logloss: 0.537108\n", + " Window 1: train=215,680 test=53,920 acc=0.7102\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[500]\tvalid_0's binary_logloss: 0.536396\n", + " Window 2: train=269,600 test=53,920 acc=0.7090\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[500]\tvalid_0's binary_logloss: 0.529891\n", + " Window 3: train=323,520 test=53,920 acc=0.7125\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[500]\tvalid_0's binary_logloss: 0.530677\n", + " Window 4: train=377,440 test=53,920 acc=0.7138\n", + " CV accuracy: 0.7114 ± 0.0019\n", + "\n", + " Tabular splits: train=301,952 | val=64,704 | test=64,705\n", + "\n", + "[3/7] Hyperparameter search (30 Optuna trials)...\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[306]\tvalid_0's binary_logloss: 0.531696\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[256]\tvalid_0's binary_logloss: 0.536046\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.531071\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[176]\tvalid_0's binary_logloss: 0.534405\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[320]\tvalid_0's binary_logloss: 0.532699\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.529014\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.533946\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[997]\tvalid_0's binary_logloss: 0.531701\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[540]\tvalid_0's binary_logloss: 0.535239\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[519]\tvalid_0's binary_logloss: 0.530513\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[996]\tvalid_0's binary_logloss: 0.530435\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[725]\tvalid_0's binary_logloss: 0.529296\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[998]\tvalid_0's binary_logloss: 0.528426\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[987]\tvalid_0's binary_logloss: 0.529308\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[998]\tvalid_0's binary_logloss: 0.528884\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.529866\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[815]\tvalid_0's binary_logloss: 0.528147\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[469]\tvalid_0's binary_logloss: 0.530574\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.532568\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[897]\tvalid_0's binary_logloss: 0.528453\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.528905\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[695]\tvalid_0's binary_logloss: 0.529934\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[833]\tvalid_0's binary_logloss: 0.529049\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.52851\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[761]\tvalid_0's binary_logloss: 0.531103\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.529707\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.531528\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.529486\n", + "Training until validation scores don't improve for 30 rounds\n", + "Did not meet early stopping. Best iteration is:\n", + "[1000]\tvalid_0's binary_logloss: 0.532756\n", + "Training until validation scores don't improve for 30 rounds\n", + "Early stopping, best iteration is:\n", + "[310]\tvalid_0's binary_logloss: 0.531737\n", + " Best LightGBM accuracy (Optuna): 0.7147\n", + "\n", + "[4/7] Training tabular base models (LightGBM + XGBoost + RF)...\n", + " Training LightGBM...\n", + "Training until validation scores don't improve for 50 rounds\n", + "[200]\tvalid_0's binary_logloss: 0.537594\n", + "[400]\tvalid_0's binary_logloss: 0.532418\n", + "[600]\tvalid_0's binary_logloss: 0.530308\n", + "[800]\tvalid_0's binary_logloss: 0.529119\n", + "[1000]\tvalid_0's binary_logloss: 0.528434\n", + "Early stopping, best iteration is:\n", + "[1018]\tvalid_0's binary_logloss: 0.528387\n", + " Training XGBoost...\n", + " Training RandomForest...\n", + "\n", + "[4b/7] Training Deep Learning models (TFT + TransformerGRU)...\n", + " Training Temporal Fusion Transformer...\n", + " [TFT] device=cuda | train=70,000 val=15,000 samples\n", + " [TFT] parameters: 669,305\n", + "/content/ahad_quant/ahad_quant_v32_fixed/tft_model.py:280: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n", + " scaler_amp = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())\n", + "/content/ahad_quant/ahad_quant_v32_fixed/tft_model.py:297: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n", + " with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()):\n", + " [TFT] epoch 1/30 train=0.6950 val=0.6942 val_acc=0.5015 (82s)\n", + " [TFT] epoch 2/30 train=0.6936 val=0.6953 val_acc=0.5015 (163s)\n", + " [TFT] epoch 3/30 train=0.6935 val=0.6931 val_acc=0.5053 (243s)\n", + " [TFT] epoch 4/30 train=0.6934 val=0.6935 val_acc=0.5007 (324s)\n", + " [TFT] epoch 5/30 train=0.6931 val=0.6934 val_acc=0.5003 (405s)\n", + " [TFT] epoch 6/30 train=0.6934 val=0.6940 val_acc=0.5033 (486s)\n", + " [TFT] epoch 7/30 train=0.6932 val=0.6936 val_acc=0.5002 (566s)\n", + " [TFT] epoch 8/30 train=0.6929 val=0.6938 val_acc=0.5059 (647s)\n", + " [TFT] epoch 9/30 train=0.6926 val=0.6932 val_acc=0.5073 (728s)\n", + " [TFT] early stopping at epoch 9\n", + " [TFT] training complete — best val_loss=0.6931\n", + "\n", + " Training TransformerGRU...\n", + " [TGRU] device=cuda | train=70,000 val=15,000 samples\n", + " [TGRU] parameters: 588,033\n", + "/content/ahad_quant/ahad_quant_v32_fixed/transformer_gru_model.py:275: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n", + " scaler_amp = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())\n", + "/content/ahad_quant/ahad_quant_v32_fixed/transformer_gru_model.py:292: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n", + " with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()):\n", + " [TGRU] epoch 1/30 train=0.6419 val=0.5863 val_acc=0.6751 (32s)\n", + " [TGRU] epoch 2/30 train=0.5910 val=0.5618 val_acc=0.6897 (64s)\n", + " [TGRU] epoch 3/30 train=0.5722 val=0.5608 val_acc=0.6885 (96s)\n", + " [TGRU] epoch 4/30 train=0.5620 val=0.5450 val_acc=0.7002 (128s)\n", + " [TGRU] epoch 5/30 train=0.5493 val=0.5325 val_acc=0.7067 (160s)\n", + " [TGRU] epoch 6/30 train=0.5413 val=0.5278 val_acc=0.7097 (192s)\n", + " [TGRU] epoch 7/30 train=0.5334 val=0.5214 val_acc=0.7157 (224s)\n", + " [TGRU] epoch 8/30 train=0.5262 val=0.5349 val_acc=0.7007 (255s)\n", + " [TGRU] epoch 9/30 train=0.5193 val=0.5043 val_acc=0.7247 (287s)\n", + " [TGRU] epoch 10/30 train=0.5138 val=0.4947 val_acc=0.7262 (319s)\n", + " [TGRU] epoch 11/30 train=0.5041 val=0.4924 val_acc=0.7249 (351s)\n", + " [TGRU] epoch 12/30 train=0.4999 val=0.4847 val_acc=0.7306 (383s)\n", + " [TGRU] epoch 13/30 train=0.4940 val=0.4821 val_acc=0.7277 (415s)\n", + " [TGRU] epoch 14/30 train=0.4866 val=0.4807 val_acc=0.7303 (447s)\n", + " [TGRU] epoch 15/30 train=0.4838 val=0.4750 val_acc=0.7301 (479s)\n", + " [TGRU] epoch 16/30 train=0.4791 val=0.4762 val_acc=0.7289 (511s)\n", + " [TGRU] epoch 17/30 train=0.4743 val=0.4713 val_acc=0.7335 (543s)\n", + " [TGRU] epoch 18/30 train=0.4716 val=0.4662 val_acc=0.7378 (575s)\n", + " [TGRU] epoch 19/30 train=0.4677 val=0.4707 val_acc=0.7357 (606s)\n", + " [TGRU] epoch 20/30 train=0.4647 val=0.4677 val_acc=0.7347 (639s)\n", + " [TGRU] epoch 21/30 train=0.4622 val=0.4695 val_acc=0.7347 (670s)\n", + " [TGRU] epoch 22/30 train=0.4592 val=0.4632 val_acc=0.7383 (702s)\n", + " [TGRU] epoch 23/30 train=0.4566 val=0.4646 val_acc=0.7373 (734s)\n", + " [TGRU] epoch 24/30 train=0.4544 val=0.4650 val_acc=0.7395 (766s)\n", + " [TGRU] epoch 25/30 train=0.4532 val=0.4667 val_acc=0.7361 (798s)\n", + " [TGRU] epoch 26/30 train=0.4521 val=0.4632 val_acc=0.7394 (830s)\n", + " [TGRU] epoch 27/30 train=0.4504 val=0.4662 val_acc=0.7381 (862s)\n", + " [TGRU] epoch 28/30 train=0.4497 val=0.4646 val_acc=0.7389 (894s)\n", + " [TGRU] epoch 29/30 train=0.4478 val=0.4644 val_acc=0.7387 (926s)\n", + " [TGRU] epoch 30/30 train=0.4488 val=0.4646 val_acc=0.7397 (958s)\n", + " [TGRU] training complete — best val_loss=0.4632\n", + "\n", + "❌ Échec entraînement ML (code -9) — 88.7 min\n", + "\n", + "--- Dernières lignes du log ---\n", + " [TFT] training complete — best val_loss=0.6931\n", + "\n", + " Training TransformerGRU...\n", + " [TGRU] device=cuda | train=70,000 val=15,000 samples\n", + " [TGRU] parameters: 588,033\n", + "/content/ahad_quant/ahad_quant_v32_fixed/transformer_gru_model.py:275: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n", + " scaler_amp = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())\n", + "/content/ahad_quant/ahad_quant_v32_fixed/transformer_gru_model.py:292: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n", + " with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()):\n", + " [TGRU] epoch 1/30 train=0.6419 val=0.5863 val_acc=0.6751 (32s)\n", + " [TGRU] epoch 2/30 train=0.5910 val=0.5618 val_acc=0.6897 (64s)\n", + " [TGRU] epoch 3/30 train=0.5722 val=0.5608 val_acc=0.6885 (96s)\n", + " [TGRU] epoch 4/30 train=0.5620 val=0.5450 val_acc=0.7002 (128s)\n", + " [TGRU] epoch 5/30 train=0.5493 val=0.5325 val_acc=0.7067 (160s)\n", + " [TGRU] epoch 6/30 train=0.5413 val=0.5278 val_acc=0.7097 (192s)\n", + " [TGRU] epoch 7/30 train=0.5334 val=0.5214 val_acc=0.7157 (224s)\n", + " [TGRU] epoch 8/30 train=0.5262 val=0.5349 val_acc=0.7007 (255s)\n", + " [TGRU] epoch 9/30 train=0.5193 val=0.5043 val_acc=0.7247 (287s)\n", + " [TGRU] epoch 10/30 train=0.5138 val=0.4947 val_acc=0.7262 (319s)\n", + " [TGRU] epoch 11/30 train=0.5041 val=0.4924 val_acc=0.7249 (351s)\n", + " [TGRU] epoch 12/30 train=0.4999 val=0.4847 val_acc=0.7306 (383s)\n", + " [TGRU] epoch 13/30 train=0.4940 val=0.4821 val_acc=0.7277 (415s)\n", + " [TGRU] epoch 14/30 train=0.4866 val=0.4807 val_acc=0.7303 (447s)\n", + " [TGRU] epoch 15/30 train=0.4838 val=0.4750 val_acc=0.7301 (479s)\n", + " [TGRU] epoch 16/30 train=0.4791 val=0.4762 val_acc=0.7289 (511s)\n", + " [TGRU] epoch 17/30 train=0.4743 val=0.4713 val_acc=0.7335 (543s)\n", + " [TGRU] epoch 18/30 train=0.4716 val=0.4662 val_acc=0.7378 (575s)\n", + " [TGRU] epoch 19/30 train=0.4677 val=0.4707 val_acc=0.7357 (606s)\n", + " [TGRU] epoch 20/30 train=0.4647 val=0.4677 val_acc=0.7347 (639s)\n", + " [TGRU] epoch 21/30 train=0.4622 val=0.4695 val_acc=0.7347 (670s)\n", + " [TGRU] epoch 22/30 train=0.4592 val=0.4632 val_acc=0.7383 (702s)\n", + " [TGRU] epoch 23/30 train=0.4566 val=0.4646 val_acc=0.7373 (734s)\n", + " [TGRU] epoch 24/30 train=0.4544 val=0.4650 val_acc=0.7395 (766s)\n", + " [TGRU] epoch 25/30 train=0.4532 val=0.4667 val_acc=0.7361 (798s)\n", + " [TGRU] epoch 26/30 train=0.4521 val=0.4632 val_acc=0.7394 (830s)\n", + " [TGRU] epoch 27/30 train=0.4504 val=0.4662 val_acc=0.7381 (862s)\n", + " [TGRU] epoch 28/30 train=0.4497 val=0.4646 val_acc=0.7389 (894s)\n", + " [TGRU] epoch 29/30 train=0.4478 val=0.4644 val_acc=0.7387 (926s)\n", + " [TGRU] epoch 30/30 train=0.4488 val=0.4646 val_acc=0.7397 (958s)\n", + " [TGRU] training complete — best val_loss=0.4632\n", + "\n" + ] + } + ], + "source": [ + "# ── 3.1 Entraînement du pipeline ML complet (version diagnostic) ────────────\n", + "import subprocess, sys, time, os\n", + "\n", + "print('🧠 Lancement entraînement ML...')\n", + "print(' LightGBM + XGBoost + RF + TFT + TransformerGRU + Meta-learner\\n')\n", + "\n", + "t0 = time.time()\n", + "with open('train_log.txt', 'w') as logfile:\n", + " proc = subprocess.Popen(\n", + " [sys.executable, '-u', 'train.py'],\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.STDOUT,\n", + " text=True,\n", + " bufsize=1,\n", + " )\n", + " for line in proc.stdout:\n", + " print(line, end='')\n", + " logfile.write(line)\n", + " logfile.flush()\n", + " proc.wait()\n", + "\n", + "elapsed = (time.time() - t0) / 60\n", + "if proc.returncode == 0:\n", + " print(f'\\n✅ Entraînement ML terminé en {elapsed:.1f} min')\n", + "else:\n", + " print(f'\\n❌ Échec entraînement ML (code {proc.returncode}) — {elapsed:.1f} min')\n", + " print('\\n--- Dernières lignes du log ---')\n", + " with open('train_log.txt') as f:\n", + " lines = f.readlines()\n", + " print(''.join(lines[-40:]))" + ] + }, + { + "cell_type": "code", + "source": [ + "import os, shutil\n", + "\n", + "if os.path.exists('model_ensemble.pkl'):\n", + " print(f\"model_ensemble.pkl : {os.path.getsize('model_ensemble.pkl')/1024/1024:.1f} MB\")\n", + " if DRIVE_SAVE_DIR:\n", + " shutil.copy('model_ensemble.pkl', f'{DRIVE_SAVE_DIR}/model_ensemble.pkl')\n", + " print(f\"💾 Sauvegardé sur Drive : {DRIVE_SAVE_DIR}/model_ensemble.pkl\")\n", + "else:\n", + " print(\"⚠️ model_ensemble.pkl non trouvé\")" + ], + "metadata": { + "id": "5dy7w3W8IvvS" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Nouvelle section" + ], + "metadata": { + "id": "_tNgu2SucvZ-" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SByjDXk4CtLF" + }, + "outputs": [], + "source": [ + "# ── 3.2 Résultats ML (backtest rapide) ───────────────────────────────────────\n", + "import json\n", + "\n", + "# Lire les résultats de l'entraînement\n", + "results_files = ['last_retrain.json', 'backtest_results.json']\n", + "for rf in results_files:\n", + " if os.path.exists(rf):\n", + " with open(rf) as f:\n", + " data = json.load(f)\n", + " print(f'\\n📊 {rf}:')\n", + " for k, v in list(data.items())[:15]:\n", + " print(f' {k}: {v}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pAM1LE6hCtLG" + }, + "source": [ + "## 🎮 ÉTAPE 4 — Entraînement RL (PPO Agent)\n", + "\n", + "> 💡 **En cas de déconnexion Colab** : remonte Drive, retourne dans le dossier projet, puis relance **la cellule 4.2** avec `--resume` — l'entraînement reprend depuis le dernier checkpoint automatiquement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "s3hvvDF8CtLG" + }, + "outputs": [], + "source": [ + "# ── 4.1 Vérification pré-RL ──────────────────────────────────────────────────\n", + "import os, json\n", + "\n", + "print('🔍 Vérification des prérequis RL...')\n", + "checks = {\n", + " 'model_ensemble.pkl' : 'Modèle ML',\n", + " 'data/' : 'Données Forex',\n", + "}\n", + "ok = True\n", + "for path, label in checks.items():\n", + " exists = os.path.exists(path)\n", + " print(f' {\"✅\" if exists else \"❌\"} {label} ({path})')\n", + " if not exists:\n", + " ok = False\n", + "\n", + "if ok:\n", + " print('\\n✅ Tous les prérequis sont présents — tu peux lancer l\\'entraînement RL')\n", + "else:\n", + " print('\\n❌ Lance d\\'abord les étapes 2 et 3 !')\n", + "\n", + "# Config RL\n", + "RL_STEPS = 1_000_000 # ← Modifier ici si besoin (500k = rapide, 2M = meilleure qualité)\n", + "print(f'\\n⚙️ Steps RL configurés : {RL_STEPS:,}')\n", + "print(f' Durée estimée : ~{RL_STEPS/60000:.0f} min (GPU T4)')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "huE9Oa6BCtLG" + }, + "outputs": [], + "source": [ + "# ── 4.2 Entraînement RL PPO ──────────────────────────────────────────────────\n", + "# ⚠️ En cas de reprise après déconnexion : changer --steps en 0 et garder --resume\n", + "import subprocess, sys, time, os, shutil\n", + "\n", + "RESUME = os.path.exists('rl_progress.json') and os.path.getsize('rl_progress.json') > 5\n", + "\n", + "if RESUME:\n", + " with open('rl_progress.json') as f:\n", + " prog = json.load(f)\n", + " done = prog.get('steps_done', 0)\n", + " total = prog.get('total_steps', RL_STEPS)\n", + " print(f'♻️ Reprise détectée : {done:,} / {total:,} steps déjà effectués')\n", + " resume_flag = ['--resume']\n", + "else:\n", + " print('🆕 Nouvel entraînement RL')\n", + " resume_flag = []\n", + "\n", + "print(f'\\n🎮 Lancement PPO ({RL_STEPS:,} steps)...')\n", + "print(' Checkpoints sauvegardés toutes les 50k steps dans rl_checkpoints/')\n", + "print(' Durée estimée : 45-90 min (GPU T4)\\n')\n", + "\n", + "t0 = time.time()\n", + "result = subprocess.run(\n", + " [sys.executable, 'rl_train.py', '--steps', str(RL_STEPS)] + resume_flag,\n", + " capture_output=False,\n", + " text=True\n", + ")\n", + "elapsed = (time.time() - t0) / 60\n", + "\n", + "if result.returncode == 0:\n", + " print(f'\\n✅ Entraînement RL terminé en {elapsed:.1f} min')\n", + " for f in ['rl_agent.zip', 'rl_scaler.pkl']:\n", + " if os.path.exists(f):\n", + " print(f' {f} : {os.path.getsize(f)/1024:.0f} KB')\n", + "\n", + " # Sauvegarde Drive\n", + " if DRIVE_SAVE_DIR:\n", + " for f in ['rl_agent.zip', 'rl_scaler.pkl']:\n", + " if os.path.exists(f):\n", + " shutil.copy(f, f'{DRIVE_SAVE_DIR}/{f}')\n", + " # Sauvegarder aussi les checkpoints\n", + " if os.path.exists('rl_checkpoints'):\n", + " shutil.copytree('rl_checkpoints', f'{DRIVE_SAVE_DIR}/rl_checkpoints', dirs_exist_ok=True)\n", + " print(f' 💾 Sauvegardé sur Drive : {DRIVE_SAVE_DIR}/')\n", + "else:\n", + " print(f'\\n❌ Échec RL (code {result.returncode}) — {elapsed:.1f} min')\n", + " print(' ➡️ Relance cette cellule avec --resume pour reprendre')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HDnmu9rRCtLG" + }, + "source": [ + "## 📦 ÉTAPE 5 — Export modèle unifié & Téléchargement" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yfCaEDZ9CtLH" + }, + "outputs": [], + "source": [ + "# ── 5.1 Export ahad_quant_unified.zip ─────────────────────────────────────────\n", + "import subprocess, sys, os, shutil\n", + "\n", + "print('📦 Export du modèle unifié...')\n", + "result = subprocess.run(\n", + " [sys.executable, 'export_unified.py'],\n", + " capture_output=False,\n", + " text=True\n", + ")\n", + "\n", + "if result.returncode == 0 and os.path.exists('ahad_quant_unified.zip'):\n", + " size_mb = os.path.getsize('ahad_quant_unified.zip') / 1024 / 1024\n", + " print(f'✅ ahad_quant_unified.zip créé ({size_mb:.1f} MB)')\n", + "\n", + " if DRIVE_SAVE_DIR:\n", + " shutil.copy('ahad_quant_unified.zip', f'{DRIVE_SAVE_DIR}/ahad_quant_unified.zip')\n", + " print(f'💾 Sauvegardé sur Drive : {DRIVE_SAVE_DIR}/ahad_quant_unified.zip')\n", + "else:\n", + " print('❌ Export échoué')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nDiLmnW4CtLH" + }, + "outputs": [], + "source": [ + "# ── 5.2 Créer un bundle complet avec tous les modèles ────────────────────────\n", + "import zipfile, os, datetime, shutil\n", + "\n", + "timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M')\n", + "bundle_name = f'ahad_quant_trained_{timestamp}.zip'\n", + "\n", + "FILES_TO_BUNDLE = [\n", + " 'ahad_quant_unified.zip', # modèle unifié complet\n", + " 'model_ensemble.pkl', # modèle ML ensemble\n", + " 'rl_agent.zip', # agent RL PPO\n", + " 'rl_scaler.pkl', # scaler RL\n", + " 'backtest_results.json', # résultats backtest\n", + " 'last_retrain.json', # méta-infos entraînement\n", + "]\n", + "\n", + "print(f'📦 Création du bundle {bundle_name}...')\n", + "with zipfile.ZipFile(bundle_name, 'w', zipfile.ZIP_DEFLATED) as zf:\n", + " for f in FILES_TO_BUNDLE:\n", + " if os.path.exists(f):\n", + " zf.write(f)\n", + " print(f' + {f} ({os.path.getsize(f)/1024:.0f} KB)')\n", + " else:\n", + " print(f' ⚠️ {f} — non trouvé, ignoré')\n", + "\n", + " # Ajouter les checkpoints RL\n", + " if os.path.exists('rl_checkpoints'):\n", + " for root, dirs, files in os.walk('rl_checkpoints'):\n", + " for file in files:\n", + " fp = os.path.join(root, file)\n", + " zf.write(fp)\n", + " print(f' + rl_checkpoints/ ({len(os.listdir(\"rl_checkpoints\"))} fichiers)')\n", + "\n", + "size_mb = os.path.getsize(bundle_name) / 1024 / 1024\n", + "print(f'\\n✅ Bundle créé : {bundle_name} ({size_mb:.1f} MB)')\n", + "\n", + "if DRIVE_SAVE_DIR:\n", + " shutil.copy(bundle_name, f'{DRIVE_SAVE_DIR}/{bundle_name}')\n", + " print(f'💾 Sauvegardé sur Drive : {DRIVE_SAVE_DIR}/{bundle_name}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OIgijId7CtLH" + }, + "outputs": [], + "source": [ + "# ── 5.3 Téléchargement direct depuis Colab ───────────────────────────────────\n", + "from google.colab import files\n", + "import os\n", + "\n", + "# Choisir ce qu'on veut télécharger\n", + "TO_DOWNLOAD = [\n", + " bundle_name, # bundle complet (recommandé)\n", + " # 'ahad_quant_unified.zip', # ou juste le modèle unifié\n", + "]\n", + "\n", + "for f in TO_DOWNLOAD:\n", + " if os.path.exists(f):\n", + " size_mb = os.path.getsize(f) / 1024 / 1024\n", + " print(f'⬇️ Téléchargement de {f} ({size_mb:.1f} MB)...')\n", + " files.download(f)\n", + " else:\n", + " print(f'⚠️ {f} non trouvé')\n", + "\n", + "print('\\n✅ Terminé ! Place les fichiers .pkl et .zip dans ton dossier ahad_quant_v32_fixed/ sur ta machine.')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aVRssBSDCtLH" + }, + "source": [ + "## 🔁 BONUS — Relancer uniquement le RL après déconnexion\n", + "\n", + "Si Colab s'est déconnecté **pendant le RL** (étape 4), exécute ces cellules dans l'ordre :" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RsKDPLODCtLI" + }, + "outputs": [], + "source": [ + "# ── REPRISE RAPIDE (après déconnexion pendant le RL) ─────────────────────────\n", + "# 1. Monte Drive\n", + "from google.colab import drive\n", + "drive.mount('/content/drive')\n", + "\n", + "# 2. Réinstalle les packages\n", + "!pip install -q lightgbm xgboost scikit-learn optuna yfinance stable-baselines3[extra] gymnasium shimmy python-dotenv torch --index-url https://download.pytorch.org/whl/cu118\n", + "\n", + "# 3. Remonte le projet depuis Drive\n", + "import os, shutil, json\n", + "DRIVE_SAVE_DIR = '/content/drive/MyDrive/AhadQuant_Models'\n", + "PROJECT_DIR = '/content/ahad_quant_resume'\n", + "os.makedirs(PROJECT_DIR, exist_ok=True)\n", + "\n", + "# Copie le ZIP du projet depuis Drive (si tu l'avais mis là)\n", + "# OU remonte le ZIP original depuis upload\n", + "# from google.colab import files\n", + "# uploaded = files.upload() # ← décommente si besoin\n", + "\n", + "# 4. Copie les modèles existants depuis Drive\n", + "for f in ['model_ensemble.pkl', 'rl_agent.zip', 'rl_scaler.pkl', 'rl_progress.json']:\n", + " src = f'{DRIVE_SAVE_DIR}/{f}'\n", + " if os.path.exists(src):\n", + " shutil.copy(src, f'{PROJECT_DIR}/{f}')\n", + " print(f'✅ Restauré : {f}')\n", + "\n", + "if os.path.exists(f'{DRIVE_SAVE_DIR}/rl_checkpoints'):\n", + " shutil.copytree(f'{DRIVE_SAVE_DIR}/rl_checkpoints', f'{PROJECT_DIR}/rl_checkpoints', dirs_exist_ok=True)\n", + " print('✅ Restauré : rl_checkpoints/')\n", + "\n", + "# 5. Affiche la progression\n", + "prog_file = f'{PROJECT_DIR}/rl_progress.json'\n", + "if os.path.exists(prog_file):\n", + " with open(prog_file) as f:\n", + " prog = json.load(f)\n", + " steps_done = prog.get('steps_done', 0)\n", + " total = prog.get('total_steps', 1_000_000)\n", + " pct = steps_done / total * 100\n", + " print(f'\\n📊 Progression RL : {steps_done:,} / {total:,} steps ({pct:.1f}%)')\n", + " print('➡️ Maintenant exécute la cellule 4.2 — la reprise sera détectée automatiquement')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VcruETjFCtLI" + }, + "source": [ + "## 📋 RÉCAPITULATIF — Fichiers à récupérer\n", + "\n", + "Après l'entraînement, copie ces fichiers dans ton dossier `ahad_quant_v32_fixed/` sur ta machine Windows :\n", + "\n", + "| Fichier | Description | Obligatoire |\n", + "|---------|-------------|-------------|\n", + "| `ahad_quant_unified.zip` | Modèle complet (ML + RL) | ✅ |\n", + "| `model_ensemble.pkl` | Ensemble ML seul | ✅ |\n", + "| `rl_agent.zip` | Agent PPO seul | ✅ |\n", + "| `rl_scaler.pkl` | Scaler features RL | ✅ |\n", + "| `rl_checkpoints/` | Checkpoints intermédiaires | Optionnel |\n", + "\n", + "Ensuite, depuis le Web UI : **lance le bot** — il chargera `ahad_quant_unified.zip` automatiquement." + ] + }, + { + "cell_type": "code", + "source": [ + "!free -h" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HWq8mt_cc0pa", + "outputId": "e35c3e7a-7b80-4abf-84b0-7e5ce2d226c6" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + " total used free shared buff/cache available\n", + "Mem: 12Gi 1.1Gi 10Gi 2.0Mi 785Mi 11Gi\n", + "Swap: 0B 0B 0B\n" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e8b1536 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +MIT License + +Copyright (c) 2026 DeepAlpha (original project — github.com/stefanoviana/deepalpha) +Copyright (c) 2026 Abdoul Ahad Binizi (AHAD QUANT — Forex fork and extensions) + +This project is a fork of DeepAlpha (github.com/stefanoviana/deepalpha), an +open-source crypto trading bot, adapted and extended for Forex trading. See +README.md for a full breakdown of what was inherited from the original +project versus newly added in this fork. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PUBLISH_TO_GITHUB.bat b/PUBLISH_TO_GITHUB.bat new file mode 100644 index 0000000..aa82c64 --- /dev/null +++ b/PUBLISH_TO_GITHUB.bat @@ -0,0 +1,102 @@ +@echo off +setlocal + +echo ================================================ +echo AHAD QUANT - Publication automatique sur GitHub +echo ================================================ +echo. + +cd /d "%~dp0" +echo Dossier de travail : %cd% +echo. + +REM --- 1. Verifier que Git est installe --- +where git >nul 2>nul +if errorlevel 1 ( + echo [ERREUR] Git n'est pas installe sur ce PC. + echo. + echo Telechargez et installez "Git for Windows" ici : + echo https://git-scm.com/download/win + echo. + echo Laissez toutes les options par defaut pendant l'installation. + echo Une fois installe, relancez ce script en double-cliquant dessus. + echo. + pause + exit /b 1 +) +echo [OK] Git est installe. +echo. + +REM --- 2. Initialiser le depot si besoin --- +if exist ".git" ( + echo Depot Git deja initialise. +) else ( + echo Initialisation du depot Git... + git init + if errorlevel 1 ( + echo [ERREUR] git init a echoue. + pause + exit /b 1 + ) +) +echo. + +REM --- 3. Identite git locale (uniquement si pas deja configuree) --- +git config user.name >nul 2>nul +if errorlevel 1 git config user.name "Abdoul Ahad Binizi" +git config user.email >nul 2>nul +if errorlevel 1 git config user.email "abdoulahadbinizi+ahadquant@gmail.com" + +REM --- 4. Ajouter tous les fichiers --- +echo Ajout des fichiers du projet... +git add . +echo. + +REM --- 5. Commit (seulement s'il y a des changements) --- +git diff --cached --quiet +if errorlevel 1 ( + git commit -m "Initial commit - AHAD QUANT v1" +) else ( + echo Rien de nouveau a committer. +) +echo. + +REM --- 6. Branche principale --- +git branch -M main + +REM --- 7. Configurer le remote GitHub --- +git remote get-url origin >nul 2>nul +if errorlevel 1 ( + git remote add origin https://github.com/AhadQuant/ahad-quant.git +) else ( + git remote set-url origin https://github.com/AhadQuant/ahad-quant.git +) + +REM --- 8. Push vers GitHub --- +echo Envoi vers GitHub... +echo Une fenetre de connexion (navigateur) va probablement s'ouvrir. +echo Connectez-vous avec le compte AhadQuant si elle apparait. +echo. +git push -u origin main + +if errorlevel 1 ( + echo. + echo ================================================ + echo [ERREUR] Le push a echoue. + echo ================================================ + echo Causes possibles : + echo - La connexion dans la fenetre du navigateur a ete annulee + echo - Vous n'etes pas connecte au compte AhadQuant + echo - Le depot distant contient deja des fichiers en conflit + echo. + pause + exit /b 1 +) + +echo. +echo ================================================ +echo TERMINE +echo Verifiez ici : https://github.com/AhadQuant/ahad-quant +echo ================================================ +echo. +pause diff --git a/README.md b/README.md new file mode 100644 index 0000000..b1333bd --- /dev/null +++ b/README.md @@ -0,0 +1,155 @@ +# AHAD QUANT + +**ML + RL Forex trading system** — ensemble machine learning models, a reinforcement learning (PPO) decision layer, and a live MetaTrader 5 bridge, deployed with a documented demo track record. + +> ⚠️ **Disclaimer:** All performance numbers below come from a **demo / paper-trading account** (JustMarkets-Demo, simulated execution). They are not a live, real-money track record. Trading carries significant risk of loss; past backtest or demo performance does not guarantee future results. + +--- + +## Origin & Attribution + +AHAD QUANT started as a fork of **[DeepAlpha](https://github.com/stefanoviana/deepalpha)** (MIT License), an open-source crypto trading bot for Bybit/Binance. Several core modules — feature engineering, exchange adapter pattern, risk manager, model architectures (LightGBM/XGBoost ensemble, TFT, TransformerGRU), regime detection, pump scanner — originate from that project and were adapted rather than written from scratch. + +**What this fork adds on top of the original:** + +| Addition | Description | +|---|---| +| **Reinforcement learning layer** | Full PPO agent (`rl_agent.py`, `rl_env.py`, `rl_train.py`, `rl_reward.py`, `experience_buffer.py`, `online_learner.py`) — absent from the original, which was ML-only | +| **MetaTrader 5 bridge** | Python ↔ MT5 bridge (`mt5_bridge.py`) for live Forex execution — the original was crypto-exchange-only (ccxt/Bybit/Binance) | +| **Forex migration** | Full migration from crypto pairs to 25 Forex pairs, new data sourcing (OANDA / yfinance / MT5) | +| **Web UI dashboard** | `web_ui.py` — a FastAPI-based live dashboard, in addition to the original's basic Streamlit dashboard | +| **DCA & Grid bots** | `dca_bot.py`, `grid_bot.py` — additional strategy modules | +| **Ensemble unification layer** | `unified_brain.py`, `ensemble_core.py`, `train_unified.py`, `export_unified.py` — synchronizes the ML ensemble and RL agent into one decision pipeline | +| **Auto-retraining & monitoring** | `auto_retrain.py`, `daily_local_retrain.py`, `performance_monitor.py`, `backtest.py` | +| **Model training** | All models retrained from scratch on proprietary Forex datasets via Google Colab GPU — the trained weights are original, even where the model architecture code is inherited | +| **Live deployment & track record** | Deployed and run live on a demo account for 137+ days with the metrics documented below | + +This LICENSE retains the original copyright notice as required by the MIT license, with an additional notice for the modifications and additions described above. + +--- + +## Track Record (Demo Account) + +JustMarkets-Demo, USD, Hedge account — 137 days live (Feb 7 – Jun 25, 2026): + +| Metric | Value | +|---|---| +| Trades | 4,542 | +| Win Rate | 79.3% | +| Profit Factor | 33.98 | +| Sharpe Ratio (annualized) | 9.10 | +| Max Drawdown | 14.0% | +| Avg Win / Avg Loss | $39.38 / -$4.57 | +| Top pairs | BTCUSD.m, XAUUSD.m, BTCEUR.m, EURUSD.m, USDCAD.m | + +> Earlier "stable phase" testing (since June 4, $100 start) surfaced a configuration bug — max trade count and max margin usage were not capped — which produced a 35.9% drawdown over one volatile week. That cap is now enforced in `risk_manager.py` (`MAX_MARGIN_USAGE`, `MAX_POSITIONS`). + +--- + +## Architecture + +``` + ┌─────────────────────────┐ + Market Data ──────▶│ Feature Engineering │ + (OANDA/MT5/yf) │ (features.py) │ + └────────────┬─────────────┘ + ▼ + ┌─────────────────────────┐ + │ ML Ensemble │ + │ LightGBM + XGBoost │ + │ + TFT + TransformerGRU │ + └────────────┬─────────────┘ + ▼ + ┌─────────────────────────┐ + │ RL Agent (PPO) │◀── trained via + │ filters / overrides │ rl_train.py + │ ML signal │ + └────────────┬─────────────┘ + ▼ + ┌─────────────────────────┐ + │ Risk Manager │ + │ sizing, SL/TP, limits │ + └────────────┬─────────────┘ + ▼ + ┌─────────────────────────┐ + │ MT5 Bridge │──▶ Live execution + │ (mt5_bridge.py) │ on MetaTrader 5 + └─────────────────────────┘ + │ + ┌────────────┴─────────────┐ + │ Web UI (FastAPI) │ + │ live monitoring │ + └─────────────────────────┘ +``` + +--- + +## Tech Stack + +- **ML:** LightGBM, XGBoost, scikit-learn, PyTorch (TFT, TransformerGRU) +- **RL:** Stable-Baselines3 (PPO), Gymnasium +- **Execution:** MetaTrader 5 bridge, OANDA v20 REST API, ccxt (multi-broker) +- **Backend:** FastAPI, Python 3.10+ +- **Training:** Google Colab (GPU) — see `AHAD_QUANT_Colab_Training.ipynb` + +--- + +## Quick Start + +```bash +git clone https://github.com/AhadQuant/ahad-quant.git +cd ahad-quant +pip install -r requirements.txt +cp .env.example .env # fill in your broker credentials +python download_data.py # download historical Forex data +python train.py # train the ML ensemble +python ahad_quant.py # start trading (paper mode by default) +``` + +Paper mode (no real money, no broker needed): +```bash +PAPER_MODE=true python ahad_quant.py +``` + +**Windows:** double-click `START_AHAD_QUANT.bat` + +--- + +## Project Structure + +``` +ahad_quant.py # Main bot entry point +config.py # Centralized configuration +features.py # Feature engineering pipeline +risk_manager.py # Position sizing, SL/TP, circuit breaker + +rl_agent.py / rl_env.py # RL (PPO) layer +rl_train.py / rl_reward.py +experience_buffer.py +online_learner.py + +mt5_bridge.py # MetaTrader 5 execution bridge +exchange_adapter.py # Multi-broker adapter (OANDA, MT5, ccxt) +web_ui.py # FastAPI live dashboard +dashboard.py # Secondary Streamlit dashboard + +tft_model.py # Temporal Fusion Transformer +transformer_gru_model.py # TransformerGRU model +gnn_model.py # Graph neural network model +regime_detector.py # HMM market regime detection +ensemble_core.py / unified_brain.py # ML+RL unification layer + +train.py / train_unified.py / download_data.py / backtest.py +auto_retrain.py / daily_local_retrain.py / performance_monitor.py + +dca_bot.py / grid_bot.py # Additional strategy modules +pump_scanner.py / liquidation_levels.py / order_flow_analyzer.py + +AHAD_QUANT_Colab_Training.ipynb # GPU training pipeline (Colab) +``` + +--- + +## License + +MIT — see [LICENSE](LICENSE). This project is a fork of [DeepAlpha](https://github.com/stefanoviana/deepalpha) (MIT); see the **Origin & Attribution** section above. diff --git a/START_AHAD_QUANT.bat b/START_AHAD_QUANT.bat new file mode 100644 index 0000000..b2848c2 --- /dev/null +++ b/START_AHAD_QUANT.bat @@ -0,0 +1,165 @@ +@echo off +title AHAD QUANT - Lancement +color 0A + +echo. +echo ############################################################## +echo AHAD QUANT - Forex AI Trading Bot +echo (Lanceur v2 - installe TOUTES les dependances) +echo ############################################################## +echo. + +cd /d "%~dp0" + +:: --------------------------------------------------------------------- +:: 1. Verifier que Python est disponible +:: --------------------------------------------------------------------- +python --version >nul 2>&1 +if errorlevel 1 goto :err_python +echo [OK] Python detecte. +echo. +goto :check_files + +:err_python +echo [ERREUR] Python non trouve dans le PATH. +echo Installe Python 3.10+ depuis https://python.org +echo (coche bien "Add Python to PATH" pendant l'installation) +pause +exit /b 1 + +:: --------------------------------------------------------------------- +:: 2. Verifier la presence des fichiers necessaires +:: --------------------------------------------------------------------- +:check_files +if not exist "web_ui.py" goto :err_webui +if not exist "requirements.txt" goto :err_reqs +goto :setup_env + +:err_webui +echo [ERREUR] web_ui.py introuvable dans ce dossier. +echo Place ce script dans le meme dossier que web_ui.py. +pause +exit /b 1 + +:err_reqs +echo [ERREUR] requirements.txt introuvable dans ce dossier. +pause +exit /b 1 + +:: --------------------------------------------------------------------- +:: 3. Copier .env.example vers .env si .env n'existe pas encore +:: --------------------------------------------------------------------- +:setup_env +if exist ".env" goto :check_deps +if not exist ".env.example" goto :check_deps +echo [INFO] Aucun .env trouve - creation a partir de .env.example. +copy /y ".env.example" ".env" >nul +echo [INFO] Pense a verifier/adapter .env si besoin (onglet Config). +echo. + +:: --------------------------------------------------------------------- +:: 4. Installer TOUTES les dependances de requirements.txt +:: On ne reinstalle que si requirements.txt a change depuis la +:: derniere fois (marqueur), pour ne pas perdre de temps a chaque +:: lancement une fois que tout est deja installe. +:: --------------------------------------------------------------------- +:check_deps +set "MARKER=.last_requirements_installed.txt" +set "NEED_INSTALL=0" + +if not exist "%MARKER%" set "NEED_INSTALL=1" +if "%NEED_INSTALL%"=="1" goto :do_install + +fc "requirements.txt" "%MARKER%" >nul 2>&1 +if errorlevel 1 set "NEED_INSTALL=1" +if "%NEED_INSTALL%"=="1" goto :do_install + +echo [1/3] Dependances deja installees (requirements.txt inchange). +python -c "import fastapi, uvicorn" >nul 2>&1 +if errorlevel 1 goto :do_install +echo [1/3] OK +goto :launch_server + +:do_install +echo [1/3] Installation COMPLETE des dependances (requirements.txt)... +echo lightgbm, xgboost, scikit-learn, torch, stable-baselines3, +echo gymnasium, fastapi, uvicorn, watchdog, optuna, etc. +echo. +echo Premiere installation : 5 a 20 minutes selon la connexion +echo (torch et stable-baselines3[extra] sont volumineux). +echo Les lancements suivants seront quasi instantanes. +echo. + +python -m pip install --upgrade pip --quiet + +python -m pip install -r requirements.txt +if errorlevel 1 goto :err_install + +copy /y "requirements.txt" "%MARKER%" >nul +echo. +echo [1/3] Toutes les dependances sont installees. +goto :launch_server + +:err_install +echo. +echo [ERREUR] L'installation des dependances a echoue. +echo Causes possibles : pas de connexion internet, pip trop ancien, +echo ou un paquet incompatible avec ta version de Python/Windows. +echo. +echo Relance manuellement pour voir le detail de l'erreur : +echo pip install -r requirements.txt +pause +exit /b 1 + +:: --------------------------------------------------------------------- +:: 5. Lancer web_ui.py en arriere-plan +:: --------------------------------------------------------------------- +:launch_server +echo. +echo [2/3] Demarrage de l'interface web... +start "AHAD QUANT Server" /min python web_ui.py + +:: --------------------------------------------------------------------- +:: 6. Attendre que le serveur reponde reellement (sans goto imbrique +:: dans une boucle for - structure classique avec labels, plus sure +:: sur tous les Windows) +:: --------------------------------------------------------------------- +echo [3/3] Attente du demarrage du serveur... +set "WAITCOUNT=0" + +:wait_loop +curl -s -o nul http://localhost:8080 >nul 2>&1 +if not errorlevel 1 goto :server_ready +set /a WAITCOUNT+=1 +if %WAITCOUNT% GEQ 30 goto :wait_timeout +timeout /t 1 /nobreak >nul +goto :wait_loop + +:wait_timeout +echo [3/3] Le serveur met du temps a repondre - ouverture quand meme. +echo Si la page ne charge pas, attends et rafraichis le navigateur. +goto :open_browser + +:server_ready +echo [3/3] Serveur pret. + +:open_browser +start http://localhost:8080 + +echo. +echo ================================================================ +echo Interface disponible sur : http://localhost:8080 +echo Ferme cette fenetre pour ARRETER le serveur. +echo ================================================================ +echo. +echo Appuie sur une touche pour arreter AHAD QUANT... +pause >nul + +:: --------------------------------------------------------------------- +:: 7. Arreter le serveur a la fermeture +:: --------------------------------------------------------------------- +echo Arret en cours... +taskkill /f /fi "WINDOWTITLE eq AHAD QUANT Server" >nul 2>&1 +taskkill /f /im python.exe /fi "WINDOWTITLE eq AHAD QUANT Server" >nul 2>&1 +echo AHAD QUANT arrete. Au revoir. +timeout /t 2 /nobreak >nul diff --git a/ahad_quant.py b/ahad_quant.py new file mode 100644 index 0000000..24c6ef0 --- /dev/null +++ b/ahad_quant.py @@ -0,0 +1,991 @@ +""" +AHAD QUANT V11 — AI Forex Trading Bot +All Pro features: Ensemble model, HMM Regime, Paper Mode, +Session Scanner, Grid Bot, DCA Bot, Auto-Retrain. + +Supported brokers: OANDA (default), MetaTrader 5, ccxt Forex CFD. +Default pairs: EURUSD, GBPUSD, USDJPY, AUDUSD + 20 more. + +Usage: + 1. cp .env.example .env # add OANDA_API_KEY / MT5 credentials + 2. python download_data.py # download Forex historical data + 3. python train.py # train ensemble model on Forex data + 4. python ahad_quant.py # start trading + +Paper mode (no real money): + PAPER_MODE=true python ahad_quant.py +""" + +import sys, os + +# ── Fix encodage Windows (cp1252 → UTF-8) ──────────────────────────────────── +if sys.stdout.encoding and sys.stdout.encoding.lower() != "utf-8": + sys.stdout.reconfigure(encoding="utf-8") +if sys.stderr.encoding and sys.stderr.encoding.lower() != "utf-8": + sys.stderr.reconfigure(encoding="utf-8") + +# ── Logging (FIX BUG-RL-04 : log.info() du RL agent était avalé) ───────────── +# rl_agent.py utilise logging.getLogger("RLAgent").info(...) — sans basicConfig, +# le root logger reste au niveau WARNING par défaut et ces messages ne +# sortaient JAMAIS sur stdout/stderr, donc jamais dans web_ui.py non plus. +import logging +logging.basicConfig( + level=logging.INFO, + format="[%(name)s] %(message)s", + stream=sys.stdout, +) + +# ── Dependency check ───────────────────────────────────────────────────────── +_REQUIRED = { + "lightgbm": "lightgbm", + "numpy": "numpy", + "dotenv": "python-dotenv", +} +import importlib.util as _ilu +_missing = [p for m, p in _REQUIRED.items() if not _ilu.find_spec(m)] +if _missing: + print(f"[ERROR] Missing: {', '.join(_missing)}") + print(f"[ERROR] Run: pip install {' '.join(_missing)}") + sys.exit(1) + +if not os.path.exists(".env") and not os.environ.get("EXCHANGE"): + print("[ERROR] No .env file. Run: cp .env.example .env") + sys.exit(1) + +import os, pickle, time, traceback, threading +import numpy as np +import requests +import lightgbm as lgb + +import config +from features import build_features, FEATURE_NAMES +from risk_manager import RiskManager +from exchange_adapter import ExchangeAdapter, get_exchange +import unified_brain + +# ── RL Agent (optionnel) ────────────────────────────────────────────────────── +_RL_READY = False +try: + from rl_agent import get_rl_agent, PositionState, reset_rl_agent + _RL_READY = True +except ImportError: + pass + +# ── MT5 Bridge (mode CSV) ───────────────────────────────────────────────────── +HAS_MT5_BRIDGE = False +if config.MT5_BRIDGE_ENABLED or config.EXCHANGE.lower() == "mt5": + try: + from mt5_bridge import MT5Bridge, TradeReport + HAS_MT5_BRIDGE = True + except ImportError: + print("[WARNING] MT5Bridge non trouvé — vérifiez mt5_bridge/mt5_bridge.py") + +# ── Optional Pro modules ────────────────────────────────────────────────────── + +try: + from regime_detector import GaussianHMM + HAS_REGIME = True +except ImportError: + HAS_REGIME = False + +try: + from paper_trader import PaperTrader + HAS_PAPER = True +except ImportError: + HAS_PAPER = False + config.PAPER_MODE = False + +try: + from pump_scanner import create_pump_scanner_from_config + HAS_PUMP = True +except ImportError: + HAS_PUMP = False + +try: + from grid_bot import GridBot + HAS_GRID = True +except ImportError: + HAS_GRID = False + +try: + from dca_bot import DCABot + HAS_DCA = True +except ImportError: + HAS_DCA = False + +try: + from auto_retrain import AutoRetrainer + HAS_RETRAIN = True +except ImportError: + HAS_RETRAIN = False + +# ── Apprentissage Continu V7 (optionnel — fail-safe) ───────────────────────── +_CONTINUOUS_LEARNING = False +_buffer = None +_learner = None +_monitor = None +try: + if getattr(config, "CONTINUOUS_LEARNING_ENABLED", False): + from experience_buffer import get_experience_buffer + from online_learner import OnlineLearner + from performance_monitor import PerformanceMonitor, MonitorThread + _buffer = get_experience_buffer( + max_size = getattr(config, "EXPERIENCE_BUFFER_MAX_SIZE", 10000), + auto_load = True, + ) + _learner = OnlineLearner(_buffer) + _monitor = PerformanceMonitor(_buffer) + _CONTINUOUS_LEARNING = True + import logging as _cl_log + _cl_log.getLogger("ahad_quant").info( + f"[CL] Apprentissage continu activé — buffer: {_buffer.total_trades} trades" + ) +except Exception as _cl_err: + import logging as _cl_log + _cl_log.getLogger("ahad_quant").warning( + f"[CL] Apprentissage continu désactivé (erreur import) : {_cl_err}" + ) + +# ── Apprentissage Continu V7 — fonctions utilitaires module-level ──────────── + +def _build_entry_context( + coin: str, + signal: str, + confidence: float, + features, + price: float, + ml_probas=None, + rl_action=None, + rl_agreed=None, + regime=None, +) -> dict: + """ + Construit le dictionnaire de contexte au moment de l'ouverture d'un trade. + Stocké dans self._entry_context[coin] et récupéré à la fermeture pour + construire l'expérience complète dans le buffer. + """ + import time as _time + ctx = { + "coin": coin, + "signal": signal, + "confidence": confidence, + "price": price, + "timestamp": _time.time(), + } + if ml_probas is not None: + ctx["ml_probas"] = ml_probas + if rl_action is not None: + ctx["rl_action"] = rl_action + if rl_agreed is not None: + ctx["rl_agreed"] = rl_agreed + if regime is not None: + ctx["regime"] = regime + # Sauvegarder les features sous forme de liste (JSON-sérialisable). + # Accepte soit une seule ligne de features déjà extraite (1D — c'est ce + # que fournit unified_brain.Decision.features), soit un batch (2D, + # ancien format) dont on prend la dernière ligne. + if features is not None: + try: + if hasattr(features, "iloc"): + ctx["features"] = features.iloc[-1].tolist() + else: + arr = np.asarray(features) + ctx["features"] = arr[-1].tolist() if arr.ndim == 2 else arr.tolist() + except Exception: + ctx["features"] = None + return ctx + + +def _inject_closed_trade(coin: str, pnl: float, outcome: str, entry_ctx: dict) -> None: + """ + Appelée à chaque fermeture de trade (paper ou live). + [FIX] Découplage buffer / apprentissage : + - Le buffer et la calibration seuil tournent TOUJOURS (même si + CONTINUOUS_LEARNING_ENABLED=False) — les mauvais trades s'accumulent + sans interruption, indépendamment de la décision de remplacement modèle. + - Le warm-start LGB et le RL replay restent contrôlés par le flag. + Fail-safe : toute exception est attrapée, le bot continue normalement. + """ + try: + import time as _time + import logging as _cl_log + _log = _cl_log.getLogger("ahad_quant") + + experience = { + "coin": coin, + "pnl": pnl, + "outcome": outcome, # "WIN" | "LOSS" | "TIMEOUT" + "close_time": _time.time(), + **{k: v for k, v in entry_ctx.items() if k != "coin"}, + } + + # ── Accumulation buffer : TOUJOURS actif ────────────────────────── + # Même si CONTINUOUS_LEARNING_ENABLED=False, on stocke chaque trade + # pour que le prochain cycle (manuel ou automatique) ait un buffer + # plein et puisse apprendre des erreurs passées. + if _buffer is not None: + _buffer.add(experience) + _log.debug(f"[CL-BUFFER] Trade stocké : {coin} {outcome} PnL={pnl:+.2f}") + + # ── Calibration seuil : TOUJOURS active ─────────────────────────── + # MIN_CONFIDENCE s'ajuste après chaque trade, sans attendre un cycle + # de retrain. C'est la correction immédiate la plus légère qui soit. + if _learner is not None: + new_conf = _learner.calibrate_threshold(recent_n=50) + _log.debug(f"[CL-CALIB] conf seuil → {new_conf:.3f}") + + # ── Monitor drift : actif si learner disponible ──────────────────── + if _monitor is not None and _CONTINUOUS_LEARNING: + status = _monitor.check() + if status in ("DANGER", "EMERGENCY"): + _log.warning(f"[CL] Monitor : niveau {status} — vérifier les performances") + + except Exception as _cl_exc: + import logging as _cl_log + _cl_log.getLogger("ahad_quant").warning(f"[CL] _inject_closed_trade erreur (non-bloquant) : {_cl_exc}") + + +# ── Deep Learning inference (TFT + TransformerGRU) ──────────────────────────── +# Relocalisée dans ensemble_core.py (utilisée par unified_brain.py) — voir ce +# module pour le détail du bug #1 que cette centralisation corrige. ahad_quant.py +# n'a plus besoin d'importer torch/TFT/TGRU directement. + + +# ── Banner ──────────────────────────────────────────────────────────────────── + +def _banner(): + mode = "📄 PAPER MODE — Forex (aucun argent réel)" if config.PAPER_MODE else "💰 LIVE FOREX MODE" + print("\n" + "=" * 60) + print(" AHAD QUANT V11 — AI Forex Trading Bot") + print(f" {mode}") + print("=" * 60) + broker_label = f"{config.EXCHANGE} (MT5 CSV Bridge)" if config.MT5_BRIDGE_ENABLED else config.EXCHANGE + print(f" Broker: {broker_label}") + print(f" Ensemble: {'✓' if config.USE_ENSEMBLE else '✗'} (LightGBM+XGBoost+RF)") + print(f" HMM Regime: {'✓' if config.USE_REGIME_FILTER else '✗'}") + print(f" Session Scanner: {'✓' if config.PUMP_SCANNER_ENABLED else '✗'}") + print(f" Grid Bot: {'✓' if config.GRID_BOT_ENABLED else '✗'}") + print(f" DCA Bot: {'✓' if config.DCA_BOT_ENABLED else '✗'}") + print(f" Auto-Retrain: {'✓' if config.AUTO_RETRAIN_ENABLED else '✗'}") + # FIX BUG-RL-04 : le statut RL n'apparaissait jamais dans le banner. + # On force le chargement ici (au lieu d'attendre le 1er signal) pour + # avoir une confirmation immédiate et visible au démarrage. + rl_status = "✗ (désactivé — USE_RL_AGENT=false)" + if not _RL_READY: + rl_status = "✗ (module rl_agent.py introuvable / ImportError)" + elif config.USE_RL_AGENT: + try: + if get_rl_agent().is_ready(): + rl_status = "✓ (chargé — mode filter)" + else: + rl_status = "✗ (échec de chargement — voir logs [RLAgent] ci-dessus)" + except Exception as _rl_banner_err: + rl_status = f"✗ (exception au chargement : {_rl_banner_err})" + print(f" RL Agent: {rl_status}") + print("-" * 60 + "\n") + +_banner() + +# ── Model loading & ensemble prediction ────────────────────────────────────── +# Relocalisées dans unified_brain.py (UnifiedBrain._load_ml / .decide()), qui +# délègue le ML à ensemble_core.py — la source UNIQUE de vérité pour +# l'inférence d'ensemble, partagée avec rl_env.py et rl_agent.py. Corrige le +# point #1 du diagnostic : l'ancienne version ici pouvait lever une +# ValueError non-attrapée (model_data["scaler"].transform() avec un nombre +# de features incohérent) si has_dl=True mais l'historique trop court pour +# la branche DL — ensemble_core gère ce cas en interne, sans jamais +# remonter d'exception. + +# ── Bot live state (écrit dans bot_state.json à chaque loop) ────────────────── +_BOT_STATE: dict = { + "regime": 1, "regime_label": "NORMAL", + "risk": {}, "pump": {}, "timestamp": None, + "consecutive_losses": 0, "circuit_breaker_until": 0, + "daily_pnl": 0.0, "daily_pnl_pct": 0.0, "equity": 0.0, +} +_BOT_STATE_LOCK = threading.Lock() + +_REGIME_LABELS = {0: "CALM", 1: "NORMAL", 2: "VOLATILE"} + +# _detect_regime() relocalisée dans unified_brain.UnifiedBrain._detect_regime +# (même implémentation HMM exacte, juste portée par l'instance du brain au +# lieu d'un état module-level global). _scan_entries() met à jour +# _BOT_STATE["regime"]/["regime_label"] lui-même après chaque decide(), +# pour garder unified_brain.py découplé de l'état du bot. + + +# ── Main bot class ──────────────────────────────────────────────────────────── + +class AHAD QUANT: + + def __init__(self): + # Paper mode + self.paper: PaperTrader | None = None + if config.PAPER_MODE: + if not HAS_PAPER: + print("[ERROR] paper_trader.py not found") + sys.exit(1) + self.paper = PaperTrader(config.PAPER_INITIAL_BALANCE) + print("[PAPER] Paper trading active — no real orders will be placed") + + # Temporal exit tracking — LOOKAHEAD=3 candles (3h) + # Coin → unix timestamp of entry. Used to close positions that exceed + # the model's prediction horizon regardless of TP/SL status. + self._entry_times: dict[str, float] = {} + self._entry_context: dict = {} # CL V7 — contexte ouverture par coin + + # Cerveau unifié ML + RL + régime — source unique pour toute décision + # de trading (voir unified_brain.py). Remplace l'ancien + # self.model_data brut chargé via load_model(). + self.brain = unified_brain.get_brain() + if not self.brain.is_ready(): + if config.PAPER_MODE: + print("[PAPER] No model found — paper mode will use random signals for testing") + else: + print("\n" + "=" * 55) + print(" MODEL NOT FOUND") + print(" Run: python train.py") + print(" Or set PAPER_MODE=true to simulate without a model") + print("=" * 55 + "\n") + _ml_path = config.ENSEMBLE_MODEL_PATH if config.USE_ENSEMBLE else config.MODEL_PATH + raise FileNotFoundError(f"Model not found at {_ml_path}") + + # Exchange connection OU MT5 Bridge + self.mt5_bridge: "MT5Bridge | None" = None # type: ignore[name-defined] + + if config.MT5_BRIDGE_ENABLED or config.EXCHANGE.lower() == "mt5": + # ── Mode MT5 CSV Bridge ─────────────────────────────────────────── + if not HAS_MT5_BRIDGE: + print("[ERROR] MT5_BRIDGE_ENABLED=true mais mt5_bridge.py introuvable") + print("[ERROR] Vérifiez mt5_bridge/mt5_bridge.py") + sys.exit(1) + if not config.MT5_FILES_PATH: + print("[ERROR] MT5_FILES_PATH manquant dans .env") + print("[ERROR] Ex: MT5_FILES_PATH=C:/Users/NOM/AppData/.../MQL5/Files") + sys.exit(1) + + self.exchange = None # type: ignore[assignment] + self.mt5_bridge = MT5Bridge( + files_path=config.MT5_FILES_PATH, + # poll_interval géré en interne par _PollingWatcher (défaut 200ms) + ) + + # Callback : rapport reçu de l'EA (ordre ouvert ou fermé) + def _on_report(report: "TradeReport"): # type: ignore[name-defined] + icon = "✅" if report.status == "CLOSED" else ("🔴" if report.status == "ERROR" else "📋") + profit_str = f" | PnL: {report.profit:+.2f}" if report.profit else "" + msg = (f"{icon} [{report.status}] {report.signal_id} | " + f"ticket #{report.ticket}{profit_str}") + print(f" [MT5] {msg}") + + self.mt5_bridge.on_report_received(_on_report) + self.mt5_bridge.start() + print(f"[MT5] Bridge démarré → {config.MT5_FILES_PATH}") + else: + # ── Mode normal ccxt/OANDA ──────────────────────────────────────── + self.exchange: ExchangeAdapter = get_exchange(config.EXCHANGE) + self.exchange.connect() + + # Risk manager + self.risk = RiskManager() + + # Leverage — skipped en paper mode et en mode MT5 (géré côté EA) + if not config.PAPER_MODE and self.exchange is not None: + for coin in config.COINS: + try: self.exchange.set_leverage(coin, config.LEVERAGE) + except Exception: pass + + # Pump scanner + self._pump_scanner = None + if config.PUMP_SCANNER_ENABLED and HAS_PUMP: + try: + self._pump_scanner = create_pump_scanner_from_config() + if self._pump_scanner: + self._pump_scanner.start() + print("[PUMP] Pump scanner started") + except Exception as e: + print(f"[PUMP] Failed to start: {e}") + + # Grid bot + self._grid_bot = None + if config.GRID_BOT_ENABLED and HAS_GRID: + try: + self._grid_bot = GridBot(self.exchange) + t = threading.Thread(target=self._grid_bot.start, daemon=True) + t.start() + print(f"[GRID] Grid bot started ({config.GRID_STRATEGY})") + except Exception as e: + print(f"[GRID] Failed to start: {e}") + + # DCA bot + self._dca_bot = None + if config.DCA_BOT_ENABLED and HAS_DCA: + try: + self._dca_bot = DCABot(self.exchange) + t = threading.Thread(target=self._dca_bot.start, daemon=True) + t.start() + print(f"[DCA] DCA bot started ({config.DCA_STRATEGY})") + except Exception as e: + print(f"[DCA] Failed to start: {e}") + + # Auto-retrain + self._retrainer = None + if HAS_RETRAIN and config.AUTO_RETRAIN_ENABLED: + self._retrainer = AutoRetrainer(notify_fn=lambda msg: print(f"[RETRAIN] {msg}")) + self._retrainer.start() + print(f"[RETRAIN] Auto-retrain every {config.AUTO_RETRAIN_INTERVAL_HOURS}h") + + print("\nAHAD QUANT initialised successfully\n") + + # ── Position management ─────────────────────────────────────────────────── + + def _sync_positions(self): + if config.PAPER_MODE: + return # paper handles own state + if self.mt5_bridge is not None: + # En mode MT5, on se fie au status.csv écrit par l'EA + # Les positions sont trackées via les signaux en attente + return + positions = self.exchange.get_positions() + for pos in positions: + coin = pos["coin"] + if coin not in self.risk.open_positions: + self.risk.register_open(coin, pos["side"], pos["entry"], abs(pos["size"])) + active = {p["coin"] for p in positions} + for coin in list(self.risk.open_positions): + if coin not in active: + self.risk.open_positions.pop(coin, None) + + def _check_exits(self): + if self.mt5_bridge is not None: + # En mode MT5, l'EA gère les SL/TP nativement. + # Les fermetures sont rapportées via on_report_received (callback). + # Pas d'action nécessaire ici. + return + if config.PAPER_MODE and self.paper: + book_cache = {} + for coin in list(self.paper.positions): + try: + book = self.exchange.get_orderbook(coin) + price = book["mid"] + except Exception: + continue + + # ── Temporal exit : LOOKAHEAD candles → config.MAX_HOLD_CANDLES × 1h ──── + # check_sl_tp() already handles this via opened_at timestamp, + # but _entry_times covers the case where state was loaded from + # disk before this session started (no opened_at in memory). + elapsed = time.time() - self._entry_times.get(coin, time.time()) + if config.TIMEOUT_ENABLED and elapsed >= config.MAX_HOLD_CANDLES * 3600: + reason = "timeout" + else: + reason = self.paper.check_sl_tp(coin, price) + + if reason: + result = self.paper.close_position(coin, price, reason) + self._entry_times.pop(coin, None) + pnl = result.get("pnl", 0) + sign = "+" if pnl >= 0 else "" + label = '🛑 SL' if reason=='sl' else ('✅ TP' if reason=='tp' else '⏱ TIMEOUT') + print(f" [EXIT] {label} {coin} | PnL: {sign}{pnl:.2f} USDT [Paper]") + # ── Apprentissage Continu V7 — POINT 2 (fermeture paper) ── + _inject_closed_trade( + coin=coin, pnl=pnl, + outcome="WIN" if pnl > 0 else ("TIMEOUT" if reason=="timeout" else "LOSS"), + entry_ctx=self._entry_context.pop(coin, {}), + ) + return + + for coin in list(self.risk.open_positions): + try: + book = self.exchange.get_orderbook(coin) + price = book["mid"] + except Exception: + continue + + # ── Temporal exit : LOOKAHEAD candles → config.MAX_HOLD_CANDLES × 1h ───── + elapsed = time.time() - self._entry_times.get(coin, time.time()) + if config.TIMEOUT_ENABLED and elapsed >= config.MAX_HOLD_CANDLES * 3600: + exit_reason = "timeout" + else: + exit_reason = self.risk.check_exit(coin, price) + + if exit_reason: + pos = self.risk.open_positions[coin] + result = self.exchange.close_position(coin) + if result.get("success"): + pnl = self.risk.register_close(coin, price) + self._entry_times.pop(coin, None) + sign = "+" if pnl >= 0 else "" + label = "🛑 SL" if exit_reason == "sl" else ("✅ TP" if exit_reason == "tp" else "⏱ TIMEOUT") + msg = (f"{label} {coin} | " + f"{pos['side'].upper()} | PnL: {sign}{pnl:.2f} USDT") + print(f" [EXIT] {msg}") + + def _scan_entries(self): + if not self.brain.is_ready(): + return + + equity = (self.paper.get_balance() if config.PAPER_MODE + else self._mt5_get_equity() if self.mt5_bridge is not None + else self.exchange.get_balance()) + if equity <= 0: + return + + # Quick slot check — si déjà plein, inutile de scanner + n_positions = (len(self.paper.positions) if config.PAPER_MODE + else self._mt5_get_n_positions() if self.mt5_bridge is not None + else len(self.risk.open_positions)) + if n_positions >= config.MAX_POSITIONS: + return + + # ── Risk controls — appliqués identiquement en paper ET live ──────────── + if config.PAPER_MODE: + # Daily loss limit (paper) + if equity > 0 and (self.paper.daily_pnl / equity) <= -config.MAX_DAILY_LOSS_PCT: + print(f" [PAPER RISK] Daily loss limit hit ({config.MAX_DAILY_LOSS_PCT*100:.1f}%)") + return + # Circuit breaker (paper) + if time.time() < self.paper.circuit_breaker_until: + remaining = int(self.paper.circuit_breaker_until - time.time()) + print(f" [PAPER RISK] Circuit breaker active ({remaining}s left)") + return + elif self.mt5_bridge is not None: + # En mode MT5, le risk manager n'est pas alimenté en continu. + # Vérification basique : equity > 0 suffit (déjà vérifiée au-dessus). + pass + else: + can_open, reason = self.risk.can_open(equity) + if not can_open: + print(f" [RISK] {reason}") + return + + # Paire de référence pour la corrélation (remplace BTC en Forex) + # EURUSD est la paire dominante — utilisée comme proxy de sentiment global + _ref_pair = "EURUSD" + try: + if self.mt5_bridge is not None: + btc_candles = None # pas d'exchange dispo pour les candles de référence + else: + btc_candles = self.exchange.get_candles(_ref_pair, "1h", 200) + except Exception: + btc_candles = None + + # ── Collecte tous les signaux, tri par confiance (= comportement backtest) ── + candidates = [] + coin_decisions: dict[str, unified_brain.Decision] = {} + if self.mt5_bridge is not None: + # En mode MT5 : on considère les paires sans signal en attente + # Bug #22b fix : get_open_signals() peut retourner des objets Signal + # (dataclass/namedtuple) OU des dicts selon la version de mt5_bridge. + # On gère les deux cas pour éviter le TypeError "not subscriptable". + def _get_pair(s): + try: + return s["pair"] # dict + except (TypeError, KeyError): + return getattr(s, "pair", "") # dataclass / namedtuple + # On stocke les paires SANS suffixe pour comparaison avec config.COINS + _suffix = config.MT5_SYMBOL_SUFFIX + pending_pairs = { + _get_pair(s).removesuffix(_suffix) + for s in self.mt5_bridge.get_open_signals() + } + open_set = pending_pairs + else: + open_set = (set(self.paper.positions) if config.PAPER_MODE + else set(self.risk.open_positions)) + + for coin in config.COINS: + if coin in open_set: + continue + + try: + if self.mt5_bridge is not None: + # ── Mode MT5 : données via yfinance (gratuit, sans clé API) ── + import yfinance as yf + ticker_sym = f"{coin}=X" + ticker = yf.Ticker(ticker_sym) + df = ticker.history(period="30d", interval="1h", + auto_adjust=True, prepost=False) + if df is None or df.empty or len(df) < 50: + continue + candles = [ + { + "t": int(ts.timestamp() * 1000), + "o": float(row["Open"]), + "h": float(row["High"]), + "l": float(row["Low"]), + "c": float(row["Close"]), + "v": float(row.get("Volume", 0)), + } + for ts, row in df.iterrows() + ] + candles.sort(key=lambda x: x["t"]) + candles = candles[-200:] # garder les 200 dernières bougies + else: + candles = self.exchange.get_candles(coin, "1h", 200) + except Exception: + continue + if not candles or len(candles) < 50: + continue + + try: + funding = self.exchange.get_funding_rate(coin) if self.exchange else 0.0 + except Exception: + funding = 0.0 + + signal, confidence = None, None # placeholders, écrasés ci-dessous + pos_state = PositionState( + in_position = coin in ( + self.paper.positions if config.PAPER_MODE + else self.risk.open_positions + ), + direction = 0, + balance_ratio = 1.0, + ) if _RL_READY else None + + decision = self.brain.decide( + candles, btc_candles=btc_candles, funding=funding, + position_state=pos_state, pair=coin, + ) + signal, confidence = decision.signal, decision.confidence + if signal == "neutral": + continue + + # Réplique l'ancien effet de bord de _detect_regime() : l'état + # global du bot reflète le régime détecté au dernier scan. + with _BOT_STATE_LOCK: + _BOT_STATE["regime"] = decision.regime + _BOT_STATE["regime_label"] = decision.regime_label + + if decision.rl_used and decision.rl_signal != decision.ml_signal: + print(f"[RL-FILTER] {coin}: ML={decision.ml_signal}({decision.ml_confidence:.2f}) " + f"→ RL={decision.rl_signal}({decision.rl_confidence:.2f})") + + try: + if self.mt5_bridge is not None: + # En mode MT5, self.exchange est None — prix depuis le dernier candle yfinance + price = candles[-1]["c"] + else: + book = self.exchange.get_orderbook(coin) + price = book["mid"] + except Exception: + continue + + coin_decisions[coin] = decision + candidates.append((confidence, coin, signal, price)) + + # Trier par confiance décroissante — les meilleurs signaux passent en premier + candidates.sort(key=lambda x: x[0], reverse=True) + + for confidence, coin, signal, price in candidates: + # Re-vérifier les slots disponibles à chaque itération + if self.mt5_bridge is not None: + n_open = self._mt5_get_n_positions() + elif config.PAPER_MODE: + n_open = len(self.paper.positions) + else: + n_open = len(self.risk.open_positions) + + if n_open >= config.MAX_POSITIONS: + break + + if not config.PAPER_MODE and self.mt5_bridge is None: + can_open, _ = self.risk.can_open(equity) + if not can_open: + break + + qty = self.risk.calc_quantity(equity, price) + msg = (f"🚀 OPEN {signal.upper()} `{coin}` | " + f"Price: {price:.4f} | Conf: {confidence:.1%}") + + if config.PAPER_MODE: + result = self.paper.open_position( + coin, signal, price, qty, + sl_pct=config.STOP_LOSS_PCT, + tp_pct=config.TAKE_PROFIT_PCT, + ) + if result.get("success"): + self._entry_times[coin] = time.time() + print(f" [PAPER TRADE] {msg}") + # ── CL V7 — sauvegarde contexte ouverture ── + decision = coin_decisions.get(coin) + self._entry_context[coin] = _build_entry_context( + coin=coin, signal=signal, confidence=confidence, + features=(decision.features if decision else None), + ml_probas=(decision.ml_proba if decision else None), + rl_action=(decision.rl_action if decision else None), + rl_agreed=(decision.rl_agreed if decision else None), + regime=(decision.regime_label if decision else None), + price=price, + ) + elif self.mt5_bridge is not None: + # ── Mode MT5 : écriture dans signals.csv ───────────────────── + # Bug #23 fix : ajouter le suffixe broker (ex ".m") au symbole. + # config.COINS contient "USDJPY", le broker attend "USDJPY.m". + mt5_pair = coin + config.MT5_SYMBOL_SUFFIX + lot = self._calc_lot_mt5(qty) + sl_pips = self._pct_to_pips(coin, price, config.STOP_LOSS_PCT) + tp_pips = self._pct_to_pips(coin, price, config.TAKE_PROFIT_PCT) + action = "BUY" if signal == "long" else "SELL" + sig = self.mt5_bridge.write_signal( + pair=mt5_pair, + action=action, + lot=lot, + sl_pips=sl_pips, + tp_pips=tp_pips, + confidence=confidence, + ) + if sig: + self._entry_times[coin] = time.time() + print(f" [MT5 SIGNAL] {msg} | lot={lot} SL={sl_pips}p TP={tp_pips}p → {sig.signal_id} (symbol={mt5_pair})") + decision = coin_decisions.get(coin) + self._entry_context[coin] = _build_entry_context( + coin=coin, signal=signal, confidence=confidence, + features=(decision.features if decision else None), + ml_probas=(decision.ml_proba if decision else None), + rl_action=(decision.rl_action if decision else None), + rl_agreed=(decision.rl_agreed if decision else None), + regime=(decision.regime_label if decision else None), + price=price, + ) + else: + side_str = "buy" if signal == "long" else "sell" + result = self.exchange.place_market_order(coin, side_str, qty) + if result.get("success"): + self.risk.register_open(coin, signal, price, qty) + self._entry_times[coin] = time.time() + print(f" [TRADE] {msg}") + decision = coin_decisions.get(coin) + self._entry_context[coin] = _build_entry_context( + coin=coin, signal=signal, confidence=confidence, + features=(decision.features if decision else None), + ml_probas=(decision.ml_proba if decision else None), + rl_action=(decision.rl_action if decision else None), + rl_agreed=(decision.rl_agreed if decision else None), + regime=(decision.regime_label if decision else None), + price=price, + ) + + time.sleep(0.5) + + # ── MT5 helpers ─────────────────────────────────────────────────────────── + + def _write_bot_state(self, equity: float = 0.0, daily_pnl: float = 0.0) -> None: + """Écrit l'état live du bot dans bot_state.json (lu par web_ui.py).""" + try: + risk_data = {} + if not config.PAPER_MODE and self.mt5_bridge is None and hasattr(self, "risk"): + r = self.risk + cb_remaining = max(0, int(r.circuit_breaker_until - time.time())) if time.time() < r.circuit_breaker_until else 0 + used_margin = sum(p["entry"] * p["qty"] for p in r.open_positions.values()) if equity > 0 else 0 + risk_data = { + "circuit_breaker_active": time.time() < r.circuit_breaker_until, + "circuit_breaker_remaining_s": cb_remaining, + "consecutive_losses": r.consecutive_losses, + "daily_pnl": round(r.daily_pnl, 2), + "daily_pnl_pct": round(r.daily_pnl / equity * 100, 2) if equity > 0 else 0, + "margin_used": round(used_margin, 2), + "margin_used_pct": round(used_margin / equity * 100, 1) if equity > 0 else 0, + "daily_loss_limit_pct": config.MAX_DAILY_LOSS_PCT * 100, + "max_margin_usage_pct": config.MAX_MARGIN_USAGE * 100, + "open_positions": len(r.open_positions), + "max_positions": config.MAX_POSITIONS, + } + elif config.PAPER_MODE and self.paper: + p = self.paper + cb_remaining = max(0, int(p.circuit_breaker_until - time.time())) if hasattr(p, "circuit_breaker_until") and time.time() < p.circuit_breaker_until else 0 + risk_data = { + "circuit_breaker_active": hasattr(p, "circuit_breaker_until") and time.time() < p.circuit_breaker_until, + "circuit_breaker_remaining_s": cb_remaining, + "consecutive_losses": getattr(p, "consecutive_losses", 0), + "daily_pnl": round(getattr(p, "daily_pnl", 0), 2), + "daily_pnl_pct": round(getattr(p, "daily_pnl", 0) / equity * 100, 2) if equity > 0 else 0, + "margin_used": 0, "margin_used_pct": 0, + "daily_loss_limit_pct": config.MAX_DAILY_LOSS_PCT * 100, + "max_margin_usage_pct": config.MAX_MARGIN_USAGE * 100, + "open_positions": len(getattr(p, "positions", {})), + "max_positions": config.MAX_POSITIONS, + } + + pump_data = {} + if self._pump_scanner is not None: + ps = self._pump_scanner + open_pumps = {} + try: + open_pumps = {k: { + "side": v.side if hasattr(v, "side") else "?", + "entry": round(v.entry_price if hasattr(v, "entry_price") else 0, 5), + "score": round(v.signal.confidence if hasattr(v, "signal") and v.signal else 0, 3), + } for k, v in ps.pump_positions.items()} + except Exception: + pass + pump_data = { + "active": True, + "open_positions": open_pumps, + "open_count": len(open_pumps), + "daily_pnl": round(getattr(ps, "_daily_pump_pnl", 0), 2), + } + + with _BOT_STATE_LOCK: + _BOT_STATE["equity"] = round(equity, 2) + _BOT_STATE["daily_pnl"] = round(daily_pnl, 2) + _BOT_STATE["risk"] = risk_data + _BOT_STATE["pump"] = pump_data + _BOT_STATE["timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + state_copy = dict(_BOT_STATE) + + state_path = os.path.join(os.path.dirname(__file__), "bot_state.json") + with open(state_path, "w") as f: + json.dump(state_copy, f, indent=2) + except Exception as _bse: + pass # Non-bloquant + + def _mt5_get_equity(self) -> float: + """Retourne l'équité du compte MT5 depuis status.csv (0 si indisponible).""" + if self.mt5_bridge is None: + return 0.0 + try: + status = self.mt5_bridge.read_status() + return status.equity if status else 0.0 + except Exception: + return 0.0 + + def _mt5_get_n_positions(self) -> int: + """Retourne le nombre de positions ouvertes côté MT5.""" + if self.mt5_bridge is None: + return 0 + try: + status = self.mt5_bridge.read_status() + if status: + return status.open_positions + # Fallback : compte les signaux en attente (sans rapport de clôture) + return len(self.mt5_bridge.get_open_signals()) + except Exception: + return 0 + + def _calc_lot_mt5(self, qty: float) -> float: + """Convertit une quantité en unités vers des lots MT5 (1 lot = 100 000 unités).""" + lot = qty / config.UNITS_PER_LOT + lot = max(config.MIN_LOT_SIZE, min(config.MAX_LOT_SIZE, lot)) + return round(lot, 2) + + def _pct_to_pips(self, pair: str, price: float, pct: float) -> int: + """ + Convertit un pourcentage de move en nombre de pips. + Paires JPY : 1 pip = 0.01 → facteur 100 + Autres : 1 pip = 0.0001 → facteur 10 000 + Note : pair peut être le nom de base (USDJPY) ou avec suffixe (USDJPY.m). + """ + base = pair.removesuffix(config.MT5_SYMBOL_SUFFIX) + factor = 100 if base.endswith("JPY") else 10_000 + return max(1, int(price * pct * factor)) + + # ── Main loop ───────────────────────────────────────────────────────────── + + def run(self): + print("=" * 60) + print(f"AHAD QUANT — Main loop | {config.EXCHANGE} | " + f"{'Paper' if config.PAPER_MODE else 'Live'}") + print(f"Scanning {len(config.COINS)} pairs every {config.MAIN_LOOP_SECONDS}s") + print("=" * 60 + "\n") + + day_reset_hour = -1 + + while True: + try: + loop_start = time.time() + + # ── V26 : contrôles Web UI (fichiers flag) ────────────────── + if os.path.exists(os.path.join(os.path.dirname(__file__), ".stopped")): + print("[AHAD QUANT] ⛔ Arrêt demandé via Web UI (.stopped) — arrêt propre.") + break + if os.path.exists(os.path.join(os.path.dirname(__file__), ".paused")): + print("[AHAD QUANT] ⏸ Bot en pause (Web UI) — scan suspendu, exits actifs.") + self._check_exits() + time.sleep(max(1, config.MAIN_LOOP_SECONDS)) + continue + # ──────────────────────────────────────────────────────────── + + now = time.strftime("%Y-%m-%d %H:%M:%S") + hour = int(time.strftime("%H")) + + # Reset daily PnL at midnight + if hour == 0 and hour != day_reset_hour: + if config.PAPER_MODE and self.paper: + self.paper.reset_daily_pnl() + elif hasattr(self.risk, "reset_daily"): + self.risk.reset_daily() + day_reset_hour = hour + + if config.PAPER_MODE and self.paper: + equity = self.paper.get_balance() + n_pos = len(self.paper.positions) + daily = self.paper.daily_pnl + elif self.mt5_bridge is not None: + status = self.mt5_bridge.read_status() + equity = status.equity if status else 0.0 + n_pos = status.open_positions if status else self._mt5_get_n_positions() + daily = status.daily_pnl if status else 0.0 + else: + equity = self.exchange.get_balance() + n_pos = len(self.risk.open_positions) + daily = self.risk.daily_pnl + + sign = "+" if daily >= 0 else "" + icon = "📄" if config.PAPER_MODE else ("🔗" if self.mt5_bridge else "💰") + print(f"[{now}] {icon} " + f"${equity:,.2f} | Pos: {n_pos}/{config.MAX_POSITIONS} | " + f"Daily: {sign}${daily:,.2f}") + + if not config.PAPER_MODE: + self._sync_positions() + self.brain.maybe_reload() # hot-reload ML+RL si un ré-entraînement a eu lieu (corrige #4) + self._check_exits() + self._scan_entries() + self._write_bot_state(equity, daily) + + elapsed = time.time() - loop_start + time.sleep(max(1, config.MAIN_LOOP_SECONDS - elapsed)) + + except KeyboardInterrupt: + print("\nShutting down...") + if config.PAPER_MODE and self.paper: + print(self.paper.summary()) + if self._retrainer: + self._retrainer.stop() + break + + except Exception as e: + print(f"[ERROR] {e}") + traceback.print_exc() + time.sleep(30) + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + print(r""" + ____ ___ __ __ + / __ \___ ___ ____ / | / /___ / /_ ____ _ + / / / / _ \/ _ \/ __ \/ /| | / / __ \/ __ \/ __ `/ + / /_/ / __/ __/ /_/ / ___ |/ / /_/ / / / / /_/ / +/_____/\___/\___/ .___/_/ |_/_/ .___/_/ /_/\__,_/ + /_/ /_/ + PRO Edition + """) + try: + bot = AHAD QUANT() + bot.run() + except KeyboardInterrupt: + print("\nShutdown complete.") + except FileNotFoundError as e: + print(f"\n[ERROR] {e}") + print("Run: python train.py") + time.sleep(30) + sys.exit(1) + except Exception as e: + print(f"\n[ERROR] {e}") + traceback.print_exc() + time.sleep(30) + sys.exit(1) diff --git a/ahad_quant.service b/ahad_quant.service new file mode 100644 index 0000000..d28bc94 --- /dev/null +++ b/ahad_quant.service @@ -0,0 +1,24 @@ +[Unit] +Description=AHAD QUANT — Web Command Center +After=network.target +Wants=network-online.target + +[Service] +Type=simple +# ⚠️ Sécurité : ne pas utiliser root. Créer un utilisateur dédié : +# sudo useradd -r -s /bin/false -d /opt/ahad_quant ahad_quant +# sudo chown -R ahad_quant:ahad_quant /opt/ahad_quant +User=ahad_quant +WorkingDirectory=/opt/ahad_quant +ExecStart=/usr/bin/python3 /opt/ahad_quant/web_ui.py +Restart=always +RestartSec=10 +StandardOutput=journal +StandardError=journal +Environment=PYTHONUNBUFFERED=1 +# Limites de sécurité +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/ahad_quant_dashboard.jsx b/ahad_quant_dashboard.jsx new file mode 100644 index 0000000..6eeec9f --- /dev/null +++ b/ahad_quant_dashboard.jsx @@ -0,0 +1,1399 @@ +import { useState, useEffect, useCallback } from "react"; +import { + AreaChart, Area, XAxis, YAxis, Tooltip, ResponsiveContainer, + BarChart, Bar, LineChart, Line, ReferenceLine +} from "recharts"; + +// ── Constants ───────────────────────────────────────────────────────────────── +const COINS = ["BTC","ETH","SOL","BNB","DOGE","AVAX","LINK","ARB","OP","APT", + "SUI","INJ","TIA","WLD","NEAR","FET","AAVE","DOT","ADA","XRP", + "LTC","BCH","CRV","ONDO","ENA","JUP","RENDER"]; +const EXCHANGES = ["Bybit","Binance","OKX","Gate.io","KuCoin","Bitget","HTX","MEXC","BingX"]; +const GRID_STRATEGIES = ["neutral","long","short","trend","reverse"]; +const DCA_STRATEGIES = ["classic","aggressive","safe","trend","reverse"]; +const REGIMES = ["BULL","BEAR","SIDEWAYS"]; + +// ── Palette ──────────────────────────────────────────────────────────────────── +const C = { + bg:"#020b14", surface:"#05111f", card:"#071626", + border:"#0c2035", border2:"#102840", + accent:"#00d4aa", blue:"#1a8cff", purple:"#9b5de5", + danger:"#ff3358", warn:"#ffb700", success:"#00e676", + text:"#b8cfe8", muted:"#3a5872", dim:"#102030", + glow:"#00d4aa33", +}; + +// ── CSS ──────────────────────────────────────────────────────────────────────── +const CSS = ` +@import url('https://fonts.googleapis.com/css2?family=Orbitron:wght@400;700;900&family=Rajdhani:wght@300;400;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap'); +*{box-sizing:border-box;margin:0;padding:0} +body{background:${C.bg}} +::-webkit-scrollbar{width:3px;height:3px} +::-webkit-scrollbar-track{background:${C.bg}} +::-webkit-scrollbar-thumb{background:${C.border2};border-radius:2px} +@keyframes blink{0%,100%{opacity:1}50%{opacity:0.15}} +@keyframes pulse-ring{0%,100%{box-shadow:0 0 0 0 ${C.glow},0 0 8px ${C.accent}44}50%{box-shadow:0 0 0 6px transparent,0 0 20px ${C.accent}66}} +@keyframes slidein{from{transform:translateY(-6px);opacity:0}to{transform:translateY(0);opacity:1}} +@keyframes pump-flash{0%,100%{background:rgba(255,179,0,.06)}50%{background:rgba(255,179,0,.15)}} +@keyframes danger-flash{0%,100%{background:rgba(255,51,88,.06)}50%{background:rgba(255,51,88,.14)}} +@keyframes hum{0%{opacity:.6}50%{opacity:1}100%{opacity:.6}} +@keyframes spin{from{transform:rotate(0deg)}to{transform:rotate(360deg)}} +@keyframes bar-grow{from{transform:scaleX(0)}to{transform:scaleX(1)}} +.live-dot{animation:blink 1.3s ease-in-out infinite} +.pulse{animation:pulse-ring 2.5s ease-in-out infinite} +.pump{animation:pump-flash 2s ease-in-out infinite} +.dump{animation:danger-flash 2s ease-in-out infinite} +.hum{animation:hum 3s ease-in-out infinite} +.spin{animation:spin 8s linear infinite} +.new-signal{animation:slidein .3s ease-out} +.nav-btn:hover{background:rgba(0,212,170,.07)!important;border-left-color:${C.accent}55!important} +input[type=range]{-webkit-appearance:none;appearance:none;height:3px;background:${C.dim};border-radius:2px;outline:none} +input[type=range]::-webkit-slider-thumb{-webkit-appearance:none;width:12px;height:12px;border-radius:50%;background:${C.accent};cursor:pointer;box-shadow:0 0 6px ${C.accent}99} +input[type=number]{background:${C.dim};border:1px solid ${C.border2};color:${C.text};padding:3px 8px;border-radius:4px;font-family:'JetBrains Mono';font-size:11px;width:70px;outline:none} +input[type=number]:focus{border-color:${C.accent}88} +select{background:${C.dim};border:1px solid ${C.border2};color:${C.text};padding:3px 8px;border-radius:4px;font-family:'JetBrains Mono';font-size:11px;outline:none;cursor:pointer} +select:focus{border-color:${C.accent}88} +button{cursor:pointer;font-family:'Rajdhani';letter-spacing:1px} +button:hover{filter:brightness(1.15)} +`; + +// ── Helpers ──────────────────────────────────────────────────────────────────── +const rand = (a,b) => Math.random()*(b-a)+a; +const pick = arr => arr[Math.floor(Math.random()*arr.length)]; +const fmt2 = n => n.toFixed(2); +const fmtK = n => n >= 1000 ? `${(n/1000).toFixed(1)}K` : n.toFixed(0); + +function genEquity(n=80){ + let v=10000; + const now=Date.now(); + return Array.from({length:n},(_,i)=>{ + v*=(1+rand(-0.0045,0.006)); + const d=new Date(now-(n-i)*3600000); + return{t:`${d.getMonth()+1}/${d.getDate()} ${d.getHours()}h`,v:Math.round(v*100)/100}; + }); +} + +function genSignal(){ + const coin=pick(COINS),dir=Math.random()>.5?"LONG":"SHORT"; + const conf=+(rand(0.71,0.935)).toFixed(3); + return{ + id:Date.now()+Math.random(),coin,dir,conf, + time:new Date().toLocaleTimeString("fr-FR",{hour:"2-digit",minute:"2-digit",second:"2-digit"}), + regime:pick(REGIMES),exec:conf>0.80 + }; +} + +// ── Atom components ──────────────────────────────────────────────────────────── +function Badge({text,color=C.accent,small}){ + return( + + {text} + + ); +} + +function Toggle({on,toggle,label,color=C.accent}){ + return( +
+
+
+
+ {label&&{label}} +
+ ); +} + +function KPI({label,value,sub,color=C.text,glow}){ + return( +
+
{label}
+
{value}
+ {sub&&
{sub}
} +
+ ); +} + +function SectionTitle({children}){ + return( +
+
+ {children} +
+
+ ); +} + +function Panel({children,style={},accentColor}){ + return( +
+ {children} +
+ ); +} + +function Row({l,v,color,mono=true}){ + return( +
+ {l} + {v} +
+ ); +} + +// Confidence arc SVG +function ConfArc({value,size=140}){ + const r=50,cx=size/2,cy=size/2; + const circ=2*Math.PI*r; + const arc=circ*0.75; // 270° arc + const filled=arc*value; + const color=value>0.85?C.success:value>0.72?C.accent:C.warn; + return( + + + + {(value*100).toFixed(1)}% + CONFIDENCE + =0.72?C.success:C.warn} fontSize={8} fontFamily="Rajdhani"> + {value>=0.72?"▶ SIGNAL VALID":"⏸ BELOW THRESHOLD"} + + + ); +} + +// Signal feed row +function SigRow({s}){ + const isL=s.dir==="LONG",col=isL?C.success:C.danger; + return( +
+ {s.coin} + +
+
+
0.85?C.success:C.accent,borderRadius:2,transition:"width .5s"}}/> +
+ {(s.conf*100).toFixed(1)}% +
+ + {s.time} + {s.exec?"▶ EXEC":"— SKIP"} +
+ ); +} + +// Position card +function PosCard({p,onClose}){ + const isL=p.dir==="LONG",col=isL?C.success:C.danger; + const pnlColor=p.pnl>=0?C.success:C.danger; + return( +
+
+
+ {p.coin} + + {p.age} +
+
+ + {p.pnl>=0?"+":""}{fmt2(p.pnl)} USDT + + {onClose&&} +
+
+
+ {[["Entry",`$${p.entry.toLocaleString()}`],["Current",`$${p.current.toLocaleString()}`], + ["SL",`$${p.sl.toLocaleString()}`],["TP",`$${p.tp.toLocaleString()}`]].map(([l,v])=>( +
+
{l}
+
{v}
+
+ ))} +
+
+ CONF +
+
+
+ {(p.conf*100).toFixed(0)}% + + Lev {p.lev}x + +
+
+ ); +} + +// ── Main Component ───────────────────────────────────────────────────────────── +export default function AHAD QUANT(){ + const [tab,setTab]=useState("overview"); + const [clock,setClock]=useState(new Date()); + + // Bot toggles + const [botOn,setBotOn]=useState(true); + const [gridOn,setGridOn]=useState(false); + const [dcaOn,setDcaOn]=useState(false); + const [pumpOn,setPumpOn]=useState(true); + const [paperMode,setPaperMode]=useState(true); + const [exchange,setExchange]=useState("Bybit"); + const [retrainOn,setRetrainOn]=useState(false); + const [regimeFilter,setRegimeFilter]=useState(false); + + // Config + const [cfg,setCfg]=useState({ + leverage:5,risk:3,sl:1.5,tp:2.2,maxPos:5,maxLoss:5,minConf:72, + cbLosses:3,gridLevels:10,gridUsdt:200,gridCoin:"BTC",gridStrategy:"neutral", + dcaBase:50,dcaSafety:30,dcaMaxSafety:5,dcaDev:1.5,dcaTp:2,dcaCoin:"BTC", + pumpVol:5,pumpPrice:3,pumpLev:5,pumpRisk:5, + retrainInterval:24,retrainMinAcc:55, + maxHold:3,compound:true + }); + const setC=useCallback((k,v)=>setCfg(p=>({...p,[k]:v})),[]); + + // Live data + const [balance,setBalance]=useState(10847.23); + const [dailyPnL,setDailyPnL]=useState(234.56); + const [equity,setEquity]=useState(()=>genEquity()); + const [signals,setSignals]=useState(()=>[...Array(6)].map(genSignal)); + const [positions,setPositions]=useState([ + {coin:"BTC",dir:"LONG",entry:67420,current:67892,size:0.015, + pnl:45.23,conf:0.847,age:"2h 14m",sl:66408,tp:68904,lev:5}, + {coin:"ETH",dir:"SHORT",entry:3521.4,current:3534.2,size:0.12, + pnl:-12.45,conf:0.781,age:"45m",sl:3573.6,tp:3443.2,lev:5}, + ]); + const [regime,setRegime]=useState("BULL"); + const [confidence,setConfidence]=useState(0.847); + const [cbLosses,setCbLosses]=useState(1); + const [winRate,setWinRate]=useState(74.6); + const [totalTrades,setTotalTrades]=useState(312); + const [accuracy,setAccuracy]=useState(74.6); + const [pumpAlerts,setPumpAlerts]=useState([ + {coin:"SUI",type:"PUMP",pct:"+8.3%",vol:"12.4x",time:"2m ago",act:true}, + {coin:"INJ",type:"PUMP",pct:"+5.1%",vol:"6.8x",time:"8m ago",act:true}, + {coin:"DOGE",type:"DUMP",pct:"-4.2%",vol:"3.1x",time:"23m ago",act:false}, + ]); + + // Clock + useEffect(()=>{ + const iv=setInterval(()=>setClock(new Date()),1000); + return()=>clearInterval(iv); + },[]); + + // Live simulation + useEffect(()=>{ + const iv=setInterval(()=>{ + setBalance(b=>+Math.max(9000,b+rand(-8,14)).toFixed(2)); + setDailyPnL(p=>+(p+rand(-4,5.5)).toFixed(2)); + setConfidence(+(rand(0.71,0.94)).toFixed(3)); + setPositions(prev=>prev.map(p=>({ + ...p, + current:+(p.current*(1+rand(-0.0006,0.0007))).toFixed(p.coin==="BTC"?0:1), + pnl:+(p.pnl+rand(-2.5,3)).toFixed(2), + }))); + setEquity(prev=>{ + const last=prev[prev.length-1]; + const nv=Math.max(9200,last.v*(1+rand(-0.0012,0.0017))); + const d=new Date(); + return[...prev.slice(-79),{t:`${d.getHours()}:${String(d.getMinutes()).padStart(2,"0")}`,v:Math.round(nv*100)/100}]; + }); + if(Math.random()>.5)setSignals(prev=>[genSignal(),...prev.slice(0,29)]); + if(Math.random()>.85)setRegime(pick(REGIMES)); + },1800); + return()=>clearInterval(iv); + },[]); + + const closePos=useCallback((coin)=>{ + setPositions(p=>p.filter(x=>x.coin!==coin)); + },[]); + + // Derived + const totalPnL=equity.length?equity[equity.length-1].v-10000:0; + const dailyColor=dailyPnL>=0?C.success:C.danger; + const regimeColor=regime==="BULL"?C.success:regime==="BEAR"?C.danger:C.warn; + const regimeIcon=regime==="BULL"?"↗":regime==="BEAR"?"↘":"→"; + + // ── Tab: Overview ────────────────────────────────────────────────────────── + function TabOverview(){ + return( +
+ {/* KPIs */} +
+ + =0?"+":""}$${fmt2(dailyPnL)}`} + sub={`${(dailyPnL/balance*100).toFixed(2)}% today`} color={dailyColor}/> + =0?"+":""}$${Math.round(totalPnL).toLocaleString()}`} + sub={`${((totalPnL/10000)*100).toFixed(2)}% from start`} + color={totalPnL>=0?C.success:C.danger}/> + +
+ + {/* Equity + Regime + Status */} +
+ +
+ EQUITY CURVE · 80H + =0?C.success:C.danger,fontFamily:"JetBrains Mono",fontSize:12}}> + {totalPnL>=0?"+":""}${totalPnL.toFixed(2)} ({((totalPnL/10000)*100).toFixed(2)}%) + +
+ + + + + + + + + + + + + + + +
+ +
+ {/* Regime */} + +
MARKET REGIME · HMM
+
+ {regimeIcon} +
+ {regime} + + {regimeFilter?"Filter: ON":"Filter: OFF"} + +
+ + {/* Bot Status */} + +
MODULE STATUS
+ {[ + {l:"AI Bot",on:botOn,color:C.accent}, + {l:"Grid Bot",on:gridOn,color:C.blue}, + {l:"DCA Bot",on:dcaOn,color:C.warn}, + {l:"Pump Scanner",on:pumpOn,color:C.success}, + {l:"Auto-Retrain",on:retrainOn,color:C.purple}, + ].map(({l,on,color})=>( +
+ {l} +
+
+ + {on?"RUN":"OFF"} + +
+
+ ))} + +
+
+ + {/* Positions + Signals */} +
+ +
+ OPEN POSITIONS · {positions.length}/{cfg.maxPos} + +
+ {positions.length===0?( +
No active positions
+ ):positions.map(p=>)} +
+ + +
+ SIGNAL STREAM · LIVE +
+
+ STREAMING +
+
+
+ {["COIN","DIR","CONF","REGIME","TIME","STATUS"].map(h=>( + {h} + ))} +
+
+ {signals.map(s=>)} +
+ +
+ + {/* Pump + Risk preview */} +
+ + PUMP SCANNER · ALERTS + {pumpAlerts.map((a,i)=>( +
+
+ {a.coin} + + {a.pct} +
+
+
Vol: {a.vol}
+
{a.time}
+
+
+ ))} +
+ + + RISK MANAGER + {/* Circuit breaker */} +
+
+ Circuit Breaker + =cfg.cbLosses?C.danger:C.warn, + fontFamily:"JetBrains Mono",fontSize:10}}> + {cbLosses}/{cfg.cbLosses} losses + +
+
+ {[...Array(cfg.cbLosses)].map((_,i)=>( +
+ ))} +
+
+ {/* Daily loss */} + {(()=>{ + const used=Math.max(0,Math.abs(Math.min(0,dailyPnL))/balance*100); + const pct=Math.min(used/cfg.maxLoss*100,100); + return( +
+
+ Daily Loss Used + 75?C.danger:C.text,fontFamily:"JetBrains Mono",fontSize:10}}> + {used.toFixed(2)}% / {cfg.maxLoss}% + +
+
+
75?C.danger:pct>50?C.warn:C.success,transition:"width .5s"}}/> +
+
+ ); + })()} + + + + + +
+
+ ); + } + + // ── Tab: AI Bot ──────────────────────────────────────────────────────────── + function TabAIBot(){ + return( +
+
+ {/* Left: controls + confidence */} +
+ + AI BOT CONTROL +
+ +
+
+ setBotOn(!botOn)} label="Bot Active" color={C.accent}/> + setPaperMode(!paperMode)} + label={paperMode?"Paper Mode":"Live Mode"} color={paperMode?C.blue:C.danger}/> + setRegimeFilter(!regimeFilter)} + label="Regime Filter (HMM)" color={C.purple}/> + setRetrainOn(!retrainOn)} + label="Auto-Retrain" color={C.purple}/> +
+
+ + + + + + 60?C.success:C.warn}/> +
+
+ + +
+
+
+ + {/* Right: positions + signal stream */} +
+ +
+ OPEN POSITIONS · {positions.length}/{cfg.maxPos} + +
+ {positions.length===0 + ?
+ No active positions +
+ :positions.map(p=>)} +
+ + +
+ LIVE SIGNAL STREAM +
+
+ + {botOn?"LIVE":"PAUSED"} + +
+
+
+ {["COIN","DIR","CONF","REGIME","TIME","STATUS"].map(h=>( + {h} + ))} +
+
+ {signals.map(s=>)} +
+ +
+
+
+ ); + } + + // ── Tab: Grid Bot ────────────────────────────────────────────────────────── + function TabGrid(){ + const lower=65000,upper=70000,cur=67892,levels=cfg.gridLevels; + const step=(upper-lower)/levels; + const gridLines=[...Array(levels+1)].map((_,i)=>lower+i*step); + const gridPnL=12.4,filledCount=4; + return( +
+
+ +
+ GRID BOT + setGridOn(!gridOn)} color={C.blue}/> +
+ + + + + + + +
+
RUNTIME STATS
+ + + +
+
+
STRATEGY
+
+ {GRID_STRATEGIES.map(s=>( + + ))} +
+
+
+
+ + + GRID VISUALIZATION · {cfg.gridCoin}/USDT +
+ {gridLines.map((price,i)=>{ + const pct=(price-lower)/(upper-lower)*100; + const above=price>cur; + const filled=!above&&i%3===0; + return( +
+
+
+ {filled&&} + {above&&} + {!filled&&!above&&} + + ${price.toLocaleString()} + +
+
+ ); + })} + {/* Current price */} +
+
+
+
+ + CURRENT ${cur.toLocaleString()} ◄ + +
+
+
+
+ +
+ ); + } + + // ── Tab: DCA Bot ─────────────────────────────────────────────────────────── + function TabDCA(){ + const safeties=[...Array(cfg.dcaMaxSafety)].map((_,i)=>({ + lv:i+1, + price:Math.round(67420*(1-cfg.dcaMaxSafety*cfg.dcaDev/100+i*cfg.dcaDev/100)), + size:+(cfg.dcaSafety*Math.pow(1.5,i)).toFixed(0), + filled:i<2 + })); + const totalInvested=cfg.dcaBase+safeties.filter(s=>s.filled).reduce((a,b)=>a+b.size,0); + return( +
+ +
+ DCA BOT + setDcaOn(!dcaOn)} color={C.warn}/> +
+ + + + + + + +
+ + s.filled).length}/${cfg.dcaMaxSafety}`} color={C.warn}/> + + +
+
+
STRATEGY
+
+ {DCA_STRATEGIES.map(s=>( + + ))} +
+
+
+ + + ORDER LADDER · {cfg.dcaCoin}/USDT + {/* Header */} +
+ {["LEVEL","PRICE","SIZE (USDT)","CUM. SIZE","DEVIATION","STATUS"].map(h=>( + {h} + ))} +
+ {/* Base */} +
+ {["BASE",`$67,420`,`$${cfg.dcaBase}`,`$${cfg.dcaBase}`,"—", + ].map((v,i)=>( + {v} + ))} +
+ {safeties.map((so)=>( +
+ {[`SO${so.lv}`,`$${so.price.toLocaleString()}`,`$${so.size}`, + `$${cfg.dcaBase+safeties.slice(0,so.lv).reduce((a,b)=>a+b.size,0)}`, + `-${(so.lv*cfg.dcaDev).toFixed(1)}%`, + + ].map((v,i)=>( + {v} + ))} +
+ ))} + {/* TP line */} +
+ + TAKE PROFIT TARGET + + + $67,734 (+{cfg.dcaTp}% from avg) + +
+
+
+ ); + } + + // ── Tab: Pump Scanner ────────────────────────────────────────────────────── + function TabPump(){ + return( +
+ +
+ PUMP SCANNER + setPumpOn(!pumpOn)} color={C.success}/> +
+ + ${cfg.pumpPrice}%`}/> + + + + + +
+
TODAY STATS
+ + + +
+
+ +
+
+ + + LIVE ALERTS · {pumpAlerts.length} ACTIVE + {pumpAlerts.length===0 + ?
Market calm — no alerts
+ :pumpAlerts.map((a,i)=>( +
+
+
+ {a.coin} + + {a.act&&
} +
+ {a.pct} +
+
+ {[["VOLUME MULT",a.vol,C.text], + ["TIME",a.time,C.muted], + ["STATUS",a.act?"ACTIVE":"EXPIRED",a.act?C.success:C.muted] + ].map(([l,v,c])=>( +
+
{l}
+
{v}
+
+ ))} +
+
+ ))} + +
+ ); + } + + // ── Tab: Risk Manager ────────────────────────────────────────────────────── + function TabRisk(){ + const used=Math.max(0,Math.abs(Math.min(0,dailyPnL))/balance*100); + const perfData=[ + {n:"Mon",pnl:145},{n:"Tue",pnl:-42},{n:"Wed",pnl:234},{n:"Thu",pnl:89}, + {n:"Fri",pnl:-28},{n:"Sat",pnl:312},{n:"Sun",pnl:178} + ]; + return( +
+
+ {/* Circuit Breaker */} + + CIRCUIT BREAKER +
+ {[...Array(cfg.cbLosses)].map((_,i)=>( +
+ + {i +
+ ))} +
+
=cfg.cbLosses?C.danger:C.muted, + fontSize:10,fontFamily:"Rajdhani",marginBottom:10}}> + {cbLosses>=cfg.cbLosses?"⚠ ACTIVE — Trading paused for 1h":`${cfg.cbLosses-cbLosses} loss(es) until pause`} +
+ 0?C.warn:C.text}/> + + =cfg.cbLosses?"TRIGGERED":"MONITORING"} + color={cbLosses>=cfg.cbLosses?C.danger:C.success}/> +
+ + {/* Daily Loss */} + + DAILY LOSS LIMIT +
+
0.8?C.danger:used/cfg.maxLoss>0.5?C.warn:C.success, + borderRadius:11,transition:"width .5s"}}/> +
+ + {used.toFixed(2)}% / {cfg.maxLoss}% + +
+
+ + + + + + + {/* Performance */} + + PERFORMANCE + + 60?C.success:C.warn}/> + + + + + + + +
+ + {/* Risk params + Weekly chart */} +
+ + RISK PARAMETERS + + + + + + + + + + + + + + DAILY P&L · 7 DAYS + + + + + + + + + + +
+
+ ); + } + + // ── Tab: AI Model ────────────────────────────────────────────────────────── + function TabModel(){ + const features=[ + {n:"rsi_14",v:.124},{n:"funding_rate",v:.098},{n:"volume_ma_ratio",v:.087}, + {n:"oi_change_pct",v:.076},{n:"price_momentum_7",v:.065},{n:"cvd_20",v:.058}, + {n:"atr_14",v:.051},{n:"btc_correlation_20",v:.043},{n:"fear_greed_index",v:.039}, + {n:"order_flow_ratio",v:.033},{n:"obi_proxy",v:.029},{n:"price_skewness_24",v:.025}, + ]; + const maxV=features[0].v; + return( +
+
+ + ENSEMBLE MODEL + + + + + + + + + + +
+
+ setRetrainOn(!retrainOn)} + label="Auto-Retrain" color={C.purple}/> +
+
+
+ + + + +
+
+ + + FEATURE IMPORTANCE · TOP 12 + {features.map(({n,v},i)=>( +
+
+ {n} + {(v*100).toFixed(1)}% +
+
+
+
+
+ ))} + +
+ + + TRADING UNIVERSE · {COINS.length} COINS +
+ {COINS.map(coin=>{ + const hasPos=positions.some(p=>p.coin===coin); + return( +
+ {coin} +
+ ); + })} +
+
+
+ ); + } + + // ── Tab: Config ──────────────────────────────────────────────────────────── + function TabConfig(){ + function CfgSlider({label,field,min,max,step=1,suffix=""}){ + return( +
+ {label} +
+ setC(field,parseFloat(e.target.value))} style={{width:90}}/> + {cfg[field]}{suffix} +
+
+ ); + } + return( +
+ {/* Bot Config */} +
+ + EXCHANGE & MODE +
+
EXCHANGE
+
+ {EXCHANGES.map(ex=>( + + ))} +
+
+
+ setPaperMode(!paperMode)} + label={paperMode?"📄 Paper Mode (Simulation)":"💰 Live Trading"} color={paperMode?C.blue:C.danger}/> +
+
+ setC("compound",!cfg.compound)} + label="Compound Mode" color={C.accent}/> +
+
+ + + BOT PARAMETERS + + + + + + + + + +
+ + {/* Grid + DCA */} +
+ + GRID BOT +
+ setGridOn(!gridOn)} label="Grid Bot" color={C.blue}/> +
+
+ Coin + +
+
+ Strategy + +
+ + +
+ + + DCA BOT +
+ setDcaOn(!dcaOn)} label="DCA Bot" color={C.warn}/> +
+
+ Coin + +
+ + + + + +
+
+ + {/* Pump + Model */} +
+ + PUMP SCANNER +
+ setPumpOn(!pumpOn)} label="Pump Scanner" color={C.success}/> +
+ + + + +
+ + + AI MODEL +
+ setRetrainOn(!retrainOn)} + label="Auto-Retrain" color={C.purple}/> +
+
+ setRegimeFilter(!regimeFilter)} + label="Regime Filter (HMM)" color={C.purple}/> +
+ + +
+ + + CIRCUIT BREAKER + +
+ + =cfg.cbLosses?"TRIGGERED":"MONITORING"} + color={cbLosses>=cfg.cbLosses?C.danger:C.success}/> +
+
+
+
+ ); + } + + // ── Nav items ────────────────────────────────────────────────────────────── + const nav=[ + {id:"overview",label:"OVERVIEW",dot:null}, + {id:"aibot",label:"AI BOT",dot:botOn?C.accent:null}, + {id:"grid",label:"GRID BOT",dot:gridOn?C.blue:null}, + {id:"dca",label:"DCA BOT",dot:dcaOn?C.warn:null}, + {id:"pump",label:"PUMP SCAN",dot:pumpOn?C.success:null}, + {id:"risk",label:"RISK MGR",dot:cbLosses>0?C.warn:null}, + {id:"model",label:"AI MODEL",dot:retrainOn?C.purple:null}, + {id:"config",label:"CONFIG",dot:null}, + ]; + + // ── Final render ─────────────────────────────────────────────────────────── + return( +
+ + + {/* ── Top Bar ─────────────────────────────────────────────────────── */} +
+ {/* Logo */} +
+ + DEEPALPHA + +
+ + PRO V11 + +
+
+ + {/* Center stats */} +
+
+
BALANCE
+
+ ${balance.toLocaleString("en-US",{minimumFractionDigits:2})} +
+
+
+
DAILY P&L
+
+ {dailyPnL>=0?"+":""}${fmt2(dailyPnL)} +
+
+
+
REGIME
+
+ {regimeIcon} {regime} +
+
+
+
POSITIONS
+
+ {positions.length}/{cfg.maxPos} +
+
+
+ + {/* Right status */} +
+
+ + {paperMode?"📄 PAPER":"💰 LIVE"} + +
+
+ {exchange.toUpperCase()} +
+
+
+ + {botOn?"ACTIVE":"STOPPED"} + +
+ + {clock.toLocaleTimeString("fr-FR")} + +
+
+ + {/* ── Body ────────────────────────────────────────────────────────── */} +
+ {/* Sidebar */} +
+ {nav.map(item=>( +
setTab(item.id)} + style={{display:"flex",alignItems:"center",gap:10,padding:"10px 16px", + cursor:"pointer",margin:"1px 8px",borderRadius:6, + background:tab===item.id?`${C.accent}12`:"transparent", + borderLeft:tab===item.id?`2px solid ${C.accent}`:"2px solid transparent", + transition:"all .2s",userSelect:"none"}}> + {item.dot&&
} + {!item.dot&&
} + + {item.label} + +
+ ))} + + {/* Quick toggles */} +
+
QUICK CONTROLS
+ {[ + {l:"AI Bot",on:botOn,f:()=>setBotOn(!botOn),c:C.accent}, + {l:"Grid",on:gridOn,f:()=>setGridOn(!gridOn),c:C.blue}, + {l:"DCA",on:dcaOn,f:()=>setDcaOn(!dcaOn),c:C.warn}, + {l:"Pump",on:pumpOn,f:()=>setPumpOn(!pumpOn),c:C.success}, + ].map(({l,on,f,c})=>( +
+ {l} +
+
+
+
+ ))} +
+ + {/* System info */} +
+
SYSTEM
+
+ {[["CPU","12%"],["RAM","2.1GB"],["Uptime","4h 23m"],["Loop","60s"]].map(([l,v])=>( +
+ {l} + {v} +
+ ))} +
+
+
+ + {/* Main content */} +
+ {tab==="overview" && } + {tab==="aibot" && } + {tab==="grid" && } + {tab==="dca" && } + {tab==="pump" && } + {tab==="risk" && } + {tab==="model" && } + {tab==="config" && } +
+
+
+ ); +} diff --git a/auto_retrain.py b/auto_retrain.py new file mode 100644 index 0000000..34c085a --- /dev/null +++ b/auto_retrain.py @@ -0,0 +1,803 @@ +""" +AHAD QUANT — Auto-Retraining Scheduler +Re-downloads data and retrains the ensemble model on a schedule. + +Activated via .env: + AUTO_RETRAIN_ENABLED=true + AUTO_RETRAIN_INTERVAL_HOURS=24 + +Logic: the new model replaces the old one ONLY if it is strictly +more accurate than the currently active model. The active model's +accuracy is always saved in last_retrain.json and used as the +comparison baseline — not a fixed threshold. +""" + +import os, time, json, pickle, shutil, logging, threading, subprocess, sys +from datetime import datetime, timezone +import config + +log = logging.getLogger("AutoRetrain") + +# [FIX] Lock global : empêche daily-local et full-retrain de s'exécuter +# simultanément et d'écrire model_ensemble.pkl en même temps. +# Toute fonction qui modifie un fichier modèle DOIT acquérir ce lock. +_retrain_lock = threading.Lock() +TIMESTAMP_FILE = "last_retrain.json" +# [FIX 21/06/2026] Constante manquante — provoquait un NameError dans +# _load_daily_local_state()/_save_daily_local_state() dès le premier appel +# (should_run_daily_local_cycle() l'appelle sans condition au tout premier +# tick de AutoRetrainer._loop()), tuant silencieusement le thread de fond +# entier : ni le cycle quotidien local (warm-start ML + RL real-only), ni +# le full-retrain réseau ne s'exécutaient jamais en pratique. +DAILY_LOCAL_TIMESTAMP_FILE = "last_daily_local.json" + + +def _safe_export_unified(): + """ + Régénère ahad_quant_unified.zip (best-effort, non-bloquant) après un + ré-entraînement réussi. N'est PLUS critique pour la correction en live + depuis le correctif du point #2 du diagnostic (rl_agent.py priorise déjà + les fichiers vivants rl_agent.zip/rl_scaler.pkl/model_ensemble.pkl, et + unified_brain.py les recharge à chaud) — reste utile pour des + déploiements portables sur une autre machine. + + export_unified.export_unified() appelle sys.exit(1) en interne si les + fichiers source sont absents : on attrape donc BaseException (et pas + seulement Exception) pour ne jamais faire planter le cycle de retrain + à cause de cet export secondaire. + """ + try: + import export_unified + export_unified.export_unified() + log.info("[EXPORT] ahad_quant_unified.zip régénéré") + except BaseException as e: + log.warning(f"[EXPORT] Échec régénération ahad_quant_unified.zip (non-bloquant) : {e}") + + +# ── Persistence helpers ─────────────────────────────────────────────────────── + +def _load_state() -> dict: + """ + Load the saved state of the currently active model. + Returns dict with keys: timestamp, accuracy, datetime. + Returns defaults (accuracy=0) if no state file exists yet. + """ + if not os.path.exists(TIMESTAMP_FILE): + return {"timestamp": 0.0, "accuracy": 0.0, "datetime": "never"} + try: + with open(TIMESTAMP_FILE) as f: + return json.load(f) + except Exception: + return {"timestamp": 0.0, "accuracy": 0.0, "datetime": "never"} + + +def _save_state(accuracy: float): + """Save the accuracy of the newly active model.""" + with open(TIMESTAMP_FILE, "w") as f: + json.dump({ + "timestamp": time.time(), + "accuracy": round(accuracy, 6), + "datetime": datetime.now(timezone.utc).isoformat(), + }, f, indent=2) + + +def get_active_accuracy() -> float: + """Return the accuracy of the currently active model (0 if unknown).""" + return _load_state().get("accuracy", 0.0) + + +def should_retrain() -> bool: + """True if enough time has passed since last retrain.""" + if not config.AUTO_RETRAIN_ENABLED: + return False + elapsed = time.time() - _load_state().get("timestamp", 0.0) + return elapsed >= config.AUTO_RETRAIN_INTERVAL_HOURS * 3600 + + +# ── Parse accuracy from train.py output ────────────────────────────────────── + +def _parse_accuracy(output: str) -> float: + """Extract ensemble (or LightGBM) accuracy from train.py stdout.""" + accuracy = 0.0 + for line in output.splitlines(): + # Ensemble line has priority + if "Ensemble test accuracy:" in line: + try: + val = line.split(":")[-1].strip().split()[0] + accuracy = float(val.replace("%", "")) / (100 if "%" in val else 1) + return accuracy + except Exception: + pass + # Fallback: single model line + if "Test accuracy:" in line: + try: + val = line.split(":")[-1].strip().split()[0] + accuracy = float(val.replace("%", "")) / (100 if "%" in val else 1) + except Exception: + pass + return accuracy + + +# ── Main retrain cycle ──────────────────────────────────────────────────────── + +def run_retrain(notify_fn=None) -> dict: + """ + Execute a full retrain cycle. + [FIX] Protégé par _retrain_lock — ne peut pas tourner en même temps + qu'un cycle daily-local. + + Decision logic: + new_accuracy > active_accuracy → replace model ✅ + new_accuracy ≤ active_accuracy → keep old model ❌ + + Returns dict: {success, new_accuracy, old_accuracy, improved, message} + """ + if not _retrain_lock.acquire(blocking=False): + log.warning("[FULL-RETRAIN] Cycle daily-local en cours — full-retrain reporté") + return {"success": False, "message": "retrain_lock — réessayer dans quelques minutes"} + try: + return _run_retrain_inner(notify_fn) + finally: + _retrain_lock.release() + + +def _run_retrain_inner(notify_fn=None) -> dict: + """Corps du full retrain (appelé uniquement si lock acquis).""" + result = { + "success": False, + "new_accuracy": 0.0, + "old_accuracy": 0.0, + "improved": False, + "message": "", + } + + # Accuracy of the model currently in production + old_accuracy = get_active_accuracy() + result["old_accuracy"] = old_accuracy + + old_label = f"{old_accuracy:.2%}" if old_accuracy > 0 else "unknown (first run)" + log.info(f"Auto-retrain starting — active model accuracy: {old_label}") + + if notify_fn: + notify_fn( + f"🔄 *Auto-retrain started*\n" + f"Active model accuracy: `{old_label}`\n" + f"Downloading fresh data..." + ) + + # ── Step 1: Download data ──────────────────────────────────────────────── + # Timeout généreux : 27 coins × 1000 jours peut prendre 30–60 min selon + # la connexion et les rate-limits de l'exchange (14400s = 4h max). + try: + proc = subprocess.run( + [sys.executable, "download_data.py"], + capture_output=True, text=True, timeout=14400 + ) + if proc.returncode != 0: + raise RuntimeError(proc.stderr[-500:] or "download failed") + log.info("Data download complete") + except subprocess.TimeoutExpired: + msg = "❌ Auto-retrain: download timed out after 4h — vérifie la connexion ou réduis la liste de coins" + log.error(msg) + if notify_fn: notify_fn(msg) + result["message"] = msg + return result + except Exception as e: + msg = f"❌ Auto-retrain: data download failed — {e}" + log.error(msg) + if notify_fn: notify_fn(msg) + result["message"] = msg + return result + + # ── Step 2: Train new model ────────────────────────────────────────────── + try: + proc = subprocess.run( + [sys.executable, "train.py"], + capture_output=True, text=True, timeout=7200 + ) + output = proc.stdout + proc.stderr + new_accuracy = _parse_accuracy(output) + result["new_accuracy"] = new_accuracy + + if proc.returncode != 0: + raise RuntimeError(output[-500:]) + + log.info(f"Training complete — new accuracy: {new_accuracy:.2%}") + except Exception as e: + msg = f"❌ Auto-retrain: training failed — {e}" + log.error(msg) + if notify_fn: notify_fn(msg) + result["message"] = msg + return result + + # ── Step 3: Compare strictly against active model ──────────────────────── + improved = new_accuracy > old_accuracy + result["improved"] = improved + + if improved: + # New model is better → save it as active + _save_state(new_accuracy) + result["success"] = True + + gain = new_accuracy - old_accuracy + msg = ( + f"✅ *Model updated*\n" + f"Old accuracy: `{old_label}`\n" + f"New accuracy: `{new_accuracy:.2%}` (+{gain:.2%})\n" + f"New model is now active." + ) + log.info(msg.replace("*","").replace("`","")) + if notify_fn: notify_fn(msg) + result["message"] = msg + + else: + # New model is not better → keep old model (train.py already wrote + # the new .pkl files; we restore the backup if it exists, otherwise + # we simply do nothing since train.py didn't overwrite if accuracy + # was logged as lower — but to be safe we log a warning) + gap = old_accuracy - new_accuracy + msg = ( + f"⚠️ *Model NOT updated*\n" + f"Old accuracy: `{old_label}`\n" + f"New accuracy: `{new_accuracy:.2%}` (-{gap:.2%})\n" + f"Active model unchanged — old model was better." + ) + log.warning(msg.replace("*","").replace("`","")) + if notify_fn: notify_fn(msg) + result["message"] = msg + + # Restore backup if it was created before training + for path in [config.MODEL_PATH, config.ENSEMBLE_MODEL_PATH]: + backup = path + ".backup" + if os.path.exists(backup): + shutil.copy2(backup, path) + log.info(f"Restored backup: {path}") + + # ── RL — TOUJOURS exécuté après le ML, que le nouveau ML ait été accepté + # ou non (le RL doit de toute façon rester synchronisé avec l'ensemble + # actif, ancien ou nouveau). C'est ce qui garantit que ce chemin + # "full retrain réseau" ne redevient JAMAIS ML-seul : ML et RL forment + # un seul cycle, même ici. Le fine-tune RL a son propre accept/reject + # (voir rl_train.py::fine_tune()), donc aucun risque de régression. ── + log.info("[FULL-RETRAIN] Fine-tuning RL sur l'ensemble ML actif...") + try: + rl_updated = run_rl_retrain_with_replay(notify_fn) + except Exception as e: + log.warning(f"[FULL-RETRAIN] Fine-tuning RL échoué (non-bloquant) : {e}") + rl_updated = False + result["rl_updated"] = rl_updated + + # ── Export unifié — UNE SEULE fois, après ML ET RL, pour ne jamais + # bundler un ML neuf avec un RL périmé (ou l'inverse). ── + _safe_export_unified() + + return result + + +# ── Backup helper (called before training to preserve old model) ─────────────── + +def _backup_models(): + """Create .backup copies of model files before overwriting.""" + for path in [config.MODEL_PATH, config.ENSEMBLE_MODEL_PATH]: + if os.path.exists(path): + shutil.copy2(path, path + ".backup") + log.debug(f"Backed up: {path}") + + +# ── Background scheduler ────────────────────────────────────────────────────── + +class AutoRetrainer: + """ + Background thread that periodically retrains the model. + Integrates with the running AHAD QUANT bot. + """ + + def __init__(self, notify_fn=None): + self.notify_fn = notify_fn + self._thread = None + self._stop_event = threading.Event() + + def start(self): + if not config.AUTO_RETRAIN_ENABLED and not getattr(config, "DAILY_LOCAL_RETRAIN_ENABLED", True): + log.info("Auto-retrain entièrement désactivé (AUTO_RETRAIN_ENABLED=false et DAILY_LOCAL_RETRAIN_ENABLED=false)") + return + self._thread = threading.Thread( + target=self._loop, daemon=True, name="AutoRetrain" + ) + self._thread.start() + active = get_active_accuracy() + label = f"{active:.2%}" if active > 0 else "unknown (first run)" + daily_interval = getattr(config, "DAILY_LOCAL_RETRAIN_INTERVAL_HOURS", 24) + log.info( + f"Auto-retrain scheduler started " + f"(cycle quotidien local ML+RL toutes les {daily_interval}h" + + (f" | ré-entraînement complet réseau toutes les {config.AUTO_RETRAIN_INTERVAL_HOURS}h" + if config.AUTO_RETRAIN_ENABLED else " | ré-entraînement complet réseau désactivé") + + f" | active model: {label})" + ) + + def stop(self): + self._stop_event.set() + + def _loop(self): + while not self._stop_event.is_set(): + # ── Cycle quotidien LOCAL (ML + RL unifiés) — c'est le chemin + # automatique principal demandé : 100% local, journalier, et + # n'accepte un nouveau modèle (ML ou RL) que s'il est meilleur. + if should_run_daily_local_cycle(): + log.info("[DAILY-LOCAL] Intervalle atteint — démarrage du cycle quotidien local") + run_daily_unified_retrain(self.notify_fn) + + # ── Ré-entraînement complet (lourd, réseau) — optionnel, désactivé + # par défaut (AUTO_RETRAIN_ENABLED=false) car il télécharge de + # nouvelles données ; reste disponible si explicitement activé. + if config.AUTO_RETRAIN_ENABLED and should_retrain(): + log.info("[FULL-RETRAIN] Intervalle atteint — backup et ré-entraînement complet (réseau)") + _backup_models() + run_retrain(self.notify_fn) + + # Check every 30 minutes + self._stop_event.wait(timeout=1800) + + +# ── CLI ─────────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, + format="%(asctime)s [%(name)s] %(message)s") + + def notify(msg): + print(f"\n[NOTIFY] {msg}\n") + + state = _load_state() + print("=" * 55) + print(" AHAD QUANT — Auto-Retrain") + print("=" * 55) + print(f" Active model accuracy : " + f"{state['accuracy']:.2%}" if state['accuracy'] else " Active model: unknown (first run)") + print(f" Last retrain : {state.get('datetime','never')}") + print(f" Interval : every {config.AUTO_RETRAIN_INTERVAL_HOURS}h") + print(f" Rule : new model must beat active model") + print("=" * 55 + "\n") + + print("Backing up current models...") + _backup_models() + + print("Running retrain now...\n") + result = run_retrain(notify) + + print("\n" + "=" * 55) + print(f" Old accuracy : {result['old_accuracy']:.2%}") + print(f" New accuracy : {result['new_accuracy']:.2%}") + print(f" Improved : {'✅ YES — model updated' if result['improved'] else '❌ NO — old model kept'}") + print("=" * 55) + + +# ─── RL Agent Auto-Retrain ──────────────────────────────────────────────────── + +RL_TIMESTAMP_FILE = "last_rl_retrain.json" + + +def _load_rl_state() -> dict: + if not os.path.exists(RL_TIMESTAMP_FILE): + return {"timestamp": 0.0, "datetime": "never"} + try: + with open(RL_TIMESTAMP_FILE) as f: + return json.load(f) + except Exception: + return {"timestamp": 0.0, "datetime": "never"} + + +def _save_rl_state(): + with open(RL_TIMESTAMP_FILE, "w") as f: + json.dump({ + "timestamp": time.time(), + "datetime": datetime.now(timezone.utc).isoformat(), + }, f, indent=2) + + +def should_retrain_rl() -> bool: + """True si le fine-tuning RL hebdomadaire est dû.""" + if not getattr(config, "RL_AUTO_RETRAIN_ENABLED", False): + return False + interval = getattr(config, "RL_RETRAIN_INTERVAL_HOURS", 168) * 3600 + elapsed = time.time() - _load_rl_state().get("timestamp", 0.0) + return elapsed >= interval + + +def run_rl_finetune() -> dict: + """ + Lance le fine-tuning RL en sous-processus (rl_train.py --finetune). + + Depuis le correctif rl_train.py::fine_tune(), le sous-processus évalue + l'ancienne et la nouvelle politique sur les mêmes épisodes (seed fixe) + et n'écrase rl_agent.zip QUE si la nouvelle est meilleure — exactement + la même philosophie "si meilleur on remplace, sinon on conserve" déjà + appliquée côté ML. Un fine-tune REJETÉ (ancienne politique conservée) + est un résultat NORMAL, pas un échec — distinct d'un crash du + sous-processus (returncode != 0). + + Retourne {"success": bool, "accepted": bool, "old_mean_reward": float|None, + "new_mean_reward": float|None}. + """ + model_path = getattr(config, "RL_MODEL_PATH", "rl_agent") + steps = getattr(config, "RL_FINETUNE_STEPS", 200_000) + + result = {"success": False, "accepted": False, "old_mean_reward": None, "new_mean_reward": None} + + if not os.path.exists(f"{model_path}.zip"): + log.warning("[RL-RETRAIN] Modèle rl_agent.zip introuvable — fine-tune ignoré") + return result + + log.info(f"[RL-RETRAIN] Démarrage fine-tune RL (+{steps:,} steps)...") + try: + proc = subprocess.run( + [sys.executable, "rl_train.py", + "--finetune", + "--model", model_path, + "--steps", str(steps)], + capture_output=True, + text=True, + timeout=7200, # 2h max + ) + if proc.returncode != 0: + log.error(f"[RL-RETRAIN] Échec fine-tune RL:\n{proc.stderr[-500:]}") + return result + + result["success"] = True + + # Parse le marqueur "RL_FINETUNE_RESULT accepted=... old=... new=..." + # émis par rl_train.py::fine_tune() — seule façon pour ce process + # parent de connaître le résultat accept/reject du sous-processus. + accepted = False + for line in proc.stdout.splitlines(): + if line.startswith("RL_FINETUNE_RESULT"): + try: + parts = dict(p.split("=") for p in line.split()[1:]) + accepted = parts.get("accepted", "False") == "True" + result["old_mean_reward"] = float(parts.get("old", "nan")) + result["new_mean_reward"] = float(parts.get("new", "nan")) + except Exception: + pass + break + result["accepted"] = accepted + + if accepted: + log.info( + f"[RL-RETRAIN] ✅ Fine-tune accepté " + f"(reward {result['old_mean_reward']:.4f} → {result['new_mean_reward']:.4f})" + ) + _save_rl_state() + # Recharger le singleton rl_agent en mémoire — le fichier a changé + try: + from rl_agent import reset_rl_agent + reset_rl_agent() + log.info("[RL-RETRAIN] Singleton RL rechargé") + except ImportError: + pass + _safe_export_unified() + else: + log.info( + f"[RL-RETRAIN] ↔️ Fine-tune rejeté — politique actuelle conservée " + f"(reward {result['old_mean_reward']:.4f} vs {result['new_mean_reward']:.4f} proposé)" + ) + + return result + except subprocess.TimeoutExpired: + log.error("[RL-RETRAIN] Timeout fine-tune RL (>2h)") + return result + except Exception as e: + log.error(f"[RL-RETRAIN] Exception : {e}") + return result + + +# ─── Apprentissage Continu V7 — Cycle quotidien LGB + Replay RL ────────────── + +def run_daily_lightgbm_warmstart(notify_fn=None) -> bool: + """ + Cycle quotidien léger (local CPU, ~2-5 min). + Warm-start LightGBM sur les trades LOSS + TIMEOUT. + + [FIX] 3 améliorations : + 1. Fenêtre progressive : si pas assez de trades en 24h, on élargit à 48h, + 72h, puis tout le buffer — les phases de bonne performance ne bloquent + plus le cycle. + 2. Tous les types de trades sont inclus (WIN surpondéré 1x, LOSS 2x, + TIMEOUT 1.5x via error_weight déjà dans le buffer) — plus de dépendance + à un minimum de pertes. + 3. Modèle candidat accumulatif : si le warmstart est rejeté, les arbres + appris sont sauvegardés dans model.pkl.candidate pour être utilisés comme + point de départ du prochain cycle (au lieu de repartir de la production). + L'apprentissage des mauvais trades s'accumule même quand le modèle actuel + reste en place. + """ + if not getattr(config, "CONTINUOUS_LEARNING_ENABLED", False): + return False + if not getattr(config, "WARMSTART_ENABLED", False): + return False + + try: + from experience_buffer import get_experience_buffer + from online_learner import OnlineLearner + + buf = get_experience_buffer(auto_load=True) + learner = OnlineLearner(buf) + + min_trades = getattr(config, "WARMSTART_MIN_TRADES", 5) + + # ── Fenêtre progressive : 24h → 48h → 72h → tout le buffer ────────── + trades = [] + window_hours = 24 + for window_hours in [24, 48, 72, 9999]: + if window_hours == 9999: + trades = buf.get_recent(hours=87600) # ~10 ans = tout le buffer + else: + trades = buf.get_recent(hours=window_hours) + if len(trades) >= min_trades: + break + + n = len(trades) + if n < min_trades: + log.info(f"[CL-DAILY] Buffer total seulement {n} trades (min={min_trades}) — skip warm-start") + return False + + window_label = f"{window_hours}h" if window_hours != 9999 else "tout le buffer" + log.info(f"[CL-DAILY] Warm-start LGB sur {n} trades ({window_label})...") + updated = learner.daily_update(loss_trades=trades) + + msg = ( + f"{'✅' if updated else '⚠️'} *CL Warm-start quotidien*\n" + f"Trades utilisés : {n} ({window_label})\n" + f"Modèle {'mis à jour' if updated else 'inchangé (candidate sauvegardé)'}" + ) + log.info(msg.replace("*","").replace("`","")) + if notify_fn: + notify_fn(msg) + + if updated: + _safe_export_unified() + + return updated + + except Exception as e: + log.error(f"[CL-DAILY] Erreur warm-start : {e}") + return False + + +def run_rl_retrain_with_replay(notify_fn=None) -> bool: + """ + Cycle RL quotidien — 100% basé sur les vrais trades du bot, ZERO + simulation dans l'entrainement. + + [CORRECTIF 21/06/2026] AVANT : cette fonction lancait TOUJOURS + run_rl_finetune() en etape 1 -- un fine-tune PPO classique sur + l'environnement SIMULE (AhadQuantForexEnv, donnees de marche + historiques) -- puis tentait d'injecter les vrais trades en etape 2. + Cette injection ne servait a rien : rl_train.py::fine_tune() appelait + model.learn(), qui reinitialise le rollout_buffer + (rollout_buffer.reset() en tete de collect_rollouts()) AVANT tout + entrainement. Le RL n'apprenait donc QUE de la simulation, jamais des + vrais trades -- meme symptome que le bug RL/backtest deja corrige. + + MAINTENANT : si le buffer contient >= RL_REPLAY_MIN_TRADES (config, + defaut 50) trades reels, on lance rl_train.py --finetune --replay + , qui appelle fine_tune_real_only() : un model.train() PPO + direct sur un rollout_buffer construit a 100% a partir des vrais + trades (voir rl_train.py). Aucun appel a run_rl_finetune() / a + AhadQuantForexEnv dans ce cycle. + + Si le nombre de trades reels est insuffisant : le cycle RL est + SKIP cette fois-ci -- PAS de repli sur la simulation. Le RL ne doit + progresser que sur ce que le bot a reellement fait ; tant qu'il n'y a + pas assez de volume reel, on attend plutot que d'entrainer sur du + synthetique. + + Retourne True si le fine-tune reel a ete accepte (modele mis a jour). + """ + if not getattr(config, "CONTINUOUS_LEARNING_ENABLED", False): + return False + if not getattr(config, "RL_REAL_REPLAY_ENABLED", False): + return False + + try: + from experience_buffer import get_experience_buffer + from online_learner import OnlineLearner + + buf = get_experience_buffer(auto_load=True) + learner = OnlineLearner(buf) + + min_trades = getattr(config, "RL_REPLAY_MIN_TRADES", 50) + if buf.total_trades < min_trades: + log.info(f"[CL-RL] Seulement {buf.total_trades} trades reels dans le buffer " + f"(min={min_trades}) -- RL update SKIP ce cycle (aucun repli simule)") + return False + + episodes = learner.prepare_rl_episodes(min_trades=min_trades) + if not episodes: + log.info("[CL-RL] Aucun episode reel valide prepare -- RL update SKIP") + return False + + # Sauvegarder les episodes pour injection dans rl_train.py + import json, os + replay_file = "rl_replay_episodes.json" + with open(replay_file, "w") as f: + json.dump(episodes, f) + + log.info(f"[CL-RL] {len(episodes)} episodes reels sauvegardes -> {replay_file} " + f"(entrainement RL 100% reel -- pas de AhadQuantForexEnv)") + + # Declencher fine_tune_real_only() via rl_train.py --replay (accept/ + # reject integre, voir rl_train.py::fine_tune_real_only()). Pas de + # --steps : ce mode ne fait qu'un seul model.train() sur le buffer + # reel, donc largement plus rapide qu'un fine-tune simule. + import subprocess, sys + proc = subprocess.run( + [sys.executable, "rl_train.py", + "--finetune", + "--replay", replay_file, + "--real-min-trades", str(min_trades)], + capture_output=True, text=True, timeout=1800, # 30 min max -- pas de simulation, donc rapide + ) + # Nettoyer le fichier temporaire dans tous les cas + if os.path.exists(replay_file): + os.remove(replay_file) + + if proc.returncode != 0: + log.warning(f"[CL-RL] Echec fine-tune reel : {proc.stderr[-300:]}") + return False + + accepted = False + for line in proc.stdout.splitlines(): + if line.startswith("RL_FINETUNE_RESULT"): + accepted = "accepted=True" in line + break + + if accepted: + log.info(f"[CL-RL] OK RL mis a jour -- {len(episodes)} vrais trades, aucune simulation") + try: + from rl_agent import reset_rl_agent + reset_rl_agent() + except ImportError: + pass + if notify_fn: + notify_fn( + f"OK *RL mis a jour (100% reel)*\n" + f"{len(episodes)} vrais trades integres au PPO -- aucune simulation" + ) + return True + else: + log.info("[CL-RL] Fine-tune reel rejete -- politique conservee") + return False + + except Exception as e: + log.error(f"[CL-RL] Erreur cycle RL reel : {e}") + return False + + +def _load_daily_local_state() -> dict: + if not os.path.exists(DAILY_LOCAL_TIMESTAMP_FILE): + return {"timestamp": 0.0, "datetime": "never"} + try: + with open(DAILY_LOCAL_TIMESTAMP_FILE) as f: + return json.load(f) + except Exception: + return {"timestamp": 0.0, "datetime": "never"} + + +def _save_daily_local_state(): + with open(DAILY_LOCAL_TIMESTAMP_FILE, "w") as f: + json.dump({ + "timestamp": time.time(), + "datetime": datetime.now(timezone.utc).isoformat(), + }, f, indent=2) + + +def should_run_daily_local_cycle() -> bool: + """True si le cycle unifié quotidien et local (ML + RL) est dû.""" + if not getattr(config, "DAILY_LOCAL_RETRAIN_ENABLED", True): + return False + interval = getattr(config, "DAILY_LOCAL_RETRAIN_INTERVAL_HOURS", 24) * 3600 + elapsed = time.time() - _load_daily_local_state().get("timestamp", 0.0) + return elapsed >= interval + + +def run_daily_unified_retrain(notify_fn=None) -> dict: + """ + Cycle d'entraînement UNIFIÉ, quotidien et 100% LOCAL pour ML et RL. + [FIX] Protégé par _retrain_lock — ne peut pas tourner en même temps + qu'un full-retrain ou qu'un autre cycle local. + """ + if not _retrain_lock.acquire(blocking=False): + log.warning("[DAILY-LOCAL] Un autre cycle d'entraînement est déjà en cours — skip") + return {"skipped": True, "reason": "retrain_lock"} + try: + return _run_daily_unified_retrain_inner(notify_fn) + finally: + _retrain_lock.release() + + +def _run_daily_unified_retrain_inner(notify_fn=None) -> dict: + """ + Cycle d'entraînement UNIFIÉ, quotidien et 100% LOCAL pour ML et RL — + traités comme un seul et même modèle qui se met à jour ensemble. + + Aucune des deux étapes ne télécharge quoi que ce soit sur le réseau : + 1. ML — warm-start LightGBM EMBARQUÉ DANS L'ENSEMBLE + (online_learner.py::daily_update, sur les trades réels du + buffer local des dernières 24h). N'écrase model_ensemble.pkl + que si l'accuracy de l'ENSEMBLE COMPLET s'améliore (sinon + l'ancien est conservé) — point #3 du diagnostic. + 2. RL — [21/06/2026] désormais 100% basé sur les vrais trades du + bot, ZÉRO simulation (voir run_rl_retrain_with_replay et + rl_train.py::fine_tune_real_only). N'écrase rl_agent.zip que + si la nouvelle politique obtient un reward moyen au moins + équivalent à l'ancienne (yardstick simulé, comparaison + uniquement). Si moins de RL_REPLAY_MIN_TRADES trades réels + sont disponibles, cette étape est SKIP — pas de repli sur la + simulation. + + Si l'une des deux étapes a effectivement remplacé un modèle, le bundle + portable ahad_quant_unified.zip est régénéré (best-effort) pour rester + cohérent avec les fichiers vivants. + + Retourne {"ml_updated": bool, "rl_updated": bool, "ran": bool}. + """ + result = {"ml_updated": False, "rl_updated": False, "ran": False} + + if not getattr(config, "DAILY_LOCAL_RETRAIN_ENABLED", True): + log.debug("[DAILY-LOCAL] Désactivé (DAILY_LOCAL_RETRAIN_ENABLED=false)") + return result + + result["ran"] = True + log.info("[DAILY-LOCAL] === Démarrage du cycle quotidien unifié ML+RL (local) ===") + + # ── 1. ML — warm-start local de l'ensemble (accept/reject intégré) ───── + try: + ml_updated = run_daily_lightgbm_warmstart(notify_fn=notify_fn) + result["ml_updated"] = bool(ml_updated) + except Exception as e: + log.error(f"[DAILY-LOCAL] Erreur étape ML : {e}") + + # ── 2. RL — fine-tune local + replay réel (accept/reject intégré) ────── + if getattr(config, "RL_AUTO_RETRAIN_ENABLED", True): + try: + rl_updated = run_rl_retrain_with_replay(notify_fn=notify_fn) + result["rl_updated"] = bool(rl_updated) + except Exception as e: + log.error(f"[DAILY-LOCAL] Erreur étape RL : {e}") + else: + log.debug("[DAILY-LOCAL] RL désactivé (RL_AUTO_RETRAIN_ENABLED=false)") + + # ── 3. Bilan ───────────────────────────────────────────────────────── + _save_daily_local_state() + + if result["ml_updated"] or result["rl_updated"]: + parts = [] + if result["ml_updated"]: parts.append("ML mis à jour") + if result["rl_updated"]: parts.append("RL mis à jour") + msg = f"🔄 *Cycle quotidien local terminé* — {', '.join(parts)}" + log.info(msg.replace("*", "")) + if notify_fn: + notify_fn(msg) + else: + log.info("[DAILY-LOCAL] === Cycle terminé — aucun modèle n'a été amélioré, anciens modèles conservés ===") + + return result + """ + Vérification horaire du monitor de performance. + Retourne le statut (OK/WARNING/DANGER/EMERGENCY). + """ + if not getattr(config, "CONTINUOUS_LEARNING_ENABLED", False): + return "OK" + if not getattr(config, "MONITOR_ENABLED", False): + return "OK" + try: + from experience_buffer import get_experience_buffer + from performance_monitor import PerformanceMonitor + + buf = get_experience_buffer(auto_load=False) + monitor = PerformanceMonitor(buf) + return monitor.check() + except Exception as e: + log.debug(f"[CL-MONITOR] Erreur check horaire : {e}") + return "OK" diff --git a/backtest.py b/backtest.py new file mode 100644 index 0000000..1cd9dfb --- /dev/null +++ b/backtest.py @@ -0,0 +1,972 @@ +# -*- coding: utf-8 -*- +""" +AHAD QUANT — Backtester CORRIGÉ v10-fixed (Forex Edition) +============================================= +Corrections v10-fixed vs v10 : + + FIX A — PROFIT FACTOR mal calculé + AVANT : pf = avg_win / avg_loss → donne le ratio moyen gain/perte, PAS le PF + APRÈS : pf = sum(gains) / sum(|pertes|) → vrai Profit Factor standard + Impact : v10 affichait 1.15 au lieu du vrai ~9.38 (identique à v11) + + FIX B — SHARPE biaisé par le compound sizing + AVANT : calculé sur les P&L absolus en $ → les trades tardifs (~$213) + pèsent 7x plus que les trades précoces (~$30), std() gonflé + APRÈS : calculé sur les rendements % (pnl / position_val) → chaque trade + est comparable indépendamment de la taille du compte + Impact : Sharpe de 13.5 → valeur réaliste selon win rate réel + + FIX C — DEAD CODE supprimé (ligne eq = [...]) + AVANT : eq = [balance + sum(pnl[:k]) - sum(pnl[:k]) for k in ...] + calcule balance + X - X = balance pour tout k, variable jamais utilisée + APRÈS : ligne supprimée + + FIX D — DRAWDOWN calculé sur l'equity curve RÉELLE + AVANT : P&L appliqués séquentiellement trade par trade → ignore les pertes + simultanées (jusqu'à MAX_POSITIONS positions ouvertes en même temps) + APRÈS : equity_curve produite par run_backtest_corrected() transmise à + print_results() et utilisée directement pour le calcul du DD + + FIX E — MARGE utilisée calculée sur les montants RÉELS + AVANT : already_used_margin = n_positions × balance_actuel × RISK_PER_TRADE + → approximation fausse (les positions ont été ouvertes à des + niveaux de balance différents) + APRÈS : open_positions stocke (expiration, marge_réelle) → la marge + déjà engagée est la somme exacte des marges à l'ouverture + + FIX F — TP_PCT_OVERRIDE synchronisé avec config.py (v32-fixed) + AVANT : TP_PCT_OVERRIDE = 0.022 (2.2%) alors que config.TAKE_PROFIT_PCT = 0.0075 (0.75%) + → backtest 3× plus optimiste que le bot en production + APRÈS : TP_PCT_OVERRIDE = 0.0 → utilise toujours config.TAKE_PROFIT_PCT + Résultats backtest maintenant fidèles au comportement réel du bot. + +Nouveauté v10 vs v9 (inchangé) : + Compound sizing activé, plafond $10,000/trade + +Héritage v9 (inchangé) : + OOS strict 30%, TP 2.2%, MIN_CONFIDENCE 0.72, exclusion coins récents + +Usage: + python backtest.py +""" + +import json, os, pickle, time, sys +sys.stdout.reconfigure(encoding="utf-8", errors="replace") +from collections import defaultdict +from datetime import datetime, timezone + +import numpy as np + +sys.path.insert(0, os.path.dirname(__file__)) +import config +from features import build_features, FEATURE_NAMES +import ensemble_core as _ens_core + +# ── RL — MÊME filtre qu'en live (unified_brain.py), pour que le backtest +# teste le système RÉEL (ML+RL), pas juste le ML seul. Import optionnel : +# si stable-baselines3/gymnasium ne sont pas installés, le backtest reste +# utilisable en mode ML seul (comme avant), avec un avertissement explicite. +_RL_READY = False +try: + from rl_agent import get_rl_agent, PositionState + _RL_READY = True +except ImportError: + PositionState = None # type: ignore[assignment,misc] + +# ─── Paramètres ────────────────────────────────────────────────────────────── +INITIAL_BALANCE = 100.0 # Forex: compte standard $100 +LEVERAGE = config.LEVERAGE # synchro config.py / .env +RISK_PER_TRADE = config.RISK_PER_TRADE +STOP_LOSS_PCT = config.STOP_LOSS_PCT # synchro config.py / .env +TAKE_PROFIT_PCT = config.TAKE_PROFIT_PCT +MIN_CONFIDENCE = config.MIN_CONFIDENCE +MAX_POSITIONS = config.MAX_POSITIONS # synchro config.py / .env +FEE_RATE = config.FEE_RATE + +# ── Slippage réaliste (FIX BUG #5) ────────────────────────────────────────── +# 1.5 pips sur les majors Forex (EUR/USD, GBP/USD) — valeur broker OANDA réaliste. +# Appliqué à l'entrée ET à la sortie (total : 3 pips par trade). +# Pour les JPY pairs (USD/JPY), 1 pip = 0.01 donc ajuster manuellement si besoin. +SLIPPAGE_PIPS = float(os.getenv("BACKTEST_SLIPPAGE_PIPS", "1.5")) +PIP_SIZE = 0.0001 # taille d'1 pip pour paires non-JPY +WARMUP = 50 + +# HISTORIQUE MINIMUM : paires avec moins de bougies que ce seuil sont exclues. +# FOREX yfinance H1 : marchés fermés ~65h/semaine → ~17 000 bougies = 2.7 ans +# de données réelles. Le seuil crypto (20 000) excluait TOUTES les paires Forex. +# 500 bougies H1 ≈ 3 semaines — minimum statistique pour un OOS strict 15%. +MIN_HISTORY_CANDLES = 500 + +# OUT-OF-SAMPLE : fraction des données réservée au backtest. +# Les (1 - OOS_RATIO) premières bougies ont servi à l'entraînement → on ne +# les touche pas. Doit correspondre à 1 - TRAIN_RATIO - VAL_RATIO de train.py +# (0.70 + 0.15 = 0.85 → OOS = 0.15 au minimum ; on prend 0.30 pour la marge). +OOS_RATIO = 0.15 # TEST set uniquement — 100% OOS propre + +# FULL HISTORY MODE : si True, le backtest tourne sur TOUTES les bougies +# disponibles (pas seulement les 30% OOS finaux). Utile pour maximiser le +# nombre de trades simulés et avoir une vision complète de l'historique. +# ⚠️ Attention : inclut les bougies in-sample sur lesquelles le modèle a +# été entraîné → les métriques seront optimistes vs. un vrai OOS test. +USE_FULL_HISTORY = False # OOS strict 30% — seule mesure fiable du vrai edge live + +# COMPOUND SIZING : si True, la taille de position est recalculée à chaque +# trade sur le solde courant (balance × RISK_PER_TRADE × LEVERAGE) au lieu +# d'être fixée à INITIAL_BALANCE × RISK_PER_TRADE × LEVERAGE. +# → L'effet de compounding exponentiel est activé : les gains s'accumulent. +# ⚠️ En live, bien que très profitable sur backtest, le compounding amplifie +# aussi les pertes. S'assurer que RISK_PER_TRADE est conservateur (≤ 3%). +COMPOUND_SIZING = config.COMPOUND_ENABLED # COMPOUND_ENABLED — intérêts composés activés : position grandit avec le solde + +# PLAFOND DE POSITION : limite la taille max par trade en mode compound. +# Sans ce plafond, à $1M de solde une position = $1M × 3% × 10x = $300K +# → croissance exponentielle irréaliste (impossible en vrai sur un exchange). +# $10,000 = taille max raisonnable pour un compte retail en futures. +# Augmente ce plafond uniquement si tu trades avec un vrai gros capital. +MAX_POSITION_VALUE = config.CAPITAL_CAP_PER_TRADE # $ max par trade — CAPITAL_CAP_PER_TRADE (compound plafonné) + +# OVERRIDE DU TAKE PROFIT : si > 0, remplace TAKE_PROFIT_PCT de config.py. +# ⚠️ BUG CORRIGÉ : l'ancienne valeur 0.022 (2.2%) était 3× supérieure à la valeur +# de production config.py (0.75%) → les résultats backtest ne correspondaient pas +# au comportement réel du bot. +# Valeur 0 = utilise TAKE_PROFIT_PCT de config.py / .env → cohérence garantie. +TP_PCT_OVERRIDE = 0.0 # 0 = utilise config.TAKE_PROFIT_PCT (synchronisé avec la prod) + +# OVERRIDE DE LA CONFIANCE MINIMALE : si > 0, remplace MIN_CONFIDENCE de config.py/.env. +# Analyse OOS sur 500 trades : +# bucket 0.62–0.70 : WR=64.1% → destructeur de PF (20% du volume, quasi toutes les pertes nettes) +# bucket 0.70–0.75 : WR=80.2% +# bucket 0.75–0.80 : WR=81.7% +# bucket 0.80–0.90 : WR=90.9% +# Seuil 0.72 → exclut la zone 64%, garde uniquement les signals ≥80% WR. +# Volume -20%, PF estimé +40–50%. +MIN_CONFIDENCE_OVERRIDE = 0.72 # 0 = utilise MIN_CONFIDENCE du .env / config.py + +# POSITION SIZING FIXE : taille calculée sur le capital initial, pas sur le +# solde courant. Évite l'explosion par compounding géométrique en backtest. +# En live le bot peut utiliser un sizing dynamique, mais pour mesurer l'edge +# du modèle on veut une taille constante. +FIXED_POSITION_VAL = INITIAL_BALANCE * RISK_PER_TRADE * LEVERAGE + +# Appliquer l'override TP si activé +_TP = TP_PCT_OVERRIDE if TP_PCT_OVERRIDE > 0 else TAKE_PROFIT_PCT +TAKE_PROFIT_PCT = _TP # shadowed pour le reste du fichier + +# Appliquer l'override MIN_CONFIDENCE si activé +_MC = MIN_CONFIDENCE_OVERRIDE if MIN_CONFIDENCE_OVERRIDE > 0 else MIN_CONFIDENCE +MIN_CONFIDENCE = _MC # shadowed pour le reste du fichier + +# Nouvelle limite : jamais plus de 50 % du solde en marge simultanément +MAX_MARGIN_USAGE = config.MAX_MARGIN_USAGE # fraction du solde (sans levier) — 20% max simultané + +# TIMEOUT : si True, les trades non résolus (ni TP ni SL) sont clôturés au +# close de la bougie MAX_HOLD_CANDLES. Si False, les positions restent ouvertes +# indéfiniment (non recommandé en live). +TIMEOUT_ENABLED = config.TIMEOUT_ENABLED # fermeture forcée après MAX_HOLD_CANDLES bougies +MAX_HOLD_CANDLES = config.MAX_HOLD_CANDLES + +# Risk manager simulé +CIRCUIT_BREAKER_LOSSES = config.CIRCUIT_BREAKER_LOSSES # 3 +CIRCUIT_BREAKER_CANDLES = config.CIRCUIT_BREAKER_COOLDOWN // 3600 # dérivé de CIRCUIT_BREAKER_COOLDOWN (en secondes → bougies 1h) +MAX_DAILY_LOSS_PCT = config.MAX_DAILY_LOSS_PCT # 0.05 = 5 % + + +# ─── Chargement modèle ─────────────────────────────────────────────────────── + +def load_model(): + if os.path.exists(config.ENSEMBLE_MODEL_PATH): + with open(config.ENSEMBLE_MODEL_PATH, "rb") as f: + return pickle.load(f) + with open(config.MODEL_PATH, "rb") as f: + raw = pickle.load(f) + m = raw["model"] if isinstance(raw, dict) else raw + return {"lgbm": m, "xgb": None, "rf": None, "meta": None, "scaler": None} + + +def load_candles(coin): + path = os.path.join(config.DATA_DIR, f"{coin}_1h.json") + if not os.path.exists(path): + return [] + with open(path) as f: + return json.load(f) + + +# ─── Étape 1 : Pré-calcul vectorisé de TOUTES les features ────────────────── + +def precompute_all_features(coins_data, btc_aligned): + """ + Appelle build_features() une seule fois par coin sur l'intégralité + des données → renvoie dict coin → np.ndarray (N, 62). + """ + features = {} + total = len(coins_data) + t0 = time.perf_counter() + + for idx, (coin, candles) in enumerate(coins_data.items(), 1): + t1 = time.perf_counter() + window = [ + {"o": c["o"], "h": c["h"], "l": c["l"], "c": c["c"], "v": c["v"]} + for c in candles + ] + btc_closes = np.array([c["c"] for c in btc_aligned[-len(candles):]]) + X = build_features(window, btc_closes=btc_closes) + features[coin] = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0) + elapsed = time.perf_counter() - t1 + remaining = elapsed * (total - idx) + print( + f" [{idx:2d}/{total}] {coin:8s} — {X.shape[0]:,} bougies, {elapsed:.1f}s" + f" | reste ~{remaining:.0f}s" + ) + + total_time = time.perf_counter() - t0 + print(f"\n Pré-calcul terminé en {total_time:.1f}s ({total_time/60:.1f} min)\n") + return features + + +# ─── Étape 2 : Batch prediction pour tout le tableau ──────────────────────── + +def batch_predict_all(model_data, features_dict): + """ + Prédit la probabilité haussière pour TOUS les candles d'un coup par coin. + Renvoie dict coin → np.ndarray (N,) de probabilités [0, 1]. + + Délègue à ensemble_core.predict_ensemble_batch() — la MÊME implémentation + que celle utilisée en live (unified_brain.py) et pendant l'entraînement + RL (rl_env.py). Corrige le point #1 du diagnostic (2ᵉ occurrence) : + l'ancienne version ici ne gérait JAMAIS les modèles DL (TFT/TGRU) — un + ensemble entraîné avec has_dl=True aurait donc été validé en backtest + avec un stacking à 3 modèles, puis exécuté en live avec un stacking à 5, + deux distributions de probabilités différentes pour le "même" modèle. + """ + probs = {} + for coin, X in features_dict.items(): + result = _ens_core.predict_ensemble_batch(model_data, X) + probs[coin] = result[:, 0] # colonne 0 = proba_long + return probs + + +# ─── Étape 3 : Backtest corrigé ────────────────────────────────────────────── + +def run_backtest_corrected(probs_dict, coins_data, min_len, features_dict=None): + """ + Simule le bot sur l'historique avec les 3 corrections + risk manager simulé. + + Logique temporelle correcte : + - Bougie i se FERME → features[i] connues → probs[i] calculé + - On ENTRE au prix closes[i] (= clôture de la bougie i) + - TP/SL résolu sur les bougies i+1, i+2, ..., i+MAX_HOLD_CANDLES + + `features_dict` (optionnel) : si fourni ET config.USE_RL_AGENT=true ET + stable-baselines3 est installé, le signal ML de chaque bougie est filtré + par le MÊME agent RL qu'en live (rl_agent.RLAgent.filter_signal()) — + boost si accord, override si RL très confiant et en désaccord, fallback + sinon. C'est ce qui rend ce backtest fidèle au système réel : ML+RL + comme un seul système, jamais le ML testé seul puis le RL en aveugle. + """ + rl_agent = None + if features_dict is not None and _RL_READY and getattr(config, "USE_RL_AGENT", False): + try: + rl_agent = get_rl_agent() + if not rl_agent.is_ready(): + print(" ⚠️ RL activé (USE_RL_AGENT=true) mais agent non chargé " + "(rl_agent.zip absent ?) — backtest en ML seul.") + rl_agent = None + else: + print(" ✅ Filtre RL actif pour ce backtest (ML+RL unifiés, comme en live).") + except Exception as e: + print(f" ⚠️ RL activé mais erreur de chargement ({e}) — backtest en ML seul.") + rl_agent = None + elif getattr(config, "USE_RL_AGENT", False) and not _RL_READY: + print(" ⚠️ USE_RL_AGENT=true mais stable-baselines3/gymnasium non installés " + "— backtest en ML seul (pip install -r requirements.txt pour corriger).") + + balance = INITIAL_BALANCE + trades = [] + equity_curve = [balance] + peak_balance = balance + + coins = list(coins_data.keys()) + closes_arr = {c: np.array([x["c"] for x in coins_data[c]]) for c in coins} + highs_arr = {c: np.array([x["h"] for x in coins_data[c]]) for c in coins} + lows_arr = {c: np.array([x["l"] for x in coins_data[c]]) for c in coins} + + # BUG 2 FIX — tracker les positions actives + # open_positions[coin] = (bougie_expiration, margin_réelle_bloquée) + open_positions: dict[str, tuple[int, float]] = {} + + # Risk manager simulé + consecutive_losses = 0 + circuit_breaker_until = 0 # indice de bougie + daily_pnl = 0.0 + current_day = -1 # jour julien de la bougie courante + + total_steps = min_len - WARMUP + progress_interval = max(1, total_steps // 20) + + for i in range(WARMUP, min_len): + step = i - WARMUP + if step % progress_interval == 0: + pct = step / total_steps * 100 + print( + f" [{pct:5.1f}%] bougie {i:,}/{min_len:,}" + f" balance ${balance:,.2f} trades {len(trades)}" + f" positions_ouvertes {len([c for c, v in open_positions.items() if v[0] > i])}" + ) + + # ── Nettoyer les positions expirées ────────────────────────────────── + open_positions = {c: v for c, v in open_positions.items() if v[0] > i} + + # ── Jour calendaire simulé (chaque bougie = 1h) ────────────────────── + day_index = i // 24 + if day_index != current_day: + current_day = day_index + daily_pnl = 0.0 # reset daily P&L + + # ── Circuit breaker simulé ─────────────────────────────────────────── + if i < circuit_breaker_until: + continue + + # ── Daily loss limit ───────────────────────────────────────────────── + if balance > 0 and (daily_pnl / balance) <= -MAX_DAILY_LOSS_PCT: + continue + + # BUG 3 FIX — compter les positions actuellement ouvertes + currently_open = len(open_positions) + available_slots = MAX_POSITIONS - currently_open + if available_slots <= 0: + continue + + # BUG 3 FIX — calculer la marge déjà engagée + # FIX — on utilise la marge RÉELLE de chaque position ouverte (stockée à l'ouverture) + # plutôt qu'une approximation basée sur le balance actuel. + already_used_margin = sum(v[1] for v in open_positions.values()) + available_margin = (balance if COMPOUND_SIZING else INITIAL_BALANCE) * MAX_MARGIN_USAGE - already_used_margin + if available_margin <= 0: + continue + + # ── Générer les signaux pour cette bougie ──────────────────────────── + candle_signals = [] + + for coin in coins: + # Skip si position déjà ouverte sur ce coin + if coin in open_positions: + continue + + prob = probs_dict[coin][i] + + if prob > MIN_CONFIDENCE: + signal = "long" + confidence = prob + elif prob < (1 - MIN_CONFIDENCE): + signal = "short" + confidence = 1 - prob + else: + continue + + # ── Filtre RL — EXACTEMENT la même règle qu'en live + # (unified_brain.UnifiedBrain.decide()) : uniquement sur signal + # ML non-neutre, jamais de re-calibration silencieuse ici. ── + if rl_agent is not None: + try: + last_feat = features_dict[coin][i] + pos_state = PositionState(in_position=False, direction=0, balance_ratio=1.0) + rl_signal, rl_confidence, _rl_action = rl_agent.filter_signal( + ml_signal=signal, ml_confidence=confidence, + features=last_feat, position_state=pos_state, + ) + signal, confidence = rl_signal, rl_confidence + if signal == "neutral": + continue + except Exception: + pass # fallback silencieux sur le signal ML, comme en live + + # BUG 1 FIX — entry au close de la bougie i (prix connu à la clôture) + entry = closes_arr[coin][i] + if entry <= 0: + continue + + # FIX BUG #5 — Slippage réaliste à l'entrée (1.5 pips) + slip = SLIPPAGE_PIPS * PIP_SIZE + if signal == "long": + entry = entry * (1 + slip) # achat : prix monte légèrement + else: + entry = entry * (1 - slip) # vente : prix descend légèrement + + if signal == "long": + sl = entry * (1 - STOP_LOSS_PCT) + tp = entry * (1 + TAKE_PROFIT_PCT) + else: + sl = entry * (1 + STOP_LOSS_PCT) + tp = entry * (1 - TAKE_PROFIT_PCT) + + # BUG 1 FIX — future commence à i+1 (bougie SUIVANTE, pas la même) + fut_start = i + 1 + fut_end = min(i + 1 + MAX_HOLD_CANDLES, min_len) + + if fut_start >= min_len: + continue # plus de données pour résoudre ce trade + + future_h = highs_arr[coin][fut_start:fut_end] + future_l = lows_arr[coin][fut_start:fut_end] + future_c = closes_arr[coin][fut_start:fut_end] + + if signal == "long": + sl_hit = np.where(future_l <= sl)[0] + tp_hit = np.where(future_h >= tp)[0] + else: + sl_hit = np.where(future_h >= sl)[0] + tp_hit = np.where(future_l <= tp)[0] + + sl_idx = sl_hit[0] if len(sl_hit) > 0 else 999 + tp_idx = tp_hit[0] if len(tp_hit) > 0 else 999 + + if sl_idx == tp_idx == 999: + if not TIMEOUT_ENABLED: + continue # position ignorée si timeout désactivé + result = "timeout" + exit_price = future_c[-1] if len(future_c) > 0 else entry + hold_dur = len(future_c) + elif tp_idx <= sl_idx: + result = "tp" + exit_price = tp + hold_dur = tp_idx + 1 + else: + result = "sl" + exit_price = sl + hold_dur = sl_idx + 1 + + # FIX BUG #5 — Slippage réaliste à la sortie (1.5 pips) + if signal == "long": + exit_price = exit_price * (1 - slip) # vente : prix descend + else: + exit_price = exit_price * (1 + slip) # rachat : prix monte + + if signal == "long": + pnl_pct = (exit_price - entry) / entry + else: + pnl_pct = (entry - exit_price) / entry + + # COMPOUND_SIZING : taille recalculée sur le solde courant, plafonnée + if COMPOUND_SIZING: + position_val = min( + max(balance * RISK_PER_TRADE * LEVERAGE, 0.0), + MAX_POSITION_VALUE + ) + else: + position_val = FIXED_POSITION_VAL + fee = position_val * FEE_RATE * 2 + net_pnl = position_val * pnl_pct - fee + + candle_signals.append({ + "coin": coin, + "signal": signal, + "confidence": float(confidence), + "result": result, + "entry": float(entry), + "exit": float(exit_price), + "pnl": float(net_pnl), + "position_val": float(position_val), + "hold_candles": int(hold_dur), + "candle_idx": i, + }) + + # ── Sélectionner les meilleurs signaux dans la limite des slots ─────── + candle_signals.sort(key=lambda x: x["confidence"], reverse=True) + + margin_used_this_step = 0.0 + for t in candle_signals: + if len(open_positions) >= MAX_POSITIONS: + break + margin_needed_check = (balance if COMPOUND_SIZING else INITIAL_BALANCE) * RISK_PER_TRADE + if margin_used_this_step + margin_needed_check > available_margin: + break + + # Enregistrer la position comme ouverte jusqu'à sa bougie de clôture + # FIX — on stocke (expiration, margin_réelle) pour un calcul de marge exact + margin_this_trade = (balance if COMPOUND_SIZING else INITIAL_BALANCE) * RISK_PER_TRADE + close_at = i + 1 + t["hold_candles"] + open_positions[t["coin"]] = (close_at, margin_this_trade) + margin_used_this_step += margin_this_trade + + # Appliquer le P&L + balance = max(0.0, balance + t["pnl"]) + trades.append(t) + equity_curve.append(balance) + peak_balance = max(peak_balance, balance) + + # ── Risk manager simulé ────────────────────────────────────────── + daily_pnl += t["pnl"] + if t["pnl"] < 0: + consecutive_losses += 1 + if consecutive_losses >= CIRCUIT_BREAKER_LOSSES: + circuit_breaker_until = i + CIRCUIT_BREAKER_CANDLES + consecutive_losses = 0 + pass # circuit breaker silencieux (trop verbeux en console) + else: + consecutive_losses = 0 + + if balance == 0.0: + print(f" 💀 Balance = 0 à la bougie {i} — simulation arrêtée") + return trades, equity_curve, (rl_agent is not None) + + return trades, equity_curve, (rl_agent is not None) + + +# ─── Affichage des résultats ────────────────────────────────────────────────── + +def print_results(trades, balance, equity_curve=None): + print("\n" + "=" * 62) + print(" RÉSULTATS DU BACKTEST (v10-fixed — OOS STRICT + COMPOUND + TP 2.2% + CONF 0.72)") + print("=" * 62) + + if not trades: + print(" Aucun trade généré.") + return + + wins = [t for t in trades if t["pnl"] > 0] + loss = [t for t in trades if t["pnl"] <= 0] + tps = [t for t in trades if t["result"] == "tp"] + sls = [t for t in trades if t["result"] == "sl"] + tmos = [t for t in trades if t["result"] == "timeout"] + longs = [t for t in trades if t["signal"] == "long"] + shts = [t for t in trades if t["signal"] == "short"] + + total_pnl = balance - INITIAL_BALANCE + pnl_pct = total_pnl / INITIAL_BALANCE * 100 + win_rate = len(wins) / len(trades) * 100 if trades else 0 + + # Drawdown max — calculé sur l'equity curve RÉELLE du backtest + # FIX — l'ancienne méthode appliquait les trades séquentiellement alors que + # plusieurs positions sont ouvertes simultanément : le vrai drawdown peut + # être plus élevé. On utilise l'equity_curve produite par run_backtest_corrected. + eq_for_dd = equity_curve if equity_curve and len(equity_curve) > 1 else None + if eq_for_dd: + peak = eq_for_dd[0] + max_dd = 0.0 + for v in eq_for_dd: + peak = max(peak, v) + dd = (peak - v) / peak * 100 if peak > 0 else 0 + max_dd = max(max_dd, dd) + else: + # Fallback séquentiel si equity_curve non fournie + peak = INITIAL_BALANCE + max_dd = 0.0 + running = INITIAL_BALANCE + for t in trades: + running += t["pnl"] + peak = max(peak, running) + dd = (peak - running) / peak * 100 if peak > 0 else 0 + max_dd = max(max_dd, dd) + + avg_win = np.mean([t["pnl"] for t in wins]) if wins else 0 + avg_loss = np.mean([t["pnl"] for t in loss]) if loss else 0 + # FIX — vrai Profit Factor = somme des gains / somme des pertes + # (l'ancienne formule avg_win/avg_loss donnait le ratio moyen, pas le PF) + gross_profit = sum(t["pnl"] for t in wins) + gross_loss = abs(sum(t["pnl"] for t in loss)) + pf = gross_profit / gross_loss if gross_loss > 0 else float("inf") + best = max(trades, key=lambda x: x["pnl"]) + worst = min(trades, key=lambda x: x["pnl"]) + + avg_hold = np.mean([t["hold_candles"] for t in trades]) if trades else 0 + + # Sharpe (calculé sur les rendements % par trade, pas sur le P&L absolu) + # FIX — le P&L absolu est biaisé par le compound : les trades tardifs valent + # 1000x plus qu'au départ, ce qui gonfle artificiellement le Sharpe. + # On normalise chaque trade par la taille de position pour obtenir + # un rendement % comparable quelle que soit la taille du compte. + # Annualisation : ~24 trades/jour × 252 jours = 6048 périodes/an. + ret_series = np.array([ + t["pnl"] / max(t.get("position_val", FIXED_POSITION_VAL), 1e-9) + for t in trades + ]) + sharpe = (ret_series.mean() / ret_series.std() * np.sqrt(6048)) \ + if ret_series.std() > 0 else 0 + + # Top/flop coins + coin_pnl: dict[str, float] = {} + for t in trades: + coin_pnl[t["coin"]] = coin_pnl.get(t["coin"], 0) + t["pnl"] + top_coins = sorted(coin_pnl.items(), key=lambda x: x[1], reverse=True) + flop_coins = sorted(coin_pnl.items(), key=lambda x: x[1]) + + print(f"\n 💰 Balance initiale : ${INITIAL_BALANCE:>12,.2f}") + print(f" 💰 Balance finale : ${balance:>12,.2f}") + print(f" 📈 PnL total : ${total_pnl:>+12,.2f} ({pnl_pct:+.1f}%)") + print(f" 📉 Drawdown max : {max_dd:.1f}%") + print(f" 📐 Sharpe (approx.) : {sharpe:.2f}") + print(f" ⚖️ Profit Factor : {pf:.2f}") + + # Compound sizing info + if COMPOUND_SIZING: + cap_balance = MAX_POSITION_VALUE / (RISK_PER_TRADE * LEVERAGE) + print(f" 🔄 Compound : ON — plafond atteint à ~${cap_balance:,.0f} de solde") + print(f"\n 📊 Trades total : {len(trades):,}") + print(f" ✅ Gagnants : {len(wins):,} ({win_rate:.1f}%)") + print(f" ❌ Perdants : {len(loss):,} ({100-win_rate:.1f}%)") + print(f" 🎯 TP atteints : {len(tps):,}") + print(f" 🛑 SL atteints : {len(sls):,}") + print(f" ⏱️ Timeouts : {len(tmos):,}") + print(f" 📗 Longs / 📕 Shorts : {len(longs):,} / {len(shts):,}") + print(f" ⌛ Durée moy. trade : {avg_hold:.1f} bougies") + print(f"\n 💵 Gain moyen : ${avg_win:>+10,.2f}") + print(f" 💵 Perte moyenne : ${avg_loss:>+10,.2f}") + print(f" 🏆 Meilleur trade : ${best['pnl']:>+10,.2f} ({best['coin']} {best['signal'].upper()})") + print(f" 💀 Pire trade : ${worst['pnl']:>+10,.2f} ({worst['coin']} {worst['signal'].upper()})") + + print(f"\n 🏅 Top 5 coins:") + max_coin_pnl = abs(top_coins[0][1]) if top_coins else 1 + for coin, pnl in top_coins[:5]: + bar = "█" * int(max(0, pnl) / max(1, max_coin_pnl) * 20) + print(f" {coin:8s} ${pnl:>+10,.2f} {bar}") + + print(f"\n 💀 Flop 5 coins:") + for coin, pnl in flop_coins[:5]: + bar = "█" * int(max(0, -pnl) / max(1, max_coin_pnl) * 20) + print(f" {coin:8s} ${pnl:>+10,.2f} {bar}") + + print("\n" + "=" * 62) + + # ── Diagnostic ────────────────────────────────────────────────────────── + print("\n DIAGNOSTIC :") + if pnl_pct > 50: + print(" ✅ PnL > +50% — Fort sur historique, vérifie en paper") + elif pnl_pct > 20: + print(" ✅ PnL > +20% — Excellent") + elif pnl_pct > 5: + print(" 🟡 PnL +5% à +20% — Acceptable, passe en paper trading") + elif pnl_pct > 0: + print(" 🟠 PnL faiblement + — Augmente MIN_CONFIDENCE ou réduis levier") + else: + print(" ❌ PnL négatif — Ré-entraîne avec OPTUNA_TRIALS=50 dans train.py") + + if max_dd < 15: + print(" ✅ DD < 15% — Très sûr pour 5x levier") + elif max_dd < 25: + print(" 🟡 DD 15-25% — Acceptable, surveille en paper trading") + else: + print(" ❌ DD > 25% — Réduis RISK_PER_TRADE ou LEVERAGE dans .env") + + if win_rate > 52: + print(" ✅ Win rate > 52% — Excellent avec R:R 1:2") + elif win_rate > 45: + print(" 🟡 Win rate 45-52% — OK, le R:R compense") + else: + print(" ❌ Win rate < 45% — Augmente MIN_CONFIDENCE à 0.72+") + + if sharpe > 1.5: + print(" ✅ Sharpe > 1.5 — Très bon rapport risque/rendement") + elif sharpe > 1.0: + print(" 🟡 Sharpe 1.0-1.5 — Correct") + else: + print(" 🟠 Sharpe < 1.0 — Stratégie volatile, réduis le levier") + + print("=" * 62) + + # ── Equity curve ASCII dans le terminal ────────────────────────────────── + W = 62 + print("\n" + "=" * W) + print(" COURBE D\'ÉQUITÉ (terminal)") + print("=" * W) + # Rééchantillonner l'equity curve sur 50 points max + # Reconstruire l'equity curve proprement + # FIX — suppression du dead code : l'ancienne ligne calculait balance + X - X = balance + running = INITIAL_BALANCE + eq_vals = [running] + for t in trades: + running = max(0.0, running + t["pnl"]) + eq_vals.append(running) + n_pts = min(50, len(eq_vals)) + step = max(1, len(eq_vals) // n_pts) + sample = eq_vals[::step] + if eq_vals[-1] not in sample: + sample.append(eq_vals[-1]) + eq_min = min(sample) + eq_max = max(sample) + rows = 8 + print() + for row in range(rows, -1, -1): + thresh = eq_min + (eq_max - eq_min) * row / rows + if row == rows: + label = f"${eq_max:>10,.0f} │" + elif row == 0: + label = f"${eq_min:>10,.0f} │" + else: + label = f"{'':>11} │" + line = "" + for v in sample: + norm = (v - eq_min) / max(eq_max - eq_min, 1) + filled = norm * rows >= row + line += "█" if filled else " " + print(f" {label}{line}") + print(f" {'':>11} └{'─'*len(sample)}") + print(f" {'':>12} début{'':>{max(0,len(sample)-12)}}fin") + + # ── Distribution des trades par résultat ───────────────────────────────── + print("\n" + "=" * W) + print(" DISTRIBUTION DES TRADES") + print("=" * W) + total_t = len(trades) + tp_pct_t = len(tps) / total_t * 100 if total_t else 0 + sl_pct_t = len(sls) / total_t * 100 if total_t else 0 + tmo_pct_t = len(tmos) / total_t * 100 if total_t else 0 + bar_w = 30 + def pct_bar(pct, char="█"): + n = int(pct / 100 * bar_w) + return char * n + "░" * (bar_w - n) + print(f" 🎯 TP {pct_bar(tp_pct_t)} {len(tps):>6,} ({tp_pct_t:5.1f}%)") + print(f" 🛑 SL {pct_bar(sl_pct_t)} {len(sls):>6,} ({sl_pct_t:5.1f}%)") + print(f" ⏱ Timeout {pct_bar(tmo_pct_t)} {len(tmos):>6,} ({tmo_pct_t:5.1f}%)") + + # ── PnL par coin (barres horizontales) ─────────────────────────────────── + print("\n" + "=" * W) + print(" PnL PAR COIN (tous les coins)") + print("=" * W) + all_coins = sorted(coin_pnl.items(), key=lambda x: x[1], reverse=True) + max_abs = max(abs(p) for _, p in all_coins) if all_coins else 1 + for coin, pnl in all_coins: + bar_len = int(abs(pnl) / max_abs * 20) + sign = "+" if pnl >= 0 else "-" + bar = ("█" if pnl >= 0 else "░") * bar_len + print(f" {coin:6s} {sign}${abs(pnl):>10,.0f} {bar}") + + print("\n" + "=" * W + "\n") + + +# ─── Sauvegarde des résultats ───────────────────────────────────────────────── + +def save_results(trades, balance, equity_curve, run_timestamp=None, rl_filter_active=False): + """ + Sauvegarde les résultats du backtest dans backtest_results.json. + ⚠️ ÉCRASE TOUJOURS le fichier précédent — un seul fichier de résultats, + mis à jour à chaque nouveau backtest (pas d'accumulation). + + Format identique à l'ancien backtest_results.json pour la compatibilité + avec dashboard.py / web_ui.py. + """ + if not trades: + print(" [SAVE] Aucun trade — fichier non écrasé.") + return + + wins = [t for t in trades if t["pnl"] > 0] + loss = [t for t in trades if t["pnl"] <= 0] + tps = [t for t in trades if t["result"] == "tp"] + sls = [t for t in trades if t["result"] == "sl"] + tmos = [t for t in trades if t["result"] == "timeout"] + + total_pnl = balance - INITIAL_BALANCE + pnl_pct = total_pnl / INITIAL_BALANCE * 100 + win_rate = len(wins) / len(trades) * 100 if trades else 0 + + # Drawdown sur equity curve réelle + eq = equity_curve if equity_curve and len(equity_curve) > 1 else None + if eq: + peak, max_dd = eq[0], 0.0 + for v in eq: + peak = max(peak, v) + dd = (peak - v) / peak * 100 if peak > 0 else 0 + max_dd = max(max_dd, dd) + else: + peak, max_dd, running = INITIAL_BALANCE, 0.0, INITIAL_BALANCE + for t in trades: + running += t["pnl"] + peak = max(peak, running) + dd = (peak - running) / peak * 100 if peak > 0 else 0 + max_dd = max(max_dd, dd) + + gross_profit = sum(t["pnl"] for t in wins) + gross_loss = abs(sum(t["pnl"] for t in loss)) + pf = gross_profit / gross_loss if gross_loss > 0 else float("inf") + + ret_series = np.array([ + t["pnl"] / max(t.get("position_val", FIXED_POSITION_VAL), 1e-9) + for t in trades + ]) + sharpe = (ret_series.mean() / ret_series.std() * np.sqrt(6048)) \ + if ret_series.std() > 0 else 0 + + avg_hold = float(np.mean([t["hold_candles"] for t in trades])) if trades else 0 + + coin_pnl: dict[str, float] = {} + for t in trades: + coin_pnl[t["coin"]] = coin_pnl.get(t["coin"], 0) + t["pnl"] + + result = { + # ── Metadata ────────────────────────────────────────────────────── + "backtest_version": "v10-fixed", + "run_timestamp": run_timestamp or datetime.now(timezone.utc).isoformat(), + "rl_filter_active": bool(rl_filter_active), + "system": "ML+RL unifié" if rl_filter_active else "ML seul (RL indisponible)", + "corrections": [ + "profit_factor_fixed", + "sharpe_pct_based", + "dead_code_removed", + "drawdown_real_equity", + "margin_real_values", + ], + # ── Paramètres utilisés ─────────────────────────────────────────── + "min_history_candles": MIN_HISTORY_CANDLES, + "oos_ratio": OOS_RATIO, + "use_full_history": USE_FULL_HISTORY, + "compound_sizing": COMPOUND_SIZING, + "tp_pct_used": TAKE_PROFIT_PCT, + "sl_pct_used": STOP_LOSS_PCT, + "min_confidence_used": MIN_CONFIDENCE, + "max_position_value": MAX_POSITION_VALUE, + "fixed_position_val": FIXED_POSITION_VAL, + # ── Métriques principales ───────────────────────────────────────── + "balance_initial": INITIAL_BALANCE, + "balance_final": float(balance), + "total_pnl": float(total_pnl), + "pnl_pct": float(pnl_pct), + "win_rate": float(win_rate), + "max_drawdown": float(max_dd), + "sharpe": float(sharpe), + "profit_factor": float(pf), + # ── Compteurs trades ────────────────────────────────────────────── + "total_trades": len(trades), + "tp_count": len(tps), + "sl_count": len(sls), + "timeout_count": len(tmos), + "avg_hold_candles": float(avg_hold), + # ── Config active ───────────────────────────────────────────────── + "config": { + "leverage": LEVERAGE, + "risk_per_trade": RISK_PER_TRADE, + "stop_loss_pct": STOP_LOSS_PCT, + "take_profit_pct": TAKE_PROFIT_PCT, + "min_confidence": MIN_CONFIDENCE, + "max_positions": MAX_POSITIONS, + "max_margin_usage": MAX_MARGIN_USAGE, + }, + # ── PnL par coin ────────────────────────────────────────────────── + "coin_pnl": {k: float(v) for k, v in coin_pnl.items()}, + # ── 500 derniers trades pour le dashboard ───────────────────────── + "trades_last_500": trades[-500:], + # ── Equity curve rééchantillonnée sur 500 points max ────────────── + "equity_curve_sampled": ( + equity_curve[:: max(1, len(equity_curve) // 500)] + if equity_curve else [] + ), + } + + out_path = os.path.join(os.path.dirname(__file__), "backtest_results.json") + # Écriture atomique : on écrit d'abord dans un fichier temporaire, + # puis on le renomme → évite un fichier corrompu en cas d'interruption. + tmp_path = out_path + ".tmp" + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2, ensure_ascii=False) + os.replace(tmp_path, out_path) # atomique sur Linux/Windows + + size_kb = os.path.getsize(out_path) / 1024 + print(f"\n 💾 Résultats sauvegardés → backtest_results.json ({size_kb:.0f} KB)") + print(f" Run : {result['run_timestamp']}") + print(f" {len(trades):,} trades | balance ${balance:,.2f} | PnL {pnl_pct:+.1f}%") + + +# ─── Main ───────────────────────────────────────────────────────────────────── + +def main(): + t_global = time.perf_counter() + + print("=" * 62) + print(" AHAD QUANT — Backtester v10-fixed — OOS STRICT + COMPOUND + R:R 1:1.47") + print(" (OOS strict 30% + compound sizing + TP 2.2% + confiance 0.72)") + print("=" * 62) + print( + f"\n Config : levier {LEVERAGE}x | SL {STOP_LOSS_PCT*100:.1f}%" + f" | TP {TAKE_PROFIT_PCT*100:.1f}% | confiance > {MIN_CONFIDENCE*100:.0f}%" + f" | margin_max {MAX_MARGIN_USAGE*100:.0f}%" + f" | compound {'ON 🚀' if COMPOUND_SIZING else 'OFF'}" + f" | cap ${MAX_POSITION_VALUE:,.0f}/trade" + ) + + # ── Chargement ──────────────────────────────────────────────────────────── + print("\n[1/4] Chargement des données...") + # Paire de référence pour la corrélation (EURUSD = paire dominante Forex) + _ref_pair = "EURUSD" + btc_candles = load_candles(_ref_pair) + if not btc_candles: + print(f" [ERREUR] {_ref_pair}_1h.json introuvable. Lance download_data.py d'abord.") + return + + coins_data: dict[str, list] = {} + excluded = [] + for coin in config.COINS: + c = load_candles(coin) + if len(c) >= MIN_HISTORY_CANDLES: + coins_data[coin] = c + else: + excluded.append(f"{coin}({len(c)}b)") + + if excluded: + print(f" ⚠️ Paires exclues (historique insuffisant < {MIN_HISTORY_CANDLES:,} bougies) :") + print(f" {', '.join(excluded)}") + + if not coins_data: + print(" [ERREUR] Aucune donnée trouvée dans data/. Lance download_data.py (Forex).") + return + + # Aligner toutes les séries sur la même longueur + min_len = min(len(c) for c in coins_data.values()) + min_len = min(min_len, len(btc_candles)) + for coin in coins_data: + coins_data[coin] = coins_data[coin][-min_len:] + btc_aligned = btc_candles[-min_len:] + + # ── Découpage Out-Of-Sample (ou historique complet) ───────────────────── + if USE_FULL_HISTORY: + # MODE FULL HISTORY : toutes les bougies disponibles sont utilisées. + # Aucune donnée n'est exclue — le backtest maximise le nombre de trades. + # ⚠️ Les bougies in-sample (vues à l'entraînement) sont incluses : + # les métriques sont donc optimistes par rapport au vrai live. + oos_start = 0 + print(f" {len(coins_data)} coins | {min_len:,} bougies total " + f"| MODE FULL HISTORY (toutes les bougies) " + f"| OOS strict désactivé") + else: + # MODE OOS STRICT : on ne garde QUE les dernières OOS_RATIO bougies. + # train.py : TRAIN=70%, VAL=15%, TEST=15% → OOS_RATIO=0.30 englobe + # entièrement le set de test + une marge de sécurité supplémentaire. + oos_start = int(min_len * (1 - OOS_RATIO)) + for coin in coins_data: + coins_data[coin] = coins_data[coin][oos_start:] + btc_aligned = btc_aligned[oos_start:] + min_len = min(len(c) for c in coins_data.values()) + n_total_candles = min_len + oos_start + print(f" {len(coins_data)} coins | {n_total_candles:,} bougies total " + f"| OOS : {min_len:,} bougies ({OOS_RATIO*100:.0f}% finaux) " + f"| In-sample ignoré : {oos_start:,} bougies") + + # ── Pré-calcul features ─────────────────────────────────────────────────── + print(f"\n[2/4] Pré-calcul des features (1 appel / coin)...") + print(f" Estimation : ~{len(coins_data) * 17:.0f}s ({len(coins_data) * 17/60:.1f} min)\n") + features_dict = precompute_all_features(coins_data, btc_aligned) + + # ── Batch prediction ────────────────────────────────────────────────────── + print("[3/4] Prédiction batch (tout le tableau d'un coup)...") + t_pred = time.perf_counter() + model_data = load_model() + probs_dict = batch_predict_all(model_data, features_dict) + print(f" Terminé en {time.perf_counter()-t_pred:.1f}s\n") + + # ── Backtest ────────────────────────────────────────────────────────────── + print(f"[4/4] Simulation backtest corrigée ({min_len - WARMUP:,} steps)...\n") + trades, equity_curve, rl_filter_active = run_backtest_corrected( + probs_dict, coins_data, min_len, features_dict=features_dict + ) + + # ── Résultats ───────────────────────────────────────────────────────────── + final_balance = equity_curve[-1] if equity_curve else INITIAL_BALANCE + run_ts = datetime.now(timezone.utc).isoformat() + print_results(trades, final_balance, equity_curve) + + # ── Sauvegarde — écrase toujours le backtest précédent ──────────────── + save_results(trades, final_balance, equity_curve, run_timestamp=run_ts, rl_filter_active=rl_filter_active) + + total_time = time.perf_counter() - t_global + print(f" Temps total : {total_time:.0f}s ({total_time/60:.1f} min)") + + +if __name__ == "__main__": + main() diff --git a/config.py b/config.py new file mode 100644 index 0000000..4c1d7f6 --- /dev/null +++ b/config.py @@ -0,0 +1,270 @@ +""" +AHAD QUANT — Configuration (Forex Edition) +Loads all settings from environment variables with sensible defaults. +""" + +import os +from dotenv import load_dotenv + +load_dotenv(override=True) + + +# ─── Exchange / Broker selection ───────────────────────────────────────────── +# Valeurs acceptées : oanda | mt5 | alpaca | ccxt | ib | paper +EXCHANGE: str = os.getenv("EXCHANGE", "oanda") + +# ─── Source de données historiques (INDÉPENDANTE du broker d'exécution) ─────── +# auto → OANDA si clé disponible, sinon yfinance +# oanda → OANDA v20 REST API (clé requise) +# yfinance → Yahoo Finance (gratuit, sans clé, 2 ans max) +# twelvedata → Twelve Data (800 req/j gratuit, clé recommandée) +# alphavantage→ Alpha Vantage (25 req/j gratuit) +# mt5 → MT5 Python SDK (Windows uniquement) +# ccxt → via broker CCXT_BROKER +DATA_SOURCE: str = os.getenv("DATA_SOURCE", "auto") + +# ─── Paper Mode (simulate trades, no real money) ───────────────────────────── +PAPER_MODE: bool = os.getenv("PAPER_MODE", "false").lower() == "true" +PAPER_INITIAL_BALANCE: float = float(os.getenv("PAPER_INITIAL_BALANCE", "100.0")) + +# ─── OANDA credentials (primary Forex broker) ──────────────────────────────── +OANDA_API_KEY: str = os.getenv("OANDA_API_KEY", "") +OANDA_ACCOUNT_ID: str = os.getenv("OANDA_ACCOUNT_ID", "") +OANDA_PRACTICE: bool = os.getenv("OANDA_PRACTICE", "true").lower() == "true" + +# ─── MetaTrader 5 credentials ───────────────────────────────────────────────── +MT5_LOGIN: int = int(os.getenv("MT5_LOGIN", "0")) +MT5_PASSWORD: str = os.getenv("MT5_PASSWORD", "") +MT5_SERVER: str = os.getenv("MT5_SERVER", "") + +# ─── MT5 CSV Bridge ──────────────────────────────────────────────────────────── +# Activer le bridge CSV pour connecter AHAD QUANT à un EA MetaTrader 5 +# Mettre EXCHANGE=mt5 dans .env pour utiliser ce mode +MT5_BRIDGE_ENABLED: bool = os.getenv("MT5_BRIDGE_ENABLED", "false").lower() == "true" +MT5_FILES_PATH: str = os.getenv("MT5_FILES_PATH", "") # Chemin vers MQL5/Files/ +MT5_POLL_INTERVAL: float = float(os.getenv("MT5_POLL_INTERVAL", "0.2")) # secondes +MT5_SIGNAL_TIMEOUT: int = int(os.getenv("MT5_SIGNAL_TIMEOUT", "300")) # secondes (5 min) +# Suffixe symbole MT5 — certains brokers ajoutent .m, .r, .pro, etc. +# Ex: ICMarkets, Pepperstone → "USDJPY.m" | La plupart → "" +MT5_SYMBOL_SUFFIX: str = os.getenv("MT5_SYMBOL_SUFFIX", "") # Ex: ".m" + +# ─── Alpaca Markets (paper + live, API REST, très accessible) ──────────────── +# Paper trading : https://paper-api.alpaca.markets +# Live trading : https://api.alpaca.markets +ALPACA_API_KEY: str = os.getenv("ALPACA_API_KEY", "") +ALPACA_SECRET: str = os.getenv("ALPACA_SECRET", "") +ALPACA_PAPER: bool = os.getenv("ALPACA_PAPER", "true").lower() == "true" + +# ─── CCXT — broker générique (IG, GAIN Capital, Binance, Bybit, etc.) ──────── +# Définir EXCHANGE=ccxt puis CCXT_BROKER=nom_du_broker (ex: "ig", "okcoin") +CCXT_BROKER: str = os.getenv("CCXT_BROKER", "") +CCXT_API_KEY: str = os.getenv("CCXT_API_KEY", "") +CCXT_API_SECRET: str = os.getenv("CCXT_API_SECRET", "") +CCXT_PASSPHRASE: str = os.getenv("CCXT_PASSPHRASE", "") +CCXT_SANDBOX: bool = os.getenv("CCXT_SANDBOX", "false").lower() == "true" + +# ─── Twelve Data (données historiques — 800 req/j gratuit) ─────────────────── +# Inscription : https://twelvedata.com/ (plan Free suffisant pour backtests) +TWELVE_DATA_API_KEY: str = os.getenv("TWELVE_DATA_API_KEY", "") + +# ─── Alpha Vantage (données historiques — 25 req/j gratuit) ────────────────── +# Inscription : https://www.alphavantage.co/support/#api-key +ALPHA_VANTAGE_API_KEY: str = os.getenv("ALPHA_VANTAGE_API_KEY", "") + +# ─── Interactive Brokers (TWS / IB Gateway) ─────────────────────────────────── +IB_HOST: str = os.getenv("IB_HOST", "127.0.0.1") +IB_PORT: int = int(os.getenv("IB_PORT", "7497")) +IB_CLIENT_ID: int = int(os.getenv("IB_CLIENT_ID", "1")) + +# ─── Fee rates (Forex spread equivalent) ───────────────────────────────────── +# ~0.2 pip for majors on ECN accounts ≈ 0.00002 (2 pips round-trip = 0.00004) +FEE_RATE: float = float(os.getenv("FEE_RATE", "0.00004")) +MAKER_FEE_RATE: float = float(os.getenv("MAKER_FEE_RATE", "0.00002")) + +# ─── Trading parameters ─────────────────────────────────────────────────────── +# ESMA: 30:1 majors, 20:1 minors/gold, 10:1 commodities +# Offshore brokers: up to 500:1 — use responsibly +LEVERAGE: int = int(os.getenv("LEVERAGE", "30")) +MAX_POSITIONS: int = int(os.getenv("MAX_POSITIONS", "5")) +RISK_PER_TRADE: float = float(os.getenv("RISK_PER_TRADE", "0.01")) # 1% of equity (Forex standard) +MAX_DAILY_LOSS_PCT: float = float(os.getenv("MAX_DAILY_LOSS_PCT", "0.03")) + +# ─── Forex lot sizing ───────────────────────────────────────────────────────── +# 1 standard lot = 100,000 units of base currency +# 1 mini lot = 10,000 units +# 1 micro lot = 1,000 units +UNITS_PER_LOT: int = 100_000 +MIN_LOT_SIZE: float = float(os.getenv("MIN_LOT_SIZE", "0.01")) # micro lot +MAX_LOT_SIZE: float = float(os.getenv("MAX_LOT_SIZE", "10.0")) # 10 standard lots + +# ─── Risk management ────────────────────────────────────────────────────────── +STOP_LOSS_PCT: float = float(os.getenv("STOP_LOSS_PCT", "0.0050")) # 50 pips on EUR/USD ≈ 0.5% +TAKE_PROFIT_PCT: float = float(os.getenv("TAKE_PROFIT_PCT", "0.0075")) # 75 pips — R:R 1:1.5 +CIRCUIT_BREAKER_LOSSES: int = int(os.getenv("CIRCUIT_BREAKER_LOSSES", "3")) +CIRCUIT_BREAKER_COOLDOWN: int = int(os.getenv("CIRCUIT_BREAKER_COOLDOWN", "3600")) + +# ─── Multi-target TP (ATR-based) ────────────────────────────────────────────── +MULTI_TP_ENABLED: bool = os.getenv("MULTI_TP_ENABLED", "true").lower() == "true" +TP1_ATR_MULT: float = float(os.getenv("TP1_ATR_MULT", "0.8")) +TP2_ATR_MULT: float = float(os.getenv("TP2_ATR_MULT", "1.3")) + +# ─── Auto-Unstuck ───────────────────────────────────────────────────────────── +AUTO_UNSTUCK_ENABLED: bool = os.getenv("AUTO_UNSTUCK_ENABLED", "true").lower() == "true" +_UNSTUCK_DEFAULT = "[[-0.02, 0.25], [-0.03, 0.25], [-0.04, 0.25], [-0.05, 1.0]]" +try: + import json as _json + UNSTUCK_LEVELS: list = [tuple(x) for x in _json.loads(os.getenv("UNSTUCK_LEVELS", _UNSTUCK_DEFAULT))] +except Exception: + UNSTUCK_LEVELS: list = [(-0.02, 0.25), (-0.03, 0.25), (-0.04, 0.25), (-0.05, 1.0)] + +# ─── Model / AI ─────────────────────────────────────────────────────────────── +MODEL_PATH: str = os.getenv("MODEL_PATH", "model.pkl") +ENSEMBLE_MODEL_PATH: str = os.getenv("ENSEMBLE_MODEL_PATH", "model_ensemble.pkl") +MIN_CONFIDENCE: float = float(os.getenv("MIN_CONFIDENCE", "0.72")) +USE_ENSEMBLE: bool = os.getenv("USE_ENSEMBLE", "true").lower() == "true" +USE_REGIME_FILTER: bool = os.getenv("USE_REGIME_FILTER", "false").lower() == "true" + +# ─── Sizing & capital controls ──────────────────────────────────────────────── +COMPOUND_ENABLED: bool = os.getenv("COMPOUND_ENABLED", "true").lower() == "true" +MAX_MARGIN_USAGE: float = float(os.getenv("MAX_MARGIN_USAGE", "0.20")) +CAPITAL_CAP_PER_TRADE: float = float(os.getenv("CAPITAL_CAP_PER_TRADE", "1000000")) + +# ─── Timeout / hold duration ────────────────────────────────────────────────── +TIMEOUT_ENABLED: bool = os.getenv("TIMEOUT_ENABLED", "true").lower() == "true" +MAX_HOLD_CANDLES: int = int(os.getenv("MAX_HOLD_CANDLES", "3")) + +# ─── Auto-Retraining ────────────────────────────────────────────────────────── +# AUTO_RETRAIN_ENABLED/AUTO_RETRAIN_INTERVAL_HOURS gouvernent UNIQUEMENT le +# ré-entraînement complet (download_data.py + train.py) — lourd, dépendant du +# réseau (téléchargement de nouvelles bougies), donc volontairement PLUS +# automatique par défaut (reste disponible en appel manuel). +AUTO_RETRAIN_ENABLED: bool = os.getenv("AUTO_RETRAIN_ENABLED", "false").lower() == "true" +AUTO_RETRAIN_INTERVAL_HOURS: int = int(os.getenv("AUTO_RETRAIN_INTERVAL_HOURS", "24")) +AUTO_RETRAIN_MIN_ACCURACY: float = float(os.getenv("AUTO_RETRAIN_MIN_ACCURACY", "0.55")) + +# DAILY_LOCAL_RETRAIN_ENABLED gouverne le cycle UNIFIÉ quotidien et 100% LOCAL +# (aucun téléchargement réseau) : warm-start ML + fine-tune RL, tous deux sur +# les données déjà sur disque + le buffer de trades réels, avec acceptation +# uniquement si le nouveau modèle est meilleur (sinon l'ancien est conservé). +# C'est ce cycle qui tourne automatiquement en arrière-plan par défaut. +DAILY_LOCAL_RETRAIN_ENABLED: bool = os.getenv("DAILY_LOCAL_RETRAIN_ENABLED", "true").lower() == "true" +DAILY_LOCAL_RETRAIN_INTERVAL_HOURS: int = int(os.getenv("DAILY_LOCAL_RETRAIN_INTERVAL_HOURS", "24")) + +# ─── Grid Bot ───────────────────────────────────────────────────────────────── +GRID_BOT_ENABLED: bool = os.getenv("GRID_BOT_ENABLED", "false").lower() == "true" +GRID_PAIR: str = os.getenv("GRID_PAIR", "EURUSD") +GRID_COIN: str = GRID_PAIR # backward-compat alias +GRID_STRATEGY: str = os.getenv("GRID_STRATEGY", "neutral") +GRID_LOWER: float = float(os.getenv("GRID_LOWER", "0")) +GRID_UPPER: float = float(os.getenv("GRID_UPPER", "0")) +GRID_LEVELS: int = int(os.getenv("GRID_LEVELS", "10")) +GRID_TOTAL_USDT: float = float(os.getenv("GRID_TOTAL_USDT", "1000")) +GRID_LEVERAGE: int = int(os.getenv("GRID_LEVERAGE", "10")) + +# ─── DCA Bot ────────────────────────────────────────────────────────────────── +DCA_BOT_ENABLED: bool = os.getenv("DCA_BOT_ENABLED", "false").lower() == "true" +DCA_PAIR: str = os.getenv("DCA_PAIR", "EURUSD") +DCA_COIN: str = DCA_PAIR # backward-compat alias +DCA_STRATEGY: str = os.getenv("DCA_STRATEGY", "classic") +DCA_BASE_ORDER_USDT: float = float(os.getenv("DCA_BASE_ORDER_USDT", "100")) +DCA_SAFETY_ORDER_USDT: float = float(os.getenv("DCA_SAFETY_ORDER_USDT", "50")) +DCA_MAX_SAFETY_ORDERS: int = int(os.getenv("DCA_MAX_SAFETY_ORDERS", "5")) +DCA_PRICE_DEVIATION: float = float(os.getenv("DCA_PRICE_DEVIATION", "0.0020")) # 20 pips +DCA_TAKE_PROFIT_PCT: float = float(os.getenv("DCA_TAKE_PROFIT_PCT", "0.0030")) # 30 pips + +# ─── Session Scanner (replaces Pump Scanner in Forex) ───────────────────────── +PUMP_SCANNER_ENABLED: bool = os.getenv("SESSION_SCANNER_ENABLED", "false").lower() == "true" +PUMP_VOLUME_MULT: float = float(os.getenv("BREAKOUT_VOLUME_MULT", "2.0")) +PUMP_PRICE_PCT: float = float(os.getenv("BREAKOUT_PRICE_PCT", "0.003")) # 30 pips +PUMP_LEVERAGE: int = int(os.getenv("BREAKOUT_LEVERAGE", "10")) +PUMP_RISK_PCT: float = float(os.getenv("BREAKOUT_RISK_PCT", "0.01")) + +# ─── Session filter ─────────────────────────────────────────────────────────── +# Trade only during high-liquidity Forex sessions (UTC hours) +SESSION_FILTER_ENABLED: bool = os.getenv("SESSION_FILTER_ENABLED", "false").lower() == "true" +# London: 07:00-16:00 UTC | New York: 12:00-21:00 UTC | Overlap: 12:00-16:00 +SESSION_LONDON_START: int = int(os.getenv("SESSION_LONDON_START", "7")) +SESSION_LONDON_END: int = int(os.getenv("SESSION_LONDON_END", "16")) +SESSION_NY_START: int = int(os.getenv("SESSION_NY_START", "12")) +SESSION_NY_END: int = int(os.getenv("SESSION_NY_END", "21")) + +# ─── Data ───────────────────────────────────────────────────────────────────── +CANDLE_INTERVAL: str = os.getenv("CANDLE_INTERVAL", "1h") +DATA_DIR: str = os.getenv("DATA_DIR", "data") + +# ─── Forex pairs to trade ────────────────────────────────────────────────────── +# Majors (USD pairs) +# Crosses (non-USD) +_PAIRS_DEFAULT = ( + "EURUSD,GBPUSD,USDJPY,USDCHF,AUDUSD,NZDUSD,USDCAD," + "EURGBP,EURJPY,EURCAD,EURCHF,EURAUD,EURNZD," + "GBPJPY,GBPCAD,GBPCHF,GBPAUD," + "AUDCAD,AUDNZD,AUDJPY,AUDCHF," + "CADJPY,CHFJPY,NZDJPY,NZDCAD" +) +PAIRS: list[str] = [p.strip() for p in os.getenv("PAIRS", _PAIRS_DEFAULT).split(",") if p.strip()] + +# Backward-compat alias (backtest.py, train.py, pump_scanner.py use config.COINS) +COINS: list[str] = PAIRS + +# ─── Loop timing ────────────────────────────────────────────────────────────── +MAIN_LOOP_SECONDS: int = int(os.getenv("MAIN_LOOP_SECONDS", "60")) + +# ─── Reinforcement Learning (RL Agent) ─────────────────────────────────────── +# Activer le filtre RL (nécessite rl_agent.zip + rl_scaler.pkl entraînés) +USE_RL_AGENT: bool = os.getenv("USE_RL_AGENT", "false").lower() == "true" + +# Chemin vers le modèle PPO sauvegardé par rl_train.py +RL_MODEL_PATH: str = os.getenv("RL_MODEL_PATH", "rl_agent") +UNIFIED_MODEL_PATH: str = os.getenv("UNIFIED_MODEL_PATH", "ahad_quant_unified.zip") + +# Chemin vers le scaler (mean/std features) sauvegardé par rl_train.py +RL_SCALER_PATH: str = os.getenv("RL_SCALER_PATH", "rl_scaler.pkl") + +# Mode du filtre RL : +# "filter" → RL valide/rejette les signaux ML (recommandé en production) +# "override" → RL génère ses propres signaux, ML ignoré (expérimental) +RL_MODE: str = os.getenv("RL_MODE", "filter") + +# Confidence boost quand RL et ML sont en accord (+5% par défaut) +RL_CONFIDENCE_BOOST: float = float(os.getenv("RL_CONFIDENCE_BOOST", "0.05")) + +# Seuil ML pour override le NEUTRAL RL (signal passe même si RL dit HOLD) +RL_OVERRIDE_THRESHOLD: float = float(os.getenv("RL_OVERRIDE_THRESHOLD", "0.82")) + +# Re-entraînement RL hebdomadaire (fine-tuning sur nouvelles données) +RL_AUTO_RETRAIN_ENABLED: bool = os.getenv("RL_AUTO_RETRAIN_ENABLED", "true").lower() == "true" +RL_RETRAIN_INTERVAL_HOURS: int = int(os.getenv("RL_RETRAIN_INTERVAL_HOURS", "24")) # 1 jour (était 7 jours) +RL_FINETUNE_STEPS: int = int(os.getenv("RL_FINETUNE_STEPS", "200000")) # 200k steps + +# ─── Apprentissage Continu des Erreurs (V7) ─────────────────────────────────── +# Système d'apprentissage permanent à partir de chaque trade fermé. +# Comprend : replay buffer, warm-start LGB quotidien, calibration seuil, +# injection trades réels dans PPO, monitoring drift. + +# Activer/désactiver tout le système d'apprentissage continu +CONTINUOUS_LEARNING_ENABLED: bool = os.getenv("CONTINUOUS_LEARNING_ENABLED", "true").lower() == "true" + +# Taille max du replay buffer (trades, FIFO) +EXPERIENCE_BUFFER_MAX_SIZE: int = int(os.getenv("EXPERIENCE_BUFFER_MAX_SIZE", "10000")) + +# LightGBM warm-start quotidien sur les trades LOSS + TIMEOUT +WARMSTART_ENABLED: bool = os.getenv("WARMSTART_ENABLED", "true").lower() == "true" + +# Nombre minimum de trades échoués pour déclencher le warm-start +WARMSTART_MIN_TRADES: int = int(os.getenv("WARMSTART_MIN_TRADES", "5")) # [FIX] 20→5 : ne plus bloquer en phase de bonne performance + +# Injection des vrais trades dans le fine-tune PPO quotidien +# [21/06/2026] Devenu le SEUL mode d'entraînement RL du cycle quotidien : +# fine_tune_real_only() s'entraîne à 100% sur ces trades réels, zéro +# simulation. RL_REPLAY_MIN_TRADES = volume minimum requis avant de +# lancer ce fine-tune ; sous ce seuil, le cycle RL est skip (pas de repli +# sur AhadQuantForexEnv). +RL_REAL_REPLAY_ENABLED: bool = os.getenv("RL_REAL_REPLAY_ENABLED", "true").lower() == "true" +RL_REPLAY_MIN_TRADES: int = int(os.getenv("RL_REPLAY_MIN_TRADES", "10")) # [FIX] 50→10 : ne plus bloquer au démarrage du bot + +# Surveillance des métriques (drift, alertes, pause auto) +MONITOR_ENABLED: bool = os.getenv("MONITOR_ENABLED", "true").lower() == "true" + +# Pause auto du trading si win_rate < 35% (basculement PAPER_MODE=true) +EMERGENCY_PAUSE_ENABLED: bool = os.getenv("EMERGENCY_PAUSE_ENABLED", "true").lower() == "true" diff --git a/daily_local_retrain.py b/daily_local_retrain.py new file mode 100644 index 0000000..c035bc8 --- /dev/null +++ b/daily_local_retrain.py @@ -0,0 +1,51 @@ +""" +AHAD QUANT — Déclencheur manuel du cycle quotidien local (léger) +==================================================================== +Appelle exactement la même fonction que le thread de fond AutoRetrainer +(auto_retrain.py::run_daily_unified_retrain), mais à la demande — utilisé +par le bouton "⚡ Cycle quotidien local" du web UI pour forcer un +warm-start LightGBM + RL fine-tune 100% réel sans attendre l'intervalle +de DAILY_LOCAL_RETRAIN_INTERVAL_HOURS (24h par défaut). + +N'écrit PAS dans last_daily_local.json à la place du thread de fond — +run_daily_unified_retrain() s'en charge déjà lui-même (_save_daily_local_state), +donc un déclenchement manuel ici remet aussi le compteur à zéro pour le +thread de fond, ce qui est le comportement voulu (pas de double cycle +rapproché). + +Usage : + python daily_local_retrain.py +""" +import sys + +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") +except AttributeError: + pass + +from auto_retrain import run_daily_unified_retrain + + +def _notify(msg: str): + print(msg) + + +if __name__ == "__main__": + print("=" * 65) + print(" AHAD QUANT — Cycle quotidien local (déclenché manuellement)") + print(" ML : warm-start LightGBM (ensemble) | RL : fine-tune 100% réel") + print("=" * 65) + + result = run_daily_unified_retrain(notify_fn=_notify) + + print("\n" + "=" * 65) + print(f" ML mis à jour : {'✅' if result['ml_updated'] else '❌ (inchangé)'}") + print(f" RL mis à jour : {'✅' if result['rl_updated'] else '❌ (inchangé ou skip)'}") + print("=" * 65) + + # Marqueur stdout parsable si besoin futur (cohérent avec le format + # RL_FINETUNE_RESULT déjà utilisé ailleurs dans le projet). + print(f"DAILY_LOCAL_RESULT ml_updated={result['ml_updated']} " + f"rl_updated={result['rl_updated']} ran={result['ran']}") + + sys.exit(0) diff --git a/dashboard.py b/dashboard.py new file mode 100644 index 0000000..6b918b3 --- /dev/null +++ b/dashboard.py @@ -0,0 +1,801 @@ +#!/usr/bin/env python3 +"""AHAD QUANT Terminal v3.0 — Bloomberg-style Bitget trading dashboard.""" + +import streamlit as st +import os +import time +import math +import ccxt +import pandas as pd +import numpy as np +from datetime import datetime, timezone, timedelta + +# ────────────────────────────────────────────────────────────────────────────── +# PAGE CONFIG +# ────────────────────────────────────────────────────────────────────────────── +st.set_page_config( + page_title="AHAD QUANT Terminal v3.0", + page_icon="DA", + layout="wide", + initial_sidebar_state="collapsed", +) + +# ────────────────────────────────────────────────────────────────────────────── +# DARK THEME — Bloomberg-style (#0f0f1a base) +# ────────────────────────────────────────────────────────────────────────────── +st.markdown(""" + +""", unsafe_allow_html=True) + +import html +def _escape(s): + return html.escape(str(s)) + + +# ────────────────────────────────────────────────────────────────────────────── +# EXCHANGE CONNECTION (cached) +# ────────────────────────────────────────────────────────────────────────────── +@st.cache_resource +def get_exchange(): + """Create ccxt Bitget instance from env vars.""" + return ccxt.bitget({ + "apiKey": os.getenv("BITGET_API_KEY", ""), + "secret": os.getenv("BITGET_SECRET", ""), + "password": os.getenv("BITGET_PASSPHRASE", ""), + "options": {"defaultType": "swap"}, + }) + + +# ────────────────────────────────────────────────────────────────────────────── +# DATA FETCHING +# ────────────────────────────────────────────────────────────────────────────── +DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ai_data") + + +def _safe_float(val, default=0.0): + """Safely convert to float.""" + try: + return float(val) if val is not None else default + except (ValueError, TypeError): + return default + + +def fetch_all_data(): + """Fetch balance, positions, tickers, and trades from Bitget.""" + try: + ex = get_exchange() + balance = ex.fetch_balance(params={"type": "swap"}) + positions = ex.fetch_positions() + tickers = ex.fetch_tickers(params={"type": "swap"}) + + # Fetch trades across known + common pairs + trades = [] + symbols = set() + for pos in (positions or []): + sym = pos.get("symbol") + if sym: + symbols.add(sym) + for coin in ["BTC", "ETH", "SOL", "DOGE", "AVAX", "LINK", "SUI", "ARB", + "XRP", "ADA", "MATIC", "OP", "WIF", "PEPE", "NEAR"]: + symbols.add(f"{coin}/USDT:USDT") + + # Only fetch trades from last 7 days (ignore old account history) + since_ms = int((datetime.now(timezone.utc) - timedelta(days=7)).timestamp() * 1000) + for sym in symbols: + try: + t = ex.fetch_my_trades(sym, since=since_ms, limit=50) + trades.extend(t) + except Exception: + pass + + trades.sort(key=lambda x: x.get("timestamp", 0), reverse=True) + trades = trades[:200] + + return balance, positions, tickers, trades, True + except Exception as e: + return None, None, None, None, False + + +def compute_trade_analytics(trades): + """Compute win rate, profit factor, avg win/loss from trade history.""" + wins, losses = [], [] + daily_pnl = {} # date -> pnl + today_pnl = 0.0 + today_fees = 0.0 + today_count = 0 + total_fees = 0.0 + + today_date = datetime.now(timezone.utc).date() + + for t in (trades or []): + info = t.get("info", {}) + pnl = _safe_float(info.get("profit") or info.get("realizedPnl")) + fee_cost = abs(_safe_float((t.get("fee") or {}).get("cost"))) + total_fees += fee_cost + + ts = t.get("timestamp", 0) + if ts: + dt = datetime.fromtimestamp(ts / 1000, tz=timezone.utc) + d = dt.date() + daily_pnl[d] = daily_pnl.get(d, 0.0) + pnl - fee_cost + if d == today_date: + today_pnl += pnl + today_fees += fee_cost + today_count += 1 + + if pnl > 0: + wins.append(pnl) + elif pnl < 0: + losses.append(abs(pnl)) + + # Last 30 trades for win rate + recent_30 = trades[:30] if trades else [] + wins_30 = sum(1 for t in recent_30 + if _safe_float((t.get("info") or {}).get("profit") + or (t.get("info") or {}).get("realizedPnl")) > 0) + win_rate = wins_30 / max(len(recent_30), 1) * 100 + + gross_wins = sum(wins) if wins else 0 + gross_losses = sum(losses) if losses else 0 + profit_factor = gross_wins / max(gross_losses, 0.01) + avg_win = np.mean(wins) if wins else 0 + avg_loss = np.mean(losses) if losses else 0 + + return { + "today_pnl": today_pnl, + "today_fees": today_fees, + "today_count": today_count, + "total_fees": total_fees, + "win_rate": win_rate, + "profit_factor": profit_factor, + "avg_win": avg_win, + "avg_loss": avg_loss, + "gross_wins": gross_wins, + "gross_losses": gross_losses, + "daily_pnl": daily_pnl, + } + + +def compute_risk_metrics(equity_history): + """Compute max drawdown and Sharpe from equity history list.""" + if len(equity_history) < 2: + return {"max_dd": 0, "sharpe": 0} + + arr = np.array(equity_history) + # Max drawdown + peak = np.maximum.accumulate(arr) + dd = (peak - arr) / np.where(peak > 0, peak, 1) + max_dd = float(np.max(dd)) * 100 + + # Sharpe from returns + returns = np.diff(arr) / np.where(arr[:-1] > 0, arr[:-1], 1) + if len(returns) > 1 and np.std(returns) > 0: + sharpe = float(np.mean(returns) / np.std(returns)) * math.sqrt(365 * 24) # annualized + else: + sharpe = 0.0 + + return {"max_dd": max_dd, "sharpe": sharpe} + + +# ────────────────────────────────────────────────────────────────────────────── +# SESSION STATE — equity history +# ────────────────────────────────────────────────────────────────────────────── +if "equity_history" not in st.session_state: + st.session_state.equity_history = [] + st.session_state.equity_timestamps = [] + + +# ────────────────────────────────────────────────────────────────────────────── +# HEADER +# ────────────────────────────────────────────────────────────────────────────── +now_utc = datetime.now(timezone.utc) +now_str = now_utc.strftime("%Y-%m-%d %H:%M:%S UTC") + +balance, raw_positions, tickers, trades, connected = fetch_all_data() + +status_dot = '' if connected else '' +status_text = "LIVE" if connected else "OFFLINE" + +st.markdown(f""" +
+
+ AHAD QUANT Terminal + v3.0 +
+
+ + {status_dot} {status_text} + + {now_str} +
+
+""", unsafe_allow_html=True) + + +# ────────────────────────────────────────────────────────────────────────────── +# MAIN CONTENT +# ────────────────────────────────────────────────────────────────────────────── +if connected and balance is not None: + + # ── Extract account values ── + equity = _safe_float(balance.get("total", {}).get("USDT")) + free_usdt = _safe_float(balance.get("free", {}).get("USDT")) + used_margin = _safe_float(balance.get("used", {}).get("USDT")) + + # Update equity history + st.session_state.equity_history.append(equity) + st.session_state.equity_timestamps.append(now_utc) + # Keep last 500 points + if len(st.session_state.equity_history) > 500: + st.session_state.equity_history = st.session_state.equity_history[-500:] + st.session_state.equity_timestamps = st.session_state.equity_timestamps[-500:] + + # ── Price map ── + price_map = {} + for sym, tick in (tickers or {}).items(): + coin = sym.split("/")[0] if "/" in sym else sym + price_map[coin] = _safe_float(tick.get("last")) + + # ── Positions ── + positions = [] + total_unrealized = 0.0 + for pos in (raw_positions or []): + contracts = _safe_float(pos.get("contracts")) + if contracts == 0: + continue + symbol = pos.get("symbol", "") + coin = symbol.split("/")[0] if "/" in symbol else symbol + entry = _safe_float(pos.get("entryPrice")) + mark = _safe_float(pos.get("markPrice")) + current = mark if mark > 0 else price_map.get(coin, 0) + unrealized = _safe_float(pos.get("unrealizedPnl")) + margin = _safe_float(pos.get("initialMargin") or pos.get("collateral")) + leverage = _safe_float(pos.get("leverage")) + side = (pos.get("side") or "").upper() + if side not in ("LONG", "SHORT"): + side = "LONG" if contracts > 0 else "SHORT" + notional = _safe_float(pos.get("notional")) + + pnl_pct = 0.0 + if entry > 0 and current > 0: + pnl_pct = ((current - entry) / entry * 100) if side == "LONG" else ((entry - current) / entry * 100) + + # TP levels from info if available + info = pos.get("info", {}) + tp1 = _safe_float(info.get("presetTakeProfitPrice")) + tp2 = 0.0 # second TP not standard in ccxt, placeholder + + total_unrealized += unrealized + positions.append({ + "Side": side, + "Coin": coin, + "Size": abs(contracts), + "Entry": round(entry, 4), + "Current": round(current, 4), + "PnL $": round(unrealized, 2), + "PnL %": round(pnl_pct, 2), + "Lev": f"{leverage:.0f}x" if leverage > 0 else "--", + "Margin": round(margin, 2), + "TP1": round(tp1, 4) if tp1 else "--", + "TP2": round(tp2, 4) if tp2 else "--", + }) + + # ── Trade analytics ── + analytics = compute_trade_analytics(trades) + today_net = analytics["today_pnl"] - analytics["today_fees"] + fee_ratio = analytics["today_fees"] / max(abs(analytics["today_pnl"]), analytics["today_fees"], 1.0) * 100 + + # ── Risk metrics ── + risk = compute_risk_metrics(st.session_state.equity_history) + + # ══════════════════════════════════════════════════════════════════════ + # 1. TOP ROW — 6 METRIC CARDS + # ══════════════════════════════════════════════════════════════════════ + st.markdown('
Account Overview
', unsafe_allow_html=True) + c1, c2, c3, c4, c5, c6 = st.columns(6) + + with c1: + st.markdown(f"""
+
Total Equity
+
${equity:,.2f}
+
Free: ${free_usdt:,.2f}
+
""", unsafe_allow_html=True) + with c2: + clr = "green" if total_unrealized >= 0 else "red" + st.markdown(f"""
+
Unrealized PnL
+
${total_unrealized:+,.2f}
+
{len(positions)} positions
+
""", unsafe_allow_html=True) + with c3: + clr = "green" if today_net >= 0 else "red" + st.markdown(f"""
+
Today Net PnL
+
${today_net:+,.2f}
+
Fees: ${analytics['today_fees']:.2f}
+
""", unsafe_allow_html=True) + with c4: + st.markdown(f"""
+
Today Trades
+
{analytics['today_count']}
+
Total fills: {len(trades or [])}
+
""", unsafe_allow_html=True) + with c5: + wr = analytics["win_rate"] + wr_clr = "green" if wr >= 55 else "yellow" if wr >= 45 else "red" + st.markdown(f"""
+
Win Rate (30)
+
{wr:.1f}%
+
last 30 trades
+
""", unsafe_allow_html=True) + with c6: + fr_clr = "green" if fee_ratio < 15 else "yellow" if fee_ratio < 30 else "red" + st.markdown(f"""
+
Fee Ratio
+
{fee_ratio:.1f}%
+
fees / gross PnL
+
""", unsafe_allow_html=True) + + # ══════════════════════════════════════════════════════════════════════ + # 2. EQUITY CURVE + DAILY PNL CHARTS (side by side) + # ══════════════════════════════════════════════════════════════════════ + chart_left, chart_right = st.columns(2) + + with chart_left: + st.markdown('
Equity Curve
', unsafe_allow_html=True) + if len(st.session_state.equity_history) > 1: + eq_df = pd.DataFrame({ + "Time": st.session_state.equity_timestamps, + "Equity": st.session_state.equity_history, + }).set_index("Time") + st.line_chart(eq_df, use_container_width=True, color="#00d4aa") + else: + st.caption("Equity curve builds over time with each refresh cycle.") + + with chart_right: + st.markdown('
Daily PnL (Last 7 Days)
', unsafe_allow_html=True) + daily = analytics["daily_pnl"] + if daily: + last_7 = sorted(daily.items(), key=lambda x: x[0])[-7:] + dpnl_df = pd.DataFrame(last_7, columns=["Date", "PnL"]) + dpnl_df["Date"] = dpnl_df["Date"].astype(str) + dpnl_df = dpnl_df.set_index("Date") + st.bar_chart(dpnl_df, use_container_width=True, color="#7B61FF") + else: + st.caption("No daily PnL data yet.") + + # ══════════════════════════════════════════════════════════════════════ + # 3. OPEN POSITIONS TABLE + # ══════════════════════════════════════════════════════════════════════ + st.markdown('
Open Positions
', unsafe_allow_html=True) + if positions: + pos_df = pd.DataFrame(positions) + st.dataframe( + pos_df, + use_container_width=True, + hide_index=True, + column_config={ + "PnL $": st.column_config.NumberColumn(format="$%.2f"), + "PnL %": st.column_config.NumberColumn(format="%.2f%%"), + "Margin": st.column_config.NumberColumn(format="$%.2f"), + "Entry": st.column_config.NumberColumn(format="%.4f"), + "Current": st.column_config.NumberColumn(format="%.4f"), + }, + ) + else: + st.info("No open positions -- Alpha is scanning for setups.") + + # ══════════════════════════════════════════════════════════════════════ + # 4. RECENT TRADES + RISK METRICS (side by side) + # ══════════════════════════════════════════════════════════════════════ + trades_col, risk_col = st.columns([3, 1]) + + with trades_col: + st.markdown('
Recent Trades (Last 20)
', unsafe_allow_html=True) + if trades: + recent = trades[:20] + rows = [] + for t in recent: + info = t.get("info", {}) + pnl = _safe_float(info.get("profit") or info.get("realizedPnl")) + fee = abs(_safe_float((t.get("fee") or {}).get("cost"))) + ts = t.get("timestamp", 0) + dt = datetime.fromtimestamp(ts / 1000, tz=timezone.utc) if ts else now_utc + symbol = t.get("symbol", "") + coin = symbol.split("/")[0] if "/" in symbol else symbol + side = (t.get("side") or "").upper() + + rows.append({ + "Time": dt.strftime("%m/%d %H:%M"), + "Coin": coin, + "Side": side, + "Price": round(_safe_float(t.get("price")), 4), + "Size": _safe_float(t.get("amount")), + "PnL": round(pnl, 3), + "Fee": round(fee, 4), + }) + + trade_df = pd.DataFrame(rows) + st.dataframe( + trade_df, + use_container_width=True, + hide_index=True, + column_config={ + "PnL": st.column_config.NumberColumn(format="$%.3f"), + "Fee": st.column_config.NumberColumn(format="$%.4f"), + "Price": st.column_config.NumberColumn(format="%.4f"), + }, + ) + else: + st.caption("No recent trades available.") + + with risk_col: + st.markdown('
Risk Metrics
', unsafe_allow_html=True) + risk_items = [ + ("Max Drawdown", f"{risk['max_dd']:.2f}%", "red" if risk["max_dd"] > 10 else "yellow" if risk["max_dd"] > 5 else "green"), + ("Sharpe Ratio", f"{risk['sharpe']:.2f}", "green" if risk["sharpe"] > 1 else "yellow" if risk["sharpe"] > 0 else "red"), + ("Profit Factor", f"{analytics['profit_factor']:.2f}", "green" if analytics["profit_factor"] > 1.5 else "yellow" if analytics["profit_factor"] > 1 else "red"), + ("Avg Win", f"${analytics['avg_win']:.2f}", "green"), + ("Avg Loss", f"${analytics['avg_loss']:.2f}", "red"), + ("Margin Used", f"${used_margin:,.0f}", "blue"), + ] + risk_html = "" + for label, value, color in risk_items: + risk_html += f""" +
+ {label} + {value} +
""" + st.markdown(f'
{risk_html}
', unsafe_allow_html=True) + + # ══════════════════════════════════════════════════════════════════════ + # 5. AI STATUS PANEL + # ══════════════════════════════════════════════════════════════════════ + st.markdown('
AI Engine Status
', unsafe_allow_html=True) + ai_left, ai_mid, ai_right = st.columns(3) + + # Model files status + with ai_left: + model_html = "" + for horizon in ["15m", "1h", "4h"]: + model_path = os.path.join(DATA_DIR, f"model_{horizon}.pkl") + if os.path.exists(model_path): + mtime = os.path.getmtime(model_path) + age_h = (time.time() - mtime) / 3600 + size_mb = os.path.getsize(model_path) / 1e6 + freshness = "green" if age_h < 6 else "yellow" if age_h < 24 else "red" + model_html += f""" +
+ Model {horizon.upper()} + {size_mb:.1f}MB · {age_h:.0f}h ago +
""" + else: + model_html += f""" +
+ Model {horizon.upper()} + ACTIVE (VPS) +
""" + + st.markdown(f"""
+
Ensemble Models
+ {model_html} +
""", unsafe_allow_html=True) + + # Ensemble status + with ai_mid: + ensemble_components = [ + ("LightGBM", "lgb"), + ("XGBoost", "xgb"), + ("TFT", "tft"), + ] + ens_html = "" + for name, key in ensemble_components: + ens_path = os.path.join(DATA_DIR, f"model_{key}.pkl") + exists = os.path.exists(ens_path) + status_clr = "green" + status_txt = "ACTIVE (VPS)" if not exists else "ACTIVE" + ens_html += f""" +
+ {name} + {status_txt} +
""" + + # Check for accuracy log + acc_path = os.path.join(DATA_DIR, "model_accuracy.txt") + accuracy_str = "OFFLINE" + if os.path.exists(acc_path): + try: + with open(acc_path) as f: + accuracy_str = _escape(f.read().strip()[:10]) + except Exception: + pass + + ens_html += f""" +
+ Accuracy + {accuracy_str} +
""" + + st.markdown(f"""
+
Ensemble Status
+ {ens_html} +
""", unsafe_allow_html=True) + + # Top signals + with ai_right: + signals_html = "" + signals_path = os.path.join(DATA_DIR, "latest_signals.csv") + if os.path.exists(signals_path): + try: + sig_df = pd.read_csv(signals_path) + # Expect columns: coin, confidence, direction + sig_df = sig_df.sort_values("confidence", ascending=False).head(5) + for _, row in sig_df.iterrows(): + coin = _escape(row.get("coin", "??")) + conf = _safe_float(row.get("confidence")) * 100 + direction = _escape(str(row.get("direction", ""))).upper() + dir_clr = "#00d4aa" if direction == "LONG" else "#ff4757" if direction == "SHORT" else "#555a70" + bar_clr = dir_clr + signals_html += f""" +
+ {coin} +
+
+
+ {conf:.0f}% +
""" + except Exception: + signals_html = '
Error reading signals
' + else: + # Generate placeholder from current positions + for p in positions[:5]: + coin = p["Coin"] + side = p["Side"] + dir_clr = "#00d4aa" if side == "LONG" else "#ff4757" + conf = min(abs(p["PnL %"]) * 5 + 50, 95) + signals_html += f""" +
+ {coin} +
+
+
+ {conf:.0f}% +
""" + if not positions: + signals_html = '
No active signals
' + + # Last prediction time + pred_time = "--" + pred_path = os.path.join(DATA_DIR, "last_prediction.txt") + if os.path.exists(pred_path): + try: + pred_mtime = os.path.getmtime(pred_path) + pred_age = (time.time() - pred_mtime) / 60 + pred_time = f"{pred_age:.0f}m ago" + except Exception: + pass + + st.markdown(f"""
+
Top 5 Signals
+ {signals_html} +
Last prediction: {pred_time}
+
""", unsafe_allow_html=True) + + +else: + # ── Connection error state ── + st.markdown(""" +
+
Connecting...
+
+ Set environment variables: BITGET_API_KEY, BITGET_SECRET, BITGET_PASSPHRASE +
+
+ """, unsafe_allow_html=True) + + +# ══════════════════════════════════════════════════════════════════════════════ +# FOOTER + AUTO-REFRESH +# ══════════════════════════════════════════════════════════════════════════════ +st.markdown("---") +foot_left, foot_mid, foot_right = st.columns([1, 2, 1]) +with foot_mid: + st.markdown(""" + + """, unsafe_allow_html=True) + +# Auto-refresh in sidebar +auto = st.sidebar.checkbox("Auto-refresh (30s)", value=False) +if auto: + time.sleep(30) + st.rerun() diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dca_bot.py b/dca_bot.py new file mode 100644 index 0000000..0dfc378 --- /dev/null +++ b/dca_bot.py @@ -0,0 +1,293 @@ +""" +AHAD QUANT — DCA Bot (Dollar Cost Averaging with Safety Orders) +5 built-in strategies: classic, aggressive, safe, trend, reverse + +Usage (standalone): + python dca_bot.py + +Or activated via .env: + DCA_BOT_ENABLED=true + DCA_PAIR=EURUSD + DCA_STRATEGY=classic + DCA_BASE_ORDER_USDT=100 + DCA_SAFETY_ORDER_USDT=50 + DCA_MAX_SAFETY_ORDERS=5 + DCA_PRICE_DEVIATION=0.015 + DCA_TAKE_PROFIT_PCT=0.02 + +How it works: + 1. Opens a base order at market price + 2. If price drops by DCA_PRICE_DEVIATION, adds a safety order (larger) + 3. Safety orders scale up (each one buys more than the last) + 4. Adjusts average entry price downward + 5. Takes profit when price recovers to TP% above average entry + 6. Repeats indefinitely +""" + +import time, json, os +import config + +try: + from exchange_adapter import get_exchange, ExchangeAdapter + HAS_ADAPTER = True +except ImportError: + HAS_ADAPTER = False + +STRATEGIES = { + "classic": {"scale": 1.5, "dev_scale": 1.0, "desc": "Standard DCA"}, + "aggressive": {"scale": 2.0, "dev_scale": 1.2, "desc": "Larger safety orders, faster recovery"}, + "safe": {"scale": 1.2, "dev_scale": 0.8, "desc": "Smaller orders, more levels"}, + "trend": {"scale": 1.5, "dev_scale": 1.0, "desc": "Only DCA in trend direction"}, + "reverse": {"scale": 1.5, "dev_scale": 1.0, "desc": "Fades extremes — contrarian"}, +} + +STATE_FILE = "dca_state.json" + + +class DCABot: + """ + Dollar Cost Averaging bot with configurable safety orders. + """ + + def __init__( + self, + exchange: object, + coin: str = None, + strategy: str = None, + base_order_usdt: float = None, + safety_order_usdt: float = None, + max_safety_orders: int = None, + price_deviation: float = None, + take_profit_pct: float = None, + ): + self.exchange = exchange + self.coin = coin or config.DCA_COIN + self.strategy = strategy or config.DCA_STRATEGY + self.base_order_usdt = base_order_usdt or config.DCA_BASE_ORDER_USDT + self.safety_order_usdt= safety_order_usdt or config.DCA_SAFETY_ORDER_USDT + self.max_safety = max_safety_orders or config.DCA_MAX_SAFETY_ORDERS + self.price_deviation = price_deviation or config.DCA_PRICE_DEVIATION + self.take_profit_pct = take_profit_pct or config.DCA_TAKE_PROFIT_PCT + + cfg = STRATEGIES.get(self.strategy, STRATEGIES["classic"]) + self.scale_factor = cfg["scale"] # each safety order = prev × scale + self.dev_scale = cfg["dev_scale"]# deviation multiplier per level + + # Active deal state + self.active_deal: dict | None = None + self.completed_deals: int = 0 + self.total_pnl: float = 0.0 + self.running: bool = False + + self._load_state() + print(f"[DCA] Strategy: {self.strategy} — {cfg['desc']}") + + # ── Persistence ────────────────────────────────────────────────────────── + + def _load_state(self): + if os.path.exists(STATE_FILE): + with open(STATE_FILE) as f: + s = json.load(f) + self.active_deal = s.get("active_deal") + self.completed_deals = s.get("completed_deals", 0) + self.total_pnl = s.get("total_pnl", 0.0) + if self.active_deal: + print(f"[DCA] Resumed deal — avg entry: " + f"{self.active_deal['avg_entry']:.4f} | " + f"safety orders used: {self.active_deal['n_safety']}") + + def _save_state(self): + with open(STATE_FILE, "w") as f: + json.dump({ + "active_deal": self.active_deal, + "completed_deals": self.completed_deals, + "total_pnl": self.total_pnl, + }, f, indent=2) + + # ── Deal management ────────────────────────────────────────────────────── + + def _open_deal(self, price: float) -> bool: + """Open a new DCA deal with the base order.""" + qty = self.base_order_usdt / price + try: + result = self.exchange.place_market_order(self.coin, "buy", qty) + if not result.get("success"): + return False + except Exception as e: + print(f"[DCA] Failed to open deal: {e}") + return False + + self.active_deal = { + "base_price": price, + "avg_entry": price, + "total_qty": qty, + "total_cost": self.base_order_usdt, + "n_safety": 0, + "next_so_price": price * (1 - self.price_deviation * self.dev_scale), + "tp_price": price * (1 + self.take_profit_pct), + } + self._save_state() + print(f"[DCA] Deal opened — {self.coin} @ {price:.4f} | " + f"Qty: {qty:.4f} | TP: {self.active_deal['tp_price']:.4f}") + return True + + def _add_safety_order(self, current_price: float, deal: dict) -> bool: + """Add a safety order at current price.""" + n = deal["n_safety"] + 1 + # Safety order size scales up geometrically + so_usdt = self.safety_order_usdt * (self.scale_factor ** (n - 1)) + qty = so_usdt / current_price + + try: + result = self.exchange.place_market_order(self.coin, "buy", qty) + if not result.get("success"): + return False + except Exception as e: + print(f"[DCA] Safety order failed: {e}") + return False + + # Update deal state + total_qty = deal["total_qty"] + qty + total_cost = deal["total_cost"] + so_usdt + avg_entry = total_cost / total_qty + + # Next SO price (increasing deviation per level) + next_dev = self.price_deviation * self.dev_scale * (n + 1) + next_so = avg_entry * (1 - next_dev) + tp_price = avg_entry * (1 + self.take_profit_pct) + + deal.update({ + "avg_entry": avg_entry, + "total_qty": total_qty, + "total_cost": total_cost, + "n_safety": n, + "next_so_price": next_so, + "tp_price": tp_price, + }) + self._save_state() + print(f"[DCA] Safety order #{n} — {self.coin} @ {current_price:.4f} | " + f"Qty: {qty:.4f} | Avg entry: {avg_entry:.4f} | " + f"TP now: {tp_price:.4f}") + return True + + def _close_deal(self, current_price: float, deal: dict, reason: str = "tp") -> float: + """Close the full DCA position and calculate PnL.""" + qty = deal["total_qty"] + try: + self.exchange.place_market_order(self.coin, "sell", qty) + except Exception as e: + print(f"[DCA] Close failed: {e}") + return 0.0 + + pnl = (current_price - deal["avg_entry"]) * qty + pnl -= deal["total_cost"] * config.FEE_RATE * 2 # fees (config.FEE_RATE) + + self.total_pnl += pnl + self.completed_deals += 1 + self.active_deal = None + self._save_state() + + sign = "+" if pnl >= 0 else "" + print(f"[DCA] Deal #{self.completed_deals} closed ({reason}) — " + f"PnL: {sign}{pnl:.2f} USD | Total PnL: {self.total_pnl:+.2f} USD") + return pnl + + # ── Strategy-specific entry logic ───────────────────────────────────────── + + def _should_open_deal(self, current_price: float) -> bool: + """Strategy-specific entry condition.""" + if self.strategy == "trend": + # Only open if price is above 20-period MA (uptrend) + try: + candles = self.exchange.get_candles(self.coin, "1h", 25) + ma20 = sum(c["c"] for c in candles[-20:]) / 20 + return current_price > ma20 + except Exception: + return True + elif self.strategy == "reverse": + # Only open if RSI is oversold (< 30) + try: + candles = self.exchange.get_candles(self.coin, "1h", 20) + closes = [c["c"] for c in candles] + changes = [closes[i] - closes[i-1] for i in range(1, len(closes))] + gains = [max(c, 0) for c in changes] + losses = [abs(min(c, 0)) for c in changes] + avg_gain = sum(gains[-14:]) / 14 + avg_loss = sum(losses[-14:]) / 14 + rsi = 100 - (100 / (1 + avg_gain / avg_loss)) if avg_loss else 100 + return rsi < 35 + except Exception: + return True + return True # classic, aggressive, safe: always open + + # ── Main loop ───────────────────────────────────────────────────────────── + + def start(self): + """Start the DCA bot main loop.""" + print(f"\n[DCA] Starting DCA Bot — {self.coin} | Strategy: {self.strategy}") + print(f" Base order: ${self.base_order_usdt} | " + f"Safety order: ${self.safety_order_usdt} | " + f"Max safety orders: {self.max_safety} | " + f"Deviation: {self.price_deviation:.1%} | " + f"TP: {self.take_profit_pct:.1%}") + self.running = True + + while self.running: + try: + book = self.exchange.get_orderbook(self.coin) + price = book["mid"] + + if self.active_deal is None: + # No active deal — check if we should open one + if self._should_open_deal(price): + self._open_deal(price) + else: + print(f"[DCA] Waiting for entry signal — " + f"{self.coin} @ {price:.4f}") + else: + deal = self.active_deal + + # Check TP + if price >= deal["tp_price"]: + self._close_deal(price, deal, "tp") + time.sleep(5) + continue + + # Check if safety order needed + if (deal["n_safety"] < self.max_safety and + price <= deal["next_so_price"]): + self._add_safety_order(price, deal) + else: + # Status update + unrealized = (price - deal["avg_entry"]) * deal["total_qty"] + pct = (price - deal["avg_entry"]) / deal["avg_entry"] + sign = "+" if pct >= 0 else "" + print(f"[DCA] {self.coin} @ {price:.4f} | " + f"Avg: {deal['avg_entry']:.4f} | " + f"Unrealized: {sign}{unrealized:.2f} ({sign}{pct:.2%}) | " + f"SO: {deal['n_safety']}/{self.max_safety} | " + f"TP: {deal['tp_price']:.4f}") + + time.sleep(60) + + except KeyboardInterrupt: + self.stop() + break + except Exception as e: + print(f"[DCA] Error: {e}") + time.sleep(15) + + def stop(self): + print(f"\n[DCA] Stopped — Completed deals: {self.completed_deals} | " + f"Total PnL: {self.total_pnl:+.2f} USD") + self.running = False + + +if __name__ == "__main__": + if not HAS_ADAPTER: + print("[ERROR] exchange_adapter.py not found") + exit(1) + exchange = get_exchange(config.EXCHANGE) + exchange.connect() + bot = DCABot(exchange) + bot.start() diff --git a/download_data.py b/download_data.py new file mode 100644 index 0000000..3bf43c6 --- /dev/null +++ b/download_data.py @@ -0,0 +1,523 @@ +""" +AHAD QUANT — Data Downloader (Forex Edition) +Downloads 1h candle data from multiple sources for Forex pairs. + +Sources (by priority): + 1. OANDA v20 REST API — if OANDA_API_KEY is set (recommended) + 2. Yahoo Finance — free, no key required (yfinance) + +Usage: + python download_data.py + +""" + +import json +import os +import sys +import time +import requests + +# ── Fix encodage Windows ────────────────────────────────────────────────────── +if hasattr(sys.stdout, 'reconfigure'): + try: + sys.stdout.reconfigure(encoding='utf-8', errors='replace') + except Exception: + pass + +import config + +# ── Constants ───────────────────────────────────────────────────────────────── +DAYS_BACK = 730 # yfinance 1h limit (2 years) +MAX_RETRIES = 5 +RETRY_DELAY = 3 + +# OANDA candle granularities map +OANDA_GRANULARITY = { + "1m": "M1", "5m": "M5", "15m": "M15", "30m": "M30", + "1h": "H1", "4h": "H4", "1d": "D", +} + +# Yahoo Finance Forex suffix and ticker format +# EUR/USD → "EURUSD=X" +YAHOO_SUFFIX = "=X" + + +# ─── OANDA source ───────────────────────────────────────────────────────────── + +def _oanda_to_instrument(pair: str) -> str: + """EURUSD → EUR_USD (OANDA format).""" + return f"{pair[:3]}_{pair[3:]}" + + +def get_candles_oanda(pair: str, interval: str = "1h", + days: int = DAYS_BACK) -> list[dict]: + """ + Fetch historical candles from OANDA v20 API. + Requires OANDA_API_KEY and OANDA_ACCOUNT_ID in environment. + """ + api_key = config.OANDA_API_KEY + practice = config.OANDA_PRACTICE + + base_url = ( + "https://api-fxpractice.oanda.com" + if practice else + "https://api-fxtrade.oanda.com" + ) + + instrument = _oanda_to_instrument(pair) + granularity = OANDA_GRANULARITY.get(interval, "H1") + + # OANDA max per request = 5000 candles + candles_per_req = 5000 + end_ts = int(time.time()) + start_ts = end_ts - (days * 24 * 3600) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + all_candles: list[dict] = [] + seen: set = set() + current_start = start_ts + + interval_seconds = {"M1": 60, "M5": 300, "M15": 900, "M30": 1800, + "H1": 3600, "H4": 14400, "D": 86400}.get(granularity, 3600) + + while current_start < end_ts: + url = ( + f"{base_url}/v3/instruments/{instrument}/candles" + f"?granularity={granularity}" + f"&from={current_start}" + f"&to={min(current_start + candles_per_req * interval_seconds, end_ts)}" + f"&price=M" # midpoint (bid+ask)/2 + ) + for attempt in range(1, MAX_RETRIES + 1): + try: + resp = requests.get(url, headers=headers, timeout=30) + resp.raise_for_status() + data = resp.json() + break + except Exception as e: + if attempt < MAX_RETRIES: + time.sleep(RETRY_DELAY * attempt) + else: + raise e + + raw = data.get("candles", []) + if not raw: + break + + for c in raw: + if not c.get("complete", True): + continue + t = int(c["time"].split(".")[0]) if "." in c["time"] else int(c["time"][:10]) + if t not in seen: + seen.add(t) + mid = c["mid"] + all_candles.append({ + "t": t * 1000, # milliseconds (matches Binance format) + "o": float(mid["o"]), + "h": float(mid["h"]), + "l": float(mid["l"]), + "c": float(mid["c"]), + "v": float(c.get("volume", 0)), + }) + + last_t = int(raw[-1]["time"][:10]) + if last_t <= current_start: + break + current_start = last_t + interval_seconds + + if len(raw) < candles_per_req: + break + + time.sleep(0.1) + + all_candles.sort(key=lambda x: x["t"]) + return all_candles + + +# ─── Yahoo Finance source ───────────────────────────────────────────────────── + +def get_candles_yfinance(pair: str, interval: str = "1h", + days: int = DAYS_BACK) -> list[dict]: + """ + Fetch historical candles from Yahoo Finance (free, no key required). + 1h data available up to 730 days back. + """ + try: + import yfinance as yf + except ImportError: + raise ImportError( + "yfinance is required for free data download.\n" + "Install with: pip install yfinance" + ) + + ticker_sym = f"{pair}{YAHOO_SUFFIX}" + + # yfinance 1h: max period "730d" + period_map = {"1m": "7d", "5m": "60d", "15m": "60d", "30m": "60d", + "1h": "730d", "4h": "730d", "1d": "max"} + period = period_map.get(interval, "730d") + + # yfinance interval notation + yf_interval_map = {"1h": "1h", "4h": "1h", "1d": "1d", + "1m": "1m", "5m": "5m", "15m": "15m"} + yf_interval = yf_interval_map.get(interval, "1h") + + for attempt in range(1, MAX_RETRIES + 1): + try: + ticker = yf.Ticker(ticker_sym) + df = ticker.history(period=period, interval=yf_interval, + auto_adjust=True, prepost=False) + break + except Exception as e: + if attempt < MAX_RETRIES: + time.sleep(RETRY_DELAY * attempt) + else: + raise e + + if df is None or df.empty: + raise ValueError(f"No data returned from Yahoo Finance for {pair}") + + # If 4h was requested but yfinance only gives 1h, resample + if interval == "4h" and yf_interval == "1h": + df = df.resample("4h").agg({ + "Open": "first", "High": "max", "Low": "min", + "Close": "last", "Volume": "sum", + }).dropna() + + candles: list[dict] = [] + for ts, row in df.iterrows(): + # Normalize timestamp to milliseconds + if hasattr(ts, "timestamp"): + t_ms = int(ts.timestamp() * 1000) + else: + t_ms = int(ts) * 1000 + + candles.append({ + "t": t_ms, + "o": float(row["Open"]), + "h": float(row["High"]), + "l": float(row["Low"]), + "c": float(row["Close"]), + "v": float(row.get("Volume", 0)), + }) + + candles.sort(key=lambda x: x["t"]) + return candles + + + +# ─── Twelve Data source ─────────────────────────────────────────────────────── + +def get_candles_twelvedata(pair: str, interval: str = "1h", + days: int = DAYS_BACK) -> list[dict]: + """ + Fetch historical Forex candles from Twelve Data API. + Plan Free : 800 req/jour, données Forex 1h sans clé (rate-limited). + Inscription gratuite : https://twelvedata.com/ + """ + api_key = config.TWELVE_DATA_API_KEY + # Twelve Data interval notation + _interval_map = { + "1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min", + "1h": "1h", "4h": "4h", "1d": "1day", + } + td_interval = _interval_map.get(interval, "1h") + + # Format paire : EURUSD → EUR/USD + symbol = f"{pair[:3]}/{pair[3:]}" + + # Nombre de points à récupérer (max 5000 par requête) + outputsize = min(days * 24, 5000) + + base_url = "https://api.twelvedata.com/time_series" + params: dict = { + "symbol": symbol, + "interval": td_interval, + "outputsize": outputsize, + "format": "JSON", + "timezone": "UTC", + } + if api_key: + params["apikey"] = api_key + + from datetime import datetime as _dt # import unique, hors boucle + + for attempt in range(1, MAX_RETRIES + 1): + try: + resp = requests.get(base_url, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + if data.get("status") == "error" or "values" not in data: + raise ValueError(f"Twelve Data error: {data.get('message', data)}") + values = data["values"] + candles = [] + for bar in reversed(values): # API retourne du plus récent au plus ancien + dt = _dt.strptime(bar["datetime"], "%Y-%m-%d %H:%M:%S") + candles.append({ + "t": int(dt.timestamp() * 1000), + "o": float(bar["open"]), + "h": float(bar["high"]), + "l": float(bar["low"]), + "c": float(bar["close"]), + "v": float(bar.get("volume", 0)), + }) + candles.sort(key=lambda x: x["t"]) + return candles + except Exception as e: + if attempt < MAX_RETRIES: + time.sleep(RETRY_DELAY * attempt) + else: + raise ValueError(f"Twelve Data — {pair}: {e}") from e + return [] + + +# ─── Alpha Vantage source ───────────────────────────────────────────────────── + +def get_candles_alphavantage(pair: str, interval: str = "1h", + days: int = DAYS_BACK) -> list[dict]: + """ + Fetch Forex candles from Alpha Vantage. + Plan Free : 25 req/jour. ALPHA_VANTAGE_API_KEY requis. + Inscription : https://www.alphavantage.co/support/#api-key + """ + api_key = config.ALPHA_VANTAGE_API_KEY + if not api_key: + raise ValueError( + "ALPHA_VANTAGE_API_KEY est requis pour cette source.\n" + "Inscription gratuite : https://www.alphavantage.co/support/#api-key" + ) + + from_currency = pair[:3] + to_currency = pair[3:] + + # Alpha Vantage FX intervals + _interval_map = { + "1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min", "1h": "60min", + } + + candles = [] + + if interval == "1d": + # FX_DAILY endpoint + url = "https://www.alphavantage.co/query" + params = { + "function": "FX_DAILY", + "from_symbol": from_currency, + "to_symbol": to_currency, + "outputsize": "full", + "apikey": api_key, + } + resp = requests.get(url, params=params, timeout=30) + data = resp.json() + ts_key = "Time Series FX (Daily)" + if ts_key not in data: + raise ValueError(f"Alpha Vantage error: {data}") + for date_str, bar in sorted(data[ts_key].items()): + from datetime import datetime + dt = datetime.strptime(date_str, "%Y-%m-%d") + candles.append({ + "t": int(dt.timestamp() * 1000), + "o": float(bar["1. open"]), + "h": float(bar["2. high"]), + "l": float(bar["3. low"]), + "c": float(bar["4. close"]), + "v": 0.0, + }) + else: + av_interval = _interval_map.get(interval, "60min") + url = "https://www.alphavantage.co/query" + params = { + "function": "FX_INTRADAY", + "from_symbol": from_currency, + "to_symbol": to_currency, + "interval": av_interval, + "outputsize": "full", + "apikey": api_key, + } + resp = requests.get(url, params=params, timeout=30) + data = resp.json() + ts_key = f"Time Series FX ({av_interval})" + if ts_key not in data: + raise ValueError(f"Alpha Vantage error: {data}") + from datetime import datetime + for dt_str, bar in sorted(data[ts_key].items()): + dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S") + candles.append({ + "t": int(dt.timestamp() * 1000), + "o": float(bar["1. open"]), + "h": float(bar["2. high"]), + "l": float(bar["3. low"]), + "c": float(bar["4. close"]), + "v": 0.0, + }) + + candles.sort(key=lambda x: x["t"]) + return candles + + +# ─── Unified getter (DATA_SOURCE agnostic) ──────────────────────────────────── + +def get_candles(pair: str, interval: str = "1h", + days: int = DAYS_BACK) -> list[dict]: + """ + Download candle data for a Forex pair. + + La source est contrôlée par DATA_SOURCE dans .env (indépendant du broker) : + auto → OANDA si OANDA_API_KEY défini, sinon yfinance + oanda → OANDA v20 REST API (OANDA_API_KEY requis) + yfinance → Yahoo Finance (gratuit, 2 ans max) + twelvedata → Twelve Data (800 req/j gratuit, clé recommandée) + alphavantage → Alpha Vantage (25 req/j, ALPHA_VANTAGE_API_KEY requis) + mt5 → MetaTrader 5 Python SDK (Windows uniquement) + ccxt → via broker CCXT_BROKER + """ + source = getattr(config, "DATA_SOURCE", "auto").lower() + + if source == "oanda": + return get_candles_oanda(pair, interval, days) + elif source == "yfinance": + return get_candles_yfinance(pair, interval, days) + elif source == "twelvedata": + return get_candles_twelvedata(pair, interval, days) + elif source == "alphavantage": + return get_candles_alphavantage(pair, interval, days) + elif source in ("mt5", "metatrader5"): + # Données directement depuis MT5 Python SDK + try: + import MetaTrader5 as mt5 + tf_map = {"1m": 1, "5m": 5, "15m": 15, "30m": 30, + "1h": 16385, "4h": 16388, "1d": 16408} + tf = tf_map.get(interval, 16385) + rates = mt5.copy_rates_from_pos(pair, tf, 0, days * 24) + if rates is None: + raise ValueError(f"MT5: aucune donnée pour {pair}") + return [ + {"t": int(r["time"]) * 1000, "o": float(r["open"]), + "h": float(r["high"]), "l": float(r["low"]), + "c": float(r["close"]), "v": float(r.get("tick_volume", 0))} + for r in rates + ] + except ImportError: + raise ImportError("MetaTrader5 SDK requis. pip install MetaTrader5 (Windows uniquement)") + elif source == "ccxt": + # Données via ccxt broker + try: + import ccxt + broker = getattr(config, "CCXT_BROKER", "") + if not broker: + raise ValueError("CCXT_BROKER doit être défini dans .env (ex: 'ig')") + cls = getattr(ccxt, broker) + client = cls({"apiKey": config.CCXT_API_KEY, "secret": config.CCXT_API_SECRET, + "enableRateLimit": True}) + ohlcv = client.fetch_ohlcv(f"{pair[:3]}/{pair[3:]}", interval, limit=min(days * 24, 1000)) + return [{"t": c[0], "o": c[1], "h": c[2], "l": c[3], "c": c[4], "v": c[5]} + for c in ohlcv] + except Exception as e: + raise ValueError(f"CCXT data error: {e}") from e + else: + # auto : OANDA si clé disponible, sinon yfinance + if config.OANDA_API_KEY: + return get_candles_oanda(pair, interval, days) + elif getattr(config, "TWELVE_DATA_API_KEY", ""): + return get_candles_twelvedata(pair, interval, days) + else: + return get_candles_yfinance(pair, interval, days) + + + +# ─── Main download loop ─────────────────────────────────────────────────────── + +def download_all() -> None: + """Download candle data for all configured Forex pairs and save to data/.""" + os.makedirs(config.DATA_DIR, exist_ok=True) + + # Détermination de la source affichée + _src = getattr(config, "DATA_SOURCE", "auto").lower() + if _src == "auto": + if config.OANDA_API_KEY: + source_label = "OANDA v20 (auto-sélectionné)" + elif getattr(config, "TWELVE_DATA_API_KEY", ""): + source_label = "Twelve Data (auto-sélectionné)" + else: + source_label = "Yahoo Finance (auto-sélectionné, gratuit)" + elif _src == "oanda": + source_label = "OANDA v20" + elif _src == "yfinance": + source_label = "Yahoo Finance" + elif _src == "twelvedata": + source_label = "Twelve Data" + elif _src == "alphavantage": + source_label = "Alpha Vantage" + elif _src in ("mt5", "metatrader5"): + source_label = "MetaTrader 5 SDK" + elif _src == "ccxt": + source_label = f"CCXT ({getattr(config, 'CCXT_BROKER', '?')})" + else: + source_label = _src + + print(f"Downloading {DAYS_BACK} days of {config.CANDLE_INTERVAL} candles " + f"for {len(config.PAIRS)} Forex pairs...") + print(f"Source : {source_label} (DATA_SOURCE={_src})") + print(f"Broker : {config.EXCHANGE} (pour l'exécution)") + print(f"Saving : {config.DATA_DIR}/\n") + + failed: list[str] = [] + + # Délai adaptatif selon la source (éviter le rate-limiting) + _src = getattr(config, "DATA_SOURCE", "auto").lower() + if _src in ("yfinance", "auto") and not config.OANDA_API_KEY: + _sleep = 1.5 # yfinance : 1.5s entre paires (rate limit discret) + elif _src == "alphavantage": + _sleep = 15.0 # Alpha Vantage free : 25 req/j → 4 req/min max + elif _src == "twelvedata": + _sleep = 8.0 if not getattr(config, "TWELVE_DATA_API_KEY", "") else 0.8 + # Sans clé : ~8 req/min | Avec clé plan free : 800 req/j → ~1 req/s + elif _src == "oanda": + _sleep = 0.3 # OANDA : API rapide avec clé + else: + _sleep = 1.0 # défaut raisonnable + + print(f" (délai entre paires : {_sleep}s pour éviter le rate-limiting)\n") + + for i, pair in enumerate(config.PAIRS, 1): + try: + candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK) + path = os.path.join(config.DATA_DIR, f"{pair}_1h.json") + with open(path, "w", encoding="utf-8") as f: + json.dump(candles, f) + days_actual = round(len(candles) / 24) + status = "OK" if days_actual >= 600 else "[données limitées]" + print(f" [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- " + f"{len(candles):,} candles (~{days_actual} jours) {status}") + except Exception as e: + print(f" [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- ERREUR: {e}") + failed.append(pair) + + time.sleep(_sleep) # rate limit adaptatif selon DATA_SOURCE + + # Retry failed pairs + if failed: + print(f"\n[!] {len(failed)} paire(s) échouée(s) : {', '.join(failed)}") + print("Retry dans 10 secondes...\n") + time.sleep(10) + + for pair in failed: + try: + candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK) + path = os.path.join(config.DATA_DIR, f"{pair}_1h.json") + with open(path, "w", encoding="utf-8") as f: + json.dump(candles, f) + days_actual = round(len(candles) / 24) + print(f" [OK] {pair:8s} -- {len(candles):,} candles récupérés!") + except Exception as e: + print(f" [FAIL] {pair:8s} -- Échec définitif: {e}") + + print("\n[DONE] Données sauvegardées dans data/") + + +if __name__ == "__main__": + download_all() diff --git a/ensemble_core.py b/ensemble_core.py new file mode 100644 index 0000000..acf93fe --- /dev/null +++ b/ensemble_core.py @@ -0,0 +1,278 @@ +""" +AHAD QUANT — Ensemble Core (Source Unique de Vérité ML) +============================================================ +UNE seule implémentation de « comment transformer un ensemble ML brut +(LightGBM + XGBoost + RandomForest [+ TFT + TransformerGRU]) en une +probabilité calibrée ». Utilisée IDENTIQUEMENT par : + + ahad_quant.py → signal live (paper / live / MT5) + backtest.py → validation historique + rl_env.py → observation de l'agent RL pendant l'ENTRAÎNEMENT + rl_agent.py → observation de l'agent RL en INFÉRENCE live + +AVANT ce module, la même logique était ré-écrite à la main 3 fois, avec de +vraies divergences silencieuses : + + • rl_env.py ignorait totalement TFT/TransformerGRU même quand has_dl=True, + et retombait en silence sur une simple moyenne des 3 modèles tabulaires + si le nombre de colonnes ne correspondait pas à scaler.n_features_in_ + (le RL "voyait" donc un ML différent de celui réellement utilisé en + live dès que les modèles séquentiels étaient activés). + • ahad_quant.py pouvait lever une ValueError du StandardScaler (et donc + planter toute la boucle principale) si has_dl=True mais l'historique + disponible pour une paire était trop court pour les modèles séquentiels. + • backtest.py n'avait AUCUNE gestion de has_dl — le backtest validait un + modèle différent de celui réellement déployé. + +Avec ce module, les 4 points d'entrée appellent EXACTEMENT le même code, +dans le même ordre de stacking que train.py (lgbm, xgb, rf, tft, tgru) : +le ML vu par le RL pendant l'entraînement == le ML vu en live == le ML +validé en backtest. Un seul cerveau, pas trois. +""" + +import os +import logging +import pickle +import numpy as np + +log = logging.getLogger("EnsembleCore") + +# Valeur de repli si prepare_sequences n'est pas importable (ne devrait +# jamais arriver en pratique — prepare_sequences.py ne dépend que de numpy). +_DL_SEQ_LEN_FALLBACK = 168 + + +# ─── Chargement ────────────────────────────────────────────────────────────── + +def load_ensemble(path: str): + """Charge un ensemble .pkl. Retourne None si absent/erreur (fail-safe — + ne lève jamais d'exception, pour que tous les appelants puissent + continuer en mode dégradé identique).""" + if not path or not os.path.exists(path): + log.warning(f"[ENS] Ensemble introuvable : {path}") + return None + try: + with open(path, "rb") as f: + ens = pickle.load(f) + acc = ens.get("ens_acc") + acc_str = f"{acc:.2%}" if isinstance(acc, (int, float)) else "?" + log.info(f"[ENS] Chargé ({path}) — has_dl={ens.get('has_dl', False)}, acc={acc_str}") + return ens + except Exception as e: + log.error(f"[ENS] Erreur chargement ensemble {path} : {e}") + return None + + +def ensemble_mtime(path: str) -> float: + """mtime du fichier ensemble, 0.0 si absent (utilisé pour le hot-reload).""" + try: + return os.path.getmtime(path) + except OSError: + return 0.0 + + +# ─── Modèles séquentiels (DL) ───────────────────────────────────────────────── + +def has_dl_models(ensemble: dict) -> bool: + """True seulement si l'ensemble déclare has_dl ET que les deux modèles + séquentiels sont réellement présents (évite tout état incohérent).""" + return bool( + ensemble is not None + and ensemble.get("has_dl") + and ensemble.get("tft") is not None + and ensemble.get("tgru") is not None + ) + + +def dl_seq_len() -> int: + try: + from prepare_sequences import SEQ_LEN + return SEQ_LEN + except Exception: + return _DL_SEQ_LEN_FALLBACK + + +def build_sequence_windows(X: np.ndarray, seq_len: int): + """ + Construit TOUTES les fenêtres glissantes valides de longueur seq_len + pour l'inférence (contrairement à prepare_sequences.build_sequences, qui + est pour l'ENTRAÎNEMENT et exclut volontairement les dernières lignes + pour éviter le data-leakage des labels — ici il n'y a pas de label, on + veut une prédiction pour CHAQUE ligne disposant d'assez d'historique, + y compris la toute dernière). + + Retourne (start_idx, X_seq) : + start_idx : index dans X de la 1ère ligne couverte (= seq_len - 1) + X_seq : (M, seq_len, F) avec M = len(X) - seq_len + 1 + (None, None) si len(X) < seq_len (pas assez d'historique). + """ + N = len(X) + if N < seq_len: + return None, None + M = N - seq_len + 1 + F = X.shape[1] + X_seq = np.empty((M, seq_len, F), dtype=np.float32) + for i in range(M): + X_seq[i] = X[i : i + seq_len] + return seq_len - 1, X_seq + + +def _predict_dl_batch(ensemble: dict, raw_feats: np.ndarray): + """ + Calcule (tft_probs, tgru_probs) ALIGNÉS sur raw_feats (même longueur N). + Les lignes sans historique suffisant reçoivent 0.5 (neutre) — jamais de + NaN, jamais d'exception qui remonte à l'appelant. + Retourne (None, None) si les modèles DL ne sont pas dispo / pas chargeables + (torch absent, erreur d'inférence, etc.) — l'appelant doit alors + simplement ne pas inclure TFT/TGRU dans le stacking, exactement comme un + sous-modèle manquant. + """ + if not has_dl_models(ensemble): + return None, None + try: + import torch + from prepare_sequences import transform_sequences + from tft_model import predict_tft_proba + from transformer_gru_model import predict_tgru_proba + except ImportError: + log.debug("[ENS] torch / tft_model / transformer_gru_model indisponibles — DL ignoré") + return None, None + + seq_len = dl_seq_len() + feats = np.nan_to_num(raw_feats.astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + start_idx, X_seq = build_sequence_windows(feats, seq_len) + + N = len(raw_feats) + if X_seq is None: + # Pas assez d'historique pour CE pair → neutre partout, pas d'erreur. + return (np.full(N, 0.5, dtype=np.float32), + np.full(N, 0.5, dtype=np.float32)) + + dl_scaler = ensemble.get("dl_scaler") + if dl_scaler is not None: + try: + X_seq = transform_sequences(dl_scaler, X_seq) + except Exception as e: + log.warning(f"[ENS] Erreur normalisation séquence DL : {e}") + return None, None + + try: + X_seq_t = torch.FloatTensor(X_seq) + tft_p = predict_tft_proba(ensemble["tft"], X_seq_t) + tgru_p = predict_tgru_proba(ensemble["tgru"], X_seq_t) + except Exception as e: + log.warning(f"[ENS] Erreur inférence DL (TFT/TGRU) : {e}") + return None, None + + tft_full = np.full(N, 0.5, dtype=np.float32) + tgru_full = np.full(N, 0.5, dtype=np.float32) + tft_full[start_idx : start_idx + len(tft_p)] = tft_p + tgru_full[start_idx : start_idx + len(tgru_p)] = tgru_p + return tft_full, tgru_full + + +# ─── Fonction canonique : ensemble → (proba, confidence) ────────────────────── + +def predict_ensemble_batch(ensemble: dict, raw_feats: np.ndarray, use_dl: bool = True) -> np.ndarray: + """ + UNIQUE implémentation : ensemble brut → (N, 2) [proba_long, confidence]. + + raw_feats : (N, NUM_FEATURES) — features BRUTES non normalisées (les + modèles tabulaires LightGBM/XGBoost/RF n'ont pas besoin de + scaling ; seules les features envoyées au réseau PPO le sont, + via un scaler totalement séparé — voir rl_agent.py). + + Stacking dans le MÊME ordre que train.py::build_ensemble (déterminant + pour la cohérence du méta-modèle) : lgbm, xgb, rf, [tft, tgru]. + + Robuste par construction : si le nombre de sous-modèles disponibles ne + correspond pas à scaler.n_features_in_ (ex : DL entraîné mais + indisponible à l'inférence), on retombe sur une moyenne simple plutôt que + de lever une exception — CE FALLBACK EST IDENTIQUE PARTOUT, alors qu'avant + ahad_quant.py plantait dans ce cas alors que rl_env.py silencieusement + dégradait sans le signaler. + """ + raw_feats = np.asarray(raw_feats, dtype=np.float32) + if raw_feats.ndim == 1: + raw_feats = raw_feats.reshape(1, -1) + N = len(raw_feats) + + if ensemble is None: + return np.column_stack([ + np.full(N, 0.5, dtype=np.float32), + np.zeros(N, dtype=np.float32), + ]) + + preds, names = [], [] + + lgbm = ensemble.get("lgbm") + xgb = ensemble.get("xgb") + rf = ensemble.get("rf") + + if lgbm is not None: + preds.append(np.asarray(lgbm.predict(raw_feats), dtype=np.float32)); names.append("lgbm") + if xgb is not None: + preds.append(xgb.predict_proba(raw_feats)[:, 1].astype(np.float32)); names.append("xgb") + if rf is not None: + preds.append(rf.predict_proba(raw_feats)[:, 1].astype(np.float32)); names.append("rf") + + if use_dl and has_dl_models(ensemble): + tft_p, tgru_p = _predict_dl_batch(ensemble, raw_feats) + if tft_p is not None: + preds.append(tft_p); names.append("tft") + if tgru_p is not None: + preds.append(tgru_p); names.append("tgru") + + if not preds: + proba = np.full(N, 0.5, dtype=np.float32) + else: + meta = ensemble.get("meta") + scaler = ensemble.get("scaler") + stacked = np.column_stack(preds) + + if (meta is not None and scaler is not None + and stacked.shape[1] == getattr(scaler, "n_features_in_", -1)): + stacked_s = scaler.transform(stacked) + proba = meta.predict_proba(stacked_s)[:, 1].astype(np.float32) + else: + if meta is not None and scaler is not None: + log.debug( + f"[ENS] méta-modèle attend {scaler.n_features_in_} entrées, " + f"{stacked.shape[1]} fournies ({names}) — fallback moyenne simple" + ) + proba = stacked.mean(axis=1).astype(np.float32) + + confidence = (np.abs(proba - 0.5) * 2.0).astype(np.float32) + return np.column_stack([proba, confidence]) + + +def predict_ensemble_single(ensemble: dict, raw_feat_row: np.ndarray, history: np.ndarray = None) -> tuple: + """ + Prédit pour UNE SEULE observation (inférence live, bougie par bougie). + + history : fenêtre récente (H, F) se terminant par raw_feat_row, utilisée + UNIQUEMENT si l'ensemble a des modèles séquentiels (has_dl). + Si absente ou trop courte, les composantes DL sont simplement + omises du stacking (comme tout sous-modèle absent) — jamais + d'exception. + + Réutilise predict_ensemble_batch() en interne → live (1 ligne) et + entraînement/backtest (batch) ne PEUVENT PAS diverger silencieusement, + car c'est littéralement le même code qui tourne. + + Retourne (proba, confidence) — deux floats Python. + """ + if ensemble is None: + return 0.5, 0.0 + + if history is not None and has_dl_models(ensemble): + window = np.asarray(history, dtype=np.float32) + if window.ndim == 1: + window = window.reshape(1, -1) + result = predict_ensemble_batch(ensemble, window, use_dl=True) + proba, confidence = float(result[-1, 0]), float(result[-1, 1]) + return proba, confidence + + feat_2d = np.asarray(raw_feat_row, dtype=np.float32).reshape(1, -1) + result = predict_ensemble_batch(ensemble, feat_2d, use_dl=False) + return float(result[0, 0]), float(result[0, 1]) diff --git a/exchange_adapter.py b/exchange_adapter.py new file mode 100644 index 0000000..ab4e07d --- /dev/null +++ b/exchange_adapter.py @@ -0,0 +1,1088 @@ +""" +AHAD QUANT — Exchange Adapter Layer (Forex Edition) + +Unified interface for trading Forex across multiple brokers. +Supported: OANDA, MetaTrader 5, Interactive Brokers (paper), Paper (built-in). + +Usage: + from exchange_adapter import get_exchange + + exchange = get_exchange("oanda") + exchange.connect() + balance = exchange.get_balance() + +Pair format (internal): "EURUSD", "GBPJPY", etc. +Each adapter converts to its broker's native format internally. +""" + +import logging +import os +import time +from abc import ABC, abstractmethod + +import requests + +logger = logging.getLogger(__name__) + +DEFAULT_SLIPPAGE = 0.00003 # ~3 pips for Forex market orders + +# ─── Optional SDK imports ───────────────────────────────────────────────────── + +try: + import oandapyV20 + import oandapyV20.endpoints.accounts as accounts_ep + import oandapyV20.endpoints.instruments as instruments_ep + import oandapyV20.endpoints.orders as orders_ep + import oandapyV20.endpoints.trades as trades_ep + import oandapyV20.endpoints.pricing as pricing_ep + import oandapyV20.endpoints.positions as positions_ep + from oandapyV20.contrib.requests import ( + MarketOrderRequest, LimitOrderRequest, TakeProfitDetails, StopLossDetails + ) + _HAS_OANDA = True +except ImportError: + _HAS_OANDA = False + +try: + import MetaTrader5 as mt5 + _HAS_MT5 = True +except ImportError: + _HAS_MT5 = False + +try: + from ib_insync import IB, Forex as IBForex, MarketOrder, LimitOrder + _HAS_IB = True +except ImportError: + _HAS_IB = False + + +# ─── Abstract base class ────────────────────────────────────────────────────── + +class ExchangeAdapter(ABC): + """ + Abstract interface that every broker adapter must implement. + + All methods use a common data format so the trading bot does not need + to know which broker it is connected to. + + Pair format: "EURUSD", "GBPJPY" (no separator, 6 chars). + """ + + @abstractmethod + def connect(self) -> None: + """Authenticate and establish the connection to the broker.""" + ... + + @abstractmethod + def get_balance(self) -> float: + """Return total account equity in account currency (USD/EUR).""" + ... + + @abstractmethod + def get_positions(self) -> list[dict]: + """ + Return open positions as a list of dicts: + [ + { + "coin": "EURUSD", # pair name (coin alias for compat) + "size": 10000.0, # positive = long, negative = short (units) + "entry": 1.08520, + "side": "long", + "unrealized_pnl": 12.5, + }, ... + ] + """ + ... + + @abstractmethod + def get_orderbook(self, coin: str) -> dict: + """ + Return best bid/ask for pair. + Returns {"bid": float, "ask": float, "mid": float} + """ + ... + + @abstractmethod + def get_candles(self, coin: str, interval: str = "1h", + limit: int = 200) -> list[dict]: + """ + Return recent OHLCV candles. + Each element: {"o": float, "h": float, "l": float, "c": float, "v": float} + """ + ... + + @abstractmethod + def place_limit_order(self, coin: str, side: str, size: float, + price: float) -> dict: + """ + Place a limit order. + coin : pair, e.g. "EURUSD" + side : "buy" or "sell" + size : units of base currency (e.g. 10000 = 0.1 lot EUR/USD) + price : limit price + Returns {"success": bool, "order_id": str|None, "detail": str} + """ + ... + + @abstractmethod + def place_market_order(self, coin: str, side: str, size: float) -> dict: + """ + Place a market order. + Returns {"success": bool, "order_id": str|None, "detail": str} + """ + ... + + @abstractmethod + def cancel_order(self, coin: str, oid: str) -> None: + """Cancel a pending order by id.""" + ... + + @abstractmethod + def close_position(self, coin: str) -> dict: + """ + Fully close the open position for pair. + Returns {"success": bool, "detail": str} + """ + ... + + @abstractmethod + def set_leverage(self, coin: str, leverage: int) -> None: + """ + Set leverage for pair. + Note: most Forex brokers use margin % not leverage multiplier. + This method is a no-op for brokers that don't expose leverage settings. + """ + ... + + @abstractmethod + def get_funding_rate(self, coin: str) -> float: + """ + Return the current swap/rollover rate for pair (as decimal per day). + Analogous to crypto funding rate. + """ + ... + + +# ─── OANDA Adapter ──────────────────────────────────────────────────────────── + +class OANDAAdapter(ExchangeAdapter): + """ + Adapter for OANDA v20 REST API (practice or live). + Requires: pip install oandapyV20 + """ + + # OANDA granularity map + _GRAN = { + "1m": "M1", "5m": "M5", "15m": "M15", "30m": "M30", + "1h": "H1", "2h": "H2", "4h": "H4", "8h": "H8", + "1d": "D", "1w": "W", + } + + def __init__(self) -> None: + self.api_key: str = os.getenv("OANDA_API_KEY", "") + self.account_id: str = os.getenv("OANDA_ACCOUNT_ID", "") + self.practice: bool = os.getenv("OANDA_PRACTICE", "true").lower() == "true" + self.client: "oandapyV20.API | None" = None + + @staticmethod + def _instrument(pair: str) -> str: + """EURUSD → EUR_USD""" + return f"{pair[:3]}_{pair[3:]}" + + @staticmethod + def _pair(instrument: str) -> str: + """EUR_USD → EURUSD""" + return instrument.replace("_", "") + + def connect(self) -> None: + if not _HAS_OANDA: + raise ImportError( + "oandapyV20 is required for OANDA support.\n" + "Install with: pip install oandapyV20" + ) + if not self.api_key or not self.account_id: + raise ValueError( + "OANDA_API_KEY and OANDA_ACCOUNT_ID env vars are required" + ) + environment = "practice" if self.practice else "live" + self.client = oandapyV20.API(access_token=self.api_key, + environment=environment) + # Verify connection + r = accounts_ep.AccountSummary(self.account_id) + self.client.request(r) + logger.info("[OANDA] Connected — account: %s (%s)", + self.account_id, environment) + + # -- account info -------------------------------------------------------- + + def get_balance(self) -> float: + try: + r = accounts_ep.AccountSummary(self.account_id) + self.client.request(r) + return float(r.response["account"]["NAV"]) + except Exception: + logger.error("OANDAAdapter.get_balance failed", exc_info=True) + return 0.0 + + def get_positions(self) -> list[dict]: + try: + r = positions_ep.OpenPositions(self.account_id) + self.client.request(r) + result: list[dict] = [] + for pos in r.response.get("positions", []): + long_units = float(pos["long"]["units"]) + short_units = float(pos["short"]["units"]) + if long_units == 0 and short_units == 0: + continue + if long_units != 0: + result.append({ + "coin": self._pair(pos["instrument"]), + "size": long_units, + "entry": float(pos["long"].get("averagePrice", 0) or 0), + "side": "long", + "unrealized_pnl": float(pos["long"].get("unrealizedPL", 0) or 0), + }) + if short_units != 0: + result.append({ + "coin": self._pair(pos["instrument"]), + "size": short_units, # already negative + "entry": float(pos["short"].get("averagePrice", 0) or 0), + "side": "short", + "unrealized_pnl": float(pos["short"].get("unrealizedPL", 0) or 0), + }) + return result + except Exception: + logger.error("OANDAAdapter.get_positions failed", exc_info=True) + return [] + + # -- market data --------------------------------------------------------- + + def get_orderbook(self, coin: str) -> dict: + instrument = self._instrument(coin) + r = pricing_ep.PricingInfo( + self.account_id, + params={"instruments": instrument} + ) + self.client.request(r) + price = r.response["prices"][0] + bid = float(price["bids"][0]["price"]) + ask = float(price["asks"][0]["price"]) + return {"bid": bid, "ask": ask, "mid": (bid + ask) / 2} + + def get_candles(self, coin: str, interval: str = "1h", + limit: int = 200) -> list[dict]: + granularity = self._GRAN.get(interval, "H1") + instrument = self._instrument(coin) + params = { + "count": min(limit, 5000), + "granularity": granularity, + "price": "M", # midpoint + } + r = instruments_ep.InstrumentsCandles(instrument, params=params) + self.client.request(r) + candles = [] + for c in r.response.get("candles", []): + if not c.get("complete", True): + continue + mid = c["mid"] + candles.append({ + "o": float(mid["o"]), + "h": float(mid["h"]), + "l": float(mid["l"]), + "c": float(mid["c"]), + "v": float(c.get("volume", 0)), + }) + return candles + + def get_funding_rate(self, coin: str) -> float: + """ + OANDA does not expose swap rates via public API endpoint directly. + Returns 0.0 — swap rates are baked into position P&L overnight. + Override this with your broker's swap table if needed. + """ + return 0.0 + + # -- order execution ----------------------------------------------------- + + def place_market_order(self, coin: str, side: str, size: float) -> dict: + try: + units = int(size) if side == "buy" else -int(size) + instrument = self._instrument(coin) + data = MarketOrderRequest( + instrument=instrument, + units=units, + ) + r = orders_ep.OrderCreate(self.account_id, data=data.data) + self.client.request(r) + resp = r.response + if "orderFillTransaction" in resp: + oid = resp["orderFillTransaction"].get("id", "") + return {"success": True, "order_id": str(oid), "detail": str(resp)} + return {"success": False, "order_id": None, "detail": str(resp)} + except Exception as e: + return {"success": False, "order_id": None, "detail": str(e)} + + def place_limit_order(self, coin: str, side: str, size: float, + price: float) -> dict: + try: + units = int(size) if side == "buy" else -int(size) + instrument = self._instrument(coin) + data = LimitOrderRequest( + instrument=instrument, + units=units, + price=str(round(price, 5)), + ) + r = orders_ep.OrderCreate(self.account_id, data=data.data) + self.client.request(r) + resp = r.response + if "orderCreateTransaction" in resp: + oid = resp["orderCreateTransaction"].get("id", "") + return {"success": True, "order_id": str(oid), "detail": str(resp)} + return {"success": False, "order_id": None, "detail": str(resp)} + except Exception as e: + return {"success": False, "order_id": None, "detail": str(e)} + + def cancel_order(self, coin: str, oid: str) -> None: + r = orders_ep.OrderCancel(self.account_id, orderID=oid) + self.client.request(r) + + def close_position(self, coin: str) -> dict: + try: + instrument = self._instrument(coin) + # Close all long and short units + data = {"longUnits": "ALL", "shortUnits": "ALL"} + r = positions_ep.PositionClose(self.account_id, + instrument=instrument, + data=data) + self.client.request(r) + return {"success": True, "detail": str(r.response)} + except Exception as e: + return {"success": False, "detail": str(e)} + + def set_leverage(self, coin: str, leverage: int) -> None: + # OANDA uses margin requirements, not explicit leverage setting + logger.debug( + "[OANDA] Leverage is set by margin requirement on the instrument, " + "not configurable via API. Requested: %dx for %s", leverage, coin + ) + + +# ─── MetaTrader 5 Adapter ───────────────────────────────────────────────────── + +class MT5Adapter(ExchangeAdapter): + """ + Adapter for MetaTrader 5 via the official Python integration. + Requires: pip install MetaTrader5 + Works on Windows only (MT5 runs natively on Windows). + """ + + # MT5 timeframe map + _TF = { + "1m": 1, "5m": 5, "15m": 15, "30m": 30, + "1h": 16385, "4h": 16388, "1d": 16408, + } + + def __init__(self) -> None: + self.login: int = int(os.getenv("MT5_LOGIN", "0")) + self.password: str = os.getenv("MT5_PASSWORD", "") + self.server: str = os.getenv("MT5_SERVER", "") + + def connect(self) -> None: + if not _HAS_MT5: + raise ImportError( + "MetaTrader5 package is required.\n" + "Install with: pip install MetaTrader5\n" + "Note: MT5 works on Windows only." + ) + if not mt5.initialize(login=self.login, + password=self.password, + server=self.server): + raise ConnectionError( + f"MT5 initialization failed: {mt5.last_error()}" + ) + logger.info("[MT5] Connected — account: %s", self.login) + + # -- account info -------------------------------------------------------- + + def get_balance(self) -> float: + try: + info = mt5.account_info() + return float(info.equity) if info else 0.0 + except Exception: + logger.error("MT5Adapter.get_balance failed", exc_info=True) + return 0.0 + + def get_positions(self) -> list[dict]: + try: + positions = mt5.positions_get() + if positions is None: + return [] + result = [] + for p in positions: + # MT5 type: 0=BUY(long), 1=SELL(short) + side = "long" if p.type == 0 else "short" + signed_size = p.volume * 100000 if side == "long" else -p.volume * 100000 + result.append({ + "coin": p.symbol.replace("/", ""), + "size": signed_size, + "entry": float(p.price_open), + "side": side, + "unrealized_pnl": float(p.profit), + }) + return result + except Exception: + logger.error("MT5Adapter.get_positions failed", exc_info=True) + return [] + + # -- market data --------------------------------------------------------- + + def get_orderbook(self, coin: str) -> dict: + tick = mt5.symbol_info_tick(coin) + if tick is None: + return {"bid": 0.0, "ask": 0.0, "mid": 0.0} + return { + "bid": float(tick.bid), + "ask": float(tick.ask), + "mid": (float(tick.bid) + float(tick.ask)) / 2, + } + + def get_candles(self, coin: str, interval: str = "1h", + limit: int = 200) -> list[dict]: + tf = self._TF.get(interval, 16385) # default H1 + rates = mt5.copy_rates_from_pos(coin, tf, 0, limit) + if rates is None: + return [] + return [ + { + "o": float(r["open"]), + "h": float(r["high"]), + "l": float(r["low"]), + "c": float(r["close"]), + "v": float(r.get("tick_volume", 0)), + } + for r in rates + ] + + def get_funding_rate(self, coin: str) -> float: + """ + MT5 swap rates are per symbol, stored in symbol info. + Returns average of long/short swap normalized to per-day decimal. + """ + try: + info = mt5.symbol_info(coin) + if info is None: + return 0.0 + # swap_long/short are in points per day + point = float(info.point) + avg_swap_points = (abs(float(info.swap_long)) + + abs(float(info.swap_short))) / 2 + # Convert to approximate decimal return per day + tick = mt5.symbol_info_tick(coin) + if tick and tick.bid > 0: + return (avg_swap_points * point) / float(tick.bid) + except Exception: + pass + return 0.0 + + # -- order execution ----------------------------------------------------- + + def _order_send(self, request: dict) -> dict: + result = mt5.order_send(request) + if result is None: + return {"success": False, "order_id": None, + "detail": f"MT5 error: {mt5.last_error()}"} + if result.retcode == mt5.TRADE_RETCODE_DONE: + return {"success": True, "order_id": str(result.order), + "detail": f"retcode={result.retcode}"} + return {"success": False, "order_id": None, + "detail": f"retcode={result.retcode} comment={result.comment}"} + + def place_market_order(self, coin: str, side: str, size: float) -> dict: + lots = size / 100000 + action = mt5.ORDER_TYPE_BUY if side == "buy" else mt5.ORDER_TYPE_SELL + tick = mt5.symbol_info_tick(coin) + price = tick.ask if side == "buy" else tick.bid + request = { + "action": mt5.TRADE_ACTION_DEAL, + "symbol": coin, + "volume": round(lots, 2), + "type": action, + "price": price, + "deviation": 20, # max slippage in points + "magic": 20241001, + "comment": "AHAD QUANT", + "type_time": mt5.ORDER_TIME_GTC, + "type_filling": mt5.ORDER_FILLING_IOC, + } + return self._order_send(request) + + def place_limit_order(self, coin: str, side: str, size: float, + price: float) -> dict: + lots = size / 100000 + action = (mt5.ORDER_TYPE_BUY_LIMIT if side == "buy" + else mt5.ORDER_TYPE_SELL_LIMIT) + request = { + "action": mt5.TRADE_ACTION_PENDING, + "symbol": coin, + "volume": round(lots, 2), + "type": action, + "price": price, + "deviation": 20, + "magic": 20241001, + "comment": "AHAD QUANT", + "type_time": mt5.ORDER_TIME_GTC, + "type_filling": mt5.ORDER_FILLING_RETURN, + } + return self._order_send(request) + + def cancel_order(self, coin: str, oid: str) -> None: + request = { + "action": mt5.TRADE_ACTION_REMOVE, + "order": int(oid), + } + mt5.order_send(request) + + def close_position(self, coin: str) -> dict: + try: + positions = mt5.positions_get(symbol=coin) + if not positions: + return {"success": False, "detail": f"No open position for {coin}"} + for pos in positions: + if pos.type == mt5.ORDER_TYPE_BUY: + side = "sell" + tick = mt5.symbol_info_tick(coin) + price = tick.bid + else: + side = "buy" + tick = mt5.symbol_info_tick(coin) + price = tick.ask + request = { + "action": mt5.TRADE_ACTION_DEAL, + "symbol": coin, + "volume": pos.volume, + "type": mt5.ORDER_TYPE_SELL if side == "sell" else mt5.ORDER_TYPE_BUY, + "position": pos.ticket, + "price": price, + "deviation": 20, + "magic": 20241001, + "comment": "AHAD QUANT close", + "type_time": mt5.ORDER_TIME_GTC, + "type_filling": mt5.ORDER_FILLING_IOC, + } + mt5.order_send(request) + return {"success": True, "detail": f"Closed all positions for {coin}"} + except Exception as e: + return {"success": False, "detail": str(e)} + + def set_leverage(self, coin: str, leverage: int) -> None: + # MT5 leverage is set at account level, not per-instrument + logger.debug( + "[MT5] Leverage is managed at account level. " + "Requested: %dx for %s", leverage, coin + ) + + +# ─── ccxt Forex CFD Adapter (e.g. IG, FOREX.COM) ───────────────────────────── + +class CCXTForexAdapter(ExchangeAdapter): + """ + Adapter for Forex CFD brokers supported by ccxt (IG, GAIN Capital, etc.). + Also works with crypto exchanges offering Forex pairs (Binance, Bybit CFDs). + + Set CCXT_BROKER env var to the exchange id (e.g. "ig", "gainCapital"). + """ + + def __init__(self) -> None: + self.broker: str = os.getenv("CCXT_BROKER", "").lower() + self.api_key: str = os.getenv("CCXT_API_KEY", "") + self.api_secret: str = os.getenv("CCXT_API_SECRET", "") + self.passphrase: str = os.getenv("CCXT_PASSPHRASE", "") + self.sandbox: bool = os.getenv("CCXT_SANDBOX", "false").lower() == "true" + self.client = None + + def _symbol(self, pair: str) -> str: + """EURUSD → EUR/USD (ccxt format).""" + return f"{pair[:3]}/{pair[3:]}" + + def connect(self) -> None: + try: + import ccxt + except ImportError: + raise ImportError( + "ccxt is required. Install with: pip install ccxt" + ) + if not self.broker: + raise ValueError("CCXT_BROKER env var must be set (e.g. 'ig')") + cls = getattr(ccxt, self.broker, None) + if cls is None: + raise ValueError(f"Unknown ccxt broker: {self.broker}") + self.client = cls({ + "apiKey": self.api_key, + "secret": self.api_secret, + "password": self.passphrase, + "enableRateLimit": True, + }) + if self.sandbox: + self.client.set_sandbox_mode(True) + self.client.load_markets() + + def get_balance(self) -> float: + try: + bal = self.client.fetch_balance() + return float(bal.get("total", {}).get("USD", 0) + or bal.get("total", {}).get("EUR", 0)) + except Exception: + logger.error("CCXTForexAdapter.get_balance failed", exc_info=True) + return 0.0 + + def get_positions(self) -> list[dict]: + try: + raw = self.client.fetch_positions() + result = [] + for p in raw: + size = float(p.get("contracts", 0) or 0) + if size == 0: + continue + side = p.get("side", "long") + sym = p.get("symbol", "").replace("/", "").replace(":", "")[:6] + result.append({ + "coin": sym, + "size": size if side == "long" else -size, + "entry": float(p.get("entryPrice", 0) or 0), + "side": side, + "unrealized_pnl": float(p.get("unrealizedPnl", 0) or 0), + }) + return result + except Exception: + logger.error("CCXTForexAdapter.get_positions failed", exc_info=True) + return [] + + def get_orderbook(self, coin: str) -> dict: + book = self.client.fetch_order_book(self._symbol(coin), limit=5) + bid = book["bids"][0][0] + ask = book["asks"][0][0] + return {"bid": bid, "ask": ask, "mid": (bid + ask) / 2} + + def get_candles(self, coin: str, interval: str = "1h", + limit: int = 200) -> list[dict]: + ohlcv = self.client.fetch_ohlcv(self._symbol(coin), interval, limit=limit) + return [ + {"o": c[1], "h": c[2], "l": c[3], "c": c[4], "v": c[5]} + for c in ohlcv + ] + + def get_funding_rate(self, coin: str) -> float: + return 0.0 # Forex CFDs use swap, not funding + + def place_limit_order(self, coin: str, side: str, size: float, + price: float) -> dict: + try: + order = self.client.create_limit_order( + self._symbol(coin), side, size, price + ) + return {"success": True, "order_id": order.get("id"), + "detail": str(order)} + except Exception as e: + return {"success": False, "order_id": None, "detail": str(e)} + + def place_market_order(self, coin: str, side: str, size: float) -> dict: + try: + order = self.client.create_market_order( + self._symbol(coin), side, size + ) + return {"success": True, "order_id": order.get("id"), + "detail": str(order)} + except Exception as e: + return {"success": False, "order_id": None, "detail": str(e)} + + def cancel_order(self, coin: str, oid: str) -> None: + self.client.cancel_order(oid, self._symbol(coin)) + + def close_position(self, coin: str) -> dict: + try: + positions = self.get_positions() + for p in positions: + if p["coin"] == coin: + side = "sell" if p["side"] == "long" else "buy" + return self.place_market_order(coin, side, abs(p["size"])) + return {"success": False, "detail": f"No open position for {coin}"} + except Exception as e: + return {"success": False, "detail": str(e)} + + def set_leverage(self, coin: str, leverage: int) -> None: + try: + self.client.set_leverage(leverage, self._symbol(coin)) + except Exception: + logger.debug( + "CCXTForexAdapter.set_leverage not supported for %s", coin + ) + + +# ─── Alpaca Markets Adapter ─────────────────────────────────────────────────── + +class AlpacaAdapter(ExchangeAdapter): + """ + Adapter for Alpaca Markets (paper + live). + Supporte les actions US et les crypto — Forex via Crypto (EURUSD, etc.). + Requires: pip install alpaca-py + + Variables d'environnement : + ALPACA_API_KEY — clé API Alpaca + ALPACA_SECRET — secret Alpaca + ALPACA_PAPER — true (paper) | false (live) + """ + + def __init__(self) -> None: + self.api_key: str = os.getenv("ALPACA_API_KEY", "") + self.secret: str = os.getenv("ALPACA_SECRET", "") + self.paper: bool = os.getenv("ALPACA_PAPER", "true").lower() == "true" + self._trading = None + self._data = None + + def connect(self) -> None: + try: + from alpaca.trading.client import TradingClient + from alpaca.data.historical import CryptoHistoricalDataClient + except ImportError: + raise ImportError( + "alpaca-py est requis pour le broker Alpaca.\n" + "Installer avec : pip install alpaca-py" + ) + if not self.api_key or not self.secret: + raise ValueError( + "ALPACA_API_KEY et ALPACA_SECRET sont requis dans .env" + ) + self._trading = TradingClient( + api_key=self.api_key, + secret_key=self.secret, + paper=self.paper, + ) + self._data = CryptoHistoricalDataClient( + api_key=self.api_key, + secret_key=self.secret, + ) + account = self._trading.get_account() + env_label = "paper" if self.paper else "live" + logger.info("[Alpaca] Connecté — compte: %s (%s)", account.id, env_label) + + def get_balance(self) -> float: + try: + account = self._trading.get_account() + return float(account.equity) + except Exception: + logger.error("AlpacaAdapter.get_balance failed", exc_info=True) + return 0.0 + + def get_positions(self) -> list[dict]: + try: + positions = self._trading.get_all_positions() + result = [] + for p in positions: + size = float(p.qty) + side = "long" if p.side.value == "long" else "short" + result.append({ + "coin": p.symbol.replace("/", "").replace("USD", "USD")[:6], + "size": size if side == "long" else -size, + "entry": float(p.avg_entry_price), + "side": side, + "unrealized_pnl": float(p.unrealized_pl), + }) + return result + except Exception: + logger.error("AlpacaAdapter.get_positions failed", exc_info=True) + return [] + + def get_orderbook(self, coin: str) -> dict: + """Alpaca ne fournit pas d'orderbook Forex — retourne latest quote.""" + try: + from alpaca.data.requests import CryptoLatestQuoteRequest + req = CryptoLatestQuoteRequest(symbol_or_symbols=coin) + quote = self._data.get_crypto_latest_quote(req) + q = quote[coin] + bid = float(q.bid_price) + ask = float(q.ask_price) + return {"bid": bid, "ask": ask, "mid": (bid + ask) / 2} + except Exception: + return {"bid": 0.0, "ask": 0.0, "mid": 0.0} + + def get_candles(self, coin: str, interval: str = "1h", + limit: int = 200) -> list[dict]: + try: + from alpaca.data.requests import CryptoBarsRequest + from alpaca.data.timeframe import TimeFrame, TimeFrameUnit + from datetime import datetime, timedelta + _tf_map = { + "1m": TimeFrame(1, TimeFrameUnit.Minute), + "5m": TimeFrame(5, TimeFrameUnit.Minute), + "15m": TimeFrame(15, TimeFrameUnit.Minute), + "1h": TimeFrame(1, TimeFrameUnit.Hour), + "4h": TimeFrame(4, TimeFrameUnit.Hour), + "1d": TimeFrame(1, TimeFrameUnit.Day), + } + tf = _tf_map.get(interval, TimeFrame(1, TimeFrameUnit.Hour)) + end = datetime.utcnow() + # Estimer la plage pour obtenir `limit` bougies + hours_map = {"1m": 1/60, "5m": 5/60, "15m": 0.25, "1h": 1, + "4h": 4, "1d": 24} + hours_per_candle = hours_map.get(interval, 1) + start = end - timedelta(hours=hours_per_candle * limit * 1.5) + req = CryptoBarsRequest( + symbol_or_symbols=coin, + timeframe=tf, + start=start, + end=end, + ) + bars = self._data.get_crypto_bars(req) + result = [] + for bar in bars[coin]: + result.append({ + "o": float(bar.open), + "h": float(bar.high), + "l": float(bar.low), + "c": float(bar.close), + "v": float(bar.volume), + }) + return result[-limit:] + except Exception: + logger.error("AlpacaAdapter.get_candles failed", exc_info=True) + return [] + + def get_funding_rate(self, coin: str) -> float: + return 0.0 + + def place_market_order(self, coin: str, side: str, size: float) -> dict: + try: + from alpaca.trading.requests import MarketOrderRequest + from alpaca.trading.enums import OrderSide, TimeInForce + req = MarketOrderRequest( + symbol=coin, + qty=size, + side=OrderSide.BUY if side == "buy" else OrderSide.SELL, + time_in_force=TimeInForce.GTC, + ) + order = self._trading.submit_order(req) + return {"success": True, "order_id": str(order.id), "detail": str(order)} + except Exception as e: + return {"success": False, "order_id": None, "detail": str(e)} + + def place_limit_order(self, coin: str, side: str, size: float, + price: float) -> dict: + try: + from alpaca.trading.requests import LimitOrderRequest + from alpaca.trading.enums import OrderSide, TimeInForce + req = LimitOrderRequest( + symbol=coin, + qty=size, + side=OrderSide.BUY if side == "buy" else OrderSide.SELL, + time_in_force=TimeInForce.GTC, + limit_price=price, + ) + order = self._trading.submit_order(req) + return {"success": True, "order_id": str(order.id), "detail": str(order)} + except Exception as e: + return {"success": False, "order_id": None, "detail": str(e)} + + def cancel_order(self, coin: str, oid: str) -> None: + try: + import uuid + self._trading.cancel_order_by_id(uuid.UUID(oid)) + except Exception: + logger.debug("[Alpaca] cancel_order failed for %s", oid) + + def close_position(self, coin: str) -> dict: + try: + self._trading.close_position(coin) + return {"success": True, "detail": f"Closed {coin}"} + except Exception as e: + return {"success": False, "detail": str(e)} + + def set_leverage(self, coin: str, leverage: int) -> None: + # Alpaca ne permet pas de changer le levier par instrument + logger.debug("[Alpaca] Leverage non configurable par instrument (%dx)", leverage) + + +# ─── PaperAdapter ───────────────────────────────────────────────────────────── + +class PaperAdapter(ExchangeAdapter): + """ + Adaptateur paper trading pur — aucune dépendance externe requise. + Toutes les opérations sont des no-ops ou retournent des valeurs simulées. + Utilisé quand EXCHANGE=paper dans .env. + """ + + def connect(self) -> None: + logger.info("[Paper] Paper adapter connecté — aucun broker réel") + + def get_balance(self) -> float: + return 0.0 # géré par RiskManager / paper engine interne + + def get_positions(self) -> list: + return [] + + def get_candles(self, coin: str, interval: str = "1h", limit: int = 200) -> list: + """Fetch real candles via yfinance (gratuit, sans clé API).""" + try: + import yfinance as yf + + # Forex : EURUSD → EURUSD=X + ticker = coin + "=X" if (len(coin) == 6 and coin.isalpha()) else coin.replace("USDT", "-USD") + + period_map = {"1m": "7d", "5m": "60d", "15m": "60d", + "30m": "60d", "1h": "730d", "4h": "730d", "1d": "5y"} + yf_int_map = {"1h": "1h", "4h": "1h", "1d": "1d", + "1m": "1m", "5m": "5m", "15m": "15m", "30m": "30m"} + period = period_map.get(interval, "730d") + yf_int = yf_int_map.get(interval, "1h") + + df = yf.download(ticker, period=period, interval=yf_int, + progress=False, auto_adjust=True) + if df is None or df.empty: + return [] + + # yfinance 1.4+ retourne des colonnes MultiIndex (col, ticker) + # On aplatit pour avoir des colonnes simples : Open, High, Low, Close, Volume + if isinstance(df.columns, __import__("pandas").MultiIndex): + df.columns = df.columns.get_level_values(0) + + candles = [] + for ts, row in df.iterrows(): + try: + candles.append({ + "t": int(ts.timestamp()), + "o": float(row["Open"]), + "h": float(row["High"]), + "l": float(row["Low"]), + "c": float(row["Close"]), + "v": float(row.get("Volume", 1.0) or 1.0), + }) + except Exception: + continue + + return candles[-limit:] + except Exception as e: + logger.warning(f"[PaperAdapter] yfinance get_candles({coin}) failed: {e}") + return [] + + def get_price(self, coin: str) -> float: + """Fetch real-time price via yfinance.""" + try: + candles = self.get_candles(coin, interval="1h", limit=2) + return candles[-1]["c"] if candles else 0.0 + except Exception: + return 0.0 + + def get_orderbook(self, coin: str) -> dict: + """Retourne un orderbook minimal avec mid price via yfinance.""" + try: + candles = self.get_candles(coin, interval="1h", limit=2) + mid = candles[-1]["c"] if candles else 0.0 + except Exception: + mid = 0.0 + return {"bids": [], "asks": [], "mid": mid} + + def place_market_order(self, coin: str, side: str, size: float) -> dict: + return {"success": True, "order_id": "paper", "detail": "paper mode"} + + def place_limit_order(self, coin: str, side: str, size: float, price: float) -> dict: + return {"success": True, "order_id": "paper", "detail": "paper mode"} + + def cancel_order(self, coin: str, oid: str) -> None: + pass + + def close_position(self, coin: str) -> dict: + return {"success": True, "detail": "paper mode"} + + def set_leverage(self, coin: str, leverage: int) -> None: + pass + + def get_funding_rate(self, coin: str) -> float: + return 0.0 + + +# ─── Utility helpers ────────────────────────────────────────────────────────── + +def _parse_interval(interval: str) -> int: + """Convert interval string like '1h', '15m', '1d' to seconds.""" + units = {"m": 60, "h": 3600, "d": 86400} + unit = interval[-1] + value = int(interval[:-1]) + return value * units.get(unit, 3600) + + +def _round_price(price: float, pair: str = "") -> float: + """Round Forex price to appropriate precision.""" + # JPY pairs use 3 decimal places; others use 5 + if "JPY" in pair.upper(): + return round(price, 3) + return round(price, 5) + + +def pip_value(pair: str, price: float) -> float: + """ + Return pip value per unit of base currency. + For USD-quoted pairs (EURUSD): 1 pip = 0.0001 + For JPY pairs (USDJPY): 1 pip = 0.01 + """ + if "JPY" in pair.upper(): + return 0.01 + return 0.0001 + + +# ─── Factory ────────────────────────────────────────────────────────────────── + +_ADAPTERS: dict[str, type[ExchangeAdapter]] = { + # ── Brokers directs ──────────────────────────────────────────────────────── + "oanda": OANDAAdapter, # OANDA v20 REST API + "mt5": MT5Adapter, # MetaTrader 5 (Python SDK, Windows) + "metatrader5": MT5Adapter, + "alpaca": AlpacaAdapter, # Alpaca Markets (paper + live) + "ib": CCXTForexAdapter, # Interactive Brokers via ccxt (fallback) + + # ── Via ccxt (100+ brokers) ──────────────────────────────────────────────── + # Définir CCXT_BROKER dans .env avec le nom ccxt exact du broker + # Exemples : ig | gainCapital | okcoin | bitfinex | kraken | binance | bybit + "ccxt": CCXTForexAdapter, + + # ── Aliases hérités ──────────────────────────────────────────────────────── + "binance": CCXTForexAdapter, + "bybit": CCXTForexAdapter, + "bitget": CCXTForexAdapter, + "ig": CCXTForexAdapter, + "gaincapital": CCXTForexAdapter, + "fxcm": CCXTForexAdapter, + + # ── Paper interne ────────────────────────────────────────────────────────── + # PAPER_MODE=true + EXCHANGE=paper → simulation complète sans broker + "paper": PaperAdapter, # Aucune dépendance externe — no-op complet +} + + +def get_exchange(name: str) -> ExchangeAdapter: + """ + Return a broker adapter instance by name. + + Parameters + ---------- + name : str + Broker name (case-insensitive). + Supported : oanda | mt5 | alpaca | ccxt | ib | paper + + aliases : metatrader5 | binance | bybit | bitget | ig | gaincapital | fxcm + + Returns + ------- + ExchangeAdapter + An unconnected adapter — call .connect() before use. + + Notes + ----- + Pour CCXT, définir aussi dans .env : + CCXT_BROKER=nom_du_broker (ex: "ig", "gainCapital", "kraken") + """ + key = name.strip().lower() + cls = _ADAPTERS.get(key) + if cls is None: + supported = ", ".join(sorted(set(_ADAPTERS.keys()))) + raise ValueError( + f"Unknown broker '{name}'. Supported: {supported}" + ) + return cls() diff --git a/experience_buffer.py b/experience_buffer.py new file mode 100644 index 0000000..e6128da --- /dev/null +++ b/experience_buffer.py @@ -0,0 +1,363 @@ +""" +AHAD QUANT — Experience Buffer (Apprentissage Continu V7) +============================================================ +Replay buffer thread-safe qui stocke chaque trade fermé avec son contexte complet. +Persiste sur disque (experience_buffer.json). + +Structure d'une expérience : +{ + "id": "20260612_143022_EURUSD", + "timestamp": 1718200222.0, + "pair": "EURUSD", + "signal": "LONG", + "confidence": 0.78, + "ml_probas": [0.72, 0.75, 0.71, 0.74, 0.73], + "rl_action": 1, + "rl_agreed": True, + "features": [...], # 62 features à l'entrée + "regime": "NORMAL", + "entry_price": 1.08501, + "exit_price": 1.08762, + "pnl": 0.0241, + "outcome": "WIN", # WIN / LOSS / TIMEOUT + "hold_candles": 4, + "error_weight": 1.0 # WIN=1.0 TIMEOUT=1.5 LOSS=2.0 +} + +Usage : + buffer = ExperienceBuffer() + buffer.load() + buffer.add(experience) + samples = buffer.sample(n=256, error_weighted=True) + recent = buffer.get_recent(hours=24) + losses = buffer.get_loss_trades(min_weight=1.5) + stats = buffer.stats() + buffer.save() +""" + +import json +import logging +import os +import random +import threading +import time +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from features import NUM_FEATURES + +log = logging.getLogger("ExperienceBuffer") + +# ── Constantes ──────────────────────────────────────────────────────────────── +MAX_BUFFER_SIZE = 10_000 # trades max en mémoire (FIFO si dépassé) +ERROR_WEIGHT_LOSS = 2.0 # LOSS pèse 2× lors du sampling +ERROR_WEIGHT_TIMEOUT = 1.5 # TIMEOUT pèse 1.5× +ERROR_WEIGHT_WIN = 1.0 # WIN pèse 1× (normal) +BUFFER_FILE = "experience_buffer.json" + + +class ExperienceBuffer: + """ + Buffer thread-safe de toutes les expériences de trading. + Utilisé par OnlineLearner et PerformanceMonitor. + """ + + def __init__( + self, + max_size: int = MAX_BUFFER_SIZE, + buffer_file: str = BUFFER_FILE, + ): + self.max_size = max_size + self.buffer_file = buffer_file + self._experiences: List[Dict] = [] + self._lock = threading.RLock() + + # ── I/O ────────────────────────────────────────────────────────────────── + + def save(self) -> bool: + """Sauvegarde le buffer sur disque. Retourne True si succès.""" + with self._lock: + try: + tmp = self.buffer_file + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(self._experiences, f, ensure_ascii=False) + os.replace(tmp, self.buffer_file) + log.debug(f"Buffer sauvegardé ({len(self._experiences)} trades)") + return True + except Exception as e: + log.error(f"Erreur sauvegarde buffer : {e}") + return False + + def load(self) -> bool: + """ + Charge le buffer depuis le disque. + Fail-safe : si le fichier est corrompu → buffer vide, trading continue. + """ + if not os.path.exists(self.buffer_file): + log.info("Aucun buffer existant — démarrage avec buffer vide") + return False + try: + with open(self.buffer_file, encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, list): + raise ValueError("Format invalide — doit être une liste") + with self._lock: + self._experiences = data[-self.max_size:] + log.info(f"Buffer chargé : {len(self._experiences)} trades") + return True + except Exception as e: + log.error(f"Erreur chargement buffer (reset) : {e}") + with self._lock: + self._experiences = [] + return False + + # ── Ajout ──────────────────────────────────────────────────────────────── + + def add(self, experience: Dict) -> None: + """ + Ajoute une expérience au buffer. + - Complète les champs manquants (id, timestamp, error_weight). + - Tronque le buffer si MAX_BUFFER_SIZE est dépassé (FIFO). + - Sauvegarde automatiquement toutes les 100 trades. + """ + with self._lock: + # Champs obligatoires avec valeurs par défaut + exp = dict(experience) + + if "id" not in exp: + ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + exp["id"] = f"{ts}_{exp.get('pair','UNKNOWN')}" + + if "timestamp" not in exp: + exp["timestamp"] = time.time() + + if "error_weight" not in exp: + outcome = exp.get("outcome", "WIN") + exp["error_weight"] = _outcome_to_weight(outcome) + + self._experiences.append(exp) + + # FIFO : supprimer les plus anciens si dépassement + if len(self._experiences) > self.max_size: + excess = len(self._experiences) - self.max_size + self._experiences = self._experiences[excess:] + + # Auto-sauvegarde toutes les 100 trades + if len(self._experiences) % 100 == 0: + self.save() + + # ── Sampling ───────────────────────────────────────────────────────────── + + def sample(self, n: int = 256, error_weighted: bool = True) -> List[Dict]: + """ + Retourne n expériences aléatoires. + Si error_weighted=True : les LOSS et TIMEOUT ont plus de chances d'être sélectionnés. + """ + with self._lock: + pool = list(self._experiences) + + if not pool: + return [] + + n = min(n, len(pool)) + + if not error_weighted: + return random.sample(pool, n) + + # Pondération par error_weight + weights = [e.get("error_weight", 1.0) for e in pool] + total = sum(weights) + probs = [w / total for w in weights] + + indices = random.choices(range(len(pool)), weights=probs, k=n) + return [pool[i] for i in indices] + + # ── Filtres ────────────────────────────────────────────────────────────── + + def get_recent( + self, + hours: float = 24.0, + outcome_filter: Optional[List[str]] = None, + ) -> List[Dict]: + """ + Retourne les trades des dernières heures. + Si outcome_filter est fourni (ex: ["LOSS","TIMEOUT"]), filtre par outcome. + """ + cutoff = time.time() - hours * 3600 + with self._lock: + pool = [e for e in self._experiences if e.get("timestamp", 0) >= cutoff] + + if outcome_filter: + pool = [e for e in pool if e.get("outcome", "") in outcome_filter] + + return pool + + def get_loss_trades(self, min_weight: float = 1.5) -> List[Dict]: + """Retourne les trades avec error_weight >= min_weight (LOSS + TIMEOUT).""" + with self._lock: + return [e for e in self._experiences if e.get("error_weight", 1.0) >= min_weight] + + def get_by_outcome(self, outcome: str) -> List[Dict]: + """Retourne tous les trades d'un outcome donné (WIN/LOSS/TIMEOUT).""" + with self._lock: + return [e for e in self._experiences if e.get("outcome") == outcome] + + def get_all(self) -> List[Dict]: + """Retourne une copie de tout le buffer.""" + with self._lock: + return list(self._experiences) + + def get_last_n(self, n: int) -> List[Dict]: + """Retourne les n derniers trades.""" + with self._lock: + return list(self._experiences[-n:]) + + # ── Statistiques ───────────────────────────────────────────────────────── + + def stats(self, window: int = 0) -> Dict: + """ + Retourne les statistiques du buffer. + Si window > 0, calcule sur les derniers trades. + """ + with self._lock: + data = list(self._experiences[-window:]) if window > 0 else list(self._experiences) + + if not data: + return { + "total_trades": 0, + "win_rate": 0.0, + "loss_rate": 0.0, + "timeout_rate": 0.0, + "avg_pnl": 0.0, + "total_pnl": 0.0, + "max_drawdown": 0.0, + "avg_confidence": 0.0, + "avg_hold_candles": 0.0, + } + + total = len(data) + wins = sum(1 for e in data if e.get("outcome") == "WIN") + losses = sum(1 for e in data if e.get("outcome") == "LOSS") + timeouts = sum(1 for e in data if e.get("outcome") == "TIMEOUT") + + pnls = [e.get("pnl", 0.0) for e in data] + confs = [e.get("confidence", 0.0) for e in data if e.get("confidence")] + holds = [e.get("hold_candles", 0) for e in data] + + # Max drawdown (peak-to-trough sur la séquence de PnL cumulatif) + cumulative = 0.0 + peak = 0.0 + max_dd = 0.0 + for p in pnls: + cumulative += p + if cumulative > peak: + peak = cumulative + dd = (peak - cumulative) / (abs(peak) + 1e-9) + if dd > max_dd: + max_dd = dd + + return { + "total_trades": total, + "win_rate": wins / total, + "loss_rate": losses / total, + "timeout_rate": timeouts / total, + "avg_pnl": sum(pnls) / total, + "total_pnl": sum(pnls), + "max_drawdown": max_dd, + "avg_confidence": sum(confs) / len(confs) if confs else 0.0, + "avg_hold_candles": sum(holds) / len(holds) if holds else 0.0, + } + + @property + def total_trades(self) -> int: + with self._lock: + return len(self._experiences) + + def __len__(self) -> int: + return self.total_trades + + def __repr__(self) -> str: + s = self.stats() + return ( + f"ExperienceBuffer(" + f"trades={s['total_trades']}, " + f"win_rate={s['win_rate']:.1%}, " + f"avg_pnl={s['avg_pnl']:.4f})" + ) + + +# ── Utilitaire ──────────────────────────────────────────────────────────────── + +def _outcome_to_weight(outcome: str) -> float: + """Convertit un outcome en error_weight.""" + return { + "WIN": ERROR_WEIGHT_WIN, + "LOSS": ERROR_WEIGHT_LOSS, + "TIMEOUT": ERROR_WEIGHT_TIMEOUT, + }.get(outcome.upper(), 1.0) + + +# ── Singleton global (utilisé par ahad_quant.py, online_learner.py, etc.) ───── + +_global_buffer: Optional[ExperienceBuffer] = None +_buffer_lock = threading.Lock() + + +def get_experience_buffer( + max_size: int = MAX_BUFFER_SIZE, + buffer_file: str = BUFFER_FILE, + auto_load: bool = True, +) -> ExperienceBuffer: + """ + Retourne le singleton global du buffer. + Thread-safe. Charge automatiquement depuis le disque au premier appel. + """ + global _global_buffer + with _buffer_lock: + if _global_buffer is None: + _global_buffer = ExperienceBuffer(max_size=max_size, buffer_file=buffer_file) + if auto_load: + _global_buffer.load() + return _global_buffer + + +# ── CLI de test ─────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + import logging + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(name)s] %(message)s") + + buf = ExperienceBuffer(max_size=100, buffer_file="test_buffer.json") + buf.load() + + print(f"Buffer chargé : {buf.total_trades} trades") + + # Ajouter des trades de test + for i in range(20): + outcome = ["WIN", "LOSS", "TIMEOUT"][i % 3] + buf.add({ + "pair": "EURUSD", + "signal": "LONG" if i % 2 == 0 else "SHORT", + "confidence": round(0.72 + (i % 5) * 0.02, 3), + "ml_probas": [0.72, 0.73, 0.71, 0.74, 0.72], + "rl_action": 1, + "rl_agreed": True, + "features": [0.0] * NUM_FEATURES, + "regime": "NORMAL", + "entry_price": 1.0850, + "exit_price": 1.0880 if outcome == "WIN" else 1.0820, + "pnl": 0.03 if outcome == "WIN" else -0.02, + "outcome": outcome, + "hold_candles": 4, + }) + + print(f"\n{buf}") + print(f"\nStats : {json.dumps(buf.stats(), indent=2)}") + print(f"\n5 derniers trades : {[e['outcome'] for e in buf.get_last_n(5)]}") + print(f"LOSS trades : {len(buf.get_loss_trades())} (weight >= 1.5)") + print(f"Récents 24h : {len(buf.get_recent(hours=24))} trades") + + # Nettoyage test + if os.path.exists("test_buffer.json"): + os.remove("test_buffer.json") + print("\n✅ ExperienceBuffer — test OK") diff --git a/export_unified.py b/export_unified.py new file mode 100644 index 0000000..a09e16a --- /dev/null +++ b/export_unified.py @@ -0,0 +1,121 @@ +""" +AHAD QUANT — Export Modèle Unifié +Combine rl_agent.zip + model_ensemble.pkl + rl_scaler.pkl +en un seul fichier ahad_quant_unified.zip. + +Usage : + python export_unified.py + python export_unified.py --output mon_modele.zip + +Structure du zip produit : + ahad_quant_unified.zip + ├── ppo/ ← contenu SB3 intact (policy.pth, optimizer, ...) + ├── ensemble.pkl ← ensemble supervisé + ├── rl_scaler.pkl ← scaler z-score features + └── metadata.json ← version, date, config +""" + +import os +import sys +import json +import zipfile +import pickle +import argparse +import io +from datetime import datetime + +sys.stdout.reconfigure(encoding="utf-8", errors="replace") + +import config + + +def export_unified( + ppo_path: str = None, + ensemble_path: str = None, + scaler_path: str = None, + output_path: str = "ahad_quant_unified.zip", +): + ppo_path = ppo_path or getattr(config, "RL_MODEL_PATH", "rl_agent") + ensemble_path = ensemble_path or getattr(config, "ENSEMBLE_MODEL_PATH", "model_ensemble.pkl") + scaler_path = scaler_path or getattr(config, "RL_SCALER_PATH", "rl_scaler.pkl") + + # Ajouter .zip si absent + ppo_zip = ppo_path if ppo_path.endswith(".zip") else ppo_path + ".zip" + + # ── Vérifications ──────────────────────────────────────────────────────── + missing = [] + for label, path in [("PPO", ppo_zip), ("Ensemble", ensemble_path), ("Scaler", scaler_path)]: + if not os.path.exists(path): + missing.append(f" ❌ {label} : {path}") + if missing: + print("\n".join(missing)) + print("\nExécuter dans l'ordre :") + print(" 1. python train.py") + print(" 2. python rl_train.py") + sys.exit(1) + + print("=" * 55) + print(" AHAD QUANT — Export Modèle Unifié") + print("=" * 55) + print(f" PPO : {ppo_zip}") + print(f" Ensemble : {ensemble_path}") + print(f" Scaler : {scaler_path}") + print(f" Sortie : {output_path}") + print() + + # ── Metadata ───────────────────────────────────────────────────────────── + metadata = { + "version": "ahad_quant-v6-unified", + "exported_at": datetime.now().isoformat(), + "ppo_source": ppo_zip, + "ensemble_source": ensemble_path, + "scaler_source": scaler_path, + "coins": getattr(config, "COINS", []), + "leverage": getattr(config, "LEVERAGE", 30), + "obs_dim": 69, + } + + # ── Construction du zip unifié ──────────────────────────────────────────── + with zipfile.ZipFile(output_path, "w", compression=zipfile.ZIP_DEFLATED) as out_zip: + + # 1. Contenu PPO → sous-dossier ppo/ + with zipfile.ZipFile(ppo_zip, "r") as ppo_zip_f: + for item in ppo_zip_f.namelist(): + data = ppo_zip_f.read(item) + out_zip.writestr(f"ppo/{item}", data) + print(f" ✅ PPO intégré (ppo/)") + + # 2. Ensemble pkl + with open(ensemble_path, "rb") as f: + out_zip.writestr("ensemble.pkl", f.read()) + print(f" ✅ Ensemble intégré (ensemble.pkl)") + + # 3. Scaler pkl + with open(scaler_path, "rb") as f: + out_zip.writestr("rl_scaler.pkl", f.read()) + print(f" ✅ Scaler intégré (rl_scaler.pkl)") + + # 4. Metadata + out_zip.writestr("metadata.json", json.dumps(metadata, indent=2)) + print(f" ✅ Metadata intégrée (metadata.json)") + + size_mb = os.path.getsize(output_path) / (1024 * 1024) + print() + print(f" 💾 {output_path} ({size_mb:.1f} MB)") + print("=" * 55) + print(" Chargement :") + print(f" from rl_agent import RLAgent") + print(f" agent = RLAgent(unified_path='{output_path}')") + print("=" * 55) + return output_path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ppo", type=str, default=None) + parser.add_argument("--ensemble", type=str, default=None) + parser.add_argument("--scaler", type=str, default=None) + parser.add_argument("--output", type=str, default="ahad_quant_unified.zip") + args = parser.parse_args() + + export_unified(args.ppo, args.ensemble, args.scaler, args.output) diff --git a/features.py b/features.py new file mode 100644 index 0000000..ee60024 --- /dev/null +++ b/features.py @@ -0,0 +1,640 @@ +""" +AHAD QUANT — Feature Engineering — Forex Edition (matches train_ai_v2.py exactly) +Generates 62 features from 1h OHLCV candle data for Forex pairs. + +Key differences vs crypto version: + - funding_rate → swap_rate (overnight rollover expressed as decimal) + - oi_change_pct → tick_volume_change_pct (Forex has no OI; tick volume used) + - oi_value → tick_volume_value + - btc_correlation_20 → base_pair_correlation_20 (correlation with dominant pair, e.g. EURUSD) + - fear_greed_index → sentiment_index (any 0-100 sentiment proxy; defaults to 50) + - liquidation_pressure → momentum_pressure (candle momentum proxy) + - funding_oi_weighted → swap_volume_weighted + +All formulas are identical — only semantic meaning of inputs changes. +Pass swap_rates via funding_map={t_sec: rate}, sentiment via fear_greed_map. +If not provided, defaults produce neutral 0.0 values (safe for both training and live). + +CRITICAL: Do NOT change any formula without updating train.py simultaneously. +""" + +import numpy as np +from datetime import datetime + + +# ─── Feature names (must match FEATURE_NAMES_1H in train_ai_v2.py) ────────── + +FEATURE_NAMES = [ + 'swap_rate', 'swap_rate_delta_1h', 'swap_rate_delta_4h', 'swap_rate_delta_8h', + 'tick_volume_change_pct', 'price_change_pct', 'volume', 'volume_ma_ratio', + 'volume_spike_3x', 'price_vs_vwap', 'high_low_range', 'close_vs_open', + 'rsi_14', 'price_ma_20_ratio', 'volume_change_pct', + 'tick_volume_value', 'swap_abs', 'candle_body_ratio', + 'atr_14', 'base_pair_correlation_20', 'hour_of_day', 'day_of_week', + 'dist_from_24h_high', 'dist_from_24h_low', + 'consecutive_green', 'consecutive_red', + 'rsi_divergence', 'price_momentum_3', 'price_momentum_7', + 'volume_momentum_3', 'ema_12_26_diff', + # Order flow features + 'order_flow_ratio', # bid/ask imbalance proxy from candle structure + 'momentum_pressure', # taker momentum proxy (long-short imbalance estimate) + 'rsi_4h', # RSI calculated on 4h aggregated candles + 'obi_proxy', # candle-based order book imbalance proxy + 'cvd_5', # cumulative volume delta 5 candles + 'cvd_20', # cumulative volume delta 20 candles + 'obi_momentum', # OBI change over 3 candles + # V7.1 features + 'sentiment_index', # market sentiment proxy (0-100, normalized to 0-1) + 'swap_volume_weighted', # swap rate * tick_volume magnitude + # V8.3 advanced features + 'price_skewness_24', # rolling skewness of returns + 'price_kurtosis_24', # rolling kurtosis (fat tail detection) + 'linear_trend_slope_24', # OLS slope normalized by price + 'close_stddev_ratio', # short/long volatility ratio + 'area_ratio_24', # price position vs recent history [-1,+1] + 'range_position_48', # where in 48h range (0=bottom, 1=top) + 'atr_ratio_6_48', # short/long ATR ratio (breakout detector) + 'volume_price_trend', # cumulative volume-adjusted price changes + 'first_loc_max_24', # where did max occur in last 24 candles + 'longest_strike_below', # consecutive candles below rolling mean + # V9.0 multi-timeframe features + 'sma_4h_ratio', # price vs 4h SMA ratio + 'momentum_4h', # 4h momentum (aggregated) + 'daily_return', # daily aggregated return + 'daily_range', # daily aggregated high-low range + 'daily_volume_ratio', # daily volume vs 5-day avg + 'weekly_momentum', # 7-day momentum + # V10.0 L2 orderbook proxies + 'book_imbalance_proxy', # (close - low) / (high - low) — buy pressure proxy + 'depth_ratio_proxy', # volume / avg_volume_20 — order flow depth proxy + 'large_order_proxy', # max(high-low) / ATR — large order detection + 'book_pressure_proxy', # (close - open) / (high - low) — candle body directional pressure + 'spread_proxy', # (high - low) / close * 100 — spread proxy from range + 'flow_intensity', # abs(close - open) * volume — price impact * volume +] + +NUM_FEATURES = 62 +assert len(FEATURE_NAMES) == NUM_FEATURES, f"Expected {NUM_FEATURES} features, got {len(FEATURE_NAMES)}" + + +# ─── Helpers (identical formulas to train_ai_v2.py) ───────────────────────── + +def _compute_rsi(closes, period=14): + """RSI using simple rolling mean of gains/losses — matches train_ai_v2.py compute_rsi().""" + rsi = np.full_like(closes, 50.0, dtype=float) + pc = np.diff(closes, prepend=closes[0]) + for i in range(period, len(closes)): + g = np.maximum(pc[i - period + 1:i + 1], 0).mean() + lo = np.maximum(-pc[i - period + 1:i + 1], 0).mean() + rsi[i] = 100.0 - (100.0 / (1.0 + g / lo)) if lo != 0 else 100.0 + return rsi + + +def _compute_rsi_4h(closes_1h): + """RSI on 4h timeframe by aggregating 1h candles — matches train_ai_v2.py compute_rsi_4h().""" + n = len(closes_1h) + closes_4h = [] + for i in range(3, n, 4): + closes_4h.append(closes_1h[i]) + if len(closes_4h) < 20: + return np.full(n, 50.0) + closes_4h = np.array(closes_4h) + rsi_4h = _compute_rsi(closes_4h, period=14) + result = np.full(n, 50.0) + for idx_4h in range(len(rsi_4h)): + start_1h = idx_4h * 4 + end_1h = min(start_1h + 4, n) + for j in range(start_1h, end_1h): + result[j] = rsi_4h[idx_4h] + return result + + +def _compute_atr(highs, lows, closes, period=14): + """ATR using simple moving average — matches train_ai_v2.py compute_atr().""" + atr = np.zeros_like(closes, dtype=float) + for i in range(1, len(closes)): + tr = max(highs[i] - lows[i], abs(highs[i] - closes[i - 1]), abs(lows[i] - closes[i - 1])) + atr[i] = tr + result = np.zeros_like(closes, dtype=float) + for i in range(period, len(closes)): + result[i] = atr[i - period:i].mean() + return result + + +def _compute_ema(data, period): + """EMA — matches train_ai_v2.py compute_ema().""" + ema = np.zeros_like(data, dtype=float) + ema[0] = data[0] + k = 2.0 / (period + 1) + for i in range(1, len(data)): + ema[i] = data[i] * k + ema[i - 1] * (1 - k) + return ema + + +def _count_consecutive(closes): + """Consecutive green/red candles — matches train_ai_v2.py count_consecutive().""" + green = np.zeros(len(closes), dtype=float) + red = np.zeros(len(closes), dtype=float) + for i in range(1, len(closes)): + if closes[i] > closes[i - 1]: + green[i] = green[i - 1] + 1 + red[i] = 0 + elif closes[i] < closes[i - 1]: + red[i] = red[i - 1] + 1 + green[i] = 0 + return green, red + + +def _compute_multi_timeframe(opens, highs, lows, closes, volumes): + """Multi-timeframe features — matches train_ai_v2.py compute_multi_timeframe().""" + n = len(closes) + sma_4h_ratio = np.zeros(n, dtype=float) + momentum_4h = np.zeros(n, dtype=float) + daily_return = np.zeros(n, dtype=float) + daily_range = np.zeros(n, dtype=float) + daily_volume_ratio = np.zeros(n, dtype=float) + weekly_momentum = np.zeros(n, dtype=float) + + # Build 4h closes + closes_4h = [] + for i in range(3, n, 4): + closes_4h.append(closes[i]) + closes_4h = np.array(closes_4h) if closes_4h else np.array([0.0]) + + # 4h SMA with period 5 (=20h lookback) + sma_4h_arr = np.zeros(len(closes_4h), dtype=float) + for j in range(5, len(closes_4h)): + sma_4h_arr[j] = closes_4h[j - 5:j].mean() + + # 4h momentum + mom_4h_arr = np.zeros(len(closes_4h), dtype=float) + for j in range(1, len(closes_4h)): + if closes_4h[j - 1] > 0: + mom_4h_arr[j] = (closes_4h[j] - closes_4h[j - 1]) / closes_4h[j - 1] * 100 + + # Expand 4h features back to 1h + for idx_4h in range(len(closes_4h)): + start_1h = idx_4h * 4 + end_1h = min(start_1h + 4, n) + for j in range(start_1h, end_1h): + if sma_4h_arr[idx_4h] > 0: + sma_4h_ratio[j] = (closes[j] - sma_4h_arr[idx_4h]) / sma_4h_arr[idx_4h] * 100 + momentum_4h[j] = mom_4h_arr[idx_4h] + + # Daily features (aggregate every 24 candles) + for i in range(24, n): + if closes[i - 24] > 0: + daily_return[i] = (closes[i] - closes[i - 24]) / closes[i - 24] * 100 + dh = highs[i - 24:i].max() + dl = lows[i - 24:i].min() + if closes[i] > 0: + daily_range[i] = (dh - dl) / closes[i] * 100 + vol_24h = volumes[i - 24:i].sum() + if i >= 144: + vol_5d_avg = volumes[i - 144:i - 24].sum() / 5.0 + daily_volume_ratio[i] = vol_24h / vol_5d_avg if vol_5d_avg > 0 else 1.0 + else: + daily_volume_ratio[i] = 1.0 + + # Weekly momentum + for i in range(168, n): + if closes[i - 168] > 0: + weekly_momentum[i] = (closes[i] - closes[i - 168]) / closes[i - 168] * 100 + + return sma_4h_ratio, momentum_4h, daily_return, daily_range, daily_volume_ratio, weekly_momentum + + +# ─── Main function ───────────────────────────────────────────────────────── + +def build_features( + candles: list[dict], + *, + btc_closes: np.ndarray | None = None, + funding_map: dict | None = None, + taker_buy_volumes: np.ndarray | None = None, + taker_ratio_map: dict | None = None, + fear_greed_map: dict | None = None, +) -> np.ndarray: + """ + Build a (N, 62) feature matrix from OHLCV candle data. + + Matches train_ai_v2.py build_features_1h() exactly. + + Parameters + ---------- + candles : list[dict] + List of candle dicts with keys: t, o, h, l, c, v + OR Binance kline arrays [open_time, open, high, low, close, volume, ...] + btc_closes : np.ndarray | None + Dominant pair close prices aligned to same timestamps (e.g. EURUSD). + Used to compute base_pair_correlation_20. If None, correlation is set to 0. + funding_map : dict | None + Dict mapping timestamp_sec -> swap_rate (overnight rollover as decimal). + If None, all swap features default to 0.0. + taker_buy_volumes : np.ndarray | None + Taker buy base volume per candle. + If None, order_flow_ratio defaults to 0.5 (neutral). + taker_ratio_map : dict | None + Dict mapping timestamp_ms -> taker long/short ratio. + If None, liquidation_pressure defaults to 0.0 (neutral). + fear_greed_map : dict | None + Dict mapping timestamp_sec (hour-aligned) -> sentiment_index (0-100). + If None, sentiment_index defaults to 0.5 (neutral, i.e. 50/100). + + Returns + ------- + np.ndarray of shape (N, 62) + Feature matrix. First ~26 rows may have incomplete lookback; + the caller should use only rows from index 26+ onward. + """ + # ── Parse candles ── + times, opens, highs, lows, closes, volumes = [], [], [], [], [], [] + taker_buy_vols_parsed = [] + + for c in candles: + if isinstance(c, list): + # Binance kline array format + times.append(c[0]) + opens.append(float(c[1])) + highs.append(float(c[2])) + lows.append(float(c[3])) + closes.append(float(c[4])) + volumes.append(float(c[5])) + taker_buy_vols_parsed.append(float(c[9]) if len(c) > 9 else 0.0) + else: + t = c.get('t', c.get('T', 0)) + if isinstance(t, str): + t = int(t) + times.append(t) + opens.append(float(c.get('o', 0))) + highs.append(float(c.get('h', 0))) + lows.append(float(c.get('l', 0))) + closes.append(float(c.get('c', 0))) + volumes.append(float(c.get('v', c.get('vlm', 0)))) + taker_buy_vols_parsed.append(0.0) + + c_times = np.array(times, dtype=float) + opens = np.array(opens, dtype=float) + highs = np.array(highs, dtype=float) + lows = np.array(lows, dtype=float) + closes = np.array(closes, dtype=float) + volumes = np.array(volumes, dtype=float) + n = len(closes) + + # Taker buy volumes (for order_flow_ratio) + if taker_buy_volumes is not None: + tbv = taker_buy_volumes + else: + tbv = np.array(taker_buy_vols_parsed, dtype=float) + + # ── Order flow ratio: taker_buy_volume / total_volume ── + order_flow = np.zeros(n, dtype=float) + for i in range(n): + if volumes[i] > 0 and tbv[i] > 0: + order_flow[i] = tbv[i] / volumes[i] + else: + order_flow[i] = 0.5 # neutral default + + # ── Funding map (default empty) ── + # funding_map maps timestamp_sec -> funding_rate + # If not provided, all funding features will be 0.0 + if funding_map is None: + funding_map = {} + + # ── Taker ratio map (default empty) ── + if taker_ratio_map is None: + taker_ratio_map = {} + + # ── Fear & Greed map (default empty) ── + # If not provided, fear_greed_index defaults to 50 (neutral) -> 0.5 normalized + if fear_greed_map is None: + fear_greed_map = {} + + # ── Compute indicators ── + rsi = _compute_rsi(closes) + rsi_4h = _compute_rsi_4h(closes) + atr = _compute_atr(highs, lows, closes) + ema12 = _compute_ema(closes, 12) + ema26 = _compute_ema(closes, 26) + cons_green, cons_red = _count_consecutive(closes) + + # Multi-timeframe features + mtf_sma_4h_ratio, mtf_momentum_4h, mtf_daily_return, mtf_daily_range, \ + mtf_daily_volume_ratio, mtf_weekly_momentum = _compute_multi_timeframe( + opens, highs, lows, closes, volumes) + + # ── Rolling indicators (20-period) ── + vol_ma = np.ones(n, dtype=float) + price_ma = np.full(n, np.nan, dtype=float) + vwap = np.full(n, np.nan, dtype=float) + typical = (highs + lows + closes) / 3.0 + + for i in range(20, n): + vm = volumes[i - 20:i].mean() + vol_ma[i] = vm if vm > 0 else 1.0 + price_ma[i] = closes[i - 20:i].mean() + vs = volumes[i - 20:i].sum() + vwap[i] = (typical[i - 20:i] * volumes[i - 20:i]).sum() / vs if vs > 0 else closes[i] + + # ── BTC closes for correlation ── + # btc_closes should be aligned array same length as closes, or None + # If it's a dict (timestamp -> price), caller should convert before passing + + # ── Build feature rows (matches train_ai_v2.py build_features_1h loop) ── + feature_matrix = np.zeros((n, NUM_FEATURES), dtype=np.float64) + + for i in range(n): + t_ms = int(c_times[i]) + t_sec = int(t_ms / 1000) if t_ms > 1e12 else int(t_ms) + + # --- Funding rates and deltas --- + fr = funding_map.get(t_sec, 0.0) + fr_1h_ago = funding_map.get(t_sec - 3600, 0.0) + fr_4h_ago = funding_map.get(t_sec - 14400, 0.0) + fr_8h_ago = funding_map.get(t_sec - 28800, 0.0) + fr_delta_1h = fr - fr_1h_ago + fr_delta_4h = fr - fr_4h_ago + fr_delta_8h = fr - fr_8h_ago + + # --- OI proxy --- + oi_cur = volumes[i] * closes[i] + oi_prev = volumes[i - 1] * closes[i - 1] if (i > 0 and closes[i - 1] > 0) else 1.0 + oi_chg = ((oi_cur - oi_prev) / oi_prev * 100) if oi_prev > 0 else 0.0 + + # --- Price change % --- + price_chg = ((closes[i] - closes[i - 1]) / closes[i - 1] * 100) if (i > 0 and closes[i - 1] > 0) else 0.0 + + # --- Volume / MA ratio --- + vmr = volumes[i] / vol_ma[i] if vol_ma[i] > 0 else 1.0 + + # --- Volume spike 3x --- + vol_spike = 1.0 if vmr > 3.0 else 0.0 + + # --- Price vs VWAP (rolling 20-period) --- + pvw = ((closes[i] - vwap[i]) / vwap[i] * 100) if (not np.isnan(vwap[i]) and vwap[i] > 0) else 0.0 + + # --- High-Low range --- + hlr = ((highs[i] - lows[i]) / closes[i] * 100) if closes[i] > 0 else 0.0 + + # --- Close vs Open --- + cvo = ((closes[i] - opens[i]) / opens[i] * 100) if opens[i] > 0 else 0.0 + + # --- Price vs MA(20) --- + pma = ((closes[i] - price_ma[i]) / price_ma[i] * 100) if (not np.isnan(price_ma[i]) and price_ma[i] > 0) else 0.0 + + # --- Volume change % --- + vol_chg = ((volumes[i] - volumes[i - 1]) / volumes[i - 1] * 100) if (i > 0 and volumes[i - 1] > 0) else 0.0 + + # --- Candle body ratio --- + cbr = abs(closes[i] - opens[i]) / (highs[i] - lows[i]) if (highs[i] - lows[i]) > 0 else 0.0 + + # --- ATR normalized --- + atr_norm = (atr[i] / closes[i] * 100) if closes[i] > 0 else 0.0 + + # --- BTC correlation --- + btc_corr = 0.0 + if btc_closes is not None and len(btc_closes) == n and i >= 20: + coin_rets = [] + btc_rets = [] + for j in range(i - 19, i + 1): + if j > 0 and closes[j - 1] > 0: + coin_rets.append((closes[j] - closes[j - 1]) / closes[j - 1]) + bc = btc_closes[j] + bc_prev = btc_closes[j - 1] + if bc_prev > 0 and bc > 0: + btc_rets.append((bc - bc_prev) / bc_prev) + else: + btc_rets.append(0) + if len(coin_rets) >= 10: + cr = np.array(coin_rets) + br = np.array(btc_rets) + if cr.std() > 0 and br.std() > 0: + btc_corr = np.corrcoef(cr, br)[0, 1] + if np.isnan(btc_corr): + btc_corr = 0.0 + + # --- Time features --- + hour = 0 + dow = 0 + try: + dt = datetime.utcfromtimestamp(t_sec) + hour = dt.hour + dow = dt.weekday() + except Exception: + pass + + # --- Distance from 24h high/low --- + window_24 = min(24, i) + high_24 = highs[i - window_24:i + 1].max() + low_24 = lows[i - window_24:i + 1].min() + dist_high = ((closes[i] - high_24) / high_24 * 100) if high_24 > 0 else 0.0 + dist_low = ((closes[i] - low_24) / low_24 * 100) if low_24 > 0 else 0.0 + + # --- Consecutive green/red --- + # Already computed vectorized above + + # --- RSI divergence --- + rsi_div = 0.0 + if i >= 5: + price_dir = closes[i] - closes[i - 5] + rsi_dir = rsi[i] - rsi[i - 5] + if price_dir > 0 and rsi_dir < -3: + rsi_div = -1.0 + elif price_dir < 0 and rsi_dir > 3: + rsi_div = 1.0 + + # --- Momentum --- + mom_3 = ((closes[i] - closes[i - 3]) / closes[i - 3] * 100) if (i >= 3 and closes[i - 3] > 0) else 0.0 + mom_7 = ((closes[i] - closes[i - 7]) / closes[i - 7] * 100) if (i >= 7 and closes[i - 7] > 0) else 0.0 + vol_mom_3 = ((volumes[i] - volumes[i - 3]) / volumes[i - 3] * 100) if (i >= 3 and volumes[i - 3] > 0) else 0.0 + ema_diff = ((ema12[i] - ema26[i]) / ema26[i] * 100) if ema26[i] > 0 else 0.0 + + # --- Order flow ratio --- + oflow = order_flow[i] + + # --- Liquidation pressure --- + # taker_ratio_map: timestamp_ms -> buySellRatio + # Centered so 0 = neutral (ratio - 1.0) + liq_pressure = taker_ratio_map.get(t_ms, taker_ratio_map.get(t_sec * 1000, 1.0)) + liq_pressure = liq_pressure - 1.0 + + # --- RSI 4h --- + rsi4h = rsi_4h[i] + + # --- OBI proxy --- + obi = (closes[i] - lows[i]) / (highs[i] - lows[i]) if (highs[i] - lows[i]) > 0 else 0.5 + + # --- CVD proxy --- + cvd_5_val = sum( + volumes[max(i - 4, 0):i + 1] * ( + 2 * ((closes[max(i - 4, 0):i + 1] - lows[max(i - 4, 0):i + 1]) / + np.maximum(highs[max(i - 4, 0):i + 1] - lows[max(i - 4, 0):i + 1], 1e-10)) - 1 + ) + ) + cvd_20_val = sum( + volumes[max(i - 19, 0):i + 1] * ( + 2 * ((closes[max(i - 19, 0):i + 1] - lows[max(i - 19, 0):i + 1]) / + np.maximum(highs[max(i - 19, 0):i + 1] - lows[max(i - 19, 0):i + 1], 1e-10)) - 1 + ) + ) + avg_vol = volumes[max(i - 19, 0):i + 1].mean() + cvd_5_norm = cvd_5_val / avg_vol if avg_vol > 0 else 0 + cvd_20_norm = cvd_20_val / avg_vol if avg_vol > 0 else 0 + + # --- OBI momentum --- + obi_prev = (closes[max(i - 3, 0)] - lows[max(i - 3, 0)]) / (highs[max(i - 3, 0)] - lows[max(i - 3, 0)]) \ + if (highs[max(i - 3, 0)] - lows[max(i - 3, 0)]) > 0 else 0.5 + obi_momentum = obi - obi_prev + + # --- Fear & Greed Index --- + # Defaults to 50 (neutral) if not available, normalized to 0-1 + fg_val = 50 + if fear_greed_map: + t_hour = (t_sec // 3600) * 3600 + fg_val = fear_greed_map.get(t_hour, fear_greed_map.get(t_hour - 3600, 50)) + fear_greed_norm = fg_val / 100.0 + + # --- Funding * OI weighted --- + funding_oi = fr * (oi_cur / 1e6) if oi_cur > 0 else 0.0 + + # --- V8.3 Advanced Features --- + + # 1. Rolling skewness of returns (24 candles) + if i >= 24: + rets_24 = np.diff(closes[i - 24:i + 1]) / closes[i - 24:i] + _mean = rets_24.mean() + _std = rets_24.std() + price_skew = float(((rets_24 - _mean) ** 3).mean() / (_std ** 3)) if _std > 1e-10 else 0.0 + else: + price_skew = 0.0 + + # 2. Rolling kurtosis of returns (24 candles) + if i >= 24: + price_kurt = float(((rets_24 - _mean) ** 4).mean() / (_std ** 4) - 3.0) if _std > 1e-10 else 0.0 + else: + price_kurt = 0.0 + + # 3. Linear trend slope (24 candles, normalized) + if i >= 24: + _x = np.arange(24) + _y = closes[i - 23:i + 1] + _slope = np.polyfit(_x, _y, 1)[0] + trend_slope = _slope / closes[i] * 100 if closes[i] > 0 else 0.0 + else: + trend_slope = 0.0 + + # 4. Short/long volatility ratio + if i >= 48: + rets_s = np.diff(closes[i - 6:i + 1]) / closes[i - 6:i] + rets_l = np.diff(closes[i - 48:i + 1]) / closes[i - 48:i] + std_s = rets_s.std() + std_l = rets_l.std() + stddev_ratio = std_s / std_l if std_l > 1e-10 else 1.0 + else: + stddev_ratio = 1.0 + + # 5. Area ratio (price position vs recent 24 candles) + if i >= 24: + _window = closes[i - 23:i + 1] + _level = closes[i] + _diff = _window - _level + _total = np.sum(np.abs(_diff)) + area_ratio = (2 * np.sum(np.maximum(_diff, 0)) / _total - 1) if _total > 0 else 0.0 + else: + area_ratio = 0.0 + + # 6. Range position (where in 48h range, 0=bottom 1=top) + if i >= 48: + h48 = highs[i - 48:i + 1].max() + l48 = lows[i - 48:i + 1].min() + range_pos = (closes[i] - l48) / (h48 - l48) if (h48 - l48) > 0 else 0.5 + else: + range_pos = 0.5 + + # 7. ATR ratio short/long (breakout detector) + if i >= 48: + atr_short = np.mean([ + max(highs[j] - lows[j], abs(highs[j] - closes[j - 1]), abs(lows[j] - closes[j - 1])) + for j in range(max(1, i - 5), i + 1) + ]) + atr_long = np.mean([ + max(highs[j] - lows[j], abs(highs[j] - closes[j - 1]), abs(lows[j] - closes[j - 1])) + for j in range(max(1, i - 47), i + 1) + ]) + atr_ratio = atr_short / atr_long if atr_long > 0 else 1.0 + else: + atr_ratio = 1.0 + + # 8. Volume-price trend (normalized) + if i >= 24: + vpt = sum( + volumes[j] * ((closes[j] - closes[j - 1]) / closes[j - 1]) + for j in range(max(1, i - 23), i + 1) if closes[j - 1] > 0 + ) + vpt_norm = vpt / avg_vol if avg_vol > 0 else 0.0 + else: + vpt_norm = 0.0 + + # 9. First location of max in 24 candles (0=start, 1=end) + if i >= 24: + first_loc_max = float(np.argmax(closes[i - 23:i + 1])) / 23.0 + else: + first_loc_max = 0.5 + + # 10. Longest strike below mean (24 candles) + if i >= 24: + _win = closes[i - 23:i + 1] + _wmean = _win.mean() + _below = _win < _wmean + max_run = 0 + cur_run = 0 + for b in _below: + if b: + cur_run += 1 + max_run = max(max_run, cur_run) + else: + cur_run = 0 + longest_below = max_run / 24.0 + else: + longest_below = 0.0 + + # --- V10.0 L2 Orderbook Proxies --- + book_imb = (closes[i] - lows[i]) / (highs[i] - lows[i]) if (highs[i] - lows[i]) > 0 else 0.5 + depth_ratio = volumes[i] / vol_ma[i] if vol_ma[i] > 0 else 1.0 + large_order = (highs[i] - lows[i]) / (atr[i] if atr[i] > 0 else 1e-10) + book_pressure = (closes[i] - opens[i]) / (highs[i] - lows[i] + 0.001) if (highs[i] - lows[i]) > 0 else 0.0 + spread_proxy = (highs[i] - lows[i]) / closes[i] * 100 if closes[i] > 0 else 0.0 + flow_intensity = abs(closes[i] - opens[i]) * volumes[i] + + # ── Assemble row (order MUST match FEATURE_NAMES) ── + feature_matrix[i] = [ + fr, fr_delta_1h, fr_delta_4h, fr_delta_8h, + oi_chg, price_chg, volumes[i], vmr, + vol_spike, pvw, hlr, cvo, + rsi[i], pma, vol_chg, + oi_cur, abs(fr), cbr, + atr_norm, btc_corr, hour, dow, + dist_high, dist_low, + cons_green[i], cons_red[i], + rsi_div, mom_3, mom_7, + vol_mom_3, ema_diff, + # Order flow + oflow, liq_pressure, rsi4h, + obi, cvd_5_norm, cvd_20_norm, obi_momentum, + fear_greed_norm, funding_oi, + # V8.3 advanced + price_skew, price_kurt, trend_slope, stddev_ratio, + area_ratio, range_pos, atr_ratio, vpt_norm, + first_loc_max, longest_below, + # V9.0 multi-timeframe + mtf_sma_4h_ratio[i], mtf_momentum_4h[i], + mtf_daily_return[i], mtf_daily_range[i], + mtf_daily_volume_ratio[i], mtf_weekly_momentum[i], + # V10.0 L2 orderbook proxies + book_imb, depth_ratio, large_order, book_pressure, spread_proxy, flow_intensity, + ] + + # Replace any NaN with 0.0 for safety + np.nan_to_num(feature_matrix, copy=False, nan=0.0, posinf=0.0, neginf=0.0) + + return feature_matrix diff --git a/gnn_model.py b/gnn_model.py new file mode 100644 index 0000000..3d05dd8 --- /dev/null +++ b/gnn_model.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +""" +gnn_model.py - Graph Neural Network for Cross-Asset Trading +============================================================= +Models the crypto market as a graph where: + - Each coin is a NODE with its features + - EDGES represent correlations between coins + - Message passing captures lead-lag relationships + +Architecture: + 1. Graph construction from correlation matrix + 2. GCN (Graph Convolutional Network) layers for message passing + 3. Node-level prediction (per-coin UP/DOWN probability) + +Key insight: if BTC drops, the GNN learns that ETH follows in ~5min, +SOL in ~10min, DOGE in ~30min. This gives earlier signals. + +Dependencies: torch (PyTorch). No torch_geometric needed. +""" +import numpy as np +import pickle +import logging + +log = logging.getLogger("AHAD QUANT") + +try: + import torch + import torch.nn as nn + import torch.nn.functional as F + HAS_TORCH = True +except ImportError: + HAS_TORCH = False + + +class GraphConvLayer(nn.Module): + """Simple Graph Convolutional Layer (Kipf & Welling, 2017).""" + def __init__(self, in_features, out_features): + super().__init__() + self.weight = nn.Parameter(torch.FloatTensor(in_features, out_features)) + self.bias = nn.Parameter(torch.FloatTensor(out_features)) + nn.init.xavier_uniform_(self.weight) + nn.init.zeros_(self.bias) + + def forward(self, x, adj): + """ + Args: + x: (batch, n_nodes, in_features) + adj: (n_nodes, n_nodes) normalized adjacency matrix + + Returns: + (batch, n_nodes, out_features) + """ + # Message passing: A * X * W + b + support = torch.matmul(x, self.weight) # (batch, n_nodes, out_features) + output = torch.matmul(adj, support) # (batch, n_nodes, out_features) + return output + self.bias + + +class CryptoGNN(nn.Module): + """ + Graph Neural Network for crypto market. + + Takes features for ALL coins simultaneously and predicts + UP/DOWN probability for each coin, using cross-asset information. + """ + def __init__(self, n_features, hidden_dim=64, n_gcn_layers=3, dropout=0.2): + super().__init__() + self.n_features = n_features + + # Node feature encoder (shared across all coins) + self.encoder = nn.Sequential( + nn.Linear(n_features, hidden_dim), + nn.ReLU(), + nn.Dropout(dropout), + ) + + # GCN layers + self.gcn_layers = nn.ModuleList() + self.gcn_norms = nn.ModuleList() + for i in range(n_gcn_layers): + self.gcn_layers.append(GraphConvLayer(hidden_dim, hidden_dim)) + self.gcn_norms.append(nn.LayerNorm(hidden_dim)) + + self.dropout = nn.Dropout(dropout) + + # Node-level prediction head + self.head = nn.Sequential( + nn.Linear(hidden_dim * 2, hidden_dim), # concat local + global + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, 1), + nn.Sigmoid() + ) + + def forward(self, x, adj): + """ + Args: + x: (batch, n_nodes, n_features) - features for all coins + adj: (n_nodes, n_nodes) - normalized adjacency matrix + + Returns: + pred: (batch, n_nodes, 1) - UP probability per coin + """ + # Encode node features + h = self.encoder(x) # (batch, n_nodes, hidden) + + # GCN message passing + for gcn, norm in zip(self.gcn_layers, self.gcn_norms): + h_new = gcn(h, adj) + h_new = F.relu(h_new) + h_new = self.dropout(h_new) + h = norm(h_new + h) # residual connection + + # Global graph context (mean pooling) + global_ctx = h.mean(dim=1, keepdim=True).expand_as(h) # (batch, n_nodes, hidden) + + # Concatenate local + global for prediction + combined = torch.cat([h, global_ctx], dim=-1) # (batch, n_nodes, hidden*2) + pred = self.head(combined) # (batch, n_nodes, 1) + + return pred + + +def build_correlation_graph(returns_dict, threshold=0.3): + """ + Build adjacency matrix from return correlations. + + Args: + returns_dict: dict of {coin_name: np.array of returns} + threshold: minimum absolute correlation for edge (default 0.3) + + Returns: + adj: (n_coins, n_coins) normalized adjacency matrix + coin_order: list of coin names in matrix order + """ + coins = sorted(returns_dict.keys()) + n = len(coins) + + # Compute correlation matrix + returns_matrix = np.column_stack([returns_dict[c] for c in coins]) + min_len = min(len(returns_dict[c]) for c in coins) + returns_matrix = returns_matrix[:min_len] + + corr = np.corrcoef(returns_matrix.T) + corr = np.nan_to_num(corr, nan=0.0) + + # Build adjacency: threshold + self-loops + adj = np.zeros((n, n)) + for i in range(n): + adj[i, i] = 1.0 # self-loop + for j in range(i + 1, n): + if abs(corr[i, j]) > threshold: + adj[i, j] = abs(corr[i, j]) + adj[j, i] = abs(corr[i, j]) + + # Normalize: D^{-1/2} A D^{-1/2} + degree = adj.sum(axis=1) + d_inv_sqrt = np.zeros_like(degree) + nonzero = degree > 0 + d_inv_sqrt[nonzero] = 1.0 / np.sqrt(degree[nonzero]) + D = np.diag(d_inv_sqrt) + adj_norm = D @ adj @ D + + return adj_norm.astype(np.float32), coins + + +def build_lead_lag_graph(returns_dict, max_lag=5): + """ + Build directed graph based on lead-lag relationships. + + If coin A's returns at time t predict coin B at time t+lag, + then A→B edge exists. + + Args: + returns_dict: dict of {coin: returns_array} + max_lag: maximum lag to check (in candles) + + Returns: + adj: (n_coins, n_coins) normalized adjacency (DIRECTED) + coin_order: list of coin names + """ + coins = sorted(returns_dict.keys()) + n = len(coins) + min_len = min(len(returns_dict[c]) for c in coins) + + adj = np.eye(n, dtype=np.float32) # self-loops + + for i, ci in enumerate(coins): + ri = returns_dict[ci][:min_len] + for j, cj in enumerate(coins): + if i == j: + continue + rj = returns_dict[cj][:min_len] + + # Check if coin i leads coin j + best_corr = 0.0 + for lag in range(1, max_lag + 1): + if lag >= min_len: + break + corr = np.corrcoef(ri[:-lag], rj[lag:])[0, 1] + if not np.isnan(corr): + best_corr = max(best_corr, abs(corr)) + + if best_corr > 0.15: + adj[i, j] = best_corr + + # Row-normalize + row_sums = adj.sum(axis=1, keepdims=True) + row_sums = np.where(row_sums > 0, row_sums, 1.0) + adj_norm = adj / row_sums + + return adj_norm, coins + + +def train_gnn(features_dict, labels_dict, adj, coin_order, + n_features=None, hidden_dim=64, epochs=100, + lr=0.001, batch_size=32, patience=15, device='cpu'): + """ + Train GNN model. + + Args: + features_dict: {coin: (n_samples, n_features)} aligned by time + labels_dict: {coin: (n_samples,)} binary labels + adj: (n_coins, n_coins) adjacency matrix + coin_order: list of coin names matching adj rows + + Returns: + model: trained CryptoGNN + history: training metrics + """ + if not HAS_TORCH: + raise ImportError("PyTorch required") + + # Build aligned data matrix: (n_times, n_coins, n_features) + n_coins = len(coin_order) + min_len = min(len(features_dict[c]) for c in coin_order) + if n_features is None: + n_features = features_dict[coin_order[0]].shape[1] + + X_all = np.zeros((min_len, n_coins, n_features), dtype=np.float32) + y_all = np.zeros((min_len, n_coins), dtype=np.float32) + + for idx, coin in enumerate(coin_order): + X_all[:, idx, :] = features_dict[coin][:min_len, :n_features] + y_all[:, idx] = labels_dict[coin][:min_len] + + # Split train/val (time-based) + split = int(min_len * 0.8) + X_train = X_all[:split] + y_train = y_all[:split] + X_val = X_all[split:] + y_val = y_all[split:] + + model = CryptoGNN(n_features=n_features, hidden_dim=hidden_dim).to(device) + adj_tensor = torch.FloatTensor(adj).to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=1e-5) + criterion = nn.BCELoss() + + best_val_loss = float('inf') + best_state = None + patience_counter = 0 + history = {'train_loss': [], 'val_loss': [], 'val_acc': []} + + n_train = len(X_train) + + for epoch in range(epochs): + model.train() + train_loss = 0.0 + n_batches = 0 + + indices = np.random.permutation(n_train) + for start in range(0, n_train, batch_size): + end = min(start + batch_size, n_train) + batch_idx = indices[start:end] + + x_b = torch.FloatTensor(X_train[batch_idx]).to(device) + y_b = torch.FloatTensor(y_train[batch_idx]).unsqueeze(-1).to(device) + + optimizer.zero_grad() + pred = model(x_b, adj_tensor) + loss = criterion(pred, y_b) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + optimizer.step() + + train_loss += loss.item() + n_batches += 1 + + avg_train = train_loss / max(n_batches, 1) + + # Validation + model.eval() + with torch.no_grad(): + x_v = torch.FloatTensor(X_val).to(device) + y_v = torch.FloatTensor(y_val).unsqueeze(-1).to(device) + pred_v = model(x_v, adj_tensor) + val_loss = criterion(pred_v, y_v).item() + val_acc = ((pred_v > 0.5).float() == y_v).float().mean().item() * 100 + + history['train_loss'].append(avg_train) + history['val_loss'].append(val_loss) + history['val_acc'].append(val_acc) + + if val_loss < best_val_loss: + best_val_loss = val_loss + best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} + patience_counter = 0 + else: + patience_counter += 1 + + if (epoch + 1) % 10 == 0: + log.info(f"[GNN] Epoch {epoch+1}/{epochs} | train={avg_train:.4f} | " + f"val={val_loss:.4f} | acc={val_acc:.1f}%") + + if patience_counter >= patience: + log.info(f"[GNN] Early stopping at epoch {epoch+1}") + break + + if best_state: + model.load_state_dict(best_state) + model.eval() + + return model, history + + +def save_gnn(model, adj, coin_order, config, normalization, accuracy, path): + """Save GNN model + graph structure.""" + data = { + 'model_state': model.state_dict(), + 'adj': adj, + 'coin_order': coin_order, + 'config': config, + 'normalization': normalization, + 'accuracy': accuracy, + 'model_type': 'gnn', + } + with open(path, 'wb') as f: + pickle.dump(data, f) + + +def load_gnn(path): + """Load GNN model.""" + if not HAS_TORCH: + return None, None, None, None + try: + with open(path, 'rb') as f: + data = pickle.load(f) + cfg = data['config'] + model = CryptoGNN( + n_features=cfg['n_features'], + hidden_dim=cfg.get('hidden_dim', 64), + ) + model.load_state_dict(data['model_state']) + model.eval() + return model, data['adj'], data['coin_order'], data.get('normalization', {}) + except Exception as e: + log.warning(f"[GNN] Load failed: {e}") + return None, None, None, None + + +def predict_gnn(model, features_dict, adj, coin_order, normalization=None, target_coin=None): + """ + Predict UP probability for all coins (or a specific coin). + + Args: + model: trained CryptoGNN + features_dict: {coin: (n_features,) array} current features for each coin + adj: adjacency matrix + coin_order: list of coin names + normalization: optional {'mean': array, 'std': array} + target_coin: optional specific coin to get prediction for + + Returns: + dict of {coin: probability} or single float if target_coin specified + """ + if not HAS_TORCH or model is None: + return None + + try: + n_coins = len(coin_order) + n_features = model.n_features + + # Build input tensor + x = np.zeros((1, n_coins, n_features), dtype=np.float32) + for idx, coin in enumerate(coin_order): + if coin in features_dict: + feat = np.array(features_dict[coin][:n_features], dtype=np.float32) + if normalization: + mean = np.array(normalization['mean'][:n_features], dtype=np.float32) + std = np.array(normalization['std'][:n_features], dtype=np.float32) + std = np.where(std < 1e-8, 1.0, std) + feat = (feat - mean) / std + x[0, idx, :len(feat)] = feat + + adj_tensor = torch.FloatTensor(adj) + x_tensor = torch.FloatTensor(x) + + with torch.no_grad(): + pred = model(x_tensor, adj_tensor) # (1, n_coins, 1) + + probs = pred[0, :, 0].numpy() + result = {coin: float(probs[idx]) for idx, coin in enumerate(coin_order)} + + if target_coin: + return result.get(target_coin, 0.5) + return result + + except Exception as e: + log.warning(f"[GNN] Predict error: {e}") + return None + + +if __name__ == '__main__': + if not HAS_TORCH: + print("[GNN] PyTorch not available") + else: + print("[GNN] Graph Neural Network smoke test") + + np.random.seed(42) + coins = ['BTC', 'ETH', 'SOL', 'DOGE', 'LINK'] + n = 500 + n_features = 20 + + # Synthetic correlated returns + btc_returns = np.random.randn(n) * 0.01 + returns_dict = {'BTC': btc_returns} + for coin in coins[1:]: + lag = np.random.randint(1, 4) + noise = np.random.randn(n) * 0.005 + r = np.roll(btc_returns, lag) * (0.5 + np.random.rand() * 0.5) + noise + returns_dict[coin] = r + + # Build graph + adj, order = build_correlation_graph(returns_dict, threshold=0.1) + print(f" Adjacency matrix:\n{adj}") + + # Synthetic features and labels + features_dict = {c: np.random.randn(n, n_features).astype(np.float32) for c in coins} + labels_dict = {c: (np.random.rand(n) > 0.5).astype(np.float32) for c in coins} + + model, history = train_gnn( + features_dict, labels_dict, adj, order, + n_features=n_features, epochs=5, batch_size=32 + ) + + # Predict + current = {c: np.random.randn(n_features) for c in coins} + result = predict_gnn(model, current, adj, order) + for c, p in result.items(): + print(f" {c}: {p:.4f}") + + n_params = sum(p.numel() for p in model.parameters()) + print(f" Parameters: {n_params:,}") + + # Lead-lag graph + adj_ll, _ = build_lead_lag_graph(returns_dict, max_lag=3) + print(f" Lead-lag adjacency:\n{adj_ll}") + + print("[GNN] Smoke test passed") diff --git a/grid_bot.py b/grid_bot.py new file mode 100644 index 0000000..1ac6bb8 --- /dev/null +++ b/grid_bot.py @@ -0,0 +1,299 @@ +""" +AHAD QUANT — Grid Trading Bot +5 built-in strategies: neutral, long, short, trend, reverse + +Usage (standalone): + python grid_bot.py # paire par défaut : EURUSD + +Or activated via .env: + GRID_BOT_ENABLED=true + GRID_PAIR=EURUSD + GRID_STRATEGY=neutral + GRID_LEVELS=10 + GRID_TOTAL_USDT=200 + +How it works: + - Divides a price range into N equal levels + - Places a buy order below current price and a sell order above + - Each time a sell is filled, a new buy is placed below it + - Each time a buy is filled, a new sell is placed above it + - Profits from price oscillation within the range +""" + +import time, json, os, math +import config + +try: + from exchange_adapter import get_exchange, ExchangeAdapter + HAS_ADAPTER = True +except ImportError: + HAS_ADAPTER = False + +STATE_FILE = "grid_state.json" + +STRATEGIES = { + "neutral": {"bias": 0.0, "desc": "Equal buys and sells — best for ranging markets"}, + "long": {"bias": 0.3, "desc": "More buys than sells — bullish bias"}, + "short": {"bias": -0.3, "desc": "More sells than buys — bearish bias"}, + "trend": {"bias": 0.0, "desc": "Enters in trend direction, exits at reversal"}, + "reverse": {"bias": 0.0, "desc": "Fades extreme moves — contrarian"}, +} + + +class GridBot: + """ + Perpetual futures grid bot with 5 configurable strategies. + """ + + def __init__( + self, + exchange: object, + coin: str = None, + strategy: str = None, + lower: float = 0, + upper: float = 0, + levels: int = None, + total_usdt: float = None, + leverage: int = None, + ): + self.exchange = exchange + self.coin = coin or config.GRID_COIN + self.strategy = strategy or config.GRID_STRATEGY + self.n_levels = levels or config.GRID_LEVELS + self.total_usdt = total_usdt or config.GRID_TOTAL_USDT + self.leverage = leverage or config.GRID_LEVERAGE + self._lower = lower + self._upper = upper + + self.grid_prices: list[float] = [] + self.orders: dict[str, dict] = {} # price → order info + self.realized_pnl: float = 0.0 + self.n_fills: int = 0 + self.running: bool = False + + if self.strategy not in STRATEGIES: + raise ValueError(f"Unknown strategy '{self.strategy}'. " + f"Choose from: {list(STRATEGIES.keys())}") + print(f"[GRID] Strategy: {self.strategy} — {STRATEGIES[self.strategy]['desc']}") + + # ── Grid calculation ────────────────────────────────────────────────────── + + def _auto_range(self, current_price: float) -> tuple[float, float]: + """ + Auto-detect grid range from recent ATR if lower/upper not specified. + Uses 2x ATR above and below current price. + """ + try: + candles = self.exchange.get_candles(self.coin, "1h", 50) + highs = [c["h"] for c in candles] + lows = [c["l"] for c in candles] + closes= [c["c"] for c in candles] + atr_vals = [] + for i in range(1, len(closes)): + tr = max(highs[i] - lows[i], + abs(highs[i] - closes[i-1]), + abs(lows[i] - closes[i-1])) + atr_vals.append(tr) + atr = sum(atr_vals[-14:]) / 14 if len(atr_vals) >= 14 else current_price * 0.02 + except Exception: + atr = current_price * 0.02 + + factor = 2.5 # grid spans ±2.5 ATR + lower = current_price - factor * atr + upper = current_price + factor * atr + print(f"[GRID] Auto range — ATR: {atr:.4f} | " + f"Lower: {lower:.4f} | Upper: {upper:.4f}") + return lower, upper + + def _build_grid(self, current_price: float): + lower = self._lower + upper = self._upper + if lower == 0 or upper == 0: + lower, upper = self._auto_range(current_price) + + self.grid_prices = [ + lower + i * (upper - lower) / (self.n_levels - 1) + for i in range(self.n_levels) + ] + usdt_per_grid = self.total_usdt / self.n_levels + self.qty_per_level = (usdt_per_grid * self.leverage) / current_price + + print(f"[GRID] {self.n_levels} levels | " + f"{lower:.4f} → {upper:.4f} | " + f"Qty/level: {self.qty_per_level:.4f} {self.coin} | " + f"USDT/level: {usdt_per_grid:.2f}") + + # ── Strategy-specific order placement ──────────────────────────────────── + + def _should_buy_at(self, price: float, current_price: float) -> bool: + """Decide whether to place a buy order at this grid level.""" + if self.strategy == "neutral": + return price < current_price + elif self.strategy == "long": + # More buy levels (lower 70% of grid) + midpoint = self.grid_prices[int(self.n_levels * 0.3)] + return price < max(current_price, midpoint) + elif self.strategy == "short": + # Fewer buy levels + midpoint = self.grid_prices[int(self.n_levels * 0.7)] + return price < min(current_price, midpoint) + elif self.strategy in ("trend", "reverse"): + return price < current_price + return price < current_price + + def _should_sell_at(self, price: float, current_price: float) -> bool: + return price > current_price + + # ── Order management ───────────────────────────────────────────────────── + + def _place_initial_orders(self, current_price: float): + """Place initial grid orders around current price.""" + print(f"[GRID] Placing initial orders...") + placed = 0 + for price in self.grid_prices: + if abs(price - current_price) / current_price < 0.001: + continue # skip levels too close to market price + try: + if self._should_buy_at(price, current_price): + self.exchange.place_limit_order( + self.coin, "buy", self.qty_per_level, price + ) + self.orders[f"buy_{price:.4f}"] = { + "side": "buy", "price": price, + "qty": self.qty_per_level, "status": "open" + } + placed += 1 + elif self._should_sell_at(price, current_price): + self.exchange.place_limit_order( + self.coin, "sell", self.qty_per_level, price + ) + self.orders[f"sell_{price:.4f}"] = { + "side": "sell", "price": price, + "qty": self.qty_per_level, "status": "open" + } + placed += 1 + except Exception as e: + print(f"[GRID] Failed to place order at {price:.4f}: {e}") + time.sleep(0.1) + print(f"[GRID] {placed} orders placed") + + def _handle_fill(self, filled_order: dict, current_price: float): + """When an order fills, place the opposite order on the other side.""" + price = filled_order["price"] + side = filled_order["side"] + qty = filled_order["qty"] + + # Estimate PnL from the grid spread + grid_step = (self.grid_prices[-1] - self.grid_prices[0]) / (self.n_levels - 1) + trade_pnl = grid_step * qty * (1 if side == "sell" else -1) + self.realized_pnl += trade_pnl + self.n_fills += 1 + + # Place the opposite order + try: + if side == "buy": + # Buy filled → place sell above + new_price = price + grid_step + if new_price <= self.grid_prices[-1]: + self.exchange.place_limit_order( + self.coin, "sell", qty, new_price + ) + self.orders[f"sell_{new_price:.4f}"] = { + "side": "sell", "price": new_price, + "qty": qty, "status": "open" + } + else: + # Sell filled → place buy below + new_price = price - grid_step + if new_price >= self.grid_prices[0]: + self.exchange.place_limit_order( + self.coin, "buy", qty, new_price + ) + self.orders[f"buy_{new_price:.4f}"] = { + "side": "buy", "price": new_price, + "qty": qty, "status": "open" + } + except Exception as e: + print(f"[GRID] Failed to place opposite order: {e}") + + sign = "+" if trade_pnl >= 0 else "" + print(f"[GRID] Fill #{self.n_fills}: {side.upper()} {qty:.4f} {self.coin} " + f"@ {price:.4f} | Grid PnL: {sign}{trade_pnl:.2f} | " + f"Total: {self.realized_pnl:+.2f} USD") + + # ── Main loop ───────────────────────────────────────────────────────────── + + def start(self): + """Start the grid bot main loop.""" + print(f"\n[GRID] Starting Grid Bot — {self.coin} | Strategy: {self.strategy}") + self.running = True + + # Set leverage + try: + self.exchange.set_leverage(self.coin, self.leverage) + except Exception: + pass + + # Get current price and build grid + book = self.exchange.get_orderbook(self.coin) + current_price = book["mid"] + self._build_grid(current_price) + self._place_initial_orders(current_price) + + print("[GRID] Running... Ctrl+C to stop\n") + + while self.running: + try: + # Check for filled orders (simplified polling) + open_orders = self.exchange.get_open_orders(self.coin) + open_ids = {o.get("id") for o in open_orders} + + for key, order in list(self.orders.items()): + if order["status"] == "open": + # Detect fill by checking if order disappeared from open orders + book = self.exchange.get_orderbook(self.coin) + current_price = book["mid"] + self._handle_fill(order, current_price) + self.orders[key]["status"] = "filled" + + # Status every 60s + print(f"[GRID] {self.coin} @ {current_price:.4f} | " + f"Fills: {self.n_fills} | PnL: {self.realized_pnl:+.2f} USD | " + f"Open orders: {sum(1 for o in self.orders.values() if o['status']=='open')}") + + time.sleep(30) + except KeyboardInterrupt: + self.stop() + break + except Exception as e: + print(f"[GRID] Error: {e}") + time.sleep(10) + + def stop(self): + """Cancel all open grid orders.""" + print("\n[GRID] Stopping — cancelling all open orders...") + self.running = False + try: + self.exchange.cancel_all_orders(self.coin) + except Exception as e: + print(f"[GRID] Cancel failed: {e}") + print(f"[GRID] Final PnL: {self.realized_pnl:+.2f} USD | " + f"Total fills: {self.n_fills}") + + +if __name__ == "__main__": + if not HAS_ADAPTER: + print("[ERROR] exchange_adapter.py not found") + exit(1) + + print("AHAD QUANT — Grid Bot") + print(f" Coin: {config.GRID_COIN}") + print(f" Strategy: {config.GRID_STRATEGY}") + print(f" Levels: {config.GRID_LEVELS}") + print(f" USDT: {config.GRID_TOTAL_USDT}") + print(f" Leverage: {config.GRID_LEVERAGE}x\n") + + exchange = get_exchange(config.EXCHANGE) + exchange.connect() + bot = GridBot(exchange) + bot.start() diff --git a/install_service.sh b/install_service.sh new file mode 100644 index 0000000..5b7f40c --- /dev/null +++ b/install_service.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# AHAD QUANT — Installation service automatique (Linux VPS) +# Lance UNE SEULE FOIS en tant que root +# Apres ca : le serveur demarre automatiquement a chaque reboot + +set -e + +echo "" +echo "========================================" +echo " AHAD QUANT — Installation service" +echo "========================================" +echo "" + +# Detecter le chemin du projet +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$SCRIPT_DIR" +PYTHON_BIN="$(which python3)" +SERVICE_USER="ahad_quant" +INSTALL_DIR="/opt/ahad_quant" + +echo "[1/7] Dossier projet source : $PROJECT_DIR" +echo "[2/7] Python : $PYTHON_BIN" + +# Créer l'utilisateur dédié (sans droits root, sans shell interactif) +echo "[3/7] Création de l'utilisateur système '$SERVICE_USER'..." +if ! id "$SERVICE_USER" &>/dev/null; then + useradd -r -s /bin/false -d "$INSTALL_DIR" -m "$SERVICE_USER" + echo " ✅ Utilisateur '$SERVICE_USER' créé" +else + echo " ✅ Utilisateur '$SERVICE_USER' existe déjà" +fi + +# Copier le projet dans /opt/ahad_quant +echo "[4/7] Déploiement dans $INSTALL_DIR..." +mkdir -p "$INSTALL_DIR" +cp -r "$PROJECT_DIR"/. "$INSTALL_DIR/" +chown -R "$SERVICE_USER:$SERVICE_USER" "$INSTALL_DIR" +chmod 750 "$INSTALL_DIR" +echo " ✅ Fichiers copiés et permissions appliquées" + +# Installer les dependances +echo "[5/7] Installation des dependances Python..." +pip3 install -r "$INSTALL_DIR/requirements.txt" -q +echo " ✅ Dépendances installées" + +# Creer le fichier service systemd avec utilisateur non-root +echo "[6/7] Creation du service systemd (utilisateur: $SERVICE_USER)..." +cat > /etc/systemd/system/ahad_quant.service << EOF +[Unit] +Description=AHAD QUANT — Web Command Center +After=network.target +Wants=network-online.target + +[Service] +Type=simple +User=${SERVICE_USER} +WorkingDirectory=${INSTALL_DIR} +ExecStart=${PYTHON_BIN} ${INSTALL_DIR}/web_ui.py +Restart=always +RestartSec=10 +Environment=PYTHONUNBUFFERED=1 +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +EOF + +# Activer et demarrer le service +echo "[7/7] Activation du service..." +systemctl daemon-reload +systemctl enable ahad_quant +systemctl start ahad_quant + +echo "" +echo "========================================" +echo " Installation terminee !" +echo "========================================" +echo "" +echo " AHAD QUANT tourne sous l'utilisateur '$SERVICE_USER' (non-root)" +echo " Fichiers deployés dans : $INSTALL_DIR" +echo "" +echo " ⚠️ N'oubliez pas de configurer $INSTALL_DIR/.env :" +echo " cp $INSTALL_DIR/.env.example $INSTALL_DIR/.env" +echo " nano $INSTALL_DIR/.env" +echo "" +echo " Acces : http://$(hostname -I | awk '{print $1}'):8080" +echo "" +echo " Commandes utiles :" +echo " systemctl status ahad_quant # voir le statut" +echo " systemctl stop ahad_quant # arreter" +echo " systemctl restart ahad_quant # redemarrer" +echo " journalctl -u ahad_quant -f # voir les logs" +echo "" diff --git a/liquidation_levels.py b/liquidation_levels.py new file mode 100644 index 0000000..4b560ba --- /dev/null +++ b/liquidation_levels.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +liquidation_levels.py - Real-time Liquidation Level Estimator +============================================================== +Estimates where liquidation clusters are based on current open interest, +funding rates, and price levels. No historical data download needed. + +Uses Bybit API to estimate liquidation zones and provides: +- Liquidation cluster levels (above and below current price) +- Liquidation intensity score (-1 to +1) +- Suggested TP/SL adjustments based on liquidation zones + +Integration: call get_liquidation_signal(exchange, symbol) from pro_trader.py +""" +import logging +import numpy as np + +logger = logging.getLogger("AHAD QUANT") + + +def estimate_liquidation_levels(exchange, symbol, current_price=None): + """ + Estimate liquidation cluster levels from open interest and leverage data. + + Uses the principle that most retail traders use 5x-25x leverage, + so liquidation prices cluster at predictable distances from entry. + + Args: + exchange: ccxt exchange instance + symbol: e.g. "BTC/USDT:USDT" + current_price: current market price (fetched if None) + + Returns: + dict with: + liq_above: list of (price, intensity) for liquidation levels above + liq_below: list of (price, intensity) for liquidation levels below + nearest_liq_above: nearest liquidation cluster above current price + nearest_liq_below: nearest liquidation cluster below current price + liq_bias: -1 to +1 (positive = more longs to liquidate below) + """ + try: + if current_price is None: + ticker = exchange.fetch_ticker(symbol) + current_price = float(ticker["last"]) + + # Fetch open interest if available + oi_long = 0 + oi_short = 0 + try: + # Bybit long/short ratio + coin = symbol.split("/")[0] + import requests + r = requests.get( + f"https://api.bybit.com/v5/market/account-ratio", + params={"category": "linear", "symbol": f"{coin}USDT", "period": "1h", "limit": 1}, + timeout=5 + ) + if r.status_code == 200: + data = r.json().get("result", {}).get("list", []) + if data: + buy_ratio = float(data[0].get("buyRatio", 0.5)) + sell_ratio = float(data[0].get("sellRatio", 0.5)) + oi_long = buy_ratio + oi_short = sell_ratio + except Exception: + oi_long = 0.5 + oi_short = 0.5 + + # Common leverage levels used by retail (5x, 10x, 20x, 25x, 50x) + leverages = [5, 10, 20, 25, 50] + + # For LONG positions, liquidation = entry * (1 - 1/leverage) + # For SHORT positions, liquidation = entry * (1 + 1/leverage) + liq_below = [] # long liquidations (below current price) + liq_above = [] # short liquidations (above current price) + + for lev in leverages: + # Where longs opened near current price get liquidated + liq_price_long = current_price * (1 - 0.9 / lev) # 90% of margin = liq + distance_pct = (current_price - liq_price_long) / current_price * 100 + # Intensity based on how common this leverage is + intensity = _leverage_popularity(lev) * oi_long + liq_below.append((round(liq_price_long, 6), round(intensity, 3), f"{lev}x")) + + # Where shorts opened near current price get liquidated + liq_price_short = current_price * (1 + 0.9 / lev) + intensity_short = _leverage_popularity(lev) * oi_short + liq_above.append((round(liq_price_short, 6), round(intensity_short, 3), f"{lev}x")) + + # Sort by distance from current price + liq_below.sort(key=lambda x: -x[0]) # closest first + liq_above.sort(key=lambda x: x[0]) # closest first + + # Nearest clusters + nearest_below = liq_below[0][0] if liq_below else current_price * 0.95 + nearest_above = liq_above[0][0] if liq_above else current_price * 1.05 + + # Bias: positive = more longs to liquidate (bearish pressure) + total_below = sum(x[1] for x in liq_below) + total_above = sum(x[1] for x in liq_above) + total = total_below + total_above + liq_bias = (total_below - total_above) / total if total > 0 else 0.0 + + return { + "liq_above": liq_above, + "liq_below": liq_below, + "nearest_liq_above": nearest_above, + "nearest_liq_below": nearest_below, + "liq_bias": round(liq_bias, 4), + "long_ratio": round(oi_long, 4), + "short_ratio": round(oi_short, 4), + "current_price": current_price, + } + + except Exception as e: + logger.debug(f"[LIQ] Failed for {symbol}: {e}") + return None + + +def _leverage_popularity(leverage): + """Estimate how popular each leverage level is among retail traders.""" + # Based on Bybit/Binance data: most use 5-10x + popularity = { + 5: 0.30, # 30% of traders + 10: 0.35, # 35% most popular + 20: 0.20, # 20% + 25: 0.10, # 10% + 50: 0.05, # 5% degens + } + return popularity.get(leverage, 0.1) + + +def get_liquidation_signal(exchange, symbol, side="LONG"): + """ + Get a liquidation-based trading signal. + + Args: + exchange: ccxt instance + symbol: trading pair + side: "LONG" or "SHORT" - the side we want to trade + + Returns: + dict with: + score: -1 to +1 (positive = favorable for the given side) + nearest_target: price level where liquidation cascade helps us + nearest_danger: price level where liquidation cascade hurts us + adjust_tp: suggested TP adjustment (closer to liq cluster) + adjust_sl: suggested SL adjustment (away from liq cluster) + """ + levels = estimate_liquidation_levels(exchange, symbol) + if not levels: + return {"score": 0, "nearest_target": None, "nearest_danger": None} + + price = levels["current_price"] + bias = levels["liq_bias"] + + if side == "LONG": + # For LONG: we want short liquidations above (cascade up = good) + # and we fear long liquidations below (cascade down = bad) + score = -bias # negative bias = more shorts to squeeze = good for long + nearest_target = levels["nearest_liq_above"] + nearest_danger = levels["nearest_liq_below"] + else: + # For SHORT: we want long liquidations below (cascade down = good) + # and we fear short liquidations above (cascade up = bad) + score = bias # positive bias = more longs to liquidate = good for short + nearest_target = levels["nearest_liq_below"] + nearest_danger = levels["nearest_liq_above"] + + # TP adjustment: put TP just before the cascade target (take profit before bounce) + target_dist = abs(nearest_target - price) + adjust_tp = target_dist * 0.9 # 90% of distance to liq cluster + + # SL adjustment: put SL beyond the danger zone (don't get caught in cascade) + danger_dist = abs(nearest_danger - price) + adjust_sl = danger_dist * 0.5 # SL at 50% of distance to danger cluster + + return { + "score": round(score, 4), + "nearest_target": round(nearest_target, 6), + "nearest_danger": round(nearest_danger, 6), + "adjust_tp": round(adjust_tp, 6), + "adjust_sl": round(adjust_sl, 6), + "long_ratio": levels["long_ratio"], + "short_ratio": levels["short_ratio"], + } + + +if __name__ == "__main__": + print("[LIQ] Liquidation Levels - smoke test") + + import ccxt + ex = ccxt.bybit() + + for coin in ["BTC", "ETH", "SOL"]: + sym = f"{coin}/USDT:USDT" + levels = estimate_liquidation_levels(ex, sym) + if levels: + print(f"\n {coin}: price=${levels['current_price']}") + print(f" Long/Short ratio: {levels['long_ratio']}/{levels['short_ratio']}") + print(f" Liq bias: {levels['liq_bias']} ({'bearish' if levels['liq_bias'] > 0 else 'bullish'})") + print(f" Nearest liq below: ${levels['nearest_liq_below']:.2f}") + print(f" Nearest liq above: ${levels['nearest_liq_above']:.2f}") + + sig = get_liquidation_signal(ex, sym, "SHORT") + print(f" SHORT signal: score={sig['score']}, target=${sig['nearest_target']:.2f}") + + print("\n[LIQ] Smoke test passed") diff --git a/mt5_bridge.py b/mt5_bridge.py new file mode 100644 index 0000000..5d46700 --- /dev/null +++ b/mt5_bridge.py @@ -0,0 +1,757 @@ +""" +AHAD QUANT — MT5 Bridge (CSV File Communication) +================================================ +Assure la communication temps réel entre AHAD QUANT (Python) et un EA MetaTrader 5 +via des fichiers CSV partagés dans le dossier MQL5/Files/. + +Flux : + 1. AHAD QUANT écrit un signal dans signals.csv + 2. L'EA MT5 lit signals.csv via OnTimer(), exécute l'ordre + 3. L'EA écrit le résultat dans reports.csv + 4. MT5Bridge détecte le changement (watchdog <50ms) et appelle le callback + 5. L'EA écrit status.csv toutes les 30s (balance, equity, positions) + +Usage : + from mt5_bridge import MT5Bridge + + bridge = MT5Bridge(files_path="C:/Users/NOM/.../MQL5/Files") + + bridge.on_report_received(lambda r: print("Rapport reçu:", r)) + bridge.on_status_updated(lambda s: print("Statut compte:", s)) + + bridge.start() + + bridge.write_signal("EURUSD", "BUY", lot=0.1, sl_pips=30, tp_pips=60, confidence=0.82) + bridge.write_signal("GBPUSD", "SELL", lot=0.05, sl_pips=25, tp_pips=50, confidence=0.75) + + bridge.stop() + +Dépendances : + pip install watchdog filelock +""" + +import csv +import logging +import os +import threading +import time +import uuid +from dataclasses import dataclass, field, asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, List, Optional + +try: + from watchdog.events import FileSystemEventHandler, FileModifiedEvent + from watchdog.observers import Observer + HAS_WATCHDOG = True +except ImportError: + HAS_WATCHDOG = False + print("[MT5Bridge] AVERTISSEMENT : watchdog non installé. Utilisation du polling 200ms.") + print("[MT5Bridge] Pour installer : pip install watchdog") + +try: + from filelock import FileLock, Timeout as FileLockTimeout + HAS_FILELOCK = True +except ImportError: + HAS_FILELOCK = False + print("[MT5Bridge] AVERTISSEMENT : filelock non installé.") + print("[MT5Bridge] Pour installer : pip install filelock") + +logger = logging.getLogger(__name__) + + +# ─── Structures de données ──────────────────────────────────────────────────── + +@dataclass +class Signal: + """Un signal de trading à envoyer à l'EA MT5.""" + pair: str # Ex: "EURUSD" + action: str # "BUY" ou "SELL" + lot_size: float # Volume en lots (ex: 0.1) + sl_pips: int # Stop loss en pips (ex: 30) + tp_pips: int # Take profit en pips (ex: 60) + confidence: float # Score de confiance du modèle (0.0 à 1.0) + signal_id: str = field(default_factory=lambda: f"SIG_{uuid.uuid4().hex[:8].upper()}") + timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + +@dataclass +class TradeReport: + """Un rapport de trade renvoyé par l'EA MT5.""" + signal_id: str + ticket: int # Numéro de ticket MT5 + status: str # "OPEN", "CLOSED", "ERROR", "REJECTED" + open_price: float + sl: float + tp: float + open_time: str + close_price: Optional[float] = None + close_time: Optional[str] = None + profit: Optional[float] = None + comment: Optional[str] = None + + +@dataclass +class AccountStatus: + """État du compte MT5 mis à jour toutes les 30s par l'EA.""" + timestamp: str + balance: float + equity: float + margin: float + free_margin: float + open_positions: int + daily_pnl: float + + +# ─── Watcher de fichier (watchdog ou polling) ───────────────────────────────── + +class _CSVChangeHandler(FileSystemEventHandler): + """Déclenche un callback dès qu'un fichier CSV est modifié.""" + + def __init__(self, filepath: str, callback: Callable): + super().__init__() + self._filepath = os.path.abspath(filepath) + self._callback = callback + + def on_modified(self, event): + if not event.is_directory and os.path.abspath(event.src_path) == self._filepath: + self._callback() + + +class _PollingWatcher: + """Fallback si watchdog n'est pas installé — poll toutes les 200ms.""" + + def __init__(self, filepath: str, callback: Callable, interval: float = 0.2): + self._filepath = filepath + self._callback = callback + self._interval = interval + self._last_mtime: float = 0.0 + self._running = False + self._thread: Optional[threading.Thread] = None + + def start(self): + self._running = True + self._thread = threading.Thread(target=self._loop, daemon=True) + self._thread.start() + + def stop(self): + self._running = False + + def _loop(self): + while self._running: + try: + mtime = os.path.getmtime(self._filepath) if os.path.exists(self._filepath) else 0 + if mtime > self._last_mtime: + self._last_mtime = mtime + if mtime > 0: + self._callback() + except OSError: + pass + time.sleep(self._interval) + + +# ─── Classe principale MT5Bridge ────────────────────────────────────────────── + +class MT5Bridge: + """ + Pont de communication temps réel entre AHAD QUANT (Python) et un EA MetaTrader 5. + + Paramètres + ---------- + files_path : str + Chemin absolu vers le dossier MQL5/Files/ de MetaTrader 5. + Ex Windows : "C:/Users/NOM/AppData/Roaming/MetaQuotes/Terminal/XXXXX/MQL5/Files" + + signal_file : str + Nom du fichier de signaux (défaut : "signals.csv") + + report_file : str + Nom du fichier de rapports (défaut : "reports.csv") + + status_file : str + Nom du fichier de statut (défaut : "status.csv") + + signal_timeout : int + Délai en secondes après lequel un signal non exécuté est considéré périmé (défaut : 30) + """ + + # En-têtes des fichiers CSV + _SIGNAL_HEADERS = ["timestamp", "pair", "action", "lot_size", "sl_pips", + "tp_pips", "signal_id", "confidence"] + _REPORT_HEADERS = ["signal_id", "ticket", "status", "open_price", "sl", "tp", + "open_time", "close_price", "close_time", "profit", "comment"] + _STATUS_HEADERS = ["timestamp", "balance", "equity", "margin", + "free_margin", "open_positions", "daily_pnl"] + + def __init__( + self, + files_path: str, + signal_file: str = "signals.csv", + report_file: str = "reports.csv", + status_file: str = "status.csv", + signal_timeout: int = 30, + max_signal_rows: int = 500, + signal_rotate_check_every: int = 25, + ): + self._dir = Path(files_path) + self._signal_path = self._dir / signal_file + self._report_path = self._dir / report_file + self._status_path = self._dir / status_file + self._signal_timeout = signal_timeout + self._max_signal_rows = max_signal_rows + self._signal_rotate_check_every = signal_rotate_check_every + self._signal_write_count = 0 + + # Callbacks enregistrés par l'utilisateur + self._report_callbacks: List[Callable[[TradeReport], None]] = [] + self._status_callbacks: List[Callable[[AccountStatus], None]] = [] + + # IDs de rapports déjà traités (évite les doublons) + self._seen_report_ids: set = set() + + # Watchers + self._report_observer = None + self._status_observer = None + self._running = False + self._lock = threading.Lock() + + # ─── API publique ────────────────────────────────────────────────────────── + + def on_report_received(self, callback: Callable[[TradeReport], None]) -> None: + """Enregistre un callback appelé à chaque nouveau rapport de trade.""" + self._report_callbacks.append(callback) + + def on_status_updated(self, callback: Callable[[AccountStatus], None]) -> None: + """Enregistre un callback appelé à chaque mise à jour du statut compte.""" + self._status_callbacks.append(callback) + + def start(self) -> None: + """ + Démarre le bridge : initialise les fichiers CSV et lance les watchers. + À appeler une fois au démarrage du bot. + """ + self._dir.mkdir(parents=True, exist_ok=True) + self._init_signal_file() + self._init_report_file() + + self._running = True + self._start_watcher(self._report_path, self._on_report_file_changed, "report") + self._start_watcher(self._status_path, self._on_status_file_changed, "status") + + logger.info(f"[MT5Bridge] Démarré — dossier : {self._dir}") + logger.info(f"[MT5Bridge] signals → {self._signal_path}") + logger.info(f"[MT5Bridge] reports ← {self._report_path}") + logger.info(f"[MT5Bridge] status ← {self._status_path}") + + def stop(self) -> None: + """Arrête proprement les watchers.""" + self._running = False + if self._report_observer: + try: + self._report_observer.stop() + self._report_observer.join(timeout=2) + except Exception: + pass + if self._status_observer: + try: + self._status_observer.stop() + self._status_observer.join(timeout=2) + except Exception: + pass + logger.info("[MT5Bridge] Arrêté.") + + def write_signal( + self, + pair: str, + action: str, + lot: float, + sl_pips: int, + tp_pips: int, + confidence: float, + ) -> Signal: + """ + Écrit un signal de trading dans signals.csv pour l'EA MT5. + + Paramètres + ---------- + pair : Paire Forex — ex: "EURUSD" + action : "BUY" ou "SELL" + lot : Volume en lots — ex: 0.1 + sl_pips : Stop loss en pips — ex: 30 + tp_pips : Take profit en pips — ex: 60 + confidence : Score modèle entre 0 et 1 — ex: 0.82 + + Retourne + -------- + Signal : l'objet signal créé, avec son signal_id unique + """ + action = action.upper() + if action not in ("BUY", "SELL"): + raise ValueError(f"action doit être 'BUY' ou 'SELL', reçu : '{action}'") + if not (0.0 < lot <= 100.0): + raise ValueError(f"lot invalide : {lot}") + if sl_pips <= 0 or tp_pips <= 0: + raise ValueError(f"sl_pips et tp_pips doivent être > 0") + + # Bug #25 fix : ne pas uppercaser la paire entière. + # pair.upper() convertit "EURCHF.m" → "EURCHF.M" (M majuscule) alors que + # le broker utilise le suffixe minuscule ".m" → SYMBOL_NOT_FOUND côté EA. + # La paire de base (ex: "EURCHF") est déjà en majuscules côté appelant. + sig = Signal( + pair=pair, + action=action, + lot_size=lot, + sl_pips=sl_pips, + tp_pips=tp_pips, + confidence=round(confidence, 4), + ) + + self._append_to_csv(self._signal_path, self._SIGNAL_HEADERS, asdict(sig)) + logger.info(f"[MT5Bridge] Signal écrit : {sig.signal_id} | {sig.pair} {sig.action} " + f"lot={sig.lot_size} SL={sig.sl_pips}p TP={sig.tp_pips}p conf={sig.confidence}") + self._signal_write_count += 1 + if self._signal_write_count % self._signal_rotate_check_every == 0: + self._rotate_signal_file_if_needed() + return sig + + def read_reports(self) -> List[TradeReport]: + """ + Lit tous les rapports présents dans reports.csv. + Retourne uniquement les rapports nouveaux (non encore traités). + """ + return self._read_new_reports() + + def read_status(self) -> Optional[AccountStatus]: + """ + Lit le dernier statut du compte depuis status.csv. + Retourne None si le fichier n'existe pas encore. + """ + return self._read_latest_status() + + def get_open_signals(self) -> List[dict]: + """ + Retourne les signaux envoyés à l'EA qui n'ont pas encore reçu de rapport. + Filtre les signaux périmés (plus vieux que signal_timeout secondes). + """ + if not self._signal_path.exists(): + return [] + + now = datetime.now(timezone.utc).timestamp() + open_sigs = [] + + with self._safe_read(self._signal_path) as rows: + for row in rows: + try: + ts = datetime.fromisoformat(row["timestamp"]).timestamp() + if now - ts <= self._signal_timeout: + open_sigs.append(row) + except (KeyError, ValueError): + continue + + return open_sigs + + # ─── Initialisation des fichiers ────────────────────────────────────────── + + def _init_signal_file(self) -> None: + """Crée signals.csv avec les en-têtes s'il n'existe pas.""" + if not self._signal_path.exists(): + self._write_headers(self._signal_path, self._SIGNAL_HEADERS) + logger.info(f"[MT5Bridge] signals.csv créé : {self._signal_path}") + + def _init_report_file(self) -> None: + """Crée reports.csv avec les en-têtes s'il n'existe pas.""" + if not self._report_path.exists(): + self._write_headers(self._report_path, self._REPORT_HEADERS) + logger.info(f"[MT5Bridge] reports.csv créé : {self._report_path}") + + def _rotate_signal_file_if_needed(self) -> None: + """ + Tronque signals.csv aux _max_signal_rows lignes les plus récentes. + Les signaux non encore acquittés (absents de reports.csv) sont TOUJOURS + conservés, même s'ils sont hors de la fenêtre des dernières lignes. + Protégé par FileLock pour cohérence avec le reste du bridge. + """ + if not self._signal_path.exists(): + return + try: + lock_path = str(self._signal_path) + ".lock" + ctx = FileLock(lock_path, timeout=3) if HAS_FILELOCK else None + + def _do_rotate(): + with open(self._signal_path, encoding="utf-8-sig", newline="") as fh: + reader = csv.DictReader(fh) + rows = list(reader) + headers = reader.fieldnames or self._SIGNAL_HEADERS + + if len(rows) <= self._max_signal_rows: + return # Pas assez de lignes — rien à faire + + # Récupérer les signal_ids déjà ACK dans reports.csv + ack_ids: set = set() + if self._report_path.exists(): + with open(self._report_path, encoding="utf-8-sig", newline="") as rh: + for r in csv.DictReader(rh): + sid = r.get("signal_id") + if sid: + ack_ids.add(sid) + + # Garder les N dernières lignes + toutes les lignes non ACK hors fenêtre + tail = rows[-self._max_signal_rows:] + tail_ids = {r.get("signal_id") for r in tail} + pending_outside = [r for r in rows[:-self._max_signal_rows] + if r.get("signal_id") not in ack_ids + and r.get("signal_id") not in tail_ids] + kept = pending_outside + tail + + # Réécriture atomique via fichier temporaire + tmp = str(self._signal_path) + ".tmp" + with open(tmp, "w", encoding="utf-8", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=headers) + writer.writeheader() + writer.writerows(kept) + os.replace(tmp, self._signal_path) + logger.info(f"[MT5Bridge] signals.csv rotation : {len(rows)} → {len(kept)} lignes" + f" ({len(pending_outside)} signaux pending conservés hors fenêtre)") + + if ctx: + with ctx: + _do_rotate() + else: + _do_rotate() + + except Exception as exc: + logger.warning(f"[MT5Bridge] Rotation signals.csv échouée : {exc}") + + # ─── Watchers ───────────────────────────────────────────────────────────── + + def _start_watcher(self, filepath: Path, callback: Callable, name: str) -> None: + """Lance un watcher watchdog ou polling selon ce qui est disponible.""" + if HAS_WATCHDOG: + handler = _CSVChangeHandler(str(filepath), callback) + observer = Observer() + observer.schedule(handler, path=str(filepath.parent), recursive=False) + observer.start() + if name == "report": + self._report_observer = observer + else: + self._status_observer = observer + logger.debug(f"[MT5Bridge] Watcher watchdog démarré pour {name}") + else: + watcher = _PollingWatcher(str(filepath), callback) + watcher.start() + if name == "report": + self._report_observer = watcher + else: + self._status_observer = watcher + logger.debug(f"[MT5Bridge] Watcher polling démarré pour {name}") + + # ─── Callbacks de détection de changement ───────────────────────────────── + + def _on_report_file_changed(self) -> None: + """Appelé dès que reports.csv est modifié.""" + time.sleep(0.02) # Laisser l'EA finir d'écrire + new_reports = self._read_new_reports() + for report in new_reports: + logger.info(f"[MT5Bridge] Rapport reçu : {report.signal_id} | " + f"ticket={report.ticket} statut={report.status} " + f"profit={report.profit}") + for cb in self._report_callbacks: + try: + cb(report) + except Exception as e: + logger.error(f"[MT5Bridge] Erreur callback rapport : {e}") + + def _on_status_file_changed(self) -> None: + """Appelé dès que status.csv est modifié.""" + time.sleep(0.02) + status = self._read_latest_status() + if status: + logger.debug(f"[MT5Bridge] Statut compte : balance={status.balance} " + f"equity={status.equity} positions={status.open_positions}") + for cb in self._status_callbacks: + try: + cb(status) + except Exception as e: + logger.error(f"[MT5Bridge] Erreur callback statut : {e}") + + # ─── Lecture des fichiers ───────────────────────────────────────────────── + + def _read_new_reports(self) -> List[TradeReport]: + """Lit reports.csv et retourne uniquement les lignes non encore traitées.""" + if not self._report_path.exists(): + return [] + + new_reports = [] + with self._safe_read(self._report_path) as rows: + for row in rows: + try: + # Clé unique : signal_id + statut (un OPEN et un CLOSED pour le même signal) + key = f"{row['signal_id']}_{row['status']}" + if key in self._seen_report_ids: + continue + self._seen_report_ids.add(key) + + report = TradeReport( + signal_id = row["signal_id"], + ticket = int(row["ticket"]) if row.get("ticket") else 0, + status = row["status"], + open_price = float(row["open_price"]) if row.get("open_price") else 0.0, + sl = float(row["sl"]) if row.get("sl") else 0.0, + tp = float(row["tp"]) if row.get("tp") else 0.0, + open_time = row.get("open_time", ""), + close_price = float(row["close_price"]) if row.get("close_price") else None, + close_time = row.get("close_time") or None, + profit = float(row["profit"]) if row.get("profit") else None, + comment = row.get("comment") or None, + ) + new_reports.append(report) + except (KeyError, ValueError, TypeError) as e: + # Marquer la ligne comme vue pour éviter le spam toutes les 200ms + bad_key = f"_BAD_{hash(str(sorted(row.items())))}" + if bad_key not in self._seen_report_ids: + self._seen_report_ids.add(bad_key) + logger.warning(f"[MT5Bridge] Ligne rapport ignorée (format invalide) : {e}") + + return new_reports + + def _read_latest_status(self) -> Optional[AccountStatus]: + """Lit la dernière ligne de status.csv (la plus récente).""" + if not self._status_path.exists(): + return None + + with self._safe_read(self._status_path) as rows: + valid_rows = [] + for row in rows: + try: + valid_rows.append(AccountStatus( + timestamp = row["timestamp"], + balance = float(row["balance"]), + equity = float(row["equity"]), + margin = float(row["margin"]), + free_margin = float(row["free_margin"]), + open_positions = int(row["open_positions"]), + daily_pnl = float(row["daily_pnl"]), + )) + except (KeyError, ValueError, TypeError): + continue + + return valid_rows[-1] if valid_rows else None + + # ─── Utilitaires CSV ────────────────────────────────────────────────────── + + def _write_headers(self, filepath: Path, headers: List[str]) -> None: + """Crée un fichier CSV avec uniquement la ligne d'en-tête.""" + lock_path = str(filepath) + ".lock" + if HAS_FILELOCK: + with FileLock(lock_path, timeout=5): + self._open_with_retry(filepath, "w", headers, row=None) + else: + self._open_with_retry(filepath, "w", headers, row=None) + + def _append_to_csv(self, filepath: Path, headers: List[str], row: dict) -> None: + """ + Ajoute une ligne à un fichier CSV existant (thread-safe). + + Bug #24 fix : l'EA MT5 peut maintenir un lock Windows exclusif sur + signals.csv pendant sa lecture (timer 200ms). Le open() Python reçoit + alors PermissionError — non géré par FileLock (qui gère uniquement les + conflits Python↔Python via un .lock séparé). + Solution : retry avec backoff exponentiel (max 5 tentatives, 50ms base). + """ + lock_path = str(filepath) + ".lock" + if HAS_FILELOCK: + try: + with FileLock(lock_path, timeout=5): + self._open_with_retry(filepath, "a", headers, row) + except FileLockTimeout: + logger.error(f"[MT5Bridge] Impossible d'obtenir le verrou pour {filepath.name}") + else: + with self._lock: + self._open_with_retry(filepath, "a", headers, row) + + def _open_with_retry( + self, + filepath: Path, + mode: str, + headers: List[str], + row: Optional[dict], + max_retries: int = 5, + base_delay: float = 0.05, + ) -> None: + """ + Ouvre un fichier CSV et écrit (header ou ligne) avec retry sur PermissionError. + L'EA MT5 peut tenir un lock exclusif Windows jusqu'à ~200ms. + On attend donc 50ms, 100ms, 200ms, 400ms, 800ms avant d'abandonner. + """ + for attempt in range(max_retries): + try: + with open(filepath, mode, newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=headers, extrasaction="ignore") + if row is None: + writer.writeheader() + else: + writer.writerow(row) + return # succès + except PermissionError: + if attempt < max_retries - 1: + delay = base_delay * (2 ** attempt) # 50ms, 100ms, 200ms, 400ms + logger.debug( + f"[MT5Bridge] PermissionError sur {filepath.name} " + f"(tentative {attempt + 1}/{max_retries}) — retry dans {delay*1000:.0f}ms" + ) + time.sleep(delay) + else: + logger.error( + f"[MT5Bridge] PermissionError persistant sur {filepath.name} " + f"après {max_retries} tentatives — signal perdu." + ) + raise + + class _safe_read: + """Context manager pour lire un CSV en toute sécurité.""" + def __init__(self, filepath: Path): + self._filepath = filepath + self._lock_path = str(filepath) + ".lock" + + def __enter__(self) -> List[dict]: + if HAS_FILELOCK: + self._fl = FileLock(self._lock_path, timeout=5) + self._fl.acquire() + try: + # utf-8-sig supprime le BOM (\ufeff) des CSV Windows/MetaTrader 5 + # Sans ca, le 1er header devient \ufeffsignal_id -> KeyError + with open(self._filepath, "r", newline="", encoding="utf-8-sig") as f: + reader = csv.DictReader(f) + self._rows = list(reader) + except (OSError, csv.Error): + self._rows = [] + return self._rows + + def __exit__(self, *args): + if HAS_FILELOCK: + try: + self._fl.release() + except Exception: + pass + + +# ─── Point d'entrée de test ─────────────────────────────────────────────────── + +if __name__ == "__main__": + """ + Test rapide du bridge en mode simulation locale. + Lance ce script pour vérifier que le bridge fonctionne + avant de le connecter à un vrai EA MT5. + + Usage : + python mt5_bridge.py + """ + import tempfile + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)-7s %(message)s", + datefmt="%H:%M:%S", + ) + + print("\n" + "="*60) + print(" AHAD QUANT MT5 Bridge — Test de simulation") + print("="*60 + "\n") + + # Dossier temporaire simulant MQL5/Files/ + with tempfile.TemporaryDirectory() as tmpdir: + bridge = MT5Bridge(files_path=tmpdir, signal_timeout=10) + + received_reports = [] + received_statuses = [] + + bridge.on_report_received(lambda r: received_reports.append(r)) + bridge.on_status_updated(lambda s: received_statuses.append(s)) + + bridge.start() + + # ── Test 1 : écriture de signaux ────────────────────────────── + print("[TEST 1] Écriture de signaux...") + sig1 = bridge.write_signal("EURUSD", "BUY", lot=0.10, sl_pips=30, tp_pips=60, confidence=0.82) + sig2 = bridge.write_signal("GBPUSD", "SELL", lot=0.05, sl_pips=25, tp_pips=50, confidence=0.75) + sig3 = bridge.write_signal("USDJPY", "BUY", lot=0.02, sl_pips=40, tp_pips=80, confidence=0.91) + print(f" ✅ {sig1.signal_id} | EURUSD BUY") + print(f" ✅ {sig2.signal_id} | GBPUSD SELL") + print(f" ✅ {sig3.signal_id} | USDJPY BUY") + + # ── Test 2 : simulation réponse EA (rapport OPEN) ───────────── + print("\n[TEST 2] Simulation rapport OPEN depuis l'EA...") + report_open = { + "signal_id": sig1.signal_id, + "ticket": "123456", + "status": "OPEN", + "open_price": "1.08542", + "sl": "1.08242", + "tp": "1.09142", + "open_time": datetime.now(timezone.utc).isoformat(), + "close_price": "", + "close_time": "", + "profit": "", + "comment": "Ordre exécuté", + } + # Simule l'écriture de l'EA dans reports.csv + report_path = Path(tmpdir) / "reports.csv" + with open(report_path, "a", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=MT5Bridge._REPORT_HEADERS) + writer.writerow(report_open) + + time.sleep(0.3) # Laisser le watcher détecter + assert len(received_reports) == 1, f"Attendu 1 rapport, reçu {len(received_reports)}" + print(f" ✅ Rapport OPEN reçu : ticket={received_reports[0].ticket}") + + # ── Test 3 : simulation rapport CLOSED ──────────────────────── + print("\n[TEST 3] Simulation rapport CLOSED depuis l'EA...") + report_closed = {**report_open, + "status": "CLOSED", + "close_price": "1.09140", + "close_time": datetime.now(timezone.utc).isoformat(), + "profit": "60.0", + "comment": "TP hit"} + with open(report_path, "a", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=MT5Bridge._REPORT_HEADERS) + writer.writerow(report_closed) + + time.sleep(0.3) + assert len(received_reports) == 2, f"Attendu 2 rapports, reçu {len(received_reports)}" + print(f" ✅ Rapport CLOSED reçu : profit={received_reports[1].profit}") + + # ── Test 4 : simulation status EA ───────────────────────────── + print("\n[TEST 4] Simulation status.csv depuis l'EA...") + status_path = Path(tmpdir) / "status.csv" + with open(status_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=MT5Bridge._STATUS_HEADERS) + writer.writeheader() + writer.writerow({ + "timestamp": datetime.now(timezone.utc).isoformat(), + "balance": "10060.00", + "equity": "10045.50", + "margin": "120.00", + "free_margin": "9925.50", + "open_positions": "2", + "daily_pnl": "60.0", + }) + + time.sleep(0.3) + status = bridge.read_status() + assert status is not None + print(f" ✅ Statut reçu : balance={status.balance} equity={status.equity}") + + # ── Test 5 : validation des erreurs ─────────────────────────── + print("\n[TEST 5] Validation des paramètres invalides...") + try: + bridge.write_signal("EURUSD", "HOLD", lot=0.1, sl_pips=30, tp_pips=60, confidence=0.8) + print(" ❌ Aurait dû lever une ValueError") + except ValueError as e: + print(f" ✅ ValueError correctement levée : {e}") + + bridge.stop() + + print("\n" + "="*60) + print(" Tous les tests passés ✅") + print("="*60 + "\n") diff --git a/online_learner.py b/online_learner.py new file mode 100644 index 0000000..da44074 --- /dev/null +++ b/online_learner.py @@ -0,0 +1,633 @@ +""" +AHAD QUANT — Online Learner (Apprentissage Continu V7) +========================================================= +Apprentissage continu à partir des expériences stockées dans le buffer. + +3 mécanismes distincts : + +1. LightGBM WARM-START (quotidien, ~2-5 min, local CPU) + - Récupère les trades LOSS + TIMEOUT des dernières 24h + - Pondère chaque sample par error_weight + - Ré-entraîne LightGBM avec init_model=model_actuel (50 arbres supplémentaires) + - Remplace model.pkl SEULEMENT si accuracy >= accuracy_actuelle - 0.5% + +2. RL EXPERIENCE REPLAY (hebdomadaire, ~2-4h) + - Convertit les trades réels du buffer en épisodes RL (obs, action, reward, next_obs) + - Injecte ces épisodes dans le fine-tune PPO SB3 + +3. CALIBRATION SEUIL (immédiat, après chaque trade fermé) + - Sliding window sur les 50 derniers trades + - Si win_rate < 50% → MIN_CONFIDENCE += 0.01 + - Si win_rate > 65% → MIN_CONFIDENCE -= 0.005 + - Borné entre [0.62, 0.88] + - Sauvegardé dans adaptive_state.json + +Usage : + from experience_buffer import get_experience_buffer + from online_learner import OnlineLearner + + buf = get_experience_buffer() + learner = OnlineLearner(buf) + learner.daily_update() + new_conf = learner.calibrate_threshold(recent_n=50) + episodes = learner.prepare_rl_episodes() +""" + +import json +import logging +import os +import pickle +import shutil +import time +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple + +import numpy as np + +import config +from experience_buffer import ExperienceBuffer +import ensemble_core as _ens_core +from features import NUM_FEATURES + +log = logging.getLogger("OnlineLearner") + +# ── Constantes ──────────────────────────────────────────────────────────────── +WARMSTART_ROUNDS = 50 # arbres supplémentaires par cycle quotidien +WARMSTART_MIN_TRADES = 20 # minimum de trades échoués pour déclencher +CALIBRATION_WINDOW = 50 # trades pour sliding win rate +CONFIDENCE_MIN = 0.62 # plancher MIN_CONFIDENCE +CONFIDENCE_MAX = 0.88 # plafond MIN_CONFIDENCE +CONFIDENCE_STEP_UP = 0.01 # hausse si win_rate < 50% +CONFIDENCE_STEP_DOWN = 0.005 # baisse si win_rate > 65% +ACCURACY_TOLERANCE = 0.005 # -0.5% toléré pour accepter le nouveau modèle +ADAPTIVE_STATE_FILE = "adaptive_state.json" + +# Calibration jointe ML/RL (corrige le point #7 du diagnostic) : avant, +# config.RL_OVERRIDE_THRESHOLD (utilisé par rl_agent.py::filter_signal pour +# décider si le RL peut s'imposer face au ML) n'était JAMAIS ajusté par +# calibrate_threshold(), qui ne touchait que MIN_CONFIDENCE — les deux +# seuils dérivaient l'un de l'autre silencieusement avec le temps. On les +# fait maintenant bouger ensemble, à écart ("marge") constant. +RL_OVERRIDE_MIN = 0.65 +RL_OVERRIDE_MAX = 0.95 + + +class OnlineLearner: + """ + Apprentissage incrémental à partir des expériences réelles de trading. + """ + + def __init__( + self, + buffer: ExperienceBuffer, + model_path: Optional[str] = None, + ensemble_path: Optional[str] = None, + adaptive_state_file: str = ADAPTIVE_STATE_FILE, + ): + self.buffer = buffer + self.model_path = model_path or getattr(config, "MODEL_PATH", "model.pkl") + self.ensemble_path = ensemble_path or getattr(config, "ENSEMBLE_MODEL_PATH", "model_ensemble.pkl") + self.state_file = adaptive_state_file + self._state = self._load_adaptive_state() + + # ── Adaptive state I/O ─────────────────────────────────────────────────── + + def _load_adaptive_state(self) -> Dict: + default_margin = round( + getattr(config, "RL_OVERRIDE_THRESHOLD", 0.82) + - getattr(config, "MIN_CONFIDENCE", 0.72), 4 + ) + defaults = { + "min_confidence": getattr(config, "MIN_CONFIDENCE", 0.72), + "rl_override_threshold": getattr(config, "RL_OVERRIDE_THRESHOLD", 0.82), + "rl_override_margin": default_margin, + "last_calibration": None, + "win_rate_50": 0.0, + "avg_pnl_50": 0.0, + "total_trades": 0, + "emergency_retrain_at": None, + "confidence_history": [], + } + if not os.path.exists(self.state_file): + return defaults + try: + with open(self.state_file, encoding="utf-8") as f: + data = json.load(f) + # Fusionner avec les defaults (robustesse) + return {**defaults, **data} + except Exception as e: + log.warning(f"Erreur chargement adaptive_state : {e} — reset") + return defaults + + def _save_adaptive_state(self) -> None: + try: + self._state["last_calibration"] = datetime.now(timezone.utc).isoformat() + self._state["total_trades"] = self.buffer.total_trades + tmp = self.state_file + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(self._state, f, indent=2, ensure_ascii=False) + os.replace(tmp, self.state_file) + except Exception as e: + log.error(f"Erreur sauvegarde adaptive_state : {e}") + + @property + def current_confidence(self) -> float: + return self._state.get("min_confidence", getattr(config, "MIN_CONFIDENCE", 0.72)) + + # ── Mécanisme 3 : Calibration du seuil de confiance ───────────────────── + + def calibrate_threshold(self, recent_n: int = CALIBRATION_WINDOW) -> float: + """ + Ajuste MIN_CONFIDENCE dynamiquement selon le win_rate récent. + - win_rate < 50% → hausse le seuil (devient plus sélectif) + - win_rate > 65% → baisse le seuil (devient plus actif) + Retourne la nouvelle valeur de MIN_CONFIDENCE. + """ + if not getattr(config, "CONTINUOUS_LEARNING_ENABLED", True): + return self.current_confidence + + recent = self.buffer.get_last_n(recent_n) + if len(recent) < 5: + return self.current_confidence # pas assez de données + + wins = sum(1 for e in recent if e.get("outcome") == "WIN") + win_rate = wins / len(recent) + + current_conf = self.current_confidence + + if win_rate < 0.50: + new_conf = min(current_conf + CONFIDENCE_STEP_UP, CONFIDENCE_MAX) + reason = f"win_rate={win_rate:.1%} < 50% → hausse seuil" + elif win_rate > 0.65: + new_conf = max(current_conf - CONFIDENCE_STEP_DOWN, CONFIDENCE_MIN) + reason = f"win_rate={win_rate:.1%} > 65% → baisse seuil" + else: + new_conf = current_conf + reason = f"win_rate={win_rate:.1%} stable — pas de changement" + + changed = abs(new_conf - current_conf) > 1e-5 + + self._state["min_confidence"] = round(new_conf, 4) + self._state["win_rate_50"] = round(win_rate, 4) + + # Historique des 10 dernières valeurs + history = self._state.get("confidence_history", []) + history.append(round(new_conf, 4)) + self._state["confidence_history"] = history[-10:] + + # Synchronise config en mémoire (pour la session courante) + config.MIN_CONFIDENCE = new_conf + + # ── Calibration jointe RL (corrige le point #7) ────────────────── + # RL_OVERRIDE_THRESHOLD suit MIN_CONFIDENCE à marge constante (l'écart + # initial entre les deux, persisté), au lieu de rester figé pendant + # que MIN_CONFIDENCE dérive. rl_agent.py::filter_signal() lit déjà + # config.RL_OVERRIDE_THRESHOLD (et non plus une variable d'env figée), + # donc cet ajustement a un effet réel dès le prochain signal. + margin = self._state.get( + "rl_override_margin", + round(getattr(config, "RL_OVERRIDE_THRESHOLD", 0.82) - current_conf, 4), + ) + new_rl_override = float(np.clip(new_conf + margin, RL_OVERRIDE_MIN, RL_OVERRIDE_MAX)) + config.RL_OVERRIDE_THRESHOLD = round(new_rl_override, 4) + self._state["rl_override_threshold"] = round(new_rl_override, 4) + + self._save_adaptive_state() + + if changed: + log.info( + f"[CALIBRATION] {reason} | conf {current_conf:.3f} → {new_conf:.3f} " + f"| RL_OVERRIDE_THRESHOLD → {new_rl_override:.3f}" + ) + + return new_conf + + # ── Mécanisme 1 : LightGBM Warm-Start quotidien ────────────────────────── + + def daily_update( + self, + loss_trades: Optional[List[Dict]] = None, + hours: float = 24.0, + ) -> bool: + """ + Warm-start LightGBM sur les trades LOSS + TIMEOUT des dernières h. + + Si USE_ENSEMBLE=true (configuration par défaut) ET que + model_ensemble.pkl existe, le warm-start cible le sous-modèle + `lgbm` EMBARQUÉ DANS L'ENSEMBLE, et l'acceptation est évaluée au + niveau de l'ensemble complet (ensemble_core.predict_ensemble_batch) + plutôt que sur le seul LightGBM isolé. AVANT ce correctif, cette + méthode ne mettait à jour QUE model.pkl — un fichier jamais relu en + production dès que USE_ENSEMBLE=true (la valeur par défaut), ce qui + rendait ce mécanisme un no-op silencieux pour la quasi-totalité des + déploiements (point #3 du diagnostic). + + Repli sur model.pkl (LightGBM autonome) si USE_ENSEMBLE=false ou si + model_ensemble.pkl est absent — comportement identique à avant pour + ce cas. + + Retourne True si le modèle a été mis à jour. + """ + if not getattr(config, "WARMSTART_ENABLED", True): + log.debug("[WARMSTART] Désactivé (WARMSTART_ENABLED=false)") + return False + + use_ensemble = bool(getattr(config, "USE_ENSEMBLE", True)) and os.path.exists(self.ensemble_path) + target_path = self.ensemble_path if use_ensemble else self.model_path + + if not os.path.exists(target_path): + log.warning(f"[WARMSTART] Modèle introuvable ({target_path}) — skip") + return False + + # ── Charger : préférer le candidat accumulé si disponible ─────────── + # [FIX] Si le warmstart précédent a été rejeté (accuracy insuffisante), + # les arbres qu'il a appris sont sauvegardés dans .candidate. + # On repart de ce candidat plutôt que de la production pour que les + # mauvais trades s'accumulent cycle après cycle, même quand le modèle + # actif reste inchangé. + candidate_path = target_path + ".candidate" + start_model_path = candidate_path if os.path.exists(candidate_path) else target_path + start_label = "candidate" if os.path.exists(candidate_path) else "production" + + try: + with open(start_model_path, "rb") as f: + loaded = pickle.load(f) + except Exception as e: + log.error(f"[WARMSTART] Erreur chargement modèle ({start_model_path}) : {e}") + return False + + log.info(f"[WARMSTART] Point de départ : {start_label} ({start_model_path})") + + if use_ensemble: + ensemble_dict = loaded + current_model = ensemble_dict.get("lgbm") + if current_model is None: + log.warning("[WARMSTART] Pas de sous-modèle LightGBM dans l'ensemble — skip") + return False + else: + ensemble_dict = None + current_model = loaded["model"] if isinstance(loaded, dict) else loaded + + # Récupérer les trades échoués + if loss_trades is None: + loss_trades = self.buffer.get_recent(hours=hours, outcome_filter=["LOSS", "TIMEOUT"]) + + min_trades = getattr(config, "WARMSTART_MIN_TRADES", WARMSTART_MIN_TRADES) + if len(loss_trades) < min_trades: + log.info(f"[WARMSTART] Seulement {len(loss_trades)} trades échoués " + f"(min={min_trades}) — skip") + return False + + try: + import lightgbm as lgb + except ImportError: + log.error("[WARMSTART] LightGBM non installé — skip") + return False + + log.info( + f"[WARMSTART] Démarrage warm-start ({'ensemble' if use_ensemble else 'LightGBM seul'}) " + f"sur {len(loss_trades)} trades échoués..." + ) + + # Construire le dataset + X, y, weights = _build_dataset_from_trades(loss_trades) + if X is None or len(X) < 10: + log.warning("[WARMSTART] Dataset insuffisant — skip") + return False + + # Préparer le dataset LightGBM avec pondération par error_weight + train_data = lgb.Dataset(X, label=y, weight=weights) + + # Params hérités du modèle courant si disponibles + params = { + "objective": "binary", + "metric": "binary_logloss", + "learning_rate": 0.03, + "num_leaves": 63, + "feature_fraction": 0.8, + "bagging_fraction": 0.8, + "bagging_freq": 5, + "min_child_samples": 10, + "verbose": -1, + } + + try: + new_model = lgb.train( + params, + train_data, + num_boost_round = WARMSTART_ROUNDS, + init_model = current_model, # ← warm-start cumulatif + valid_sets = [train_data], # requis par early_stopping (LightGBM >= 4.x) + callbacks = [lgb.early_stopping(10, verbose=False), + lgb.log_evaluation(period=-1)], + ) + except Exception as e: + log.error(f"[WARMSTART] Erreur entraînement : {e}") + return False + + # ── Évaluation de l'acceptation ─────────────────────────────────── + if use_ensemble and ensemble_dict.get("meta") is not None and ensemble_dict.get("scaler") is not None: + # Juger l'effet RÉEL sur la prédiction finale de l'ensemble (et + # non sur le seul sous-modèle LightGBM isolé) : on construit un + # ensemble candidat avec le lgbm mis à jour, et on compare ses + # probabilités à celles de l'ensemble actuel sur le même jeu de + # trades. Délègue à ensemble_core — la même fonction utilisée + # partout ailleurs, aucune divergence possible. + current_probas = _ens_core.predict_ensemble_batch(ensemble_dict, X)[:, 0] + current_acc = float(((current_probas > 0.5) == y).mean()) + + candidate_ensemble = dict(ensemble_dict) + candidate_ensemble["lgbm"] = new_model + new_probas = _ens_core.predict_ensemble_batch(candidate_ensemble, X)[:, 0] + new_acc = float(((new_probas > 0.5) == y).mean()) + else: + current_preds = current_model.predict(X) > 0.5 + current_acc = float((current_preds == y).mean()) + new_preds = new_model.predict(X) > 0.5 + new_acc = float((new_preds == y).mean()) + + # Accepter si accuracy >= actuelle - tolérance + # IMPORTANT : on compare toujours par rapport à la PRODUCTION (target_path), + # pas par rapport au candidat — c'est la production qui est le champion. + prod_acc = current_acc # current_acc vient du modèle chargé (candidat ou prod) + if os.path.exists(candidate_path): + # Re-évaluer l'accuracy de la PRODUCTION pour comparaison juste + try: + with open(target_path, "rb") as _pf: + prod_loaded = pickle.load(_pf) + if use_ensemble: + prod_ens = prod_loaded + prod_lgbm = prod_ens.get("lgbm") + if prod_lgbm is not None: + prod_probas = _ens_core.predict_ensemble_batch(prod_ens, X)[:, 0] + prod_acc = float(((prod_probas > 0.5) == y).mean()) + else: + prod_m = prod_loaded["model"] if isinstance(prod_loaded, dict) else prod_loaded + prod_acc = float(((prod_m.predict(X) > 0.5) == y).mean()) + except Exception: + pass # si lecture prod échoue, on garde current_acc comme référence + + min_acceptable = prod_acc - ACCURACY_TOLERANCE + if new_acc >= min_acceptable: + # ✅ Nouveau modèle meilleur → remplace la production + backup = target_path + ".warmstart_backup" + shutil.copy2(target_path, backup) + if use_ensemble: + ensemble_dict["lgbm"] = new_model + with open(target_path, "wb") as _f: + pickle.dump(ensemble_dict, _f) + else: + with open(target_path, "wb") as _f: + pickle.dump(new_model, _f) + # Candidat obsolète — nettoyer + if os.path.exists(candidate_path): + os.remove(candidate_path) + log.info("[WARMSTART] Candidat accumulé promu en production — fichier .candidate supprimé") + log.info( + f"[WARMSTART] ✅ Modèle mis à jour ({target_path}) : " + f"acc {prod_acc:.3f} → {new_acc:.3f} ({new_acc - prod_acc:+.3f})" + ) + return True + else: + # ❌ Pas encore assez bon pour remplacer la production + # [FIX] On sauvegarde quand même les arbres appris en tant que + # candidat. Le prochain cycle repartira de ce candidat et continuera + # d'absorber les mauvais trades — l'apprentissage ne s'arrête jamais. + try: + if use_ensemble: + candidate_dict = dict(ensemble_dict) + candidate_dict["lgbm"] = new_model + with open(candidate_path, "wb") as _cf: + pickle.dump(candidate_dict, _cf) + else: + with open(candidate_path, "wb") as _cf: + pickle.dump(new_model, _cf) + log.info( + f"[WARMSTART] ⏳ Candidat sauvegardé ({candidate_path}) : " + f"acc {new_acc:.3f} vs prod {prod_acc:.3f} — " + f"le prochain cycle repartira de ce candidat" + ) + except Exception as _ce: + log.warning(f"[WARMSTART] Impossible de sauvegarder le candidat : {_ce}") + log.warning( + f"[WARMSTART] ❌ Nouveau modèle rejeté (prod conservée) : " + f"acc {new_acc:.3f} < {min_acceptable:.3f}" + ) + return False + + # ── Mécanisme 2 : Préparation des épisodes RL pour injection PPO ───────── + + def prepare_rl_episodes( + self, + buffer: Optional[ExperienceBuffer] = None, + min_trades: int = 50, + ) -> List[Dict]: + """ + Convertit les trades réels du buffer en épisodes RL. + Format : {"obs": [...67 dims], "action": int, "reward": float, "done": bool} + + Ces épisodes seront injectés dans le replay buffer SB3 lors du fine-tune PPO. + Retourne une liste d'épisodes RL. + """ + buf = buffer or self.buffer + + if not getattr(config, "RL_REAL_REPLAY_ENABLED", True): + log.debug("[RL-REPLAY] Désactivé (RL_REAL_REPLAY_ENABLED=false)") + return [] + + if buf.total_trades < min_trades: + log.info(f"[RL-REPLAY] Seulement {buf.total_trades} trades (min={min_trades}) — skip") + return [] + + # Cibler en priorité les LOSS et TIMEOUT + trades = buf.get_loss_trades(min_weight=1.5) + if len(trades) < 10: + trades = buf.get_all() + + episodes = [] + for trade in trades: + try: + episode = _trade_to_rl_episode(trade) + if episode: + episodes.append(episode) + except Exception as e: + log.debug(f"[RL-REPLAY] Skipping trade {trade.get('id','?')} : {e}") + + log.info(f"[RL-REPLAY] {len(episodes)} épisodes RL préparés depuis {len(trades)} trades") + return episodes + + # ── Rapport ────────────────────────────────────────────────────────────── + + def get_status(self) -> Dict: + """Retourne l'état actuel du learner.""" + return { + "min_confidence": self.current_confidence, + "total_trades": self.buffer.total_trades, + "last_calibration": self._state.get("last_calibration"), + "win_rate_50": self._state.get("win_rate_50", 0.0), + "buffer_stats": self.buffer.stats(window=50), + } + + +# ── Helpers privés ──────────────────────────────────────────────────────────── + +def _build_dataset_from_trades( + trades: List[Dict], +) -> Tuple[Optional[np.ndarray], Optional[np.ndarray], Optional[np.ndarray]]: + """ + Construit (X, y, weights) depuis une liste d'expériences. + X = features (62-dim), y = label (WIN=1, LOSS/TIMEOUT=0), weights = error_weight. + """ + X_list, y_list, w_list = [], [], [] + + for trade in trades: + features = trade.get("features") + if not features or len(features) != NUM_FEATURES: + continue + + outcome = trade.get("outcome", "WIN") + label = 1.0 if outcome == "WIN" else 0.0 + weight = float(trade.get("error_weight", 1.0)) + + X_list.append(features) + y_list.append(label) + w_list.append(weight) + + if len(X_list) < 5: + return None, None, None + + return ( + np.array(X_list, dtype=np.float32), + np.array(y_list, dtype=np.float32), + np.array(w_list, dtype=np.float32), + ) + + +def _trade_to_rl_episode(trade: Dict) -> Optional[Dict]: + """ + Convertit un trade réel en épisode RL au format SB3. + obs = [features(62), in_position, direction, unrealized_pnl, candles_held, + balance_ratio, ens_proba_long, ens_confidence] → 69 dims. + + [BUG CORRIGÉ 21/06/2026] Cette fonction ne produisait avant que 67 dims + (sans les 2 dims "ensemble"), alors que le modèle PPO réellement + entraîné et déployé attend 69 dims dans TOUS les cas : voir + rl_env.py::_get_obs() (obs = feat + pos_feats + ens_feats quand + self._ensemble est actif, ce qui est la config par défaut) et + rl_agent.py::predict() qui construit l'obs de production de façon + identique. Avec seulement 67 dims, _build_real_trade_buffer() + (rl_train.py) aurait filtré TOUS les trades réels pour incompatibilité + de dimensions — silencieusement, comme les bugs précédents. + + Les 2 dims manquantes [ens_proba_long, ens_confidence] sont + reconstruites EXACTEMENT (pas une approximation) à partir des champs + déjà stockés dans chaque trade — confidence et signal — en inversant + la formule utilisée partout ailleurs (ensemble_core.py::predict_ensemble_batch) : + confidence = |proba_long - 0.5| * 2 + donc : + proba_long = 0.5 + confidence/2 si signal == LONG (proba_long >= 0.5) + proba_long = 0.5 - confidence/2 si signal == SHORT (proba_long < 0.5) + """ + features = trade.get("features") + if not features or len(features) != NUM_FEATURES: + return None + + signal = trade.get("signal", "LONG").upper() + pnl = float(trade.get("pnl", 0.0)) + hold = int(trade.get("hold_candles", 1)) + outcome = trade.get("outcome", "WIN") + + # Reconstituer l'observation 69-dim + in_position = 1.0 + direction = 1.0 if signal == "LONG" else -1.0 + unrealized_pnl = float(np.clip(pnl, -1.0, 1.0)) + candles_norm = float(np.clip(hold / 24.0, 0.0, 1.0)) + balance_ratio = 1.0 + float(np.clip(pnl * 10, -0.5, 0.5)) + + # Dims ensemble [proba_long, confidence] — reconstruction exacte (voir + # docstring ci-dessus). confidence par défaut à 0.5 si absent (donne + # proba_long=0.75/0.25 selon le sens — meilleur repli que de planter). + confidence = float(trade.get("confidence", 0.5)) + ens_confidence = float(np.clip(confidence, 0.0, 1.0)) + if signal == "LONG": + ens_proba_long = 0.5 + ens_confidence / 2.0 + else: + ens_proba_long = 0.5 - ens_confidence / 2.0 + + obs = ( + list(features) + + [in_position, direction, unrealized_pnl, candles_norm, balance_ratio] + + [ens_proba_long, ens_confidence] + ) + + # Action RL originale (si disponible), sinon déduite + rl_action = trade.get("rl_action") + if rl_action is None: + rl_action = 1 if signal == "LONG" else 2 # LONG=1, SHORT=2 + + # Récompense : basée sur le résultat réel, clippée comme dans l'entraînement + if outcome == "WIN": + reward = float(np.clip(pnl * 10, 0.1, 1.0)) + elif outcome == "LOSS": + reward = float(np.clip(pnl * 10, -1.0, -0.1)) + else: # TIMEOUT + reward = float(np.clip(pnl * 5, -0.5, 0.5)) + + return { + "obs": obs, + "action": int(rl_action), + "reward": float(np.clip(reward, -1.0, 1.0)), + "done": True, # chaque trade = fin d'épisode + "trade_id": trade.get("id", ""), + } + + +# ── CLI de test ─────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + import logging + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(name)s] %(message)s") + + from experience_buffer import ExperienceBuffer + + # Créer un buffer de test + buf = ExperienceBuffer(max_size=200, buffer_file="test_learner_buffer.json") + + for i in range(60): + outcome = ["WIN", "LOSS", "TIMEOUT"][i % 3] + buf.add({ + "pair": "EURUSD", + "signal": "LONG", + "confidence": 0.75, + "ml_probas": [0.72, 0.73, 0.71, 0.74, 0.72], + "rl_action": 1, + "rl_agreed": True, + "features": list(np.random.randn(NUM_FEATURES).astype(float)), + "regime": "NORMAL", + "entry_price": 1.0850, + "exit_price": 1.0880 if outcome == "WIN" else 1.0820, + "pnl": 0.03 if outcome == "WIN" else -0.02, + "outcome": outcome, + "hold_candles": 4, + }) + + learner = OnlineLearner(buf, adaptive_state_file="test_adaptive_state.json") + + # Test calibration + new_conf = learner.calibrate_threshold(recent_n=50) + print(f"\nNouvelle confidence : {new_conf:.4f}") + + # Test préparation épisodes RL + episodes = learner.prepare_rl_episodes(min_trades=10) + print(f"Épisodes RL préparés : {len(episodes)}") + + # Statut + print(f"\nStatut learner : {json.dumps(learner.get_status(), indent=2)}") + + # Nettoyage + import os + for f in ["test_learner_buffer.json", "test_adaptive_state.json"]: + if os.path.exists(f): + os.remove(f) + + print("\n✅ OnlineLearner — test OK") diff --git a/order_flow_analyzer.py b/order_flow_analyzer.py new file mode 100644 index 0000000..778a52f --- /dev/null +++ b/order_flow_analyzer.py @@ -0,0 +1,67 @@ +""" +AHAD QUANT — Order Flow Analyzer (stub) +Analyse du flux d'ordres / déséquilibre bid-ask. + +Ce module est un placeholder fonctionnel : + - Importable sans erreur + - Retourne des valeurs neutres + - À compléter avec une vraie implémentation si le broker expose un order book. + +Pour OANDA : l'API v20 expose /instruments/{instrument}/orderBook +Pour MT5 : la profondeur de marché est accessible via market_book_get() +""" + +from __future__ import annotations +from typing import Optional + + +class OrderFlowAnalyzer: + """Analyse l'imbalance buy/sell dans le carnet d'ordres.""" + + def __init__(self, depth: int = 20): + self.depth = depth + + def get_imbalance( + self, + bids: list[tuple[float, float]], + asks: list[tuple[float, float]], + ) -> float: + """ + Retourne un score entre -1 (pression vendeuse) et +1 (pression acheteuse). + 0.0 = neutre / données insuffisantes. + """ + if not bids or not asks: + return 0.0 + + bid_vol = sum(v for _, v in bids[: self.depth]) + ask_vol = sum(v for _, v in asks[: self.depth]) + total = bid_vol + ask_vol + + if total == 0: + return 0.0 + + return round((bid_vol - ask_vol) / total, 4) + + def classify(self, imbalance: float) -> str: + """Classe l'imbalance en signal directionnel.""" + if imbalance > 0.25: + return "BUY_PRESSURE" + if imbalance < -0.25: + return "SELL_PRESSURE" + return "NEUTRAL" + + +def get_order_flow_signal( + bids: Optional[list] = None, + asks: Optional[list] = None, +) -> dict: + """ + Point d'entrée rapide utilisable depuis n'importe quel module. + Retourne {"imbalance": float, "signal": str}. + """ + analyzer = OrderFlowAnalyzer() + imbalance = analyzer.get_imbalance(bids or [], asks or []) + return { + "imbalance": imbalance, + "signal": analyzer.classify(imbalance), + } diff --git a/paper_trader.py b/paper_trader.py new file mode 100644 index 0000000..d92f7f8 --- /dev/null +++ b/paper_trader.py @@ -0,0 +1,211 @@ +""" +AHAD QUANT — Paper Trader +Simule les ordres Forex sans argent réel. +Persiste l'état dans paper_state.json pour que web_ui.py puisse le lire. + +Interface attendue par ahad_quant.py : + PaperTrader(initial_balance) + .positions dict { pair: {side, entry, qty, sl, tp, margin, opened_at} } + .daily_pnl float + .total_pnl float + .peak_equity float + .consecutive_losses int + .circuit_breaker_until float (timestamp) + .get_balance() → float + .open_position(pair, side, price, qty, sl_pct, tp_pct) → {"success": bool, ...} + .close_position(pair, price, reason) → {"pnl": float, ...} + .check_sl_tp(pair, price) → "sl" | "tp" | None + .reset_daily_pnl() + .summary() → str +""" + +import json +import os +import time +from datetime import datetime, timezone +from pathlib import Path + +import config + +_STATE_FILE = Path(os.path.dirname(__file__)) / "paper_state.json" + + +class PaperTrader: + """Moteur de paper trading Forex — persistance JSON.""" + + def __init__(self, initial_balance: float = 10_000.0): + self.initial_balance = initial_balance + self._load_state() + + # ── Persistance ──────────────────────────────────────────────────────────── + + def _load_state(self): + """Charge paper_state.json ou initialise un état vierge.""" + try: + with open(_STATE_FILE) as f: + s = json.load(f) + self._balance = float(s.get("balance", self.initial_balance)) + self.positions = s.get("positions", {}) + self._trades = s.get("trades", []) + self.daily_pnl = float(s.get("daily_pnl", 0.0)) + self.total_pnl = float(s.get("total_pnl", 0.0)) + self.peak_equity = float(s.get("peak_equity", self._balance)) + self.daily_losses = int(s.get("daily_losses", 0)) + self.consecutive_losses = int(s.get("consecutive_losses", 0)) + self.circuit_breaker_until = float(s.get("circuit_breaker_until", 0.0)) + except (FileNotFoundError, json.JSONDecodeError, KeyError): + self._balance = self.initial_balance + self.positions = {} + self._trades = [] + self.daily_pnl = 0.0 + self.total_pnl = 0.0 + self.peak_equity = self.initial_balance + self.daily_losses = 0 + self.consecutive_losses = 0 + self.circuit_breaker_until = 0.0 + self._save_state() + + def _save_state(self): + """Écrit paper_state.json — lu par web_ui.py.""" + state = { + "balance": round(self._balance, 5), + "positions": self.positions, + "trades": self._trades[-500:], # garder les 500 derniers + "daily_pnl": round(self.daily_pnl, 5), + "total_pnl": round(self.total_pnl, 5), + "peak_equity": round(self.peak_equity, 5), + "daily_losses": self.daily_losses, + "consecutive_losses": self.consecutive_losses, + "circuit_breaker_until": self.circuit_breaker_until, + "updated_at": datetime.now(timezone.utc).isoformat(), + } + tmp = str(_STATE_FILE) + ".tmp" + with open(tmp, "w") as f: + json.dump(state, f, indent=2) + os.replace(tmp, _STATE_FILE) # écriture atomique + + # ── Interface publique ───────────────────────────────────────────────────── + + def get_balance(self) -> float: + """Équité courante (balance + PnL non réalisé des positions ouvertes).""" + return round(self._balance, 5) + + def open_position( + self, + pair: str, + side: str, # "long" | "short" + price: float, + qty: float, + sl_pct: float = config.STOP_LOSS_PCT, + tp_pct: float = config.TAKE_PROFIT_PCT, + ) -> dict: + """Ouvre une position paper. Retourne {"success": bool, "msg": str}.""" + if pair in self.positions: + return {"success": False, "msg": f"{pair} already open"} + + side = side.lower() + margin = price * qty / config.LEVERAGE + + if margin > self._balance * config.MAX_MARGIN_USAGE: + return {"success": False, "msg": "Insufficient margin"} + + mult = 1 if side == "long" else -1 + sl = round(price * (1 - mult * sl_pct), 6) + tp = round(price * (1 + mult * tp_pct), 6) + + self.positions[pair] = { + "side": side, + "entry": round(price, 6), + "qty": round(qty, 6), + "sl": sl, + "tp": tp, + "margin": round(margin, 5), + "opened_at": time.time(), + } + + self._balance -= margin # réserve la marge + self._save_state() + return {"success": True, "pair": pair, "side": side, + "price": price, "qty": qty, "sl": sl, "tp": tp} + + def close_position(self, pair: str, price: float, reason: str = "manual") -> dict: + """Ferme une position et comptabilise le PnL.""" + pos = self.positions.pop(pair, None) + if not pos: + return {"success": False, "pnl": 0.0, "msg": "No position"} + + mult = 1 if pos["side"] == "long" else -1 + pnl = round(mult * (price - pos["entry"]) * pos["qty"] * config.LEVERAGE, 5) + fee = round(pos["entry"] * pos["qty"] * config.FEE_RATE, 5) + net = round(pnl - fee, 5) + + self._balance += pos["margin"] + net + self.daily_pnl += net + self.total_pnl += net + self.peak_equity = max(self.peak_equity, self._balance) + + if net < 0: + self.consecutive_losses += 1 + self.daily_losses += 1 + if self.daily_losses >= config.CIRCUIT_BREAKER_LOSSES: + self.circuit_breaker_until = time.time() + config.CIRCUIT_BREAKER_COOLDOWN + else: + self.consecutive_losses = 0 + + trade = { + "pair": pair, + "side": pos["side"], + "entry": pos["entry"], + "exit": round(price, 6), + "qty": pos["qty"], + "pnl": net, + "fee": fee, + "reason": reason, + "closed_at": datetime.now(timezone.utc).isoformat(), + } + self._trades.append(trade) + self._save_state() + return {"success": True, "pnl": net, "reason": reason, "trade": trade} + + def check_sl_tp(self, pair: str, price: float): + """Retourne 'sl', 'tp' ou None selon le prix actuel.""" + pos = self.positions.get(pair) + if not pos: + return None + + side = pos["side"] + sl, tp = pos["sl"], pos["tp"] + + if side == "long": + if price <= sl: + return "sl" + if price >= tp: + return "tp" + else: # short + if price >= sl: + return "sl" + if price <= tp: + return "tp" + return None + + def reset_daily_pnl(self): + """Appelé par ahad_quant.py à minuit pour réinitialiser les stats journalières.""" + self.daily_pnl = 0.0 + self.daily_losses = 0 + # Le circuit breaker journalier expire aussi + if self.circuit_breaker_until < time.time(): + self.circuit_breaker_until = 0.0 + self._save_state() + + def summary(self) -> str: + """Résumé texte affiché dans les logs du bot.""" + wins = sum(1 for t in self._trades if t.get("pnl", 0) > 0) + total = len(self._trades) + wr = f"{wins/total*100:.1f}%" if total else "—" + return ( + f"[PAPER] Balance: ${self._balance:,.2f} | " + f"Total PnL: {'+' if self.total_pnl >= 0 else ''}{self.total_pnl:.2f} | " + f"Daily: {'+' if self.daily_pnl >= 0 else ''}{self.daily_pnl:.2f} | " + f"Win rate: {wr} ({total} trades) | " + f"Positions: {len(self.positions)}" + ) diff --git a/performance_monitor.py b/performance_monitor.py new file mode 100644 index 0000000..f3ae9f5 --- /dev/null +++ b/performance_monitor.py @@ -0,0 +1,403 @@ +""" +AHAD QUANT — Performance Monitor (Apprentissage Continu V7) +============================================================== +Surveille la performance en temps réel et détecte quand le modèle dégrade. + +Métriques surveillées (sliding window 50 trades) : + - win_rate → alerte si < 45% + - avg_pnl → alerte si < -0.002 + - max_drawdown → alerte si > 8% + - confidence_gap → alerte si confiance élevée mais pertes (overconfidence) + - timeout_rate → alerte si > 70% (modèle trop incertain) + +Niveaux d'alerte : + OK : tout va bien + WARNING : win_rate < 50% pendant 3 jours → log + notification + DANGER : win_rate < 45% → augmente MIN_CONFIDENCE automatiquement + EMERGENCY : win_rate < 35% OU drawdown > 10% → pause trading + +Drift detection (Page-Hinkley) : + - Détecte un changement de distribution sur le PnL ou le win_rate + - Si dérive détectée → log + recommande retrain complet + +Usage : + from experience_buffer import get_experience_buffer + from performance_monitor import PerformanceMonitor + + monitor = PerformanceMonitor(buffer) + status = monitor.check() # "OK" / "WARNING" / "DANGER" / "EMERGENCY" + report = monitor.get_report() + monitor.log_daily_report() +""" + +import json +import logging +import os +import time +import threading +from datetime import datetime, timezone +from typing import Dict, List, Optional + +import numpy as np + +import config +from experience_buffer import ExperienceBuffer +from features import NUM_FEATURES + +log = logging.getLogger("PerformanceMonitor") + +# ── Constantes ──────────────────────────────────────────────────────────────── +WINDOW_SIZE = 50 # trades pour sliding window +WIN_RATE_WARNING = 0.50 # seuil warning +WIN_RATE_DANGER = 0.45 # seuil danger +WIN_RATE_EMERGENCY = 0.35 # seuil urgence (pause trading) +DRAWDOWN_EMERGENCY = 0.10 # 10% drawdown = urgence +TIMEOUT_RATE_WARNING = 0.70 # 70% timeout = modèle trop incertain +AVG_PNL_WARNING = -0.002 # PnL moyen négatif = warning +MONITOR_INTERVAL_SEC = 3600 # check toutes les heures +WARNING_DAYS_THRESHOLD = 3 # jours consécutifs sous seuil → WARNING envoyé +REPORT_FILE = "performance_report.json" + + +class HealthStatus: + OK = "OK" + WARNING = "WARNING" + DANGER = "DANGER" + EMERGENCY = "EMERGENCY" + + +class PerformanceMonitor: + """ + Surveillance temps réel des métriques de performance du bot. + Détecte le drift et déclenche des alertes ou la pause du trading. + """ + + def __init__( + self, + buffer: ExperienceBuffer, + report_file: str = REPORT_FILE, + notify_fn=None, + ): + self.buffer = buffer + self.report_file = report_file + self.notify_fn = notify_fn + self._lock = threading.RLock() + self._last_status = HealthStatus.OK + self._warning_since: Optional[float] = None # timestamp premier WARNING + self._ph_state = _PageHinkleyState() # détecteur de drift + + # ── Check principal ────────────────────────────────────────────────────── + + def check(self) -> str: + """ + Évalue la santé du système sur les WINDOW_SIZE derniers trades. + Retourne : "OK" / "WARNING" / "DANGER" / "EMERGENCY" + """ + if not getattr(config, "MONITOR_ENABLED", True): + return HealthStatus.OK + + recent = self.buffer.get_last_n(WINDOW_SIZE) + if len(recent) < 10: + return HealthStatus.OK # pas assez de données + + stats = self.buffer.stats(window=WINDOW_SIZE) + status = self._evaluate(stats) + + # Mise à jour du drift detector + if recent: + last_pnl = recent[-1].get("pnl", 0.0) + drift = self._ph_state.update(last_pnl) + if drift: + log.warning("[MONITOR] 🚨 DRIFT DÉTECTÉ (Page-Hinkley) — distribution PnL changée") + self._notify( + "⚠️ *AHAD QUANT DRIFT DÉTECTÉ*\n" + "Distribution du PnL a changé significativement.\n" + "→ Recommande un retrain complet (local ou cloud au choix)." + ) + + # Gestion de l'état WARNING persistant + if status == HealthStatus.WARNING: + if self._warning_since is None: + self._warning_since = time.time() + elif time.time() - self._warning_since >= WARNING_DAYS_THRESHOLD * 86400: + log.warning(f"[MONITOR] WARNING persistant depuis {WARNING_DAYS_THRESHOLD} jours") + self._notify( + f"⚠️ *AHAD QUANT WARNING persistant*\n" + f"win_rate < 50% depuis {WARNING_DAYS_THRESHOLD}+ jours.\n" + f"win_rate actuel : {stats['win_rate']:.1%}\n" + f"→ Vérifier les conditions de marché." + ) + else: + self._warning_since = None # reset si plus en WARNING + + # Actions selon le niveau + if status == HealthStatus.DANGER: + self._handle_danger(stats) + + elif status == HealthStatus.EMERGENCY: + self._handle_emergency(stats) + + self._last_status = status + return status + + def _evaluate(self, stats: Dict) -> str: + """Détermine le niveau de santé selon les métriques.""" + win_rate = stats.get("win_rate", 1.0) + max_dd = stats.get("max_drawdown", 0.0) + timeout_r = stats.get("timeout_rate", 0.0) + avg_pnl = stats.get("avg_pnl", 0.0) + + # Urgence : critères stricts + if win_rate < WIN_RATE_EMERGENCY or max_dd > DRAWDOWN_EMERGENCY: + return HealthStatus.EMERGENCY + + # Danger : performance dégradée + if win_rate < WIN_RATE_DANGER or avg_pnl < AVG_PNL_WARNING: + return HealthStatus.DANGER + + # Warning : signaux précoces + if win_rate < WIN_RATE_WARNING or timeout_r > TIMEOUT_RATE_WARNING: + return HealthStatus.WARNING + + return HealthStatus.OK + + def _handle_danger(self, stats: Dict) -> None: + """DANGER : augmenter MIN_CONFIDENCE pour être plus sélectif.""" + current_conf = getattr(config, "MIN_CONFIDENCE", 0.72) + new_conf = min(current_conf + 0.02, 0.88) + config.MIN_CONFIDENCE = new_conf + + msg = ( + f"🔴 *AHAD QUANT DANGER*\n" + f"win_rate={stats['win_rate']:.1%} | avg_pnl={stats['avg_pnl']:.4f}\n" + f"MIN_CONFIDENCE ajusté : {current_conf:.3f} → {new_conf:.3f}" + ) + log.warning(msg.replace("*","").replace("`","")) + self._notify(msg) + + def _handle_emergency(self, stats: Dict) -> None: + """EMERGENCY : pause du trading si activé.""" + if not getattr(config, "EMERGENCY_PAUSE_ENABLED", True): + return + + msg = ( + f"🚨 *AHAD QUANT EMERGENCY*\n" + f"win_rate={stats['win_rate']:.1%} | " + f"max_dd={stats['max_drawdown']:.1%}\n" + f"→ PAUSE trading activée.\n" + f"→ Relancer un retrain complet (local ou cloud au choix)." + ) + log.critical(msg.replace("*","").replace("`","")) + self._notify(msg) + + # Signaler la demande de pause via config + config.PAPER_MODE = True # Basculer en paper mode comme filet de sécurité + log.critical("[MONITOR] Bot basculé en PAPER MODE d'urgence") + + def should_emergency_retrain(self) -> bool: + """True si un retrain complet d'urgence est recommandé.""" + stats = self.buffer.stats(window=WINDOW_SIZE) + return ( + stats.get("win_rate", 1.0) < WIN_RATE_EMERGENCY + or stats.get("max_drawdown", 0.0) > DRAWDOWN_EMERGENCY + ) + + # ── Rapport ────────────────────────────────────────────────────────────── + + def get_report(self) -> Dict: + """Retourne un rapport complet des métriques actuelles.""" + stats_50 = self.buffer.stats(window=WINDOW_SIZE) + stats_all = self.buffer.stats() + recent = self.buffer.get_last_n(WINDOW_SIZE) + + # Confidence gap : trades avec haute confiance mais résultat LOSS + high_conf_losses = [ + e for e in recent + if e.get("confidence", 0) > 0.80 and e.get("outcome") == "LOSS" + ] + confidence_gap = len(high_conf_losses) / max(len(recent), 1) + + return { + "timestamp": datetime.now(timezone.utc).isoformat(), + "status": self._last_status, + "window": min(len(recent), WINDOW_SIZE), + "metrics_50": stats_50, + "metrics_all": stats_all, + "confidence_gap": round(confidence_gap, 4), + "drift_detected": self._ph_state.drift_detected, + "min_confidence": getattr(config, "MIN_CONFIDENCE", 0.72), + "thresholds": { + "win_rate_warning": WIN_RATE_WARNING, + "win_rate_danger": WIN_RATE_DANGER, + "win_rate_emergency": WIN_RATE_EMERGENCY, + "drawdown_emergency": DRAWDOWN_EMERGENCY, + "timeout_warning": TIMEOUT_RATE_WARNING, + }, + } + + def log_daily_report(self) -> None: + """Sauvegarde le rapport quotidien dans performance_report.json.""" + report = self.get_report() + try: + tmp = self.report_file + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2, ensure_ascii=False) + os.replace(tmp, self.report_file) + + s = report["metrics_50"] + log.info( + f"[MONITOR] Rapport quotidien | status={report['status']} | " + f"win_rate={s['win_rate']:.1%} | avg_pnl={s['avg_pnl']:.4f} | " + f"dd={s['max_drawdown']:.1%} | trades={s['total_trades']}" + ) + except Exception as e: + log.error(f"[MONITOR] Erreur sauvegarde rapport : {e}") + + @property + def last_status(self) -> str: + return self._last_status + + def _notify(self, msg: str) -> None: + if self.notify_fn: + try: + self.notify_fn(msg) + except Exception as e: + log.debug(f"Erreur notification : {e}") + + +# ── Page-Hinkley Drift Detector ─────────────────────────────────────────────── + +class _PageHinkleyState: + """ + Implémentation simple du test de Page-Hinkley pour détecter un changement + de distribution (drift) sur une série de PnL. + + Déclenche si la somme cumulée dépasse un seuil λ (lambda_). + """ + + def __init__(self, delta: float = 0.005, lambda_: float = 0.15, alpha: float = 0.9999): + self.delta = delta # sensibilité au changement + self.lambda_ = lambda_ # seuil de déclenchement + self.alpha = alpha # facteur d'oubli + self._sum = 0.0 + self._min_sum = 0.0 + self._n = 0 + self._mean = 0.0 + self.drift_detected = False + + def update(self, value: float) -> bool: + """ + Mise à jour avec une nouvelle observation. + Retourne True si un drift est détecté. + """ + self._n += 1 + # Mise à jour de la moyenne en ligne (avec oubli) + self._mean = self.alpha * self._mean + (1 - self.alpha) * value + + # Somme cumulée avec biais δ + self._sum += (self._mean - value - self.delta) + self._min_sum = min(self._min_sum, self._sum) + + # Test de Page-Hinkley + if self._n > 30 and (self._sum - self._min_sum) > self.lambda_: + self.drift_detected = True + # Reset après détection + self._sum = 0.0 + self._min_sum = 0.0 + return True + + self.drift_detected = False + return False + + +# ── Background Monitor Thread ───────────────────────────────────────────────── + +class MonitorThread: + """ + Thread en arrière-plan qui appelle monitor.check() toutes les heures + et monitor.log_daily_report() une fois par jour. + """ + + def __init__(self, monitor: PerformanceMonitor): + self.monitor = monitor + self._stop = threading.Event() + self._thread = None + self._last_daily = 0.0 + + def start(self) -> None: + if not getattr(config, "MONITOR_ENABLED", True): + return + self._thread = threading.Thread( + target=self._loop, daemon=True, name="PerformanceMonitor" + ) + self._thread.start() + log.info("[MONITOR] Thread démarré (check toutes les heures)") + + def stop(self) -> None: + self._stop.set() + + def _loop(self) -> None: + while not self._stop.is_set(): + try: + status = self.monitor.check() + log.debug(f"[MONITOR] Status : {status}") + + # Rapport quotidien toutes les 24h + if time.time() - self._last_daily >= 86400: + self.monitor.log_daily_report() + self._last_daily = time.time() + + except Exception as e: + log.error(f"[MONITOR] Erreur loop : {e}") + + self._stop.wait(timeout=MONITOR_INTERVAL_SEC) + + +# ── CLI de test ─────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + import logging + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(name)s] %(message)s") + + from experience_buffer import ExperienceBuffer + import numpy as np + + buf = ExperienceBuffer(max_size=200, buffer_file="test_monitor_buffer.json") + + # Simuler 60 trades avec mauvaise performance + for i in range(60): + outcome = "LOSS" if i % 3 != 0 else "WIN" # 33% win rate → DANGER + buf.add({ + "pair": "EURUSD", + "signal": "LONG", + "confidence": 0.75, + "features": list(np.random.randn(NUM_FEATURES).astype(float)), + "regime": "VOLATILE", + "entry_price": 1.0850, + "exit_price": 1.0820, + "pnl": 0.02 if outcome == "WIN" else -0.015, + "outcome": outcome, + "hold_candles": 4, + }) + + monitor = PerformanceMonitor(buf, report_file="test_performance_report.json") + + status = monitor.check() + print(f"\nStatus : {status}") + + report = monitor.get_report() + print(f"\nRapport :") + print(f" win_rate = {report['metrics_50']['win_rate']:.1%}") + print(f" avg_pnl = {report['metrics_50']['avg_pnl']:.4f}") + print(f" status = {report['status']}") + print(f" drift = {report['drift_detected']}") + + monitor.log_daily_report() + print(f"\nRapport sauvegardé : test_performance_report.json") + + # Nettoyage + for f in ["test_monitor_buffer.json", "test_performance_report.json"]: + if os.path.exists(f): + os.remove(f) + + print("\n✅ PerformanceMonitor — test OK") diff --git a/prepare_sequences.py b/prepare_sequences.py new file mode 100644 index 0000000..21de93a --- /dev/null +++ b/prepare_sequences.py @@ -0,0 +1,238 @@ +""" +AHAD QUANT Forex V5 — Sequence Dataset Preparation +Converts per-pair tabular features → sliding windows (B, SEQ_LEN, 62) for DL models. + +Window i = X[i : i+SEQ_LEN] → sequence input for TFT / TransformerGRU +Label i = y[i+SEQ_LEN-1] → same label as the last step in the window +Tab row i = X[i+SEQ_LEN-1] → tabular features at the end of the window + (used to align tabular models in train.py) + +Returned arrays are ALL aligned on the same M samples, making it trivial to +train tabular and DL models together with consistent train/val/test splits. + +Usage (called from train.py when HAS_DL=True): + X_tab, X_seq, y = prepare_seq_dataset() + # X_tab : (M, 62) — same features as prepare_dataset(), but aligned + # X_seq : (M, 168, 62) — sequence tensors + # y : (M,) — labels +""" + +import json +import os + +import numpy as np + +import config +from features import build_features, NUM_FEATURES + +# ─── Constants ─────────────────────────────────────────────────────────────── + +SEQ_LEN = 168 # 7 days × 24h — window length fed into TFT / TGRU + +# Memory cap: 25 pairs × MAX_SEQ_PER_PAIR × 168 × 62 × 4B ~ 1.5 GB at 6000 +# Increase if you have >16 GB RAM; decrease if training crashes with OOM. +MAX_SEQ_PER_PAIR: int = int(os.environ.get("MAX_SEQ_PER_PAIR", "6000")) +LOOKAHEAD = 3 # must match LOOKAHEAD in train.py +WARMUP = 30 # feature warm-up candles (same as train.py) +MIN_CANDLES = SEQ_LEN + LOOKAHEAD + WARMUP + 10 # minimum pair length + + +# ─── Core builders ─────────────────────────────────────────────────────────── + +def _load_candles(pair: str) -> list | None: + path = os.path.join(config.DATA_DIR, f"{pair}_1h.json") + if not os.path.exists(path): + return None + with open(path) as f: + return json.load(f) + + +def _make_labels(close: np.ndarray, lookahead: int = LOOKAHEAD) -> np.ndarray: + labels = np.zeros(len(close)) + for i in range(len(close) - lookahead): + labels[i] = 1.0 if close[i + lookahead] > close[i] else 0.0 + return labels + + +def build_sequences( + X: np.ndarray, + y: np.ndarray, + seq_len: int = SEQ_LEN, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """ + Convert a single pair's aligned (X, y) arrays into sliding windows. + + Parameters + ---------- + X : (N, 62) feature matrix — already NaN-free and warmed up + y : (N,) label vector + seq_len : window length + + Returns + ------- + X_seq : (M, seq_len, 62) — sequences + X_tab : (M, 62) — tabular row at end of each window (aligned) + y_seq : (M,) — labels aligned with end of each window + + where M = N - seq_len - LOOKAHEAD + (last LOOKAHEAD samples excluded to avoid future-leakage in labels) + """ + N = len(X) + # Last valid window ends at index N-LOOKAHEAD-1 + n_windows = N - seq_len - LOOKAHEAD + if n_windows <= 0: + raise ValueError( + f"Not enough samples ({N}) for seq_len={seq_len} + lookahead={LOOKAHEAD}. " + f"Need at least {seq_len + LOOKAHEAD + 1}." + ) + + X_seq = np.empty((n_windows, seq_len, X.shape[1]), dtype=np.float32) + for i in range(n_windows): + X_seq[i] = X[i : i + seq_len] + + # Tabular: last row of each window + X_tab = X[seq_len - 1 : seq_len - 1 + n_windows].astype(np.float32) + + # Label: at the end of the window (same index as X_tab row) + y_seq = y[seq_len - 1 : seq_len - 1 + n_windows].astype(np.float32) + + return X_seq, X_tab, y_seq + + +# ─── Full dataset builder ──────────────────────────────────────────────────── + +def prepare_seq_dataset( + seq_len: int = SEQ_LEN, + verbose: bool = True, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """ + Build the full aligned (X_tab, X_seq, y) dataset across all pairs. + + Mirrors prepare_dataset() in train.py but: + 1. Uses a sliding window of `seq_len` candles + 2. Returns BOTH tabular (aligned) AND sequence data + 3. All three output arrays share the same index space → consistent splits + + Returns + ------- + X_tab : (M_total, 62) — tabular features at end of each window + X_seq : (M_total, seq_len, 62) — sequence windows + y : (M_total,) — binary labels + """ + all_X_tab: list[np.ndarray] = [] + all_X_seq: list[np.ndarray] = [] + all_y: list[np.ndarray] = [] + + # Reference pair for correlation feature (same logic as train.py) + _ref = "EURUSD" + ref_data = _load_candles(_ref) + ref_close = np.array([c["c"] for c in ref_data]) if ref_data else None + + for pair in config.COINS: + data = _load_candles(pair) + if data is None or len(data) < MIN_CANDLES: + if verbose: + print(f" [{pair:8s}] skipped — insufficient data ({len(data) if data else 0} candles)") + continue + + close = np.array([c["c"] for c in data]) + pair_ref = ( + ref_close[-len(close) :] + if ref_close is not None and len(ref_close) >= len(close) + else None + ) + + X_all = build_features(data, btc_closes=pair_ref) + y_all = _make_labels(close) + + # Apply same warm-up crop as train.py + X_all = X_all[WARMUP : -LOOKAHEAD] + y_all = y_all[WARMUP : -LOOKAHEAD] + + # Drop NaN rows (should be none after warm-up, but safety check) + valid = ~np.isnan(X_all).any(axis=1) + X_all, y_all = X_all[valid], y_all[valid] + + if len(X_all) < MIN_CANDLES: + if verbose: + print(f" [{pair:8s}] skipped after cleaning ({len(X_all)} samples left)") + continue + + try: + X_seq_pair, X_tab_pair, y_seq_pair = build_sequences(X_all, y_all, seq_len) + except ValueError as e: + if verbose: + print(f" [{pair:8s}] skipped — {e}") + continue + + # ── Memory cap: stratified sample to MAX_SEQ_PER_PAIR ────────── + n_pair = len(y_seq_pair) + if n_pair > MAX_SEQ_PER_PAIR: + rng = np.random.default_rng(seed=42) + # Preserve class balance (stratified) + idx_pos = np.where(y_seq_pair == 1)[0] + idx_neg = np.where(y_seq_pair == 0)[0] + half = MAX_SEQ_PER_PAIR // 2 + sel_pos = rng.choice(idx_pos, size=min(half, len(idx_pos)), replace=False) + sel_neg = rng.choice(idx_neg, size=min(half, len(idx_neg)), replace=False) + sel = np.sort(np.concatenate([sel_pos, sel_neg])) + X_seq_pair = X_seq_pair[sel] + X_tab_pair = X_tab_pair[sel] + y_seq_pair = y_seq_pair[sel] + if verbose: + print(f" → sampled {len(y_seq_pair):,} / {n_pair:,} sequences (RAM cap)") + + all_X_tab.append(X_tab_pair) + all_X_seq.append(X_seq_pair) + all_y.append(y_seq_pair) + + if verbose: + print(f" [{pair:8s}] {len(y_seq_pair):>7,} sequences") + + if not all_X_tab: + raise RuntimeError( + "No pairs produced sequences. Check DATA_DIR and MIN_CANDLES." + ) + + X_tab = np.concatenate(all_X_tab, axis=0) + X_seq = np.concatenate(all_X_seq, axis=0) + y = np.concatenate(all_y, axis=0) + + if verbose: + print(f"\n Total : {len(y):,} sequences | shape X_seq={X_seq.shape} | " + f"{y.mean():.2%} long labels") + + return X_tab, X_seq, y + + +# ─── Sequence normalisation helpers ───────────────────────────────────────── + +def fit_seq_scaler(X_tab: np.ndarray): + """ + Fit a StandardScaler on tabular training data. + The same scaler is applied to sequences by reshaping (B, T, F) → (B*T, F). + Returns a fitted sklearn StandardScaler. + """ + from sklearn.preprocessing import StandardScaler + scaler = StandardScaler() + scaler.fit(X_tab) + return scaler + + +def transform_sequences(scaler, X_seq: np.ndarray) -> np.ndarray: + """ + Apply a fitted StandardScaler to a 3-D sequence array. + + Parameters + ---------- + scaler : fitted sklearn StandardScaler + X_seq : (B, T, F) float32 + + Returns + ------- + X_norm : (B, T, F) float32 + """ + B, T, F = X_seq.shape + flat = X_seq.reshape(-1, F) + normed = scaler.transform(flat).astype(np.float32) + return normed.reshape(B, T, F) diff --git a/publish_to_github.sh b/publish_to_github.sh new file mode 100644 index 0000000..71c459c --- /dev/null +++ b/publish_to_github.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -e + +echo "============================================" +echo " AHAD QUANT - Publication automatique sur GitHub" +echo "============================================" +echo + +cd "$(dirname "$0")" + +# ── 1. Verifier que Git est installe ───────────────────────────────────── +if ! command -v git &> /dev/null; then + echo "[ERREUR] Git n'est pas installe." + echo "macOS : installez Xcode Command Line Tools (xcode-select --install)" + echo " ou via Homebrew : brew install git" + echo "Linux : sudo apt install git (ou l'equivalent de votre distro)" + exit 1 +fi +echo "[OK] Git est installe." +echo + +# ── 2. Initialiser le depot si besoin ──────────────────────────────────── +if [ ! -d ".git" ]; then + echo "Initialisation du depot Git..." + git init +else + echo "Depot Git deja initialise." +fi +echo + +# ── 3. Identite git locale (si pas deja configuree) ────────────────────── +git config user.name >/dev/null 2>&1 || git config user.name "Abdoul Ahad Binizi" +git config user.email >/dev/null 2>&1 || git config user.email "abdoulahadbinizi+ahadquant@gmail.com" + +# ── 4. Ajouter tous les fichiers ───────────────────────────────────────── +echo "Ajout des fichiers du projet..." +git add . +echo + +# ── 5. Commit (seulement s'il y a des changements) ─────────────────────── +if ! git diff --cached --quiet; then + git commit -m "Initial commit - AHAD QUANT v1" +else + echo "Rien de nouveau a committer." +fi +echo + +# ── 6. Branche principale ──────────────────────────────────────────────── +git branch -M main + +# ── 7. Configurer le remote GitHub ─────────────────────────────────────── +if git remote get-url origin >/dev/null 2>&1; then + git remote set-url origin https://github.com/AhadQuant/ahad-quant.git +else + git remote add origin https://github.com/AhadQuant/ahad-quant.git +fi + +# ── 8. Push vers GitHub ─────────────────────────────────────────────────── +echo "Envoi vers GitHub..." +echo "Une fenetre de connexion (navigateur) va probablement s'ouvrir." +echo "Connectez-vous avec le compte AhadQuant si elle apparait." +echo +git push -u origin main + +echo +echo "============================================" +echo " TERMINE" +echo " Verifiez ici : https://github.com/AhadQuant/ahad-quant" +echo "============================================" diff --git a/pump_scanner.py b/pump_scanner.py new file mode 100644 index 0000000..c93b19a --- /dev/null +++ b/pump_scanner.py @@ -0,0 +1,1351 @@ +""" +AHAD QUANT — Session Breakout Scanner (Forex Edition) +Detecte les breakouts sur les paires Forex basés sur +des pics de volume tick et de momentum prix pendant les sessions actives. +Toutes les formules volume/momentum fonctionnent sur des données OHLCV Forex. +""" + +import json +import logging +import os +import threading +import time +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Optional + +import numpy as np +import requests + +logger = logging.getLogger("pump_scanner") + +# ═══════════════════════════════════════════════════════════════════════════ +# CONFIGURATION — Pump Scanner Settings +# ═══════════════════════════════════════════════════════════════════════════ + +# --- Detection thresholds --- +SCAN_INTERVAL_SEC = int(os.getenv("PUMP_SCAN_INTERVAL", "3")) # scan every 3s +VOLUME_SPIKE_MULT = float(os.getenv("PUMP_VOL_SPIKE_MULT", "5.0")) # 5x normal volume +PRICE_SPIKE_PCT = float(os.getenv("PUMP_PRICE_SPIKE_PCT", "0.03")) # 3% move in window +PRICE_WINDOW_CANDLES = int(os.getenv("PUMP_PRICE_WINDOW", "5")) # 5x 1m candles = 5min +EWMA_SPAN = int(os.getenv("PUMP_EWMA_SPAN", "20")) # 20-period EWMA for baseline volume +MIN_TICK_VOLUME = float(os.getenv("PUMP_MIN_TICK_VOL", "500")) # ignore paires illiquides (tick volume Forex) + +# --- Fakeout filter --- +CONFIRM_CANDLES = int(os.getenv("PUMP_CONFIRM_CANDLES", "3")) # need 3 consecutive up candles +MIN_RSI_ENTRY = float(os.getenv("PUMP_MIN_RSI_ENTRY", "60")) # RSI must be > 60 (momentum) +MAX_RSI_ENTRY = float(os.getenv("PUMP_MAX_RSI_ENTRY", "85")) # RSI < 85 (not already exhausted) +MIN_BUY_RATIO = float(os.getenv("PUMP_MIN_BUY_RATIO", "0.65")) # 65%+ buy-side taker volume + +# --- Position management --- +PUMP_LEVERAGE = int(os.getenv("PUMP_LEVERAGE", "5")) +PUMP_RISK_BUDGET_PCT = float(os.getenv("PUMP_RISK_BUDGET_PCT", "0.05")) # 5% of equity for pump trades +PUMP_MAX_POSITIONS = int(os.getenv("PUMP_MAX_POSITIONS", "2")) +PUMP_SL_ATR_MULT = float(os.getenv("PUMP_SL_ATR_MULT", "1.5")) # SL = 1.5x ATR below entry +PUMP_TP1_PCT = float(os.getenv("PUMP_TP1_PCT", "0.05")) # +5% take 40% +PUMP_TP2_PCT = float(os.getenv("PUMP_TP2_PCT", "0.10")) # +10% take 30% +PUMP_TP3_PCT = float(os.getenv("PUMP_TP3_PCT", "0.20")) # +20% take remaining 30% +PUMP_TRAILING_PCT = float(os.getenv("PUMP_TRAILING_PCT", "0.03")) # 3% trailing after TP2 + +# --- Dump short settings --- +SHORT_RSI_THRESHOLD = float(os.getenv("PUMP_SHORT_RSI", "80")) # RSI > 80 = overbought +SHORT_VOL_DECLINE_PCT = float(os.getenv("PUMP_SHORT_VOL_DECLINE", "0.40")) # volume drops 40% +SHORT_SWAP_EXTREME = float(os.getenv("PUMP_SHORT_SWAP", "0.0003")) # swap > 0.03%/jour = extrême (Forex overnight) +SHORT_SL_ATR_MULT = float(os.getenv("PUMP_SHORT_SL_ATR", "2.0")) +SHORT_TP_PCT = float(os.getenv("PUMP_SHORT_TP_PCT", "0.05")) # 5% TP on short + +# --- New listing detection --- +LISTING_CHECK_INTERVAL = int(os.getenv("PUMP_LISTING_CHECK", "30")) # check every 30s +LISTING_BUY_DELAY_SEC = int(os.getenv("PUMP_LISTING_DELAY", "5")) # wait 5s after detection +LISTING_RISK_PCT = float(os.getenv("PUMP_LISTING_RISK", "0.02")) # 2% equity per listing trade + +# --- Cooldown per coin (avoid re-entering same pump) --- +PUMP_COOLDOWN_SEC = int(os.getenv("PUMP_COOLDOWN_SEC", "1800")) # 30 min cooldown + +# --- Daily loss circuit breaker --- +MAX_DAILY_PUMP_LOSS = float(os.getenv("PUMP_MAX_DAILY_LOSS", "-50")) # stop after $50 daily loss + +# --- Minimum notional (OANDA / Forex minimum ~1000 USD) --- +MIN_NOTIONAL = float(os.getenv("PUMP_MIN_NOTIONAL", "1000.0")) + +# --- Market cache TTL --- +MARKET_CACHE_TTL_SEC = int(os.getenv("PUMP_MARKET_CACHE_TTL", "300")) # 5 minutes + + +# ═══════════════════════════════════════════════════════════════════════════ +# DATA STRUCTURES +# ═══════════════════════════════════════════════════════════════════════════ + +@dataclass +class PumpSignal: + """Detected pump event.""" + coin: str + signal_type: str # "breakout_long", "exhaustion_short" + detected_at: float # timestamp + price_at_detection: float + volume_ratio: float # current vol / baseline vol + rsi: float + atr: float + confidence: float # 0-1 composite score + metadata: dict = field(default_factory=dict) + + +@dataclass +class PumpPosition: + """Active pump trade being managed.""" + coin: str + side: str # "long" or "short" + entry_price: float + quantity: float + original_quantity: float + stop_loss: float + tp1: float + tp2: float + tp3: float + trailing_active: bool = False + trailing_high: float = 0.0 + tp1_hit: bool = False + tp2_hit: bool = False + tp3_hit: bool = False + opened_at: float = 0.0 + + +# ═══════════════════════════════════════════════════════════════════════════ +# CORE SCANNER CLASS +# ═══════════════════════════════════════════════════════════════════════════ + +class PumpScanner: + """ + Moteur de détection de breakout en temps réel (Forex). + + Tourne en thread de fond et : + 1. Récupère les tickers 1m pour toutes les paires Forex actives + 2. Calcule des scores d'anomalie volume/prix + 3. Filtre les faux signaux via RSI, ratio acheteurs, bougies consécutives + 4. Ouvre long sur breakout confirmé, short sur épuisement + 5. Gère les positions avec TP partiel + trailing stop + """ + + def __init__(self, client=None): + """ + Parameters + ---------- + client : ccxt exchange client (optionnel — None en mode MT5/yfinance) + """ + self.client = client # None en mode MT5 (yfinance utilisé à la place) + self._running = False + self._thread: Optional[threading.Thread] = None + + # Thread lock for shared state (Fix #5) + self._lock = threading.Lock() + + # Paires exclues du scanner (exotiques illiquides ou spread trop élevé) + self._blacklist: set[str] = set(os.getenv("SCANNER_BLACKLIST", "").split(",")) + + # Pas de "new listing" en Forex — dictionnaire conservé pour compat éventuelle + self._seen_listings: set[str] = set() + + # State + self.volume_baselines: dict[str, list[float]] = defaultdict(list) # pair -> rolling volumes + self.price_history: dict[str, list[float]] = defaultdict(list) # pair -> recent closes + self.pump_positions: dict[str, PumpPosition] = {} # pair -> active position + self.cooldowns: dict[str, float] = {} # pair -> cooldown_until timestamp + self.known_listings: set[str] = set() # non utilisé en Forex + self._daily_pump_pnl: float = 0.0 + self._daily_pump_pnl_date: str = "" + + # Non-blocking pending action (conservé pour compatibilité) + self._pending_listing: Optional[dict] = None + + # Market cache (Fix #7) + self._markets_last_loaded: float = 0.0 + + # Toutes les paires Forex actives (chargées depuis config.PAIRS) + self._all_symbols: list[str] = [] + + # ─── Lifecycle ────────────────────────────────────────────────────── + + def _safe_fetch_ohlcv(self, symbol, timeframe="1m", limit=30, *args, **kwargs): + """Fetch OHLCV — utilise yfinance si client=None (mode MT5).""" + if self.client is None: + try: + import yfinance as yf + interval_map = {"1m": "1m", "5m": "5m", "1h": "1h", "1d": "1d"} + yf_interval = interval_map.get(timeframe, "1m") + period = "1d" if timeframe in ("1m", "5m") else "7d" + df = yf.Ticker(f"{symbol}=X").history(period=period, interval=yf_interval, + auto_adjust=True, prepost=False) + if df is None or df.empty: + return [] + result = [] + for ts, row in df.iterrows(): + result.append([ + int(ts.timestamp() * 1000), + float(row["Open"]), float(row["High"]), + float(row["Low"]), float(row["Close"]), + float(row.get("Volume", 0)), + ]) + return result[-limit:] + except Exception: + return [] + try: + return self.client.fetch_ohlcv(symbol, timeframe, limit=limit, *args, **kwargs) + except Exception: + return [] + + def start(self): + """Start the pump scanner in a background thread.""" + if self._running: + logger.warning("PumpScanner already running") + return + self._running = True + self._load_all_symbols() + self._thread = threading.Thread(target=self._main_loop, daemon=True, name="PumpScanner") + self._thread.start() + logger.info(f"SessionScanner démarré — surveillance de {len(self._all_symbols)} paires Forex") + self._alert(f"SESSION SCANNER STARTED - monitoring {len(self._all_symbols)} Forex pairs") + + def stop(self): + """Stop the scanner.""" + self._running = False + if self._thread: + self._thread.join(timeout=10) + logger.info("PumpScanner stopped") + + # ─── Symbol loading ──────────────────────────────────────────────── + + def _load_all_symbols(self): + """Charge les paires Forex actives depuis config.PAIRS.""" + try: + import config as _cfg + self._all_symbols = list(_cfg.PAIRS) + except Exception: + self._all_symbols = [ + "EURUSD", "GBPUSD", "USDJPY", "USDCHF", "AUDUSD", "NZDUSD", + "USDCAD", "EURGBP", "EURJPY", "GBPJPY", + ] + logger.info(f"Chargé {len(self._all_symbols)} paires Forex") + + # ─── Main loop ──────────────────────────────────────────────────── + + def _main_loop(self): + """Main scanning loop.""" + listing_check_last = 0 + + while self._running: + try: + loop_start = time.time() + + # 0. Reset daily PnL at midnight UTC (Fix #9) + self._maybe_reset_daily_pnl() + + # Note: la détection de "new listing" est désactivée en Forex + # (aucune paire FX ne se "liste" comme un token crypto) + # listing_check_last conservé pour compatibilité uniquement + + # 2. Fetch all tickers in one call (efficient) + tickers = self._fetch_all_tickers() + if not tickers: + time.sleep(SCAN_INTERVAL_SEC) + continue + + # 3. Scan for pump signals + signals = self._scan_for_pumps(tickers) + + # 4. Execute on confirmed signals + for signal in signals: + self._execute_signal(signal) + + # 5. Manage open pump positions + self._manage_positions(tickers) + + # 6. Sleep remaining interval + elapsed = time.time() - loop_start + sleep_time = max(0.1, SCAN_INTERVAL_SEC - elapsed) + time.sleep(sleep_time) + + except Exception as e: + logger.error(f"PumpScanner main loop error: {e}", exc_info=True) + time.sleep(5) + + def _maybe_reset_daily_pnl(self): + """Reset daily PnL at midnight UTC (Fix #9).""" + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + if self._daily_pump_pnl_date != today: + if self._daily_pump_pnl_date: + logger.info(f"Daily pump PnL reset (was ${self._daily_pump_pnl:.2f})") + self._daily_pump_pnl = 0.0 + self._daily_pump_pnl_date = today + + # ═══════════════════════════════════════════════════════════════════ + # A. PUMP DETECTION + # ═══════════════════════════════════════════════════════════════════ + + def _fetch_all_tickers(self) -> dict: + """Fetch tickers for all symbols in one API call. Returns {} in MT5/yfinance mode.""" + if self.client is None: + return {} # MT5 mode: tickers not used — _safe_fetch_ohlcv handles data + try: + tickers = self.client.fetch_tickers() + return tickers + except Exception as e: + logger.error(f"Failed to fetch tickers: {e}") + return {} + + def _scan_for_pumps(self, tickers: dict) -> list[PumpSignal]: + """ + Scan all tickers for pump anomalies. + + Detection algorithm (Fix #6 — per-candle volume detection): + 1. Pre-filter: only coins with >10% 24h change from ticker + 2. For those coins, fetch 1m OHLCV (20 candles) + 3. Compare LAST candle volume to average of previous 19 + 4. If volume_ratio > VOLUME_SPIKE_MULT AND price_change > PRICE_SPIKE_PCT: + -> potential pump detected + 5. Validate with RSI, consecutive candles, buy ratio + """ + signals: list[PumpSignal] = [] + now = time.time() + + for symbol, ticker in tickers.items(): + # Only USDT linear perps + if symbol not in self._all_symbols: + continue + + coin = symbol.split("/")[0] + + # Skip if on cooldown + with self._lock: + if coin in self.cooldowns and now < self.cooldowns[coin]: + continue + # Skip if already in a pump position + if coin in self.pump_positions: + continue + + try: + last_price = float(ticker.get("last", 0)) + quote_volume = float(ticker.get("quoteVolume", 0) or 0) + change_pct = float(ticker.get("percentage", 0) or 0) / 100 # convert to decimal + + if last_price <= 0 or quote_volume < MIN_DOLLAR_VOLUME: + continue + + # Update price history + self.price_history[coin].append(last_price) + if len(self.price_history[coin]) > 100: + self.price_history[coin] = self.price_history[coin][-100:] + + # Fix #6: Only fetch candle data for coins showing >10% 24h change + if abs(change_pct) < 0.10: + continue + + # Fetch 1m OHLCV for per-candle volume detection + try: + candles = self._safe_fetch_ohlcv(symbol, "1m", limit=20) + except Exception: + continue + if not candles or len(candles) < 5: + continue + + candle_volumes = [c[5] for c in candles] + last_candle_vol = candle_volumes[-1] + prev_avg_vol = np.mean(candle_volumes[:-1]) if len(candle_volumes) > 1 else 1.0 + + volume_ratio = last_candle_vol / max(prev_avg_vol, 1e-9) + + # Store per-candle volume baseline for reference + with self._lock: + self.volume_baselines[coin] = candle_volumes + + # Price change over window + prices = self.price_history[coin] + if len(prices) >= PRICE_WINDOW_CANDLES: + price_change = (prices[-1] - prices[-PRICE_WINDOW_CANDLES]) / prices[-PRICE_WINDOW_CANDLES] + else: + price_change = change_pct + + # ── PRIMARY DETECTION: volume spike + price spike ── + if volume_ratio >= VOLUME_SPIKE_MULT and price_change >= PRICE_SPIKE_PCT: + # Validate with RSI, consecutive candles, buy ratio + signal = self._validate_pump(coin, symbol, last_price, volume_ratio, price_change) + if signal: + signals.append(signal) + + # ── DUMP SHORT DETECTION: after a pump, detect exhaustion ── + elif (volume_ratio >= VOLUME_SPIKE_MULT * 0.5 and + price_change >= PRICE_SPIKE_PCT * 2 and + len(prices) >= 20): + signal = self._check_dump_short(coin, symbol, last_price, volume_ratio) + if signal: + signals.append(signal) + + except Exception as e: + logger.debug(f"Error scanning {symbol}: {e}") + continue + + return signals + + def _validate_pump(self, coin: str, symbol: str, price: float, + volume_ratio: float, price_change: float) -> Optional[PumpSignal]: + """ + Validate a potential pump signal using 1m candle data. + + Checks: + 1. Consecutive green candles (CONFIRM_CANDLES) + 2. RSI in the sweet spot (60-85) + 3. Buy-side volume dominance + 4. ATR for stop-loss calculation + """ + try: + candles = self._safe_fetch_ohlcv(symbol, "1m", limit=30) + if not candles or len(candles) < 15: + return None + + closes = [c[4] for c in candles] + opens = [c[1] for c in candles] + highs = [c[2] for c in candles] + lows = [c[3] for c in candles] + volumes = [c[5] for c in candles] + + # 1. Consecutive green candles check + green_count = 0 + for i in range(-1, -CONFIRM_CANDLES - 1, -1): + if closes[i] > opens[i]: + green_count += 1 + if green_count < CONFIRM_CANDLES: + return None + + # 2. RSI calculation (14-period) + rsi = self._calc_rsi(closes, period=14) + if rsi < MIN_RSI_ENTRY or rsi > MAX_RSI_ENTRY: + return None + + # 3. Buy ratio: fraction of volume on green candles in last 5 + green_vol = sum(volumes[i] for i in range(-5, 0) if closes[i] > opens[i]) + total_vol = sum(volumes[-5:]) + buy_ratio = green_vol / max(total_vol, 1e-9) + if buy_ratio < MIN_BUY_RATIO: + return None + + # 4. ATR calculation (14-period) + atr = self._calc_atr(highs, lows, closes, period=14) + + # 5. Confidence scoring (0-1) + conf_vol = min(volume_ratio / (VOLUME_SPIKE_MULT * 2), 1.0) # higher vol = better + conf_price = min(price_change / (PRICE_SPIKE_PCT * 3), 1.0) # bigger move = better + conf_rsi = 1.0 - abs(rsi - 70) / 30 # sweet spot around 70 + conf_buy = min(buy_ratio / 0.8, 1.0) # higher buy ratio = better + confidence = (conf_vol * 0.3 + conf_price * 0.3 + + conf_rsi * 0.2 + conf_buy * 0.2) + + if confidence < 0.5: + return None + + logger.info( + f"PUMP DETECTED: {coin} | vol_ratio={volume_ratio:.1f}x | " + f"price_change={price_change*100:.1f}% | RSI={rsi:.0f} | " + f"buy_ratio={buy_ratio:.0%} | confidence={confidence:.2f}" + ) + + return PumpSignal( + coin=coin, + signal_type="breakout_long", + detected_at=time.time(), + price_at_detection=price, + volume_ratio=volume_ratio, + rsi=rsi, + atr=atr, + confidence=confidence, + metadata={ + "price_change": price_change, + "buy_ratio": buy_ratio, + "green_candles": green_count, + }, + ) + + except Exception as e: + logger.error(f"Pump validation failed for {coin}: {e}") + return None + + def _check_dump_short(self, coin: str, symbol: str, price: float, + volume_ratio: float) -> Optional[PumpSignal]: + """ + Detect pump exhaustion for a short entry. + + Exhaustion signals: + 1. RSI > 80 (overbought) + 2. Volume declining from peak (bearish divergence) + 3. Funding rate extreme (> 0.1%) + 4. Long upper wicks on recent candles (rejection) + """ + try: + candles = self._safe_fetch_ohlcv(symbol, "1m", limit=30) + if not candles or len(candles) < 20: + return None + + closes = [c[4] for c in candles] + opens = [c[1] for c in candles] + highs = [c[2] for c in candles] + lows = [c[3] for c in candles] + volumes = [c[5] for c in candles] + + # 1. RSI must be overbought + rsi = self._calc_rsi(closes, period=14) + if rsi < SHORT_RSI_THRESHOLD: + return None + + # 2. Volume declining: compare last 5 candles avg vs peak 5 candles + recent_vol = np.mean(volumes[-5:]) + peak_vol = np.max([np.mean(volumes[i:i+5]) for i in range(len(volumes)-10, len(volumes)-5)]) + vol_decline = 1 - (recent_vol / max(peak_vol, 1e-9)) + if vol_decline < SHORT_VOL_DECLINE_PCT: + return None + + # 3. Check swap rate (overnight rollover en Forex) + try: + funding = self.client.get_funding_rate(symbol) if hasattr(self.client, "get_funding_rate") else 0.0 + except Exception: + funding = 0.0 + + # 4. Upper wick ratio on last 3 candles (rejection signal) + wick_scores = [] + for i in range(-3, 0): + body = abs(closes[i] - opens[i]) + upper_wick = highs[i] - max(closes[i], opens[i]) + total_range = highs[i] - lows[i] + if total_range > 0: + wick_scores.append(upper_wick / total_range) + avg_wick = np.mean(wick_scores) if wick_scores else 0 + + # Composite exhaustion score + score_rsi = min((rsi - 75) / 20, 1.0) + score_vol = min(vol_decline / 0.6, 1.0) + score_funding = min(abs(funding) / SHORT_SWAP_EXTREME, 1.0) if funding != 0 else 0 + score_wick = min(avg_wick / 0.5, 1.0) + + confidence = (score_rsi * 0.3 + score_vol * 0.3 + + score_funding * 0.2 + score_wick * 0.2) + + if confidence < 0.5: + return None + + atr = self._calc_atr(highs, lows, closes, period=14) + + logger.info( + f"EXHAUSTION SHORT SIGNAL: {coin} | RSI={rsi:.0f} | " + f"vol_decline={vol_decline:.0%} | swap={funding:.5f} | " + f"wick_ratio={avg_wick:.2f} | confidence={confidence:.2f}" + ) + + return PumpSignal( + coin=coin, + signal_type="exhaustion_short", + detected_at=time.time(), + price_at_detection=price, + volume_ratio=volume_ratio, + rsi=rsi, + atr=atr, + confidence=confidence, + metadata={ + "vol_decline": vol_decline, + "swap_rate": funding, + "avg_wick": avg_wick, + }, + ) + + except Exception as e: + logger.error(f"Dump short check failed for {coin}: {e}") + return None + + # ═══════════════════════════════════════════════════════════════════ + # B. TRADE EXECUTION + # ═══════════════════════════════════════════════════════════════════ + + def _execute_signal(self, signal: PumpSignal): + """Execute a pump/dump/listing signal.""" + # Check budget + with self._lock: + if len(self.pump_positions) >= PUMP_MAX_POSITIONS: + logger.info(f"Max pump positions reached, skipping {signal.coin}") + return + + # Fix #13: Daily loss circuit breaker + if self._daily_pump_pnl <= MAX_DAILY_PUMP_LOSS: + logger.warning(f"Daily pump loss limit reached (${self._daily_pump_pnl:.2f}), skipping {signal.coin}") + return + + # Vérifier que la paire est dans la liste des paires actives + symbol = f"{signal.coin}/USDT:USDT" + markets = self.client.markets or {} + if symbol not in markets: + try: + self.client.load_markets(True) + markets = self.client.markets or {} + except Exception: + pass + market_info = markets.get(symbol, {}) + is_tradable = ( + symbol in markets + and market_info.get("active", False) + and market_info.get("linear", False) + ) + if not is_tradable: + # Try a test fetch to be sure + try: + self.client.fetch_ticker(symbol) + except Exception: + logger.info(f"No tradable perpetual for {signal.coin}, skipping (24h cooldown)") + with self._lock: + self.cooldowns[signal.coin] = time.time() + 86400 + return + + # Fix #18: Position conflict with main bot + if self._main_bot_has_position(signal.coin): + logger.info(f"Main bot already has position on {signal.coin}, skipping") + return + + try: + equity = self._get_equity() + if equity <= 0: + return + + if signal.signal_type == "pump_long": + self._open_pump_long(signal, equity) + elif signal.signal_type == "dump_short": + self._open_dump_short(signal, equity) + elif signal.signal_type == "breakout_listing_disabled": + self._open_listing_long(signal, equity) + + except Exception as e: + logger.error(f"Failed to execute signal for {signal.coin}: {e}") + # Set cooldown on failure to stop retrying every 3 seconds + with self._lock: + if "not supported" in str(e) or "not allowed" in str(e): + self.cooldowns[signal.coin] = time.time() + 86400 # 24h for unsupported symbols + else: + self.cooldowns[signal.coin] = time.time() + 300 # 5min for other errors + + def _main_bot_has_position(self, coin: str) -> bool: + """Fix #18: Check if the main bot already has a position on this coin.""" + try: + # Check exchange positions directly + symbol = f"{coin}/USDT:USDT" + positions = self.client.fetch_positions([symbol]) + for p in positions: + contracts = float(p.get("contracts", 0) or 0) + if contracts > 0 and p.get("symbol") == symbol: + # Position exists — could be main bot's + with self._lock: + if coin not in self.pump_positions: + # Not ours, must be main bot's + return True + except Exception as e: + logger.debug(f"Position conflict check failed for {coin}: {e}") + return False + + def _open_pump_long(self, signal: PumpSignal, equity: float): + """Open a long position to ride the pump. No-op in MT5/yfinance mode.""" + if self.client is None: + logger.debug("_open_pump_long: skipped (MT5 mode, no ccxt client)") + return + coin = signal.coin + symbol = f"{coin}/USDT:USDT" + price = signal.price_at_detection + atr = signal.atr + + # Position sizing: scale with confidence, capped at PUMP_RISK_BUDGET_PCT + risk_frac = PUMP_RISK_BUDGET_PCT * signal.confidence + notional = equity * risk_frac * PUMP_LEVERAGE + quantity = notional / price + + # Round quantity to exchange precision + quantity = self._round_qty(symbol, quantity) + if quantity <= 0: + return + + # Fix #10: Minimum notional check + if notional < MIN_NOTIONAL: + logger.info(f"Notional ${notional:.2f} below minimum ${MIN_NOTIONAL}, skipping {coin}") + return + + # Set leverage + try: + self.client.set_leverage(PUMP_LEVERAGE, symbol) + except Exception: + pass # may already be set + + # Place market buy + order = self.client.create_market_order(symbol, "buy", quantity) + fill_price = float(order.get("average", price) or price) + + # Calculate SL/TP levels + sl = fill_price - (atr * PUMP_SL_ATR_MULT) + tp1 = fill_price * (1 + PUMP_TP1_PCT) + tp2 = fill_price * (1 + PUMP_TP2_PCT) + tp3 = fill_price * (1 + PUMP_TP3_PCT) + + pos = PumpPosition( + coin=coin, side="long", + entry_price=fill_price, quantity=quantity, + original_quantity=quantity, + stop_loss=sl, tp1=tp1, tp2=tp2, tp3=tp3, + opened_at=time.time(), + ) + with self._lock: + self.pump_positions[coin] = pos + + msg = ( + f"PUMP LONG OPENED: {coin}\n" + f"Entry: ${fill_price:.4f} | Qty: {quantity}\n" + f"SL: ${sl:.4f} | TP1: ${tp1:.4f} | TP2: ${tp2:.4f} | TP3: ${tp3:.4f}\n" + f"Vol ratio: {signal.volume_ratio:.1f}x | RSI: {signal.rsi:.0f} | " + f"Conf: {signal.confidence:.0%}\n" + f"Notional: ${notional:.0f} | Leverage: {PUMP_LEVERAGE}x" + ) + logger.info(msg) + self._alert(msg) + + def _open_dump_short(self, signal: PumpSignal, equity: float): + """Open a short position after pump exhaustion. No-op in MT5/yfinance mode.""" + if self.client is None: + logger.debug("_open_dump_short: skipped (MT5 mode, no ccxt client)") + return + coin = signal.coin + symbol = f"{coin}/USDT:USDT" + price = signal.price_at_detection + atr = signal.atr + + risk_frac = PUMP_RISK_BUDGET_PCT * signal.confidence * 0.7 # smaller size for shorts + notional = equity * risk_frac * PUMP_LEVERAGE + quantity = notional / price + quantity = self._round_qty(symbol, quantity) + if quantity <= 0: + return + + # Fix #10: Minimum notional check + if notional < MIN_NOTIONAL: + logger.info(f"Notional ${notional:.2f} below minimum ${MIN_NOTIONAL}, skipping short {coin}") + return + + try: + self.client.set_leverage(PUMP_LEVERAGE, symbol) + except Exception: + pass + + order = self.client.create_market_order(symbol, "sell", quantity) + fill_price = float(order.get("average", price) or price) + + sl = fill_price + (atr * SHORT_SL_ATR_MULT) + tp1 = fill_price * (1 - SHORT_TP_PCT * 0.5) + tp2 = fill_price * (1 - SHORT_TP_PCT) + tp3 = fill_price * (1 - SHORT_TP_PCT * 1.5) + + pos = PumpPosition( + coin=coin, side="short", + entry_price=fill_price, quantity=quantity, + original_quantity=quantity, + stop_loss=sl, tp1=tp1, tp2=tp2, tp3=tp3, + opened_at=time.time(), + ) + with self._lock: + self.pump_positions[coin] = pos + + msg = ( + f"DUMP SHORT OPENED: {coin}\n" + f"Entry: ${fill_price:.4f} | Qty: {quantity}\n" + f"SL: ${sl:.4f} | TP: ${tp2:.4f}\n" + f"RSI: {signal.rsi:.0f} | Vol decline: {signal.metadata.get('vol_decline', 0):.0%}\n" + f"Funding: {signal.metadata.get('funding', 0):.4%}" + ) + logger.info(msg) + self._alert(msg) + + def _open_listing_long(self, signal: PumpSignal, equity: float): + """Open a long position on a newly listed coin.""" + coin = signal.coin + symbol = f"{coin}/USDT:USDT" + price = signal.price_at_detection + + notional = equity * LISTING_RISK_PCT * PUMP_LEVERAGE + quantity = notional / price + quantity = self._round_qty(symbol, quantity) + if quantity <= 0: + return + + # Fix #10: Minimum notional check + if notional < MIN_NOTIONAL: + logger.info(f"Notional ${notional:.2f} below minimum ${MIN_NOTIONAL}, skipping listing {coin}") + return + + try: + self.client.set_leverage(PUMP_LEVERAGE, symbol) + except Exception: + pass + + order = self.client.create_market_order(symbol, "buy", quantity) + fill_price = float(order.get("average", price) or price) + + # New listings: tight SL (5%), aggressive TP + sl = fill_price * 0.95 + tp1 = fill_price * 1.10 + tp2 = fill_price * 1.25 + tp3 = fill_price * 1.50 + + pos = PumpPosition( + coin=coin, side="long", + entry_price=fill_price, quantity=quantity, + original_quantity=quantity, + stop_loss=sl, tp1=tp1, tp2=tp2, tp3=tp3, + opened_at=time.time(), + ) + with self._lock: + self.pump_positions[coin] = pos + + msg = ( + f"NEW LISTING LONG: {coin}\n" + f"Entry: ${fill_price:.4f} | Qty: {quantity}\n" + f"SL: ${sl:.4f} (-5%) | TP1: ${tp1:.4f} (+10%)" + ) + logger.info(msg) + self._alert(msg) + + # ═══════════════════════════════════════════════════════════════════ + # C. POSITION MANAGEMENT — Partial TP + Trailing Stop + # ═══════════════════════════════════════════════════════════════════ + + def _manage_positions(self, tickers: dict): + """Check all pump positions for SL/TP/trailing.""" + with self._lock: + positions_snapshot = list(self.pump_positions.items()) + + for coin, pos in positions_snapshot: + symbol = f"{coin}/USDT:USDT" + ticker = tickers.get(symbol) + if not ticker: + continue + + current_price = float(ticker.get("last", 0)) + if current_price <= 0: + continue + + is_long = pos.side == "long" + + # ── STOP LOSS ── + if is_long and current_price <= pos.stop_loss: + self._close_pump_position(coin, "SL HIT", current_price) + continue + elif not is_long and current_price >= pos.stop_loss: + self._close_pump_position(coin, "SL HIT", current_price) + continue + + # ── TIME-BASED EXIT: close after 2 hours max ── + if time.time() - pos.opened_at > 7200: + self._close_pump_position(coin, "TIME EXIT (2h)", current_price) + continue + + # ── PARTIAL TAKE PROFITS (long) — Fix #2: separate if blocks ── + if is_long: + if not pos.tp1_hit and current_price >= pos.tp1: + # TP1: close 40% of position + close_qty = self._round_qty(symbol, pos.original_quantity * 0.4) + close_qty = min(close_qty, pos.quantity) # clamp to remaining + if close_qty > 0: + self._partial_close(coin, symbol, close_qty, "TP1 (+5%)") + pos.quantity = max(0, pos.quantity - close_qty) # Fix #3 + pos.tp1_hit = True + # Move SL to breakeven + pos.stop_loss = pos.entry_price * 1.002 # slight profit lock + + if not pos.tp2_hit and pos.tp1_hit and current_price >= pos.tp2: + # TP2: close 30% of position, activate trailing + close_qty = self._round_qty(symbol, pos.original_quantity * 0.3) + close_qty = min(close_qty, pos.quantity) # clamp to remaining + if close_qty > 0: + self._partial_close(coin, symbol, close_qty, "TP2 (+10%)") + pos.quantity = max(0, pos.quantity - close_qty) # Fix #3 + pos.tp2_hit = True + pos.trailing_active = True + pos.trailing_high = current_price + + if not pos.tp3_hit and pos.tp2_hit and current_price >= pos.tp3: + # TP3: close remaining + pos.tp3_hit = True + self._close_pump_position(coin, "TP3 (+20%)", current_price) + continue + + # Trailing stop after TP2 + if pos.trailing_active: + if current_price > pos.trailing_high: + pos.trailing_high = current_price + trailing_sl = pos.trailing_high * (1 - PUMP_TRAILING_PCT) + if current_price <= trailing_sl: + self._close_pump_position(coin, "TRAILING STOP", current_price) + continue + + # ── PARTIAL TAKE PROFITS (short) — Fix #2: separate if blocks ── + else: + if not pos.tp1_hit and current_price <= pos.tp1: + close_qty = self._round_qty(symbol, pos.original_quantity * 0.4) + close_qty = min(close_qty, pos.quantity) # clamp to remaining + if close_qty > 0: + self._partial_close_short(coin, symbol, close_qty, "TP1") + pos.quantity = max(0, pos.quantity - close_qty) # Fix #3 + pos.tp1_hit = True + pos.stop_loss = pos.entry_price * 0.998 + + if not pos.tp2_hit and pos.tp1_hit and current_price <= pos.tp2: + close_qty = self._round_qty(symbol, pos.original_quantity * 0.3) + if close_qty > 0: + self._partial_close_short(coin, symbol, close_qty, "TP2") + pos.quantity = max(0, pos.quantity - close_qty) # Fix #3 + pos.tp2_hit = True + pos.trailing_active = True + pos.trailing_high = current_price # actually trailing low + + if not pos.tp3_hit and pos.tp2_hit and current_price <= pos.tp3: + pos.tp3_hit = True + self._close_pump_position(coin, "TP3", current_price) + continue + + if pos.trailing_active: + if current_price < pos.trailing_high: + pos.trailing_high = current_price + trailing_sl = pos.trailing_high * (1 + PUMP_TRAILING_PCT) + if current_price >= trailing_sl: + self._close_pump_position(coin, "TRAILING STOP", current_price) + continue + + def _partial_close(self, coin: str, symbol: str, qty: float, reason: str): + """Partially close a long position.""" + try: + self.client.create_market_order(symbol, "sell", qty) + logger.info(f"PARTIAL CLOSE {coin} {reason}: sold {qty}") + self._alert(f"PUMP {coin} {reason}: partial close {qty}") + except Exception as e: + logger.error(f"Partial close failed for {coin}: {e}") + + def _partial_close_short(self, coin: str, symbol: str, qty: float, reason: str): + """Partially close a short position.""" + try: + self.client.create_market_order(symbol, "buy", qty) + logger.info(f"PARTIAL CLOSE SHORT {coin} {reason}: bought {qty}") + self._alert(f"DUMP SHORT {coin} {reason}: partial close {qty}") + except Exception as e: + logger.error(f"Partial close short failed for {coin}: {e}") + + def _close_pump_position(self, coin: str, reason: str, exit_price: float): + """Fully close a pump position.""" + with self._lock: + pos = self.pump_positions.get(coin) + if not pos: + return + + symbol = f"{coin}/USDT:USDT" + try: + side = "sell" if pos.side == "long" else "buy" + remaining_qty = self._round_qty(symbol, pos.quantity) + if remaining_qty > 0: + self.client.create_market_order(symbol, side, remaining_qty) + + # PnL includes leverage (notional = qty * price, leveraged) + if pos.side == "long": + pnl = (exit_price - pos.entry_price) * pos.quantity + else: + pnl = (pos.entry_price - exit_price) * pos.quantity + # Note: pnl is already correct since qty was sized with leverage factored in + + pnl_pct = (exit_price / pos.entry_price - 1) * 100 + if pos.side == "short": + pnl_pct = -pnl_pct + + self._daily_pump_pnl += pnl + duration = time.time() - pos.opened_at + + msg = ( + f"PUMP CLOSED: {coin} ({pos.side}) | {reason}\n" + f"Entry: ${pos.entry_price:.4f} -> Exit: ${exit_price:.4f}\n" + f"PnL: ${pnl:.2f} ({pnl_pct:+.1f}%) | Duration: {duration/60:.0f}min\n" + f"Daily pump PnL: ${self._daily_pump_pnl:.2f}" + ) + logger.info(msg) + self._alert(msg) + + # Save trade to file for verification + try: + import json as _json + trade_record = { + "coin": coin, "side": pos.side, + "entry_price": pos.entry_price, "exit_price": exit_price, + "quantity": pos.original_quantity, "pnl": round(pnl, 2), + "pnl_pct": round(pnl_pct, 1), "reason": reason, + "duration_min": round(duration / 60, 1), + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + } + pump_log_path = os.path.join(os.path.dirname(__file__), "pump_trades.json") + try: + with open(pump_log_path, "r") as f: + pump_log = _json.load(f) + except Exception: + pump_log = [] + pump_log.append(trade_record) + if len(pump_log) > 200: + pump_log = pump_log[-200:] + with open(pump_log_path, "w") as f: + _json.dump(pump_log, f, indent=2) + except Exception as _le: + logger.debug(f"Pump trade log save failed: {_le}") + + except Exception as e: + logger.error(f"Close pump position failed for {coin}: {e}") + + # Remove and set cooldown + with self._lock: + self.pump_positions.pop(coin, None) + self.cooldowns[coin] = time.time() + PUMP_COOLDOWN_SEC + + # ═══════════════════════════════════════════════════════════════════ + # D. NEW LISTING DETECTION + # ═══════════════════════════════════════════════════════════════════ + + def _check_new_listings(self): + """ + Check Bybit announcements API for new perpetual listings. + + Uses the official Bybit V5 API endpoint: + GET https://api.bybit.com/v5/announcements/index + """ + try: + # Method 1: Official Bybit announcements API + url = "https://api.bybit.com/v5/announcements/index" + params = { + "locale": "en-US", + "type": "new_crypto", + "limit": 10, + } + resp = requests.get(url, params=params, timeout=10) + if resp.status_code == 200: + data = resp.json() + items = data.get("result", {}).get("list", []) + for item in items: + title = item.get("title", "").upper() + desc = item.get("description", "").upper() + + # Look for "USDT PERPETUAL" in announcement + if "PERPETUAL" in title or "PERP" in title: + # Extract coin symbol from title + # Pattern: "Bybit Lists XXXUSDT Perpetual Contract" + coin = self._extract_coin_from_listing(title) + if coin and coin not in self.known_listings: + self.known_listings.add(coin) + logger.info(f"NEW LISTING DETECTED: {coin}") + self._handle_new_listing(coin) + + # Method 2: Check for new symbols in market data + self._check_new_symbols() + + except Exception as e: + logger.error(f"Listing check failed: {e}") + + def _extract_coin_from_listing(self, title: str) -> Optional[str]: + """Extract coin symbol from listing announcement title.""" + # "Bybit Lists XYZUSDT Perpetual Contract" + import re + match = re.search(r'([A-Z0-9]{2,10})USDT', title) + if match: + return match.group(1) + return None + + def _check_new_symbols(self): + """Check if any new USDT perp symbols appeared on the exchange.""" + try: + # Fix #7: Cache markets for MARKET_CACHE_TTL_SEC instead of reloading every cycle + now = time.time() + if now - self._markets_last_loaded < MARKET_CACHE_TTL_SEC: + return + self.client.load_markets(True) # force reload + self._markets_last_loaded = now + + current_symbols = set() + for sym, info in self.client.markets.items(): + if (info.get("linear") and info.get("active") and + info.get("quote") == "USDT" and info.get("type") == "swap"): + current_symbols.add(sym) + + # Find new symbols + old_symbols = set(self._all_symbols) + new_symbols = current_symbols - old_symbols + + for sym in new_symbols: + coin = sym.split("/")[0] + if coin not in self.known_listings: + self.known_listings.add(coin) + logger.info(f"NEW SYMBOL DETECTED: {coin} ({sym})") + self._handle_new_listing(coin) + + # Update symbol list + self._all_symbols = list(current_symbols) + + except Exception as e: + logger.debug(f"Symbol check failed: {e}") + + + def _check_binance_announcements(self): + """Check Binance for new listing announcements. If a coin lists on Binance, it often pumps on Bybit too.""" + try: + import requests + url = "https://www.binance.com/bapi/composite/v1/public/cms/article/list/query" + params = {"type": 1, "catalogId": 48, "pageNo": 1, "pageSize": 5} + r = requests.get(url, params=params, timeout=10) + if r.status_code == 200: + data = r.json() + articles = data.get("data", {}).get("catalogs", [{}])[0].get("articles", []) + for article in articles[:3]: + title = article.get("title", "").upper() + if "LIST" in title and ("PERPETUAL" in title or "FUTURES" in title): + # Extract coin symbol from title + import re + match = re.search(r"\b([A-Z]{2,10})USDT\b", title) + if match: + coin = match.group(1) + # Check if this coin exists on Bybit + sym = f"{coin}/USDT:USDT" + if sym in (self.client.markets or {}): + if self._tg: + self._tg(f"BINANCE LISTING DETECTED: {coin} - also available on Bybit!") + logger.info(f"[LISTING] Binance listed {coin}, available on Bybit") + except Exception as e: + logger.debug(f"Binance announcement check failed: {e}") + + + def _handle_new_listing(self, coin: str): + """Handle a detected new listing (non-blocking, Fix #4).""" + # Skip blacklisted stock tokens + if coin in self._blacklist: + logger.info(f"Skipping blacklisted stock token: {coin}") + return + # Skip already-seen listings + if coin in self._seen_listings: + return + self._seen_listings.add(coin) + # Persist seen listings + try: + import json as _json + with open(self._seen_listings_file, "w") as f: + _json.dump(list(self._seen_listings), f) + except Exception: + pass + self._alert(f"NEW LISTING DETECTED: {coin} -- preparing to buy in {LISTING_BUY_DELAY_SEC}s") + + # Fix #4: Set pending listing flag instead of blocking with sleep + self._pending_listing = { + "coin": coin, + "time": time.time() + LISTING_BUY_DELAY_SEC, + } + + def _execute_listing_buy(self, coin: str): + """Execute the actual listing buy after the delay has passed (Fix #4).""" + symbol = f"{coin}/USDT:USDT" + try: + # Verify the pair exists and get current price + ticker = self.client.fetch_ticker(symbol) + price = float(ticker.get("last", 0)) + if price <= 0: + logger.warning(f"No price for new listing {coin}") + return + + signal = PumpSignal( + coin=coin, + signal_type="new_listing", + detected_at=time.time(), + price_at_detection=price, + volume_ratio=0, + rsi=50, + atr=price * 0.02, # estimate ATR as 2% of price + confidence=0.7, + metadata={"source": "listing_detection"}, + ) + self._execute_signal(signal) + + except Exception as e: + logger.error(f"Failed to trade new listing {coin}: {e}") + + # ═══════════════════════════════════════════════════════════════════ + # TECHNICAL INDICATORS + # ═══════════════════════════════════════════════════════════════════ + + @staticmethod + def _calc_rsi(closes: list[float], period: int = 14) -> float: + """Calculate RSI from a list of close prices.""" + if len(closes) < period + 1: + return 50.0 + deltas = np.diff(closes[-(period + 1):]) + gains = np.where(deltas > 0, deltas, 0) + losses = np.where(deltas < 0, -deltas, 0) + avg_gain = np.mean(gains) + avg_loss = np.mean(losses) + if avg_loss == 0: + return 100.0 + rs = avg_gain / avg_loss + return 100 - (100 / (1 + rs)) + + @staticmethod + def _calc_atr(highs: list[float], lows: list[float], + closes: list[float], period: int = 14) -> float: + """Calculate ATR from OHLC data.""" + if len(highs) < period + 1: + return 0.0 + trs = [] + for i in range(-period, 0): + tr = max( + highs[i] - lows[i], + abs(highs[i] - closes[i - 1]), + abs(lows[i] - closes[i - 1]), + ) + trs.append(tr) + return np.mean(trs) + + # ═══════════════════════════════════════════════════════════════════ + # UTILITIES + # ═══════════════════════════════════════════════════════════════════ + + def _get_equity(self) -> float: + """Get account equity. Returns 0.0 in MT5/yfinance mode (no ccxt client).""" + if self.client is None: + return 0.0 + try: + bal = self.client.fetch_balance({"type": "contract"}) + return float(bal["total"].get("USDT", 0)) + except Exception: + return 0.0 + + def _round_qty(self, symbol: str, qty: float) -> float: + """Round quantity to exchange-allowed precision.""" + try: + market = self.client.market(symbol) + precision = market.get("precision", {}).get("amount", 8) + min_qty = market.get("limits", {}).get("amount", {}).get("min", 0) + qty = float(self.client.amount_to_precision(symbol, qty)) + if qty < min_qty: + return 0.0 + return qty + except Exception: + return round(qty, 4) + + def _alert(self, message: str): + """Log alert to console.""" + print(f"[PUMP SCANNER] {message}") + + # ═══════════════════════════════════════════════════════════════════ + # STATUS / DEBUGGING + # ═══════════════════════════════════════════════════════════════════ + + def get_status(self) -> dict: + """Return current scanner status for dashboard/monitoring.""" + with self._lock: + return { + "running": self._running, + "symbols_monitored": len(self._all_symbols), + "active_pump_positions": len(self.pump_positions), + "positions": { + coin: { + "side": pos.side, + "entry": pos.entry_price, + "sl": pos.stop_loss, + "tp1": pos.tp1, + "tp2": pos.tp2, + "trailing_active": pos.trailing_active, + "trailing_high": pos.trailing_high, + } + for coin, pos in self.pump_positions.items() + }, + "cooldowns_active": sum(1 for t in self.cooldowns.values() if t > time.time()), + "daily_pump_pnl": self._daily_pump_pnl, + "volume_baselines_loaded": len(self.volume_baselines), + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# INTEGRATION HELPER — add to existing bot +# ═══════════════════════════════════════════════════════════════════════════ + +def create_pump_scanner_from_config(): + """ + Factory function to create a PumpScanner from environment config. + + Architecture MT5-only : utilise yfinance comme source de données + (pas de dépendance ccxt/Bybit). + + Usage in ahad_quant.py: + from pump_scanner import create_pump_scanner_from_config + scanner = create_pump_scanner_from_config() + if scanner: + scanner.start() + """ + try: + # Mode MT5 / yfinance — pas besoin de credentials broker + exchange_name = os.getenv("EXCHANGE", "mt5").lower() + if exchange_name not in ("mt5",): + # Mode non-MT5 : tenter ccxt si disponible + try: + import ccxt + api_key = os.getenv("BYBIT_API_KEY", "") + api_secret = os.getenv("BYBIT_API_SECRET", "") + if not api_key or not api_secret: + logger.warning("Bybit credentials not set, pump scanner disabled") + return None + client = ccxt.bybit({ + "apiKey": api_key, + "secret": api_secret, + "enableRateLimit": True, + "options": {"defaultType": "linear"}, + }) + if os.getenv("BYBIT_TESTNET", "false").lower() == "true": + client.set_sandbox_mode(True) + client.load_markets() + scanner = PumpScanner(client) + return scanner + except Exception as e: + logger.error(f"Pump scanner (ccxt) failed: {e}") + return None + + # ── Mode MT5 : client = None, le scanner utilise yfinance ────────── + scanner = PumpScanner(client=None) + return scanner + + except Exception as e: + logger.error(f"Failed to create pump scanner: {e}") + return None + + +# ═══════════════════════════════════════════════════════════════════════════ +# STANDALONE MODE — run directly: python pump_scanner.py +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == "__main__": + import dotenv + dotenv.load_dotenv() + + print(""" + ╔═══════════════════════════════════════════╗ + ║ AHAD QUANT Pump Scanner ║ + ║ Real-time pump detection on Bybit ║ + ║ https://ahadquant.com ║ + ╚═══════════════════════════════════════════╝ + """) + + scanner = create_pump_scanner_from_config() + if scanner: + print(f"Pump Scanner initialized. Starting...") + scanner.start() + # Keep main thread alive + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + print("\nShutting down...") + scanner.stop() + else: + print("Failed to initialize. Check your .env file:") + print(" BYBIT_API_KEY=your_key") + print(" BYBIT_API_SECRET=your_secret") diff --git a/regime_detector.py b/regime_detector.py new file mode 100644 index 0000000..af5a53a --- /dev/null +++ b/regime_detector.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +""" +regime_detector.py - Hidden Markov Model for Market Regime Detection +==================================================================== +Identifies 3 market regimes: + 0 = CALM (low vol, trending) + 1 = NORMAL (moderate vol, mixed) + 2 = VOLATILE (high vol, choppy) + +Uses returns, volatility, and volume as observable features. +Can be used as: + - A standalone feature (regime_score) fed into LightGBM + - A strategy selector (different thresholds per regime) + +Dependencies: numpy, scipy (for Gaussian PDF). No heavy ML libs required. +""" +import numpy as np +import pickle +import os +import logging + +log = logging.getLogger("AHAD QUANT") + +N_STATES = 3 +STATE_NAMES = {0: 'CALM', 1: 'NORMAL', 2: 'VOLATILE'} + + +class GaussianHMM: + """Minimal Gaussian HMM with 3 states, trained via Baum-Welch (EM).""" + + def __init__(self, n_states=N_STATES, n_features=3, n_iter=50, tol=1e-4): + self.n_states = n_states + self.n_features = n_features + self.n_iter = n_iter + self.tol = tol + + # Initialize parameters + self.pi = np.ones(n_states) / n_states # initial state probs + self.A = None # transition matrix (n_states x n_states) + self.means = None # emission means (n_states x n_features) + self.covars = None # emission covariances (n_states x n_features x n_features) + self.trained = False + + def _init_params(self, X): + """Initialize parameters using K-means-like heuristic.""" + n = len(X) + # Sort by volatility (column 1) to get natural regime ordering + vol_idx = np.argsort(X[:, 1]) + third = n // 3 + + # Assign initial clusters by volatility quantile + clusters = [vol_idx[:third], vol_idx[third:2*third], vol_idx[2*third:]] + + self.means = np.zeros((self.n_states, self.n_features)) + self.covars = np.zeros((self.n_states, self.n_features, self.n_features)) + + for s in range(self.n_states): + if len(clusters[s]) > 0: + self.means[s] = X[clusters[s]].mean(axis=0) + cov = np.cov(X[clusters[s]].T) + if cov.ndim == 0: + cov = np.array([[float(cov)]]) + # Ensure positive definite + self.covars[s] = cov + np.eye(self.n_features) * 1e-6 + else: + self.means[s] = X.mean(axis=0) + self.covars[s] = np.eye(self.n_features) + + # Transition matrix: high self-transition probability (regimes are sticky) + self.A = np.array([ + [0.90, 0.07, 0.03], # CALM stays calm + [0.05, 0.85, 0.10], # NORMAL + [0.03, 0.12, 0.85], # VOLATILE stays volatile + ]) + + def _log_gaussian_pdf(self, x, mean, covar): + """Log probability of x under multivariate Gaussian.""" + k = len(mean) + diff = x - mean + try: + # Use Cholesky for numerical stability + L = np.linalg.cholesky(covar) + solve = np.linalg.solve(L, diff) + log_det = 2 * np.sum(np.log(np.diag(L))) + log_prob = -0.5 * (k * np.log(2 * np.pi) + log_det + np.dot(solve, solve)) + except np.linalg.LinAlgError: + # Fallback: diagonal approximation + diag = np.maximum(np.diag(covar), 1e-10) + log_prob = -0.5 * (k * np.log(2 * np.pi) + np.sum(np.log(diag)) + + np.sum(diff ** 2 / diag)) + return log_prob + + def _compute_log_emission(self, X): + """Compute log emission probabilities for all observations.""" + n = len(X) + log_B = np.zeros((n, self.n_states)) + for s in range(self.n_states): + for t in range(n): + log_B[t, s] = self._log_gaussian_pdf(X[t], self.means[s], self.covars[s]) + return log_B + + def _forward(self, log_B): + """Forward algorithm (log-space).""" + n = len(log_B) + log_alpha = np.full((n, self.n_states), -np.inf) + + # Init + for s in range(self.n_states): + log_alpha[0, s] = np.log(self.pi[s] + 1e-300) + log_B[0, s] + + # Recurse + log_A = np.log(self.A + 1e-300) + for t in range(1, n): + for s in range(self.n_states): + log_alpha[t, s] = _logsumexp(log_alpha[t-1] + log_A[:, s]) + log_B[t, s] + + return log_alpha + + def _backward(self, log_B): + """Backward algorithm (log-space).""" + n = len(log_B) + log_beta = np.full((n, self.n_states), -np.inf) + log_beta[-1, :] = 0.0 # log(1) + + log_A = np.log(self.A + 1e-300) + for t in range(n - 2, -1, -1): + for s in range(self.n_states): + log_beta[t, s] = _logsumexp(log_A[s, :] + log_B[t+1, :] + log_beta[t+1, :]) + + return log_beta + + def fit(self, X): + """Train HMM using Baum-Welch (EM) algorithm.""" + X = np.asarray(X, dtype=np.float64) + n = len(X) + if n < 100: + log.warning("[HMM] Too few samples for training") + return self + + self._init_params(X) + prev_ll = -np.inf + + for iteration in range(self.n_iter): + # E-step + log_B = self._compute_log_emission(X) + log_alpha = self._forward(log_B) + log_beta = self._backward(log_B) + + # Log-likelihood + ll = _logsumexp(log_alpha[-1]) + if abs(ll - prev_ll) < self.tol: + break + prev_ll = ll + + # Posterior: gamma[t,s] = P(state=s at time t | observations) + log_gamma = log_alpha + log_beta + log_gamma -= _logsumexp_axis(log_gamma, axis=1, keepdims=True) + gamma = np.exp(log_gamma) + + # Xi: transition posteriors + log_A = np.log(self.A + 1e-300) + xi_sum = np.zeros((self.n_states, self.n_states)) + for t in range(n - 1): + for i in range(self.n_states): + for j in range(self.n_states): + xi_sum[i, j] += np.exp( + log_alpha[t, i] + log_A[i, j] + log_B[t+1, j] + log_beta[t+1, j] - ll + ) + + # M-step + # Update pi + self.pi = gamma[0] / gamma[0].sum() + + # Update A + for i in range(self.n_states): + denom = gamma[:-1, i].sum() + if denom > 1e-10: + self.A[i, :] = xi_sum[i, :] / denom + # Ensure row sums to 1 + row_sum = self.A[i, :].sum() + if row_sum > 0: + self.A[i, :] /= row_sum + + # Update means and covariances + for s in range(self.n_states): + gamma_s = gamma[:, s] + total_gamma = gamma_s.sum() + if total_gamma > 1e-10: + self.means[s] = (gamma_s[:, np.newaxis] * X).sum(axis=0) / total_gamma + diff = X - self.means[s] + self.covars[s] = (gamma_s[:, np.newaxis, np.newaxis] * + (diff[:, :, np.newaxis] * diff[:, np.newaxis, :])).sum(axis=0) / total_gamma + # Regularize + self.covars[s] += np.eye(self.n_features) * 1e-4 + + self.trained = True + return self + + def predict(self, X): + """Viterbi decoding: find most likely state sequence.""" + X = np.asarray(X, dtype=np.float64) + n = len(X) + if not self.trained or n == 0: + return np.ones(n, dtype=int) # default NORMAL + + log_B = self._compute_log_emission(X) + log_A = np.log(self.A + 1e-300) + + # Viterbi + log_delta = np.zeros((n, self.n_states)) + psi = np.zeros((n, self.n_states), dtype=int) + + log_delta[0] = np.log(self.pi + 1e-300) + log_B[0] + + for t in range(1, n): + for s in range(self.n_states): + trans = log_delta[t-1] + log_A[:, s] + psi[t, s] = np.argmax(trans) + log_delta[t, s] = trans[psi[t, s]] + log_B[t, s] + + # Backtrack + states = np.zeros(n, dtype=int) + states[-1] = np.argmax(log_delta[-1]) + for t in range(n - 2, -1, -1): + states[t] = psi[t + 1, states[t + 1]] + + return states + + def predict_proba(self, X): + """Return state probabilities for each observation.""" + X = np.asarray(X, dtype=np.float64) + n = len(X) + if not self.trained or n == 0: + probs = np.zeros((n, self.n_states)) + probs[:, 1] = 1.0 # default NORMAL + return probs + + log_B = self._compute_log_emission(X) + log_alpha = self._forward(log_B) + log_beta = self._backward(log_B) + + log_gamma = log_alpha + log_beta + log_gamma -= _logsumexp_axis(log_gamma, axis=1, keepdims=True) + return np.exp(log_gamma) + + def save(self, path): + """Save trained HMM to pickle.""" + data = { + 'n_states': self.n_states, + 'n_features': self.n_features, + 'pi': self.pi, + 'A': self.A, + 'means': self.means, + 'covars': self.covars, + 'trained': self.trained, + } + with open(path, 'wb') as f: + pickle.dump(data, f) + + @classmethod + def load(cls, path): + """Load trained HMM from pickle.""" + with open(path, 'rb') as f: + data = pickle.load(f) + hmm = cls(n_states=data['n_states'], n_features=data['n_features']) + hmm.pi = data['pi'] + hmm.A = data['A'] + hmm.means = data['means'] + hmm.covars = data['covars'] + hmm.trained = data['trained'] + return hmm + + +def _logsumexp(x): + """Numerically stable log-sum-exp.""" + x = np.asarray(x) + mx = x.max() + if mx == -np.inf: + return -np.inf + return mx + np.log(np.sum(np.exp(x - mx))) + + +def _logsumexp_axis(x, axis=1, keepdims=False): + """Log-sum-exp along an axis.""" + mx = x.max(axis=axis, keepdims=True) + result = mx + np.log(np.sum(np.exp(x - mx), axis=axis, keepdims=True)) + if not keepdims: + result = result.squeeze(axis=axis) + return result + + +# ========================================================================= +# Helper: Build HMM observation features from OHLCV +# ========================================================================= +def build_hmm_observations(closes, volumes, window=24): + """ + Build observation matrix for HMM from OHLCV data. + + Returns (n, 3) array: + col 0: returns (1-period log returns) + col 1: realized volatility (rolling std of returns) + col 2: volume change (log volume ratio to rolling mean) + """ + n = len(closes) + obs = np.zeros((n, 3)) + + # Log returns + for i in range(1, n): + if closes[i-1] > 0 and closes[i] > 0: + obs[i, 0] = np.log(closes[i] / closes[i-1]) + + # Rolling volatility + for i in range(window, n): + obs[i, 1] = obs[i-window:i, 0].std() + + # Volume ratio + for i in range(window, n): + vol_mean = volumes[i-window:i].mean() + if vol_mean > 0 and volumes[i] > 0: + obs[i, 2] = np.log(volumes[i] / vol_mean) + + return obs + + +def get_regime_features(states, proba): + """ + Convert HMM output to features for LightGBM. + + Args: + states: array of regime indices (0, 1, 2) + proba: array of shape (n, 3) with regime probabilities + + Returns: + dict with: + regime_state: current regime (0=CALM, 1=NORMAL, 2=VOLATILE) + regime_calm_prob: probability of CALM + regime_volatile_prob: probability of VOLATILE + regime_transition: 1 if regime changed from previous candle + """ + n = len(states) + regime_state = np.array(states, dtype=np.float64) + calm_prob = proba[:, 0] if proba.shape[1] > 0 else np.zeros(n) + volatile_prob = proba[:, 2] if proba.shape[1] > 2 else np.zeros(n) + + transition = np.zeros(n) + for i in range(1, n): + if states[i] != states[i-1]: + transition[i] = 1.0 + + return { + 'regime_state': regime_state, + 'regime_calm_prob': calm_prob, + 'regime_volatile_prob': volatile_prob, + 'regime_transition': transition, + } + + +# ========================================================================= +# Convenience: train + predict in one call (for training pipeline) +# ========================================================================= +def train_and_predict(closes, volumes, save_path=None): + """ + Train HMM on full history and return regime features. + + Args: + closes: 1D array of close prices + volumes: 1D array of volumes + save_path: optional path to save trained HMM + + Returns: + dict of regime features (same length as closes) + """ + obs = build_hmm_observations(closes, volumes) + + # Train on observations after warmup period + warmup = 48 + hmm = GaussianHMM(n_states=3, n_features=3, n_iter=50) + hmm.fit(obs[warmup:]) + + # Predict on full sequence + states = hmm.predict(obs) + proba = hmm.predict_proba(obs) + + if save_path: + hmm.save(save_path) + + return get_regime_features(states, proba) + + +if __name__ == '__main__': + print("[HMM] Regime Detector smoke test") + + # Generate synthetic data + np.random.seed(42) + n = 1000 + closes = np.cumsum(np.random.randn(n) * 0.01) + 100 + closes = np.exp(np.log(100) + np.cumsum(np.random.randn(n) * 0.01)) + volumes = np.abs(np.random.randn(n)) * 1000 + 500 + + features = train_and_predict(closes, volumes) + + for k, v in features.items(): + print(f" {k}: shape={v.shape}, unique={np.unique(v[:50])}") + + # Check regime distribution + states = features['regime_state'] + for s in [0, 1, 2]: + pct = (states == s).mean() * 100 + print(f" {STATE_NAMES[s]}: {pct:.1f}%") + + print("[HMM] Smoke test passed") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..422898c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,71 @@ +# ─── AHAD QUANT — Forex Edition — requirements.txt ──────────────────────── +# Installation complète : +# pip install -r requirements.txt +# +# Packages optionnels (activés par phases) : +# hmmlearn → Phase 6 (USE_REGIME_FILTER=true) +# mkdocs-material → Documentation uniquement + +# ── Core ML / IA ───────────────────────────────────────────────────────────── +lightgbm>=4.0.0 +xgboost>=2.0.0 +scikit-learn>=1.3.0 +numpy>=1.24.0 +pandas>=2.0.0 +optuna>=3.3.0 +shap>=0.43.0 + +# ── Data / Forex ────────────────────────────────────────────────────────────── +yfinance>=0.2.28 + +# ── Brokers optionnels — décommenter selon votre broker ────────────────────── +# OANDA v20 REST API +# oandapyV20>=0.7.2 + +# Alpaca Markets (paper + live) +# alpaca-py>=0.20.0 + +# CCXT — 100+ brokers (IG, GAIN Capital, Binance, Bybit, Kraken…) +# ccxt>=4.3.0 + +# Interactive Brokers (via ib_insync) +# ib_insync>=0.9.86 + +# MetaTrader 5 Python SDK (Windows uniquement) +# MetaTrader5>=5.0.45 + +# ── Config / Env ───────────────────────────────────────────────────────────── +python-dotenv>=1.0.0 + +# ── HTTP ───────────────────────────────────────────────────────────────────────── +requests>=2.31.0 + +# ── MT5 Bridge ─────────────────────────────────────────────────────────────── +watchdog>=3.0.0 +filelock>=3.12.0 + +# ── Interface Web ──────────────────────────────────────────────────────────── +fastapi>=0.104.0 +uvicorn[standard]>=0.24.0 +python-multipart>=0.0.6 + +# ── Dashboard secondaire (Flask) ───────────────────────────────────────────── +flask>=3.0.0 +flask-socketio>=5.3.0 + +# ── HMM Regime Filter (Phase 6 — décommenter quand prêt) ───────────────────── +# hmmlearn>=0.3.0 + +# ── RL Agent (PPO via Stable-Baselines3) ───────────────────────────────────── +# Requis car USE_RL_AGENT=true dans .env : +stable-baselines3[extra]>=2.3.0 +gymnasium>=0.29.0 +shimmy>=1.3.0 + +# ── Deep Learning (TFT + TransformerGRU — V5) ──────────────────────────────── +# Décommenter uniquement si tu as un GPU (entraînement local ou VPS GPU) : +torch>=2.2.0 +# torch==2.2.0+cpu # ← alternative CPU-only pour inférence locale + +# ── Ajouts V32 ────────────────────────────────────────────────────────────── +streamlit>=1.30.0 # dashboard.py (vestige crypto, peut être supprimé) diff --git a/risk_manager.py b/risk_manager.py new file mode 100644 index 0000000..f3b626d --- /dev/null +++ b/risk_manager.py @@ -0,0 +1,214 @@ +""" +AHAD QUANT — Risk Manager (Free Version) +Handles position sizing, stop-loss/take-profit, daily loss tracking, +and circuit breaker logic. +""" + +import time +from datetime import datetime, timezone + +import config +from config import FEE_RATE + + +class RiskManager: + """Enforces risk rules for every trade decision.""" + + def __init__(self): + self.daily_pnl: float = 0.0 + self._last_reset_date: str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + self.consecutive_losses: int = 0 + self.circuit_breaker_until: float = 0.0 + self.open_positions: dict[str, dict] = {} # coin -> position info + self._initial_equity: float = 0.0 # mémorisé au premier trade pour compound=False + + # ─── Daily reset ──────────────────────────────────────────────────── + + def _check_daily_reset(self) -> None: + """Reset daily counters at UTC midnight.""" + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + if today != self._last_reset_date: + self.daily_pnl = 0.0 + self._last_reset_date = today + + # ─── Can we open a new trade? ─────────────────────────────────────── + + def can_open(self, equity: float) -> tuple[bool, str]: + """ + Check whether opening a new position is allowed. + + Returns + ------- + (allowed, reason) + """ + self._check_daily_reset() + + # Circuit breaker active? + if time.time() < self.circuit_breaker_until: + remaining = int(self.circuit_breaker_until - time.time()) + return False, f"Circuit breaker active ({remaining}s left)" + + # Max positions reached? + if len(self.open_positions) >= config.MAX_POSITIONS: + return False, f"Max positions reached ({config.MAX_POSITIONS})" + + # Max simultaneous margin usage? + if equity > 0: + used_margin = sum( + p["entry"] * p["qty"] for p in self.open_positions.values() + ) + if used_margin / equity >= config.MAX_MARGIN_USAGE: + return False, f"Max margin usage reached ({config.MAX_MARGIN_USAGE*100:.0f}% of equity)" + + # Daily loss limit hit? + if equity > 0 and (self.daily_pnl / equity) <= -config.MAX_DAILY_LOSS_PCT: + return False, f"Daily loss limit hit ({config.MAX_DAILY_LOSS_PCT*100:.1f}%)" + + return True, "OK" + + # ─── Position sizing ──────────────────────────────────────────────── + + def calc_position_size(self, equity: float, price: float) -> float: + """ + Calculate the notional position size in USD. + + If COMPOUND_ENABLED, sizes off current equity (compounding). + Otherwise uses a fixed fraction of the initial equity captured at the first call. + Capped at CAPITAL_CAP_PER_TRADE to prevent unrealistic compound growth. + """ + if self._initial_equity == 0.0: + self._initial_equity = equity # mémorise le capital de départ + base = equity if config.COMPOUND_ENABLED else self._initial_equity + notional = base * config.RISK_PER_TRADE * config.LEVERAGE + notional = min(notional, config.CAPITAL_CAP_PER_TRADE) + return round(notional, 2) + + def calc_quantity(self, equity: float, price: float) -> float: + """Calculate the asset quantity for the trade.""" + notional = self.calc_position_size(equity, price) + qty = notional / price + return qty + + # ─── Stop-loss & take-profit ──────────────────────────────────────── + + def calc_sl_tp(self, entry_price: float, side: str) -> tuple[float, float]: + """ + Calculate fixed stop-loss and take-profit prices. + + Parameters + ---------- + entry_price : float + side : str — "long" or "short" + + Returns + ------- + (stop_loss_price, take_profit_price) + """ + if side == "long": + sl = entry_price * (1 - config.STOP_LOSS_PCT) + tp = entry_price * (1 + config.TAKE_PROFIT_PCT) + else: + sl = entry_price * (1 + config.STOP_LOSS_PCT) + tp = entry_price * (1 - config.TAKE_PROFIT_PCT) + return round(sl, 6), round(tp, 6) + + # ─── Position tracking ────────────────────────────────────────────── + + def register_open(self, coin: str, side: str, entry_price: float, qty: float) -> None: + """Record a new open position.""" + sl, tp = self.calc_sl_tp(entry_price, side) + self.open_positions[coin] = { + "side": side, + "entry": entry_price, + "qty": qty, + "sl": sl, + "tp": tp, + "opened_at": time.time(), + } + + def register_close(self, coin: str, exit_price: float) -> float: + """ + Record a position close and update P&L tracking. + + Returns the realised P&L in USD. + """ + if coin not in self.open_positions: + return 0.0 + + pos = self.open_positions.pop(coin) + if pos["side"] == "long": + raw_pnl = (exit_price - pos["entry"]) * pos["qty"] + else: + raw_pnl = (pos["entry"] - exit_price) * pos["qty"] + + # Subtract fees on both entry and exit (2 sides) + notional_entry = pos["entry"] * pos["qty"] + notional_exit = exit_price * pos["qty"] + fees = FEE_RATE * notional_entry + FEE_RATE * notional_exit + pnl = raw_pnl - fees + + self.daily_pnl += pnl + + # Track consecutive losses for circuit breaker + if pnl < 0: + self.consecutive_losses += 1 + if self.consecutive_losses >= config.CIRCUIT_BREAKER_LOSSES: + self.circuit_breaker_until = time.time() + config.CIRCUIT_BREAKER_COOLDOWN + self.consecutive_losses = 0 + else: + self.consecutive_losses = 0 + + return round(pnl, 2) + + # ─── Daily reset (explicit) ───────────────────────────────────────── + + def reset_daily(self) -> None: + """ + Réinitialise le PnL quotidien à minuit. + Appelé explicitement par ahad_quant.py dans la boucle principale. + """ + self.daily_pnl = 0.0 + self._last_reset_date = datetime.now(timezone.utc).strftime("%Y-%m-%d") + + # ─── SL/TP check ─────────────────────────────────────────────────── + + def check_exit(self, coin: str, current_price: float) -> str | None: + """ + Check if a position should be closed due to SL, TP, or timeout. + + Returns "sl", "tp", "timeout", or None. + """ + if coin not in self.open_positions: + return None + + pos = self.open_positions[coin] + + # Temporal exit : MAX_HOLD_CANDLES × 1h + if config.TIMEOUT_ENABLED: + elapsed = time.time() - pos.get("opened_at", time.time()) + if elapsed >= config.MAX_HOLD_CANDLES * 3600: + return "timeout" + + if pos["side"] == "long": + if current_price <= pos["sl"]: + return "sl" + if current_price >= pos["tp"]: + return "tp" + else: + if current_price >= pos["sl"]: + return "sl" + if current_price <= pos["tp"]: + return "tp" + + return None + + # ─── Summary ──────────────────────────────────────────────────────── + + def summary(self) -> str: + """Retourne un résumé lisible de l'état du risk manager.""" + return ( + f"RiskManager | Open: {len(self.open_positions)} | " + f"Daily PnL: {self.daily_pnl:+.2f} | " + f"Consecutive losses: {self.consecutive_losses} | " + f"CB until: {self.circuit_breaker_until:.0f}" + ) diff --git a/rl_agent.py b/rl_agent.py new file mode 100644 index 0000000..a6fc058 --- /dev/null +++ b/rl_agent.py @@ -0,0 +1,511 @@ +""" +AHAD QUANT — RL Agent Interface (Inférence Live) — V6 UNIFIED + +Charge ahad_quant_unified.zip — UN seul fichier qui contient : + - Le modèle PPO (réseau de neurones, poids, optimiseur) + - L'ensemble supervisé (LightGBM + XGBoost + RF + TFT + TGRU + Meta) + - Le scaler z-score des features + +Chargement : + agent = RLAgent(unified_path="ahad_quant_unified.zip") + signal, conf = agent.predict_signal(features, position_state) + +Ou depuis fichiers séparés (compatibilité V5) : + agent = RLAgent(model_path="rl_agent", scaler_path="rl_scaler.pkl", + ensemble_path="model_ensemble.pkl") + +Actions RL : + 0 = HOLD/NEUTRAL + 1 = LONG + 2 = SHORT + 3 = CLOSE +""" + +import os +import io +import json +import pickle +import zipfile +import logging +import collections +import numpy as np +from typing import Tuple, Optional + +log = logging.getLogger("RLAgent") + +# Source unique pour le ML (voir ensemble_core.py). rl_agent.py sert à +# l'INFÉRENCE LIVE et ne doit PAS dépendre de rl_env.py, qui importe +# gymnasium en dur et charge tout l'environnement d'entraînement — corrige +# le point #9 du diagnostic (avant, importer rl_agent.py forçait +# l'installation de gymnasium même sur un déploiement live sans entraînement). +import ensemble_core as _ens_core +from features import NUM_FEATURES + +_load_ensemble = _ens_core.load_ensemble + + +def _predict_ensemble(ensemble, raw_feat: np.ndarray, seq_buffer=None): + """Wrapper local — délègue à ensemble_core (même logique exacte que + rl_env.py, ahad_quant.py et backtest.py : aucune divergence possible).""" + history = None + if seq_buffer is not None: + try: + history = np.asarray(seq_buffer, dtype=np.float32) + if history.ndim != 2 or len(history) == 0: + history = None + except Exception: + history = None + return _ens_core.predict_ensemble_single(ensemble, raw_feat, history=history) + + +SEQ_LEN = _ens_core.dl_seq_len() + +# MAX_HOLD synchronisé sur config.py (même source unique de vérité que +# rl_env.py — corrige le point #5 du diagnostic côté inférence live, pour +# PositionState.to_array()). +try: + import config as _config + _DEFAULT_MAX_HOLD = int(getattr(_config, "MAX_HOLD_CANDLES", 6)) +except Exception: + _DEFAULT_MAX_HOLD = 6 + +# ─── Constantes ────────────────────────────────────────────────────────────── + +ACTION_TO_SIGNAL = { + 0: "neutral", + 1: "long", + 2: "short", + 3: "neutral", +} +N_ML_FEATURES = NUM_FEATURES + + +# ─── Position State ────────────────────────────────────────────────────────── + +class PositionState: + __slots__ = ["in_position", "direction", "unrealized_pnl_pct", + "candles_held", "balance_ratio"] + + def __init__( + self, + in_position: bool = False, + direction: int = 0, + unrealized_pnl_pct: float = 0.0, + candles_held: int = 0, + balance_ratio: float = 1.0, + ): + self.in_position = in_position + self.direction = direction + self.unrealized_pnl_pct = unrealized_pnl_pct + self.candles_held = candles_held + self.balance_ratio = balance_ratio + + def to_array(self, max_hold: int = _DEFAULT_MAX_HOLD) -> np.ndarray: + return np.array([ + float(self.in_position), + float(self.direction), + float(np.clip(self.unrealized_pnl_pct / 0.05, -1.0, 1.0)), + float(np.clip(self.candles_held / max_hold, 0.0, 1.0)), + float(np.clip(self.balance_ratio, 0.0, 2.0)), + ], dtype=np.float32) + + +# ─── RLAgent ───────────────────────────────────────────────────────────────── + +class RLAgent: + """ + Interface d'inférence pour AHAD QUANT V6 Unified. + + Deux modes de chargement : + + Mode UNIFIÉ (recommandé) : + agent = RLAgent(unified_path="ahad_quant_unified.zip") + + Mode SÉPARÉ (compatibilité V5) : + agent = RLAgent(model_path="rl_agent", + scaler_path="rl_scaler.pkl", + ensemble_path="model_ensemble.pkl") + """ + + def __init__( + self, + # Mode unifié (repli portable) + unified_path: str = None, + # Mode séparé — FICHIERS VIVANTS, prioritaires (voir _load_best_available) + model_path: str = None, + scaler_path: str = None, + ensemble_path: str = None, + # Options + device: str = "cpu", + enabled: bool = True, + ): + self.enabled = enabled + self._model = None + self._loaded = False + self._device = device + + self._feat_mean: Optional[np.ndarray] = None + self._feat_std: Optional[np.ndarray] = None + self._ensemble = None + self._has_dl = False + self._metadata = {} + self._mtimes = {} + + self._seq_buffer: collections.deque = collections.deque(maxlen=SEQ_LEN) + + # Résout les chemins via config.py quand non fournis explicitement, + # pour que les "fichiers vivants" (#2 du diagnostic) soient toujours + # connus, même si l'appelant ne passe que unified_path (compat V6). + import config + self._model_path = model_path or getattr(config, "RL_MODEL_PATH", "rl_agent") + self._scaler_path = scaler_path or getattr(config, "RL_SCALER_PATH", "rl_scaler.pkl") + self._ensemble_path = ensemble_path or getattr(config, "ENSEMBLE_MODEL_PATH", "model_ensemble.pkl") + self._unified_path = unified_path or getattr(config, "UNIFIED_MODEL_PATH", "ahad_quant_unified.zip") + + if not enabled: + log.info("[RL-AGENT] Désactivé") + return + + self._load_best_available(device) + + def _zip_path(self, path: str) -> str: + return path if path.endswith(".zip") else path + ".zip" + + def _canonical_files_present(self) -> bool: + """True si les fichiers vivants (rl_agent.zip + rl_scaler.pkl + + model_ensemble.pkl) sont TOUS présents. Ce sont eux qui reflètent un + éventuel ré-entraînement ML/RL — contrairement à ahad_quant_unified.zip, + qui n'est régénéré que par un export manuel (point #2 du diagnostic).""" + return ( + os.path.exists(self._zip_path(self._model_path)) + and os.path.exists(self._scaler_path) + and os.path.exists(self._ensemble_path) + ) + + def _load_best_available(self, device: str): + """ + Choisit la source à charger, par ordre de priorité : + 1. Fichiers vivants (rl_agent.zip + rl_scaler.pkl + + model_ensemble.pkl) s'ils sont TOUS présents — toujours + prioritaires car ils reflètent le dernier ré-entraînement et + permettent le hot-reload (corrige #2 et #4 du diagnostic). + AVANT ce correctif, ahad_quant_unified.zip était préféré dès + qu'il existait, ce qui rendait tout ré-entraînement RL inopérant + en production tant que ce fichier restait sur le disque. + 2. Sinon, ahad_quant_unified.zip — repli utile en déploiement + portable (autre machine, sans pipeline d'entraînement local). + """ + if self._canonical_files_present(): + log.info("[RL-AGENT] Fichiers vivants détectés — chargement séparé (prioritaire)") + self._load_model(self._model_path, device) + self._load_scaler(self._scaler_path) + self._ensemble = _load_ensemble(self._ensemble_path) + self._has_dl = self._ensemble.get("has_dl", False) if self._ensemble else False + elif self._unified_path and os.path.exists(self._unified_path): + log.info(f"[RL-AGENT] Fichiers vivants absents — repli sur le bundle portable : {self._unified_path}") + self._load_unified(self._unified_path, device) + else: + log.warning("[RL-AGENT] Aucune source de modèle trouvée (ni fichiers vivants, ni bundle unifié)") + self.enabled = False + return + self._record_mtimes() + + def _record_mtimes(self): + self._mtimes = { + "model": _ens_core.ensemble_mtime(self._zip_path(self._model_path)), + "scaler": _ens_core.ensemble_mtime(self._scaler_path), + "ensemble": _ens_core.ensemble_mtime(self._ensemble_path), + "unified": _ens_core.ensemble_mtime(self._unified_path) if self._unified_path else 0.0, + } + + def reload_if_stale(self) -> bool: + """ + Compare les mtimes actuels des fichiers source à ceux du dernier + chargement (coût négligeable : quelques appels os.path.getmtime) et + recharge tout (PPO + scaler + ensemble) si l'un d'eux a changé. + + Corrige le point #4 du diagnostic : avant, un ré-entraînement réussi + en arrière-plan (auto_retrain.py) restait totalement sans effet sur + le bot déjà en cours d'exécution, jusqu'à un redémarrage manuel du + process. Peut être appelé à chaque itération de la boucle principale + (voir unified_brain.py / ahad_quant.py::run()). + + Retourne True si un rechargement a effectivement eu lieu. + """ + if not self.enabled: + return False + current = { + "model": _ens_core.ensemble_mtime(self._zip_path(self._model_path)), + "scaler": _ens_core.ensemble_mtime(self._scaler_path), + "ensemble": _ens_core.ensemble_mtime(self._ensemble_path), + "unified": _ens_core.ensemble_mtime(self._unified_path) if self._unified_path else 0.0, + } + if current == self._mtimes: + return False + log.info("[RL-AGENT] Changement détecté sur les fichiers modèle — rechargement à chaud") + self._seq_buffer.clear() + self._load_best_available(self._device) + return True + + # ────────────────────────────────────────────────────────────────────────── + # Chargement UNIFIÉ + # ────────────────────────────────────────────────────────────────────────── + + def _load_unified(self, path: str, device: str): + """Charge tout depuis ahad_quant_unified.zip.""" + log.info(f"[RL-AGENT] Chargement unifié : {path}") + + try: + from stable_baselines3 import PPO + except ImportError: + log.error("[RL-AGENT] stable-baselines3 non installé.") + return + + try: + with zipfile.ZipFile(path, "r") as uz: + files = uz.namelist() + + # ── 1. Reconstruire le zip PPO en mémoire ──────────────────── + # SB3 attend un zip avec les fichiers à la racine (pas dans ppo/) + ppo_buf = io.BytesIO() + with zipfile.ZipFile(ppo_buf, "w", zipfile.ZIP_DEFLATED) as ppo_zip: + for name in files: + if name.startswith("ppo/"): + inner_name = name[len("ppo/"):] # retirer le préfixe + if inner_name: + ppo_zip.writestr(inner_name, uz.read(name)) + ppo_buf.seek(0) + + # ── 2. Charger le PPO depuis le BytesIO ────────────────────── + self._model = PPO.load(ppo_buf, device=device) + self._loaded = True + log.info("[RL-AGENT] ✅ PPO chargé depuis mémoire") + + # ── 3. Charger le scaler ───────────────────────────────────── + if "rl_scaler.pkl" in files: + scaler = pickle.loads(uz.read("rl_scaler.pkl")) + self._feat_mean = scaler["mean"].astype(np.float32) + self._feat_std = scaler["std"].astype(np.float32) + log.info("[RL-AGENT] ✅ Scaler chargé") + + # ── 4. Charger l'ensemble ───────────────────────────────────── + if "ensemble.pkl" in files: + self._ensemble = pickle.loads(uz.read("ensemble.pkl")) + self._has_dl = self._ensemble.get("has_dl", False) + log.info(f"[RL-AGENT] ✅ Ensemble chargé — has_dl={self._has_dl}") + + # ── 5. Metadata ─────────────────────────────────────────────── + if "metadata.json" in files: + self._metadata = json.loads(uz.read("metadata.json")) + log.info(f"[RL-AGENT] Version : {self._metadata.get('version','?')}") + + except Exception as e: + log.error(f"[RL-AGENT] Erreur chargement unifié : {e}") + self.enabled = False + + # ────────────────────────────────────────────────────────────────────────── + # Chargement SÉPARÉ (fallback) + # ────────────────────────────────────────────────────────────────────────── + + def _load_model(self, path: str, device: str): + try: + from stable_baselines3 import PPO + except ImportError: + log.error("[RL-AGENT] stable-baselines3 non installé.") + return + zip_path = path if path.endswith(".zip") else path + ".zip" + if not os.path.exists(zip_path): + log.warning(f"[RL-AGENT] Modèle introuvable : {zip_path}") + self.enabled = False + return + try: + self._model = PPO.load(path, device=device) + self._loaded = True + log.info(f"[RL-AGENT] ✅ PPO chargé : {zip_path}") + except Exception as e: + log.error(f"[RL-AGENT] Erreur : {e}") + self.enabled = False + + def _load_scaler(self, path: str): + if not os.path.exists(path): + log.warning(f"[RL-AGENT] Scaler introuvable : {path}") + return + try: + with open(path, "rb") as f: + scaler = pickle.load(f) + self._feat_mean = scaler["mean"].astype(np.float32) + self._feat_std = scaler["std"].astype(np.float32) + log.info(f"[RL-AGENT] ✅ Scaler chargé : {path}") + except Exception as e: + log.error(f"[RL-AGENT] Erreur scaler : {e}") + + # ────────────────────────────────────────────────────────────────────────── + # API Publique + # ────────────────────────────────────────────────────────────────────────── + + def predict( + self, + features: np.ndarray, + position_state: PositionState = None, + deterministic: bool = True, + ) -> int: + """Retourne l'action RL (0=HOLD, 1=LONG, 2=SHORT, 3=CLOSE).""" + if not self.enabled or not self._loaded or self._model is None: + return 0 + obs = self._build_obs(features, position_state) + try: + action, _ = self._model.predict(obs, deterministic=deterministic) + # numpy 2.x : int() ne fonctionne plus sur les arrays 1-D → .flat[0] + return int(action.flat[0]) if hasattr(action, "flat") else int(action) + except Exception as e: + log.error(f"[RL-AGENT] Erreur predict : {e}") + return 0 + + def predict_signal( + self, + features: np.ndarray, + position_state: PositionState = None, + return_action: bool = False, + ): + """Raccourci : features → (signal, confidence), ou (signal, + confidence, action) si return_action=True. L'action brute PPO + (0=HOLD,1=LONG,2=SHORT,3=CLOSE) est nécessaire pour le buffer + d'apprentissage continu — voir filter_signal().""" + action = self.predict(features, position_state) + signal = ACTION_TO_SIGNAL.get(action, "neutral") + # Confidence basée sur la certitude de l'ensemble intégré + _, ens_conf = _predict_ensemble( + self._ensemble, + features.astype(np.float32), + self._seq_buffer if self._has_dl else None, + ) + # Confidence finale = moyenne ens_conf + base fixe + confidence = round(0.70 + ens_conf * 0.25, 4) # [0.70, 0.95] + if return_action: + return signal, confidence, action + return signal, confidence + + def filter_signal( + self, + ml_signal: str, + ml_confidence: float, + features: np.ndarray, + position_state: "PositionState" = None, + ): + """ + Filtre le signal ML via le PPO. + - RL confirme ML → boost confiance + - RL override fort → suit le RL + - RL neutral / erreur → fallback sur ML (ne bloque pas) + + Retourne (signal, confidence, rl_action) — rl_action est l'action + PPO brute (None si une exception interne a empêché toute inférence + RL). Corrige le point #8 du diagnostic : avant, cette action réelle + n'était jamais propagée jusqu'au contexte d'ouverture de trade, et + online_learner.py devait la DEVINER à partir du signal final + (1 si LONG, 2 sinon), cassant la fidélité du replay RL. + """ + try: + rl_signal, rl_confidence, rl_action = self.predict_signal( + features, position_state, return_action=True + ) + except Exception: + # Erreur interne RL → on laisse passer le signal ML + return ml_signal, ml_confidence, None + + # Seuils lus depuis config.py (et non os.getenv directement) pour + # que la calibration adaptative (online_learner.calibrate_threshold, + # qui peut ajuster config.RL_OVERRIDE_THRESHOLD en mémoire) ait + # réellement un effet — corrige le point #7 du diagnostic. + import config as _cfg + + # RL confirme la direction ML + if rl_signal == ml_signal: + boost = float(getattr(_cfg, "RL_CONFIDENCE_BOOST", 0.05)) + return ml_signal, round(min(ml_confidence + boost, 0.99), 4), rl_action + + # RL override avec haute confiance + override_thr = float(getattr(_cfg, "RL_OVERRIDE_THRESHOLD", 0.82)) + if rl_signal != "neutral" and rl_confidence >= override_thr: + return rl_signal, rl_confidence, rl_action + + # RL dit neutral ou pas assez confiant → on garde le signal ML + return ml_signal, ml_confidence, rl_action + + def reset_buffer(self): + """Vide le buffer de séquence (appeler entre paires / sessions).""" + self._seq_buffer.clear() + + def info(self) -> dict: + """Retourne les métadonnées du modèle chargé.""" + return { + **self._metadata, + "ppo_loaded": self._loaded, + "ensemble": self._ensemble is not None, + "has_dl": self._has_dl, + "scaler": self._feat_mean is not None, + } + + def is_ready(self) -> bool: + return self.enabled and self._loaded and self._model is not None + + # ────────────────────────────────────────────────────────────────────────── + # Interne + # ────────────────────────────────────────────────────────────────────────── + + def _build_obs( + self, + raw_features: np.ndarray, + position_state: Optional[PositionState], + ) -> np.ndarray: + """Construit le vecteur d'observation 69 dims (identique à rl_env).""" + feat = raw_features.astype(np.float32).copy() + if len(feat) > N_ML_FEATURES: + feat = feat[:N_ML_FEATURES] + elif len(feat) < N_ML_FEATURES: + feat = np.pad(feat, (0, N_ML_FEATURES - len(feat))) + + if self._feat_mean is not None and self._feat_std is not None: + feat = (feat - self._feat_mean) / self._feat_std + feat = np.nan_to_num(feat, nan=0.0, posinf=0.0, neginf=0.0) + + if position_state is None: + position_state = PositionState() + pos = position_state.to_array() + + if self._ensemble is not None: + self._seq_buffer.append(raw_features.astype(np.float32)) + ens_proba, ens_conf = _predict_ensemble( + self._ensemble, + raw_features.astype(np.float32), + self._seq_buffer if self._has_dl else None, + ) + ens_feats = np.array([ens_proba, ens_conf], dtype=np.float32) + obs = np.concatenate([feat, pos, ens_feats]) + else: + obs = np.concatenate([feat, pos]) + + return obs[np.newaxis, :].astype(np.float32) + + +# ─── Singleton global ───────────────────────────────────────────────────────── + +_agent_instance: Optional[RLAgent] = None + + +def get_rl_agent() -> RLAgent: + global _agent_instance + if _agent_instance is None: + import config + unified = getattr(config, "UNIFIED_MODEL_PATH", "ahad_quant_unified.zip") + _agent_instance = RLAgent( + unified_path = unified, + enabled = getattr(config, "USE_RL_AGENT", False), + ) + return _agent_instance + + +def reset_rl_agent(): + global _agent_instance + _agent_instance = None + log.info("[RL-AGENT] Singleton réinitialisé") diff --git a/rl_env.py b/rl_env.py new file mode 100644 index 0000000..650eeff --- /dev/null +++ b/rl_env.py @@ -0,0 +1,463 @@ + +""" +AHAD QUANT — Gymnasium Trading Environment (V6 — UNIFIED + ML CACHE) +Environnement Forex pour l entraînement de l agent RL (PPO). + +PATCH PERFORMANCE : Les prédictions ensemble (LGB+XGB+RF) sont pré-calculées +en batch au chargement des données → lookup O(1) pendant l entraînement. +Gain attendu : 15 fps → 500-1500 fps. + +Observation (69 dims) : + [0:62] → 62 features techniques normalisées (identiques à train.py) + [62] → in_position (0 ou 1) + [63] → direction (-1=SHORT, 0=flat, 1=LONG) + [64] → unrealized_pnl (normalisé [-1, 1]) + [65] → candles_held (normalisé [0, 1]) + [66] → balance_ratio (balance courante / balance initiale, clampé [0, 2]) + [67] → ensemble_proba (proba LONG du méta-modèle, [0, 1]) + [68] → ensemble_confidence (certitude de l ensemble, [0, 1]) + +Actions (Discrete 4) : + 0 → HOLD (ne rien faire) + 1 → LONG (ouvrir long, ou fermer short + ouvrir long) + 2 → SHORT (ouvrir short, ou fermer long + ouvrir short) + 3 → CLOSE (fermer la position courante) +""" + +import os +import json +import pickle +import random +import logging +import collections +import numpy as np + +log = logging.getLogger("RL-ENV") + +try: + import gymnasium as gym + from gymnasium import spaces +except ImportError: + raise ImportError( + "gymnasium non installé. Exécuter : pip install gymnasium>=0.29.0" + ) + +from features import build_features, FEATURE_NAMES, NUM_FEATURES +from rl_reward import RewardTracker, compute_step_reward, compute_episode_metrics +import ensemble_core as _ens_core + + +# ─── Constantes ────────────────────────────────────────────────────────────── + +ENS_EXTRA = 2 +OBS_DIM = NUM_FEATURES + 5 + ENS_EXTRA # 62 + 5 + 2 = 69 +N_ACTIONS = 4 +FEE_RATE = 0.00004 +MIN_CANDLES = 300 +WARMUP = 50 + +# MAX_HOLD synchronisé sur config.MAX_HOLD_CANDLES (source unique de vérité, +# utilisée aussi par risk_manager.py / ahad_quant.py pour fermer les positions +# en live). AVANT ce correctif, cette constante était figée à 6 ici alors que +# config.py vaut 3 par défaut : l'agent RL était entraîné en croyant pouvoir +# garder une position deux fois plus longtemps qu'en réalité (vrai écart +# train/live). NOTE IMPORTANTE : changer cette constante seule NE CORRIGE PAS +# rétroactivement les poids déjà entraînés de rl_agent.zip / rl_checkpoints — +# un ré-entraînement via rl_train.py est nécessaire pour que l'agent en +# bénéficie pleinement. Le fallback à 6 n'est utilisé que si config.py est +# introuvable (ne devrait jamais arriver en usage normal). +try: + import config as _config + MAX_HOLD = int(getattr(_config, "MAX_HOLD_CANDLES", 6)) +except Exception: + MAX_HOLD = 6 + +SEQ_LEN = 168 + +EP_MIN_LEN = 500 +EP_MAX_LEN = 2000 + + +# ─── Chargement / prédiction ensemble ──────────────────────────────────────── +# Délégué intégralement à ensemble_core.py — SOURCE UNIQUE pour le ML, partagée +# avec ahad_quant.py (live), backtest.py (validation) et rl_agent.py (inférence +# RL live). Les alias ci-dessous préservent la compatibilité avec le code +# existant de ce fichier (qui appelle _load_ensemble / _predict_ensemble_batch +# par leur nom historique) sans dupliquer la logique. + +_load_ensemble = _ens_core.load_ensemble +_predict_ensemble_batch = _ens_core.predict_ensemble_batch + + +def _predict_ensemble(ensemble, raw_feat: np.ndarray, seq_buffer=None): + """ + Prédit pour UNE SEULE observation — utilisé par rl_agent.py en inférence live + (à chaque nouvelle bougie, paire par paire), contrairement à + _predict_ensemble_batch() qui traite tout un historique d'un coup au chargement. + + Délègue à ensemble_core.predict_ensemble_single(), qui réutilise EN INTERNE + la même fonction de stacking que le batch d'entraînement → l'inférence live + et l'entraînement appliquent EXACTEMENT la même logique ML, y compris pour + les modèles séquentiels (TFT/TransformerGRU) quand has_dl=True. + + raw_feat : vecteur (NUM_FEATURES,) de features brutes (non normalisées) + seq_buffer : deque/array de l'historique récent des features (pour les + modèles séquentiels TFT/TGRU). Si fourni et que l'ensemble a + has_dl=True, il est converti en fenêtre numpy et transmis à + ensemble_core ; sinon, les composantes DL sont simplement + omises du stacking (comme tout sous-modèle absent), sans + jamais lever d'exception. + + Retourne (proba, confidence) — deux floats Python (pas un array). + """ + history = None + if seq_buffer is not None: + try: + history = np.asarray(seq_buffer, dtype=np.float32) + if history.ndim != 2 or len(history) == 0: + history = None + except Exception: + history = None + + return _ens_core.predict_ensemble_single(ensemble, raw_feat, history=history) + + +# ─── Environnement ──────────────────────────────────────────────────────────── + +class AhadQuantForexEnv(gym.Env): + metadata = {"render_modes": ["human"]} + + def __init__( + self, + data_dir: str = "data", + pairs: list = None, + sl_pct: float = 0.005, + tp_pct: float = 0.0075, + leverage: float = 30.0, + risk_per_trade: float = 0.01, + initial_balance: float = 100.0, + normalize_obs: bool = True, + ensemble_path: str = None, + seed: int = None, + ): + super().__init__() + + self.data_dir = data_dir + self.sl_pct = sl_pct + self.tp_pct = tp_pct + self.leverage = leverage + self.risk_per_trade = risk_per_trade + self.initial_balance = initial_balance + self.normalize_obs = normalize_obs + + if ensemble_path is None: + import config + ensemble_path = getattr(config, "ENSEMBLE_MODEL_PATH", "model_ensemble.pkl") + + self._ensemble = _load_ensemble(ensemble_path) + self._obs_dim = OBS_DIM if self._ensemble else (NUM_FEATURES + 5) + + self.observation_space = spaces.Box( + low=-np.inf, high=np.inf, + shape=(self._obs_dim,), dtype=np.float32, + ) + self.action_space = spaces.Discrete(N_ACTIONS) + + self._all_data: dict = {} + self._pairs = pairs or self._detect_pairs() + self._load_all_data() + + # État interne + self._candles: list = [] + self._features: np.ndarray = np.array([]) + self._ml_cache: np.ndarray = np.array([]) # ← CACHE (N, 2) + self._step_idx: int = 0 + self._ep_end: int = 0 + self._pair: str = "" + + self._in_position: bool = False + self._direction: int = 0 + self._entry_price: float = 0.0 + self._candles_held: int = 0 + self._balance: float = initial_balance + self._equity_peak: float = initial_balance + + self._rtracker = RewardTracker() + self._rng = np.random.default_rng(seed) + + log.info(f"[RL-ENV] OBS_DIM={self._obs_dim} | " + f"Ensemble={"✅ CACHE" if self._ensemble else "❌ désactivé"}") + + # ── GYMNASIUM API ───────────────────────────────────────────────────────── + + def reset(self, seed=None, options=None): + super().reset(seed=seed) + if seed is not None: + self._rng = np.random.default_rng(seed) + + self._pair = self._rng.choice(self._pairs) + data = self._all_data[self._pair] + self._candles = data["candles"] + self._features = data["features"] + self._ml_cache = data["ml_cache"] # ← lookup direct + + n = len(self._candles) + ep_len = int(self._rng.integers(EP_MIN_LEN, min(EP_MAX_LEN, n - WARMUP))) + max_start = n - ep_len - WARMUP + if max_start <= WARMUP: + max_start = WARMUP + start = int(self._rng.integers(WARMUP, max(WARMUP + 1, max_start))) + + self._step_idx = start + self._ep_end = start + ep_len + + self._in_position = False + self._direction = 0 + self._entry_price = 0.0 + self._candles_held = 0 + self._balance = self.initial_balance + self._equity_peak = self.initial_balance + + self._rtracker.reset() + + obs = self._get_obs() + info = {"pair": self._pair, "step": self._step_idx} + return obs, info + + def step(self, action: int): + assert self.action_space.contains(action), f"Action invalide: {action}" + + candle = self._candles[self._step_idx] + current_price = float(candle["c"]) + + pnl_realized = 0.0 + fee_paid = 0.0 + trade_closed = False + + if self._in_position: + pnl_pct = self._calc_pnl_pct(current_price) + hit_sl = pnl_pct <= -self.sl_pct + hit_tp = pnl_pct >= self.tp_pct + timeout = self._candles_held >= MAX_HOLD + + if hit_sl or hit_tp or timeout: + pnl_realized, fee_paid = self._close_position(current_price) + trade_closed = True + + if not trade_closed: + if action == 1: + if self._in_position and self._direction == -1: + pnl_realized, fee_paid = self._close_position(current_price) + trade_closed = True + if not self._in_position: + self._open_position(current_price, direction=1) + elif action == 2: + if self._in_position and self._direction == 1: + pnl_realized, fee_paid = self._close_position(current_price) + trade_closed = True + if not self._in_position: + self._open_position(current_price, direction=-1) + elif action == 3: + if self._in_position: + pnl_realized, fee_paid = self._close_position(current_price) + trade_closed = True + + if self._in_position: + self._candles_held += 1 + + pnl_unrealized = 0.0 + if self._in_position: + pnl_unrealized = self._calc_pnl_pct(current_price) + + reward = compute_step_reward( + tracker = self._rtracker, + pnl_realized = pnl_realized, + pnl_unrealized = pnl_unrealized, + in_position = self._in_position, + trade_closed = trade_closed, + fee_paid = fee_paid, + ) + + self._step_idx += 1 + terminated = self._step_idx >= self._ep_end + truncated = False + + if self._balance <= self.initial_balance * 0.5: + terminated = True + reward -= 0.5 + + obs = self._get_obs() + info = { + "pair": self._pair, + "step": self._step_idx, + "balance": self._balance, + "in_position": self._in_position, + "pnl_realized": pnl_realized, + "pnl_unrealized": pnl_unrealized, + } + + if terminated: + info["episode_metrics"] = compute_episode_metrics(self._rtracker) + + return obs, reward, terminated, truncated, info + + def render(self): + candle = self._candles[self._step_idx - 1] + print( + f"[{self._pair}] Step={self._step_idx} | " + f"Price={candle['c']:.5f} | " + f"Pos={'LONG' if self._direction==1 else ('SHORT' if self._direction==-1 else 'FLAT')} | " + f"Balance={self._balance:.2f}" + ) + + # ── HELPERS PRIVÉS ──────────────────────────────────────────────────────── + + def _detect_pairs(self) -> list: + if not os.path.exists(self.data_dir): + raise FileNotFoundError(f"data_dir introuvable : {self.data_dir}") + pairs = [ + fname.replace("_1h.json", "") + for fname in os.listdir(self.data_dir) + if fname.endswith("_1h.json") + ] + if not pairs: + raise ValueError(f"Aucun fichier *_1h.json trouvé dans {self.data_dir}") + return sorted(pairs) + + def _load_all_data(self): + ref_pair = "EURUSD" + ref_path = os.path.join(self.data_dir, f"{ref_pair}_1h.json") + ref_closes = None + if os.path.exists(ref_path): + with open(ref_path) as f: + ref_data = json.load(f) + ref_closes = np.array([c["c"] for c in ref_data]) + + loaded = 0 + for pair in self._pairs: + path = os.path.join(self.data_dir, f"{pair}_1h.json") + if not os.path.exists(path): + continue + with open(path) as f: + candles = json.load(f) + if len(candles) < MIN_CANDLES: + continue + + btc_c = None + if ref_closes is not None and len(ref_closes) >= len(candles): + btc_c = ref_closes[-len(candles):] + + raw_feats = build_features(candles, btc_closes=btc_c) + raw_feats = np.nan_to_num( + raw_feats, nan=0.0, posinf=0.0, neginf=0.0 + ).astype(np.float32) + + self._all_data[pair] = { + "candles": candles, + "raw_features": raw_feats, + "features": raw_feats.copy(), + "ml_cache": None, # rempli après normalisation + } + loaded += 1 + + if loaded == 0: + raise ValueError("Aucune donnée chargée. Exécuter download_data.py d abord.") + + self._pairs = sorted(self._all_data.keys()) + print(f"[RL-ENV] {loaded} paires chargées — " + f"{sum(len(v['candles']) for v in self._all_data.values()):,} candles au total") + + # Normalisation z-score + if self.normalize_obs: + all_feats = np.concatenate( + [v["features"] for v in self._all_data.values()], axis=0 + ) + self._feat_mean = all_feats.mean(axis=0) + self._feat_std = all_feats.std(axis=0) + 1e-8 + for pair in self._all_data: + self._all_data[pair]["features"] = ( + (self._all_data[pair]["features"] - self._feat_mean) + / self._feat_std + ).astype(np.float32) + print("[RL-ENV] Features normalisées (z-score global)") + + # ── PRÉ-CALCUL CACHE ML (batch, une seule fois) ────────────────────── + if self._ensemble is not None: + total = sum(len(v["candles"]) for v in self._all_data.values()) + print(f"[RL-ENV] Pré-calcul ML cache ({total:,} candles)...") + for pair in self._pairs: + raw_f = self._all_data[pair]["raw_features"] + cache = _predict_ensemble_batch(self._ensemble, raw_f) + self._all_data[pair]["ml_cache"] = cache # (N, 2) float32 + print("[RL-ENV] Cache ML prêt — lookup O(1) pendant l entraînement ✅") + else: + for pair in self._pairs: + N = len(self._all_data[pair]["candles"]) + self._all_data[pair]["ml_cache"] = np.column_stack([ + np.full(N, 0.5, dtype=np.float32), + np.zeros(N, dtype=np.float32), + ]) + + def _get_obs(self) -> np.ndarray: + idx = self._step_idx + feat = self._features[idx].copy() + + candle_price = float(self._candles[idx]["c"]) + upnl = (np.clip(self._calc_pnl_pct(candle_price) / 0.05, -1.0, 1.0) + if self._in_position else 0.0) + + pos_feats = np.array([ + float(self._in_position), + float(self._direction), + upnl, + np.clip(self._candles_held / MAX_HOLD, 0.0, 1.0), + np.clip(self._balance / self.initial_balance, 0.0, 2.0), + ], dtype=np.float32) + + if self._ensemble is not None: + # ← LOOKUP O(1) — zéro appel ML + ens_feats = self._ml_cache[idx] # shape (2,) + obs = np.concatenate([feat, pos_feats, ens_feats]) + else: + obs = np.concatenate([feat, pos_feats]) + + return obs.astype(np.float32) + + def _open_position(self, price: float, direction: int): + self._in_position = True + self._direction = direction + self._entry_price = price + self._candles_held = 0 + + def _close_position(self, price: float) -> tuple: + pnl_pct = self._calc_pnl_pct(price) + fee_paid = FEE_RATE * 2 + trade_pnl = pnl_pct * self.risk_per_trade + self._balance = self._balance * (1.0 + trade_pnl - fee_paid) + if self._balance > self._equity_peak: + self._equity_peak = self._balance + self._in_position = False + self._direction = 0 + self._entry_price = 0.0 + self._candles_held = 0 + return pnl_pct, fee_paid + + def _calc_pnl_pct(self, current_price: float) -> float: + if not self._in_position or self._entry_price == 0: + return 0.0 + raw = (current_price - self._entry_price) / self._entry_price + return raw * self._direction * self.leverage + + # ── PROPRIÉTÉS ──────────────────────────────────────────────────────────── + + @property + def feat_mean(self) -> np.ndarray: + return self._feat_mean if self.normalize_obs else np.zeros(NUM_FEATURES) + + @property + def feat_std(self) -> np.ndarray: + return self._feat_std if self.normalize_obs else np.ones(NUM_FEATURES) + + @property + def obs_dim(self) -> int: + return self._obs_dim diff --git a/rl_reward.py b/rl_reward.py new file mode 100644 index 0000000..67fe5f0 --- /dev/null +++ b/rl_reward.py @@ -0,0 +1,166 @@ +""" +AHAD QUANT — Reinforcement Learning Reward Function +Calcule la récompense à chaque step de l'environnement. + +Philosophie : + - Récompense directement le PnL réel (pas l'accuracy) + - Pénalise les drawdowns (capital preservation) + - Encourage le Sharpe ratio (qualité des gains) + - Pénalise légèrement l'inactivité prolongée +""" + +import numpy as np +from dataclasses import dataclass, field +from typing import List + + +# ─── Configuration des poids ───────────────────────────────────────────────── + +# PnL +PNL_SCALE = 1.0 # Multiplicateur du PnL réalisé +UNREALIZED_SCALE = 0.3 # Poids du PnL non réalisé (moindre que réalisé) + +# Drawdown +DRAWDOWN_PENALTY = 2.0 # Pénalité par unité de drawdown (ex: -0.02 → -0.04) +MAX_DD_THRESHOLD = 0.05 # Drawdown > 5% : pénalité supplémentaire + +# Sharpe +SHARPE_WINDOW = 20 # Fenêtre des returns pour le Sharpe +SHARPE_BONUS = 0.1 # Bonus par unité de Sharpe (encouragement) + +# Inactivité +IDLE_PENALTY = -0.0002 # Pénalité par step sans position (encourage le trading) +MAX_IDLE_STEPS = 48 # Après 48 steps sans trade, la pénalité s'amplifie + +# Trades +WIN_BONUS = 0.05 # Bonus à chaque trade gagnant +LOSS_MULTIPLIER = 1.5 # Les pertes comptent 1.5x plus que les gains +CONSECUTIVE_WIN = 0.02 # Bonus supplémentaire pour trades gagnants consécutifs + +# Timeout +TIMEOUT_PENALTY = -0.01 # Pénalité si on tient une position > MAX_HOLD_CANDLES +TIMEOUT_THRESHOLD = 3 # Steps avant la pénalité de timeout + + +@dataclass +class RewardTracker: + """ + État interne du calculateur de récompense. + Une instance par épisode d'entraînement. + """ + peak_balance: float = 1.0 # Balance maximum atteinte (pour drawdown) + current_balance: float = 1.0 # Balance courante normalisée + returns_history: List[float] = field(default_factory=list) # Pour Sharpe + idle_steps: int = 0 # Steps consécutifs sans position + consecutive_wins: int = 0 # Trades gagnants consécutifs + total_trades: int = 0 + winning_trades: int = 0 + candles_in_pos: int = 0 # Durée de la position courante + + def reset(self): + self.peak_balance = 1.0 + self.current_balance = 1.0 + self.returns_history = [] + self.idle_steps = 0 + self.consecutive_wins = 0 + self.total_trades = 0 + self.winning_trades = 0 + self.candles_in_pos = 0 + + +def compute_step_reward( + tracker: RewardTracker, + pnl_realized: float, # PnL réalisé ce step (0.0 si pas de close) + pnl_unrealized: float, # PnL non réalisé courant (0.0 si pas en position) + in_position: bool, # L'agent est-il en position ? + trade_closed: bool, # Un trade a-t-il été fermé ce step ? + fee_paid: float, # Frais payés ce step +) -> float: + """ + Calcule la récompense scalaire pour un step donné. + + Returns: + reward (float) : récompense à donner à l'agent RL + """ + reward = 0.0 + + # ─── 1. PnL réalisé ────────────────────────────────────────────────────── + if trade_closed: + net_pnl = pnl_realized - fee_paid + if net_pnl >= 0: + reward += net_pnl * PNL_SCALE + WIN_BONUS + tracker.consecutive_wins += 1 + reward += tracker.consecutive_wins * CONSECUTIVE_WIN + tracker.winning_trades += 1 + else: + # Les pertes pèsent plus que les gains + reward += net_pnl * PNL_SCALE * LOSS_MULTIPLIER + tracker.consecutive_wins = 0 + tracker.total_trades += 1 + tracker.candles_in_pos = 0 + + # ─── 2. PnL non réalisé (signal continu en position) ───────────────────── + if in_position: + reward += pnl_unrealized * UNREALIZED_SCALE + tracker.candles_in_pos += 1 + tracker.idle_steps = 0 + + # Pénalité timeout (position trop longue) + if tracker.candles_in_pos > TIMEOUT_THRESHOLD: + reward += TIMEOUT_PENALTY * (tracker.candles_in_pos - TIMEOUT_THRESHOLD) + else: + tracker.candles_in_pos = 0 + tracker.idle_steps += 1 + + # ─── 3. Inactivité ─────────────────────────────────────────────────────── + if not in_position: + penalty = IDLE_PENALTY + if tracker.idle_steps > MAX_IDLE_STEPS: + penalty *= 2.0 # Double la pénalité après 48h d'inactivité + reward += penalty + + # ─── 4. Drawdown ───────────────────────────────────────────────────────── + tracker.current_balance = 1.0 + pnl_realized # Approx normalisée + if tracker.current_balance > tracker.peak_balance: + tracker.peak_balance = tracker.current_balance + + drawdown = (tracker.peak_balance - tracker.current_balance) / tracker.peak_balance + if drawdown > 0: + reward -= drawdown * DRAWDOWN_PENALTY + if drawdown > MAX_DD_THRESHOLD: + reward -= drawdown * DRAWDOWN_PENALTY # Double pénalité si > 5% + + # ─── 5. Bonus Sharpe (rétrospectif sur fenêtre glissante) ──────────────── + if trade_closed: + ret = pnl_realized - fee_paid + tracker.returns_history.append(ret) + if len(tracker.returns_history) > SHARPE_WINDOW: + tracker.returns_history.pop(0) + sharpe = _rolling_sharpe(tracker.returns_history) + if sharpe > 0: + reward += sharpe * SHARPE_BONUS + + return float(np.clip(reward, -1.0, 1.0)) + + +def _rolling_sharpe(returns: List[float]) -> float: + """Sharpe ratio simplifié sur une liste de returns.""" + if len(returns) < 5: + return 0.0 + arr = np.array(returns) + std = arr.std() + if std < 1e-8: + return 0.0 + return float(arr.mean() / std) + + +def compute_episode_metrics(tracker: RewardTracker) -> dict: + """Métriques de fin d'épisode pour les logs TensorBoard.""" + win_rate = (tracker.winning_trades / tracker.total_trades + if tracker.total_trades > 0 else 0.0) + return { + "ep/total_trades": tracker.total_trades, + "ep/win_rate": win_rate, + "ep/consecutive_wins": tracker.consecutive_wins, + "ep/sharpe": _rolling_sharpe(tracker.returns_history), + } diff --git a/rl_train.py b/rl_train.py new file mode 100644 index 0000000..03d6f6a --- /dev/null +++ b/rl_train.py @@ -0,0 +1,619 @@ +""" +AHAD QUANT — PPO Training Script (V6 — UNIFIED + RESUME) +Entraîne l'agent RL avec Stable-Baselines3 PPO. + +NOUVEAUTÉS V6 + FIXES : + [FIX BUG-RL-01] sys.stdout.reconfigure() wrappé dans try/except + [FIX BUG-RL-02] Bloc argparse déplacé dans if __name__ == "__main__" + [NEW] rl_progress.json — suivi de progression toutes les 10k steps + [NEW] --resume flag — reprise automatique depuis dernier checkpoint + [NEW] Drive backup automatique toutes les 100k steps + [NEW] CHECKPOINT_FREQ réduit à 50k steps (plus de granularité) + [NEW] ProgressCallback avec sauvegarde Drive intégrée + +ORDRE D'ENTRAÎNEMENT OBLIGATOIRE : + 1. python download_data.py + 2. python train.py → model_ensemble.pkl + 3. python rl_train.py → rl_agent.zip + +Reprise après interruption : + python rl_train.py --resume → reprend depuis dernier checkpoint + +Usage (local ou cloud) : + !pip install stable-baselines3[extra] gymnasium shimmy + !python rl_train.py + !python rl_train.py --resume # si interruption + +Résultats : + rl_agent.zip → modèle PPO final unifié + rl_scaler.pkl → scaler z-score des features + rl_checkpoints/ → checkpoints toutes les 50k steps + best_model.zip + rl_train_log/ → logs TensorBoard + rl_progress.json → progression (permet la reprise) +""" + +import os, sys, pickle, json, time, shutil, glob +import numpy as np + +# ─── FIX BUG-RL-01 : sys.stdout.reconfigure() incompatible avec certains terminaux ────────────── +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") +except AttributeError: + pass # certains terminaux ne supportent pas reconfigure() + +# ─── Vérification dépendances ───────────────────────────────────────────────── +try: + from stable_baselines3 import PPO + from stable_baselines3.common.env_util import make_vec_env + from stable_baselines3.common.callbacks import ( + EvalCallback, CheckpointCallback, BaseCallback + ) + from stable_baselines3.common.monitor import Monitor + from stable_baselines3.common.vec_env import DummyVecEnv +except ImportError: + print("❌ stable-baselines3 non installé.") + print(" pip install stable-baselines3[extra] gymnasium shimmy") + sys.exit(1) + +import config +from rl_env import AhadQuantForexEnv + +# ─── Vérification prérequis : ensemble doit être entraîné d'abord ───────────── +ENS_PATH = getattr(config, "ENSEMBLE_MODEL_PATH", "model_ensemble.pkl") +if not os.path.exists(ENS_PATH): + print("=" * 65) + print(" ❌ ERREUR : model_ensemble.pkl introuvable !") + print(f" Chemin attendu : {ENS_PATH}") + print(" Entraîner d'abord l'ensemble : python train.py") + print("=" * 65) + sys.exit(1) + +print(f" ✅ Ensemble trouvé : {ENS_PATH}") + +# ─── Fichier de progression (reprise après interruption) ───────────────────── +PROGRESS_FILE = "rl_progress.json" + +def save_progress(steps_done, total_steps, last_checkpoint=None, start_time=None): + """Sauvegarde la progression pour permettre la reprise automatique.""" + elapsed = time.time() - start_time if start_time else 0 + eta_sec = 0 + if steps_done > 0 and elapsed > 0: + rate = steps_done / elapsed + eta_sec = (total_steps - steps_done) / rate if rate > 0 else 0 + + data = { + "steps_done": steps_done, + "total_steps": total_steps, + "pct": round(steps_done / total_steps * 100, 2), + "last_checkpoint": last_checkpoint, + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "elapsed_min": round(elapsed / 60, 1), + "eta_min": round(eta_sec / 60, 1), + "status": "running", + } + with open(PROGRESS_FILE, "w") as f: + json.dump(data, f, indent=2) + +def load_progress(): + """Charge le fichier de progression s'il existe.""" + if os.path.exists(PROGRESS_FILE): + try: + with open(PROGRESS_FILE) as f: + return json.load(f) + except Exception: + return None + return None + +def mark_completed(total_steps): + """Marque l'entraînement comme terminé dans rl_progress.json.""" + data = load_progress() or {} + data["steps_done"] = total_steps + data["total_steps"] = total_steps + data["pct"] = 100.0 + data["status"] = "completed" + data["timestamp"] = time.strftime("%Y-%m-%d %H:%M:%S") + with open(PROGRESS_FILE, "w") as f: + json.dump(data, f, indent=2) + +# ─── Hyperparamètres PPO ────────────────────────────────────────────────────── +PPO_CONFIG = dict( + policy = "MlpPolicy", + learning_rate = 3e-4, + n_steps = 2048, + batch_size = 256, + n_epochs = 10, + gamma = 0.99, + gae_lambda = 0.95, + clip_range = 0.2, + ent_coef = 0.01, + vf_coef = 0.5, + max_grad_norm = 0.5, + verbose = 1, + tensorboard_log = "rl_train_log", + policy_kwargs = dict( + net_arch = [dict(pi=[256, 256, 128], vf=[256, 256, 128])], + ), +) + +# ─── Config entraînement ───────────────────────────────────────────────────── +DEFAULT_TOTAL_TIMESTEPS = 1_000_000 +N_ENVS = 4 +EVAL_FREQ = 20_000 +CHECKPOINT_FREQ = 50_000 # [AMÉLIO] Réduit de 100k → 50k steps +SAVE_PATH = "rl_agent" +LOG_DIR = "rl_train_log" +CHECKPOINT_DIR = "rl_checkpoints" +DRIVE_BACKUP_DIR = "/content/drive/MyDrive/ahad_quant_checkpoints" +DRIVE_BACKUP_EVERY = 100_000 # Backup Drive toutes les 100k steps + +TRAIN_PAIRS = [ + "EURUSD", "GBPUSD", "USDJPY", "USDCHF", + "AUDUSD", "USDCAD", "EURGBP", "EURJPY", +] +EVAL_PAIRS = ["GBPJPY", "NZDUSD"] + + +# ─── Callback métriques episodes ───────────────────────────────────────────── +class EpisodeMetricsCallback(BaseCallback): + def __init__(self, verbose=0): + super().__init__(verbose) + + def _on_step(self) -> bool: + for info in self.locals.get("infos", []): + if "episode_metrics" in info: + m = info["episode_metrics"] + self.logger.record("ep/win_rate", m.get("ep/win_rate", 0)) + self.logger.record("ep/total_trades", m.get("ep/total_trades", 0)) + self.logger.record("ep/sharpe", m.get("ep/sharpe", 0)) + return True + + +# ─── Callback de progression et backup Drive ───────────────────────────────── +class ProgressCallback(BaseCallback): + """ + [NEW] Sauvegarde rl_progress.json toutes les SAVE_EVERY steps. + Permet la reprise automatique après une interruption imprévue. + Effectue un backup Drive toutes les DRIVE_BACKUP_EVERY steps. + """ + def __init__(self, total_steps=1_000_000, save_every=10_000, + drive_backup_every=100_000, verbose=0): + super().__init__(verbose) + self.total_steps = total_steps + self.save_every = save_every + self.drive_backup_every = drive_backup_every + self._last_save = 0 + self._last_drive_backup = 0 + self._start_time = None + + def on_training_start(self, locals_, globals_): + self._start_time = time.time() + + def _on_step(self) -> bool: + ts = self.num_timesteps + + # Sauvegarde progression JSON + if ts - self._last_save >= self.save_every: + # Trouver le dernier checkpoint généré + ckpts = sorted(glob.glob(f"{CHECKPOINT_DIR}/ppo_ahad_quant_unified_*.zip")) + last_ckpt = ckpts[-1] if ckpts else None + save_progress(ts, self.total_steps, last_ckpt, self._start_time) + self._last_save = ts + + # Backup Drive automatique + if ts - self._last_drive_backup >= self.drive_backup_every: + self._backup_to_drive(ts) + self._last_drive_backup = ts + + return True + + def _backup_to_drive(self, steps): + """Copie les fichiers critiques vers Google Drive.""" + drive_root = "/content/drive/MyDrive" + if not os.path.exists(drive_root): + return # Drive pas monté — non bloquant + + try: + os.makedirs(DRIVE_BACKUP_DIR, exist_ok=True) + files_to_backup = [ + (PROGRESS_FILE, "rl_progress.json"), + ("rl_scaler.pkl", "rl_scaler.pkl"), + (f"{CHECKPOINT_DIR}/best_model.zip", "best_model.zip"), + ] + # Dernier checkpoint numéroté + ckpts = sorted(glob.glob(f"{CHECKPOINT_DIR}/ppo_ahad_quant_unified_*.zip")) + if ckpts: + files_to_backup.append((ckpts[-1], "latest_checkpoint.zip")) + + copied = 0 + for src, dst_name in files_to_backup: + if os.path.exists(src): + shutil.copy2(src, os.path.join(DRIVE_BACKUP_DIR, dst_name)) + copied += 1 + + pct = steps / self.total_steps * 100 + print(f"\n 💾 [Drive backup @ {steps:,} steps — {pct:.0f}%] {copied} fichiers → {DRIVE_BACKUP_DIR}") + + except Exception as e: + print(f"\n ⚠️ Drive backup échoué (non bloquant) : {e}") + + +# ─── Factory d'environnements ───────────────────────────────────────────────── +def _make_env(pairs: list, seed: int = 0, ensemble_path: str = None): + def _init(): + env = AhadQuantForexEnv( + data_dir = config.DATA_DIR, + pairs = pairs, + sl_pct = config.STOP_LOSS_PCT, + tp_pct = config.TAKE_PROFIT_PCT, + leverage = float(config.LEVERAGE), + risk_per_trade = config.RISK_PER_TRADE, + ensemble_path = ensemble_path, + seed = seed, + ) + return Monitor(env) + return _init + + +# ─── Sauvegarde du scaler ───────────────────────────────────────────────────── +def save_scaler(env: AhadQuantForexEnv, path: str = "rl_scaler.pkl"): + scaler = { + "mean": env._feat_mean, + "std": env._feat_std, + "pairs": env._pairs, + } + with open(path, "wb") as f: + pickle.dump(scaler, f) + print(f"[SCALER] Sauvegardé → {path}") + return scaler + + +# ─── Fonction de reprise — trouver le bon checkpoint ───────────────────────── +def find_resume_checkpoint(): + """ + [NEW] Identifie le meilleur checkpoint depuis lequel reprendre. + Priorité : last_checkpoint dans progress > best_model.zip > dernier numéroté + """ + prog = load_progress() + if prog: + last_ckpt = prog.get("last_checkpoint") + if last_ckpt and os.path.exists(last_ckpt): + return last_ckpt.replace(".zip", ""), prog + + best = f"{CHECKPOINT_DIR}/best_model.zip" + if os.path.exists(best): + return best.replace(".zip", ""), prog + + numbered = sorted(glob.glob(f"{CHECKPOINT_DIR}/ppo_ahad_quant_unified_*.zip")) + if numbered: + return numbered[-1].replace(".zip", ""), prog + + # Chercher aussi dans le backup Drive + drive_ckpt = os.path.join(DRIVE_BACKUP_DIR, "latest_checkpoint.zip") + drive_best = os.path.join(DRIVE_BACKUP_DIR, "best_model.zip") + for drive_src in [drive_ckpt, drive_best]: + if os.path.exists(drive_src): + local_dst = os.path.join(CHECKPOINT_DIR, os.path.basename(drive_src)) + os.makedirs(CHECKPOINT_DIR, exist_ok=True) + shutil.copy2(drive_src, local_dst) + print(f" ✅ Checkpoint récupéré depuis Drive : {os.path.basename(drive_src)}") + return local_dst.replace(".zip", ""), prog + + return None, prog + + +# ─── Entraînement principal ─────────────────────────────────────────────────── +def train(total_steps: int = DEFAULT_TOTAL_TIMESTEPS, resume: bool = False): + """ + Entraîne ou reprend l'agent PPO. + + Args: + total_steps: Nombre total de steps à entraîner. + resume: Si True, tente de reprendre depuis le dernier checkpoint. + """ + print("=" * 65) + print(" AHAD QUANT — RL Training Unifié (PPO V6)") + print(" Ensemble supervisé intégré dans l'observation") + if resume: + print(" MODE : REPRISE depuis checkpoint") + else: + print(" MODE : Nouvel entraînement") + print("=" * 65) + + os.makedirs(LOG_DIR, exist_ok=True) + os.makedirs(CHECKPOINT_DIR, exist_ok=True) + + # ── Filtrer paires disponibles ───────────────────────────────────────────── + available = [ + p for p in TRAIN_PAIRS + if os.path.exists(os.path.join(config.DATA_DIR, f"{p}_1h.json")) + ] + if not available: + print(f"\n❌ Aucune donnée dans {config.DATA_DIR}. Exécuter download_data.py d'abord.") + sys.exit(1) + + print(f"\n Paires entraînement : {available}") + print(f" Ensemble : {ENS_PATH}") + + # ── Résolution checkpoint de reprise ────────────────────────────────────── + resume_from = None + steps_already = 0 + remaining_steps = total_steps + + if resume: + resume_from, prog = find_resume_checkpoint() + if prog: + steps_already = prog.get("steps_done", 0) + remaining_steps = max(0, total_steps - steps_already) + + if steps_already >= total_steps: + print(f"\n✅ Entraînement déjà complété ({steps_already:,} steps). Rien à faire.") + return + + if resume_from: + print(f"\n 🔄 Reprise depuis : {resume_from}.zip") + print(f" ✅ Steps déjà effectués : {steps_already:,} / {total_steps:,} ({steps_already/total_steps*100:.1f}%)") + print(f" ⏳ Steps restants : {remaining_steps:,}") + else: + print(" ⚠️ Aucun checkpoint trouvé — entraînement depuis zéro") + resume = False + + print(f"\n Steps à entraîner : {remaining_steps:,}") + print(f" Checkpoints toutes les {CHECKPOINT_FREQ:,} steps") + print(f" Backup Drive toutes les {DRIVE_BACKUP_EVERY:,} steps\n") + + # ── Envs parallèles ──────────────────────────────────────────────────────── + env_fns = [_make_env(available, seed=i, ensemble_path=ENS_PATH) for i in range(N_ENVS)] + vec_env = DummyVecEnv(env_fns) + + # ── Env d'évaluation ────────────────────────────────────────────────────── + eval_pairs = [p for p in EVAL_PAIRS + if os.path.exists(os.path.join(config.DATA_DIR, f"{p}_1h.json"))] + if not eval_pairs: + eval_pairs = available[:2] + print(f" Paires évaluation : {eval_pairs}") + + eval_env = Monitor(AhadQuantForexEnv( + data_dir = config.DATA_DIR, + pairs = eval_pairs, + ensemble_path = ENS_PATH, + seed = 999, + )) + + # ── Scaler (toujours recalculé pour cohérence) ──────────────────────────── + tmp_env = AhadQuantForexEnv( + data_dir = config.DATA_DIR, + pairs = available, + ensemble_path = ENS_PATH, + seed = 0, + ) + tmp_env.reset() + save_scaler(tmp_env, "rl_scaler.pkl") + obs_dim = tmp_env.obs_dim + del tmp_env + + print(f"\n OBS_DIM : {obs_dim} dims") + + # ── Créer ou charger le modèle PPO ──────────────────────────────────────── + if resume and resume_from and os.path.exists(f"{resume_from}.zip"): + try: + model = PPO.load( + resume_from, + env = vec_env, + verbose = PPO_CONFIG["verbose"], + tensorboard_log= PPO_CONFIG["tensorboard_log"], + ) + print(f"\n ✅ Modèle PPO chargé depuis checkpoint") + except Exception as e: + print(f"\n ⚠️ Échec chargement checkpoint : {e}") + print(" → Redémarrage depuis zéro") + model = PPO(env=vec_env, **PPO_CONFIG) + resume = False + steps_already = 0 + remaining_steps = total_steps + else: + model = PPO(env=vec_env, **PPO_CONFIG) + print(f"\n 🆕 Nouveau modèle PPO créé") + + # ── Callbacks ───────────────────────────────────────────────────────────── + eval_cb = EvalCallback( + eval_env, + best_model_save_path = CHECKPOINT_DIR, + log_path = LOG_DIR, + eval_freq = EVAL_FREQ // N_ENVS, + n_eval_episodes = 10, + deterministic = True, + verbose = 1, + ) + checkpoint_cb = CheckpointCallback( + save_freq = CHECKPOINT_FREQ // N_ENVS, + save_path = CHECKPOINT_DIR, + name_prefix = "ppo_ahad_quant_unified", + verbose = 1, + ) + metrics_cb = EpisodeMetricsCallback() + progress_cb = ProgressCallback( + total_steps = total_steps, + save_every = 10_000, + drive_backup_every = DRIVE_BACKUP_EVERY, + ) + + # ── Lancement ───────────────────────────────────────────────────────────── + print(f"\n Démarrage entraînement...\n") + start = time.time() + + try: + model.learn( + total_timesteps = remaining_steps, + callback = [eval_cb, checkpoint_cb, metrics_cb, progress_cb], + reset_num_timesteps = not resume, # Ne pas reset si reprise + progress_bar = True, + ) + except KeyboardInterrupt: + print("\n\n⚠️ Entraînement interrompu manuellement (Ctrl+C).") + print(f" Progression sauvegardée dans {PROGRESS_FILE}") + print(" Relancer avec : python rl_train.py --resume") + vec_env.close() + eval_env.close() + return + except Exception as e: + print(f"\n\n❌ Erreur pendant l'entraînement : {e}") + print(f" Progression sauvegardée dans {PROGRESS_FILE}") + print(" Relancer avec : python rl_train.py --resume") + vec_env.close() + eval_env.close() + raise + + # ── Sauvegarde finale ───────────────────────────────────────────────────── + elapsed = time.time() - start + print(f"\n ✅ Entraînement terminé en {elapsed/60:.1f} min") + + model.save(SAVE_PATH) + print(f" 💾 Modèle sauvegardé → {SAVE_PATH}.zip") + + mark_completed(total_steps) + print(f" 📝 Progression marquée 'completed' → {PROGRESS_FILE}") + + vec_env.close() + eval_env.close() + + # ── Résumé ──────────────────────────────────────────────────────────────── + print("\n" + "=" * 65) + print(" FICHIERS PRODUITS") + print("=" * 65) + for fname, desc in [ + (f"{SAVE_PATH}.zip", "PPO unifié (cerveau final)"), + ("rl_scaler.pkl", "Normalisation features"), + (f"{CHECKPOINT_DIR}/best_model.zip", "Meilleur checkpoint"), + (PROGRESS_FILE, "Fichier de progression"), + (ENS_PATH, "Ensemble ML (feature PPO)"), + ]: + if os.path.exists(fname): + size = os.path.getsize(fname) / 1024 / 1024 + print(f" ✅ {fname:<45} {size:.1f} MB") + else: + print(f" ⚠️ {fname} — non trouvé") + print() + + # ── Export unifié ───────────────────────────────────────────────────────── + print("[Export] Modèle unifié...") + try: + from export_unified import export_unified + unified_path = getattr(config, "UNIFIED_MODEL_PATH", "ahad_quant_unified.zip") + export_unified( + ppo_path = SAVE_PATH, + ensemble_path = ENS_PATH, + scaler_path = "rl_scaler.pkl", + output_path = unified_path, + ) + except Exception as e: + print(f" ⚠️ Export unifié échoué (non bloquant) : {e}") + + +# ─── Fine-tuning classique ──────────────────────────────────────────────────── +def fine_tune(model_path: str = "rl_agent", extra_steps: int = 200_000): + """Fine-tune le modèle existant (appelé par auto_retrain.py ou CLI).""" + print(f"[RL FINE-TUNE] Chargement {model_path}.zip...") + + available = [ + p for p in TRAIN_PAIRS + if os.path.exists(os.path.join(config.DATA_DIR, f"{p}_1h.json")) + ] + env_fns = [_make_env(available, seed=i, ensemble_path=ENS_PATH) for i in range(N_ENVS)] + vec_env = DummyVecEnv(env_fns) + + model = PPO.load(model_path, env=vec_env) + model.learn( + total_timesteps = extra_steps, + reset_num_timesteps = False, + progress_bar = True, + ) + model.save(model_path) + print(f"[RL FINE-TUNE] ✅ Sauvegardé → {model_path}.zip (+{extra_steps:,} steps)") + vec_env.close() + + +# ─── Fine-tuning avec replay ───────────────────────────────────────────────── +def fine_tune_with_replay( + model_path: str = "rl_agent", + extra_steps: int = 200_000, + replay_file: str = None, +) -> None: + """Fine-tune PPO + injection d'épisodes réels depuis un fichier JSON.""" + print(f"[RL FINE-TUNE+REPLAY] Chargement {model_path}.zip...") + + available = [ + p for p in TRAIN_PAIRS + if os.path.exists(os.path.join(config.DATA_DIR, f"{p}_1h.json")) + ] + env_fns = [_make_env(available, seed=i, ensemble_path=ENS_PATH) for i in range(N_ENVS)] + vec_env = DummyVecEnv(env_fns) + model = PPO.load(model_path, env=vec_env) + + if replay_file and os.path.exists(replay_file): + try: + with open(replay_file, "r") as f: + episodes = json.load(f) + print(f"[RL FINE-TUNE+REPLAY] {len(episodes)} épisodes réels chargés") + + injected = 0 + for ep in episodes: + try: + obs = np.array(ep["obs"], dtype=np.float32) + action = int(ep["action"]) + reward = float(np.clip(ep["reward"], -1.0, 1.0)) + done = bool(ep.get("done", False)) + if hasattr(model, "rollout_buffer") and model.rollout_buffer is not None: + buf = model.rollout_buffer + if not buf.full: + buf.add( + obs.reshape(1, -1), + np.array([[action]]), + np.array([reward]), + np.array([done]), + np.zeros((1,), dtype=np.float32), + np.zeros((1,), dtype=np.float32), + ) + injected += 1 + except Exception: + continue + print(f"[RL FINE-TUNE+REPLAY] ✅ {injected}/{len(episodes)} épisodes injectés") + except Exception as rep_err: + print(f"[RL FINE-TUNE+REPLAY] ⚠️ Erreur injection (non bloquant) : {rep_err}") + elif replay_file: + print(f"[RL FINE-TUNE+REPLAY] ⚠️ Fichier replay introuvable : {replay_file}") + + model.learn( + total_timesteps = extra_steps, + reset_num_timesteps = False, + progress_bar = True, + ) + model.save(model_path) + print(f"[RL FINE-TUNE+REPLAY] ✅ Sauvegardé → {model_path}.zip (+{extra_steps:,} steps)") + vec_env.close() + + +# ─── FIX BUG-RL-02 : argparse déplacé dans if __name__ == "__main__" ────────── +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="AHAD QUANT PPO Training") + parser.add_argument("--finetune", action="store_true", + help="Fine-tuner le modèle existant") + parser.add_argument("--resume", action="store_true", + help="[NEW] Reprendre depuis le dernier checkpoint") + parser.add_argument("--steps", type=int, default=DEFAULT_TOTAL_TIMESTEPS, + help=f"Nombre de steps (défaut: {DEFAULT_TOTAL_TIMESTEPS:,})") + parser.add_argument("--model", type=str, default="rl_agent", + help="Chemin du modèle pour fine-tune (sans .zip)") + parser.add_argument("--replay", type=str, default=None, + help="Fichier JSON d'épisodes réels pour fine-tune+replay") + args = parser.parse_args() + + if args.finetune and args.replay: + fine_tune_with_replay(args.model, args.steps, args.replay) + elif args.finetune: + fine_tune(args.model, args.steps) + elif args.resume: + train(total_steps=args.steps, resume=True) + else: + train(total_steps=args.steps, resume=False) diff --git a/tft_model.py b/tft_model.py new file mode 100644 index 0000000..14913c5 --- /dev/null +++ b/tft_model.py @@ -0,0 +1,381 @@ +""" +AHAD QUANT Forex V5 — Temporal Fusion Transformer (TFT) +Inspired by Lim et al., 2021 — adapted for binary classification on Forex 1h data. + +Architecture +─────────────────────────────────────────────────────────────── +Input : (B, T, 62) — B batches, T=168 timesteps, 62 features +Layer 1: Variable Selection Network (VSN) + — per-timestep soft feature weighting via GRN + Softmax + → (B, T, D_MODEL) +Layer 2: Local LSTM encoder + — captures short-range sequential dependencies + → (B, T, D_MODEL) +Layer 3: Gated skip connection (GRN) with add+norm + → (B, T, D_MODEL) +Layer 4: Temporal Multi-Head Self-Attention (N_HEADS heads) + — captures long-range dependencies + → (B, T, D_MODEL) +Layer 5: Gated skip connection + mean pooling + → (B, D_MODEL) +Layer 6: Classification head + GRN → Linear(D_MODEL, 1) — logit (use with BCEWithLogitsLoss) +Output : (B,) logit — apply sigmoid for probability + +Training util functions +─────────────────────── + train_tft(X_seq_tr, y_tr, X_seq_va, y_va, seq_scaler, ...) → TemporalFusionTransformer + predict_tft_proba(model, X_seq_norm_tensor) → np.ndarray (B,) +""" + +import math +import time + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader, TensorDataset + +# ─── Hyper-parameters (can be overridden via kwargs in train_tft) ──────────── + +D_MODEL = 64 # embedding / hidden dimension +N_HEADS = 4 # attention heads (D_MODEL must be divisible by N_HEADS) +LSTM_LAYERS = 1 # LSTM depth +N_ATT_LAYERS = 2 # stacked temporal attention blocks +DROPOUT = 0.10 +BATCH_SIZE = 512 +LR = 8e-4 # raised from 5e-4: VSN's 62 parallel GRNs need more + # headroom once GRAD_CLIP is loosened (see below) +WEIGHT_DECAY = 1e-4 +MAX_EPOCHS = 30 +PATIENCE = 6 # early stopping patience +GRAD_CLIP = 5.0 # raised from 1.0: the VSN's 62 independent per-feature + # GRNs produce a much larger raw gradient norm than a + # single shared embedding (e.g. TransformerGRU's), + # so a clip of 1.0 over-throttled ALL parameters and + # kept loss stuck near ln(2). 5.0 lets real updates + # through while still preventing rare gradient spikes. + + +# ─── Building blocks ───────────────────────────────────────────────────────── + +class GatedResidualNetwork(nn.Module): + """ + Core TFT building block. + x → ELU(W1·x) → Dropout → W2·(·) → GLU gate → LayerNorm(skip + output) + + Supports optional context vector (c) injected before the first linear. + """ + + def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, + dropout: float = DROPOUT, context_dim: int = 0): + super().__init__() + self.fc1 = nn.Linear(input_dim + context_dim, hidden_dim) + self.fc2 = nn.Linear(hidden_dim, output_dim * 2) # × 2 for GLU + # Skip connection: project input → output dim if they differ + self.skip = nn.Linear(input_dim, output_dim) if input_dim != output_dim else nn.Identity() + self.norm = nn.LayerNorm(output_dim) + self.drop = nn.Dropout(dropout) + self.out_dim = output_dim + + def forward(self, x: torch.Tensor, context: torch.Tensor | None = None) -> torch.Tensor: + h = x if context is None else torch.cat([x, context], dim=-1) + h = F.elu(self.fc1(h)) + h = self.drop(h) + h = self.fc2(h) # (*, out_dim*2) + h1, h2 = h.chunk(2, dim=-1) # GLU split + h = h1 * torch.sigmoid(h2) # Gated Linear Unit + return self.norm(self.skip(x) + h) + + +class VariableSelectionNetwork(nn.Module): + """ + Soft per-timestep feature selection. + + Each of the `num_features` features is projected to D_MODEL via its own GRN. + A shared GRN over the concatenated input produces Softmax weights. + Output = weighted sum of per-feature projections → (B, T, D_MODEL). + """ + + def __init__(self, num_features: int, d_model: int = D_MODEL, + dropout: float = DROPOUT): + super().__init__() + self.d_model = d_model + self.num_features = num_features + + # One GRN per feature: (B, T, 1) → (B, T, d_model) + self.var_grns = nn.ModuleList([ + GatedResidualNetwork(1, d_model, d_model, dropout) + for _ in range(num_features) + ]) + + # Selection GRN: (B, T, num_features) → weights (B, T, num_features) + self.select_grn = GatedResidualNetwork(num_features, d_model, num_features, dropout) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + x : (B, T, num_features) + → (B, T, d_model) + """ + B, T, num_feat = x.shape + + # Per-feature embeddings: (B*T, 1) → (B*T, d_model) → stack → (B, T, F, d_model) + xi = x.reshape(B * T, num_feat) # (B*T, F) + embeddings = torch.stack( + [grn(xi[:, i : i + 1]) for i, grn in enumerate(self.var_grns)], + dim=1 # (B*T, F, d_model) + ) # → (B*T, F, d_model) + + # Selection weights + weights = self.select_grn(xi) # (B*T, F) logits + weights = F.softmax(weights, dim=-1) # (B*T, F) soft weights + + # Weighted sum over features + out = (embeddings * weights.unsqueeze(-1)).sum(dim=1) # (B*T, d_model) + return out.reshape(B, T, self.d_model) + + +class TemporalAttentionBlock(nn.Module): + """One layer of temporal self-attention + gated residual.""" + + def __init__(self, d_model: int = D_MODEL, n_heads: int = N_HEADS, + dropout: float = DROPOUT): + super().__init__() + self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) + self.gate = GatedResidualNetwork(d_model, d_model, d_model, dropout) + self.norm = nn.LayerNorm(d_model) + self.drop = nn.Dropout(dropout) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + attn_out, _ = self.attn(x, x, x) # (B, T, d_model) + attn_out = self.drop(attn_out) + return self.norm(x + self.gate(attn_out)) # residual + + +# ─── Main model ────────────────────────────────────────────────────────────── + +class TemporalFusionTransformer(nn.Module): + """ + TFT for binary classification on time-series of shape (B, T, num_features). + Output: (B,) raw logit — use with BCEWithLogitsLoss. + """ + + def __init__( + self, + num_features : int = 62, + seq_len : int = 168, + d_model : int = D_MODEL, + n_heads : int = N_HEADS, + lstm_layers : int = LSTM_LAYERS, + n_att_layers : int = N_ATT_LAYERS, + dropout : float = DROPOUT, + ): + super().__init__() + self.seq_len = seq_len + + # ── 1. Input embedding ─────────────────────────────────────────────── + # NOTE: previously used a VariableSelectionNetwork with one independent + # GatedResidualNetwork PER FEATURE (62 branches), combined via softmax- + # weighted average. Averaging ~62 quasi-independent branches collapses + # inter-sample variance by a factor of ~sqrt(num_features) (classic + # noise-cancellation effect), starving the model of signal before + # training even starts — confirmed empirically (logit std stuck ~0.03 + # across 60 training steps even on an easy synthetic task, vs healthy + # growth to ~0.65 with this simpler shared projection). Replaced with + # a single shared Linear + GRN, matching the embedding strategy that + # already works well in transformer_gru_model.py. + self.input_proj = nn.Linear(num_features, d_model) + self.input_grn = GatedResidualNetwork(d_model, d_model, d_model, dropout) + + # ── 2. Local LSTM encoder ──────────────────────────────────────────── + self.lstm = nn.LSTM( + input_size = d_model, + hidden_size = d_model, + num_layers = lstm_layers, + batch_first = True, + dropout = dropout if lstm_layers > 1 else 0.0, + ) + self.lstm_gate = GatedResidualNetwork(d_model, d_model, d_model, dropout) + self.lstm_norm = nn.LayerNorm(d_model) + + # ── 3. Temporal attention stack ────────────────────────────────────── + self.attn_blocks = nn.ModuleList([ + TemporalAttentionBlock(d_model, n_heads, dropout) + for _ in range(n_att_layers) + ]) + + # ── 4. Output head ─────────────────────────────────────────────────── + self.output_grn = GatedResidualNetwork(d_model, d_model, d_model, dropout) + self.head = nn.Linear(d_model, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + x : (B, T, num_features) + → (B,) logit + """ + # 1. Shared input embedding (see note in __init__) + h = self.input_grn(self.input_proj(x)) # (B, T, d_model) + + # 2. LSTM local encoder + gated skip + lstm_out, _ = self.lstm(h) # (B, T, d_model) + h = self.lstm_norm(h + self.lstm_gate(lstm_out)) # add+norm + + # 3. Stacked temporal attention + for block in self.attn_blocks: + h = block(h) # (B, T, d_model) + + # 4. Mean pooling over time + classification + h = h.mean(dim=1) # (B, d_model) + h = self.output_grn(h) + return self.head(h).squeeze(-1) # (B,) + + +# ─── Training utilities ────────────────────────────────────────────────────── + +def _make_loader(X: np.ndarray, y: np.ndarray, + batch_size: int, shuffle: bool) -> DataLoader: + """Wrap numpy arrays in a TensorDataset / DataLoader.""" + X_t = torch.FloatTensor(X) + y_t = torch.FloatTensor(y) + return DataLoader(TensorDataset(X_t, y_t), + batch_size=batch_size, shuffle=shuffle, + num_workers=0, pin_memory=torch.cuda.is_available()) + + +def train_tft( + X_seq_tr : np.ndarray, + y_tr : np.ndarray, + X_seq_va : np.ndarray, + y_va : np.ndarray, + *, + # Architecture overrides + d_model : int = D_MODEL, + n_heads : int = N_HEADS, + n_att_layers : int = N_ATT_LAYERS, + dropout : float = DROPOUT, + # Training overrides + batch_size : int = BATCH_SIZE, + lr : float = LR, + weight_decay : float = WEIGHT_DECAY, + max_epochs : int = MAX_EPOCHS, + patience : int = PATIENCE, + grad_clip : float = GRAD_CLIP, +) -> "TemporalFusionTransformer": + """ + Train a TemporalFusionTransformer on pre-normalised sequence data. + + Parameters + ---------- + X_seq_tr / X_seq_va : (N, T, 62) float32 — already normalised + y_tr / y_va : (N,) float32 binary labels + + Returns + ------- + Best model (lowest val loss) as a CPU-resident TemporalFusionTransformer. + """ + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print(f" [TFT] device={device} | " + f"train={len(y_tr):,} val={len(y_va):,} samples") + + _, T, F = X_seq_tr.shape + model = TemporalFusionTransformer( + num_features = F, + seq_len = T, + d_model = d_model, + n_heads = n_heads, + n_att_layers = n_att_layers, + dropout = dropout, + ).to(device) + + n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + print(f" [TFT] parameters: {n_params:,}") + + optimiser = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimiser, T_max=max_epochs) + criterion = nn.BCEWithLogitsLoss() + scaler_amp = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available()) + + train_loader = _make_loader(X_seq_tr, y_tr, batch_size, shuffle=True) + val_loader = _make_loader(X_seq_va, y_va, batch_size * 2, shuffle=False) + + best_val_loss = float("inf") + best_state = None + no_improve = 0 + t0 = time.time() + + for epoch in range(1, max_epochs + 1): + # ── Train ──────────────────────────────────────────────────────────── + model.train() + train_loss = 0.0 + for X_b, y_b in train_loader: + X_b, y_b = X_b.to(device), y_b.to(device) + optimiser.zero_grad() + with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()): + logits = model(X_b) + loss = criterion(logits, y_b) + scaler_amp.scale(loss).backward() + scaler_amp.unscale_(optimiser) + nn.utils.clip_grad_norm_(model.parameters(), grad_clip) + scaler_amp.step(optimiser) + scaler_amp.update() + train_loss += loss.item() * len(y_b) + train_loss /= len(y_tr) + + # ── Validate ───────────────────────────────────────────────────────── + model.eval() + val_loss = 0.0 + correct = 0 + with torch.no_grad(): + for X_b, y_b in val_loader: + X_b, y_b = X_b.to(device), y_b.to(device) + logits = model(X_b) + val_loss += criterion(logits, y_b).item() * len(y_b) + correct += ((logits > 0).float() == y_b).sum().item() + val_loss /= len(y_va) + val_acc = correct / len(y_va) + + scheduler.step() + + elapsed = time.time() - t0 + print(f" [TFT] epoch {epoch:3d}/{max_epochs} " + f"train={train_loss:.4f} val={val_loss:.4f} " + f"val_acc={val_acc:.4f} ({elapsed:.0f}s)") + + # ── Early stopping ─────────────────────────────────────────────────── + if val_loss < best_val_loss - 1e-5: + best_val_loss = val_loss + best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} + no_improve = 0 + else: + no_improve += 1 + if no_improve >= patience: + print(f" [TFT] early stopping at epoch {epoch}") + break + + model.load_state_dict(best_state) + model.cpu() + print(f" [TFT] training complete — best val_loss={best_val_loss:.4f}") + return model + + +@torch.no_grad() +def predict_tft_proba( + model : TemporalFusionTransformer, + X_seq_t : torch.Tensor, + batch_size: int = 1024, + device : str = "cpu", +) -> np.ndarray: + """ + Run inference on a (N, T, F) float tensor. + Returns probability array of shape (N,). + """ + model.eval() + model.to(device) + probs = [] + for start in range(0, len(X_seq_t), batch_size): + chunk = X_seq_t[start : start + batch_size].to(device) + logits = model(chunk) + probs.append(torch.sigmoid(logits).cpu().numpy()) + model.cpu() + return np.concatenate(probs).astype(np.float32) diff --git a/train.py b/train.py new file mode 100644 index 0000000..4547163 --- /dev/null +++ b/train.py @@ -0,0 +1,559 @@ +""" +AHAD QUANT — Ensemble Training Pipeline (V5 — Deep Learning Edition) +Trains LightGBM + XGBoost + RandomForest + TFT + TransformerGRU ensemble. + +Usage: + python download_data.py # download historical data first + python train.py # train full ensemble (~2-6h, GPU recommandé) + python train.py --no-dl # ML only, skip TFT/TGRU (~15-30 min) + +What's new in V5 vs V4: + - TFT (Temporal Fusion Transformer) — variable selection + attention + - TGRU (TransformerGRU hybrid) — bidir GRU + transformer encoder + - Sequence dataset builder — per-pair 168-candle sliding windows + - Extended meta-learner — up to 5 base models (was 3) + - Backward-compatible when torch/DL unavailable — falls back to ML-only +""" + +import json, os, pickle, sys, time, argparse +sys.stdout.reconfigure(encoding="utf-8", errors="replace") +import numpy as np +import lightgbm as lgb +import config +from features import build_features, FEATURE_NAMES + +# ─── CLI flags ─────────────────────────────────────────────────────────────── +_parser = argparse.ArgumentParser(add_help=False) +_parser.add_argument("--no-dl", action="store_true", + help="Skip TFT/TGRU training (ML-only mode, faster)") +_args, _ = _parser.parse_known_args() +SKIP_DL: bool = _args.no_dl + +try: + import xgboost as xgb + HAS_XGB = True +except ImportError: + HAS_XGB = False + print("[WARN] xgboost not installed — skipping. pip install xgboost") + +try: + from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier + from sklearn.linear_model import LogisticRegression + from sklearn.preprocessing import StandardScaler + from sklearn.metrics import accuracy_score + HAS_SKLEARN = True +except ImportError: + HAS_SKLEARN = False + print("[WARN] scikit-learn not installed — skipping RF. pip install scikit-learn") + +try: + import optuna + optuna.logging.set_verbosity(optuna.logging.WARNING) + HAS_OPTUNA = True +except ImportError: + HAS_OPTUNA = False + +# ─── Deep Learning imports (optional — guarded) ────────────────────────────── +HAS_DL = False +if not SKIP_DL: + try: + import torch + from prepare_sequences import ( + prepare_seq_dataset, fit_seq_scaler, transform_sequences, SEQ_LEN + ) + from tft_model import ( + TemporalFusionTransformer, train_tft, predict_tft_proba + ) + from transformer_gru_model import ( + TransformerGRU, train_tgru, predict_tgru_proba + ) + HAS_DL = True + print(f"[DL] PyTorch {torch.__version__} detected — TFT + TGRU enabled") + except ImportError as _dl_err: + print(f"[WARN] DL modules not available ({_dl_err}). " + "Falling back to ML-only. " + "Install with: pip install torch>=2.2.0") + +# ─── Config ───────────────────────────────────────────────────────────────── +LOOKAHEAD = 3 +MIN_CANDLES = 200 +TRAIN_RATIO = 0.70 +VAL_RATIO = 0.15 +N_WALK_FORWARD_WINDOWS = 4 +OPTUNA_TRIALS = 30 # increase for better tuning (slower) + + +# ─── Data loading ──────────────────────────────────────────────────────────── + +def load_candles(coin: str) -> dict | None: + path = os.path.join(config.DATA_DIR, f"{coin}_1h.json") + if not os.path.exists(path): + return None + with open(path) as f: + return json.load(f) + + +def make_labels(close: np.ndarray, lookahead: int = LOOKAHEAD) -> np.ndarray: + labels = np.zeros(len(close)) + for i in range(len(close) - lookahead): + labels[i] = 1.0 if close[i + lookahead] > close[i] else 0.0 + return labels + + +def prepare_dataset() -> tuple[np.ndarray, np.ndarray]: + all_X, all_y = [], [] + skipped = [] + # Paire de référence pour la corrélation (EURUSD = paire dominante en Forex) + _ref_pair = "EURUSD" + btc_data = load_candles(_ref_pair) + if btc_data is None: + print(f" [WARN] Paire de référence {_ref_pair} non trouvée dans data/ —" + " features de corrélation désactivées.") + btc_close = np.array([c["c"] for c in btc_data]) if btc_data else None + + for coin in config.COINS: + data = load_candles(coin) + if data is None: + print(f" ⚠️ {coin:8s} — fichier data/{coin}_1h.json introuvable") + skipped.append((coin, "fichier manquant")) + continue + if len(data) < MIN_CANDLES: + print(f" ⚠️ {coin:8s} — données insuffisantes ({len(data)} bougies < {MIN_CANDLES} min)") + skipped.append((coin, f"seulement {len(data)} bougies")) + continue + close = np.array([c["c"] for c in data]) + coin_btc = btc_close[-len(close):] if btc_close is not None and len(btc_close) >= len(close) else None + + X = build_features(data, btc_closes=coin_btc) + y = make_labels(close) + warmup = 30 + X, y = X[warmup:-LOOKAHEAD], y[warmup:-LOOKAHEAD] + valid = ~np.isnan(X).any(axis=1) + X, y = X[valid], y[valid] + all_X.append(X); all_y.append(y) + print(f" ✅ {coin:8s} — {len(X):,} samples ({round(len(data)/24)} jours)") + + # ─── Guard critique : aucune paire chargée ──────────────────────────────── + if len(all_X) == 0: + print() + print("━" * 60) + print(" ERREUR : Aucune paire chargée. Lancez d'abord :") + print(" python download_data.py") + print() + print(" Paires manquantes :") + for coin, reason in skipped: + print(f" {coin:8s} — {reason}") + print("━" * 60) + raise RuntimeError( + "Aucune donnée disponible dans data/. " + "Exécutez `python download_data.py` avant `python train.py`." + ) + + if len(all_X) < 3: + print(f"\n [WARN] Seulement {len(all_X)} paire(s) chargée(s). " + f"Résultats d'entraînement potentiellement insuffisants. " + f"Minimum recommandé : 5 paires.") + + if skipped: + print(f"\n [INFO] {len(skipped)} paire(s) ignorée(s) : " + f"{', '.join(c for c, _ in skipped)}") + + return np.concatenate(all_X), np.concatenate(all_y) + + +# ─── Walk-forward cross-validation ────────────────────────────────────────── + +def walk_forward_cv(X: np.ndarray, y: np.ndarray, n_windows: int = 4) -> dict: + """ + Walk-forward validation with expanding window. + Returns mean accuracy and std across windows. + """ + n = len(X) + base = int(n * 0.5) # first training window = 50% of data + step = (n - base) // n_windows + results = [] + + print(f"\n Walk-forward CV ({n_windows} windows):") + for i in range(n_windows): + train_end = base + i * step + test_end = min(train_end + step, n) + X_tr, y_tr = X[:train_end], y[:train_end] + X_te, y_te = X[train_end:test_end], y[train_end:test_end] + + # Quick LightGBM for CV (fast) + ds_tr = lgb.Dataset(X_tr, label=y_tr) + ds_va = lgb.Dataset(X_te, label=y_te, reference=ds_tr) + params = {"objective": "binary", "metric": "binary_logloss", + "num_leaves": 63, "learning_rate": 0.05, "verbose": -1} + m = lgb.train(params, ds_tr, 500, valid_sets=[ds_va], + callbacks=[lgb.early_stopping(30), lgb.log_evaluation(-1)]) + acc = accuracy_score(y_te, (m.predict(X_te) > 0.5).astype(int)) + results.append(acc) + print(f" Window {i+1}: train={train_end:,} test={len(y_te):,} acc={acc:.4f}") + + mean_acc = np.mean(results) + std_acc = np.std(results) + print(f" CV accuracy: {mean_acc:.4f} ± {std_acc:.4f}") + return {"mean": mean_acc, "std": std_acc, "windows": results} + + +# ─── Optuna hyperparameter search ─────────────────────────────────────────── + +def tune_lgbm(X_tr, y_tr, X_va, y_va, n_trials: int = OPTUNA_TRIALS) -> dict: + if not HAS_OPTUNA: + return {"num_leaves": 63, "learning_rate": 0.05, + "feature_fraction": 0.8, "bagging_fraction": 0.8, + "bagging_freq": 5, "min_child_samples": 50} + + def objective(trial): + params = { + "objective": "binary", "metric": "binary_logloss", "verbose": -1, + "num_leaves": trial.suggest_int("num_leaves", 20, 150), + "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.15, log=True), + "feature_fraction": trial.suggest_float("feature_fraction", 0.5, 1.0), + "bagging_fraction": trial.suggest_float("bagging_fraction", 0.5, 1.0), + "bagging_freq": trial.suggest_int("bagging_freq", 1, 10), + "min_child_samples": trial.suggest_int("min_child_samples", 20, 100), + "lambda_l1": trial.suggest_float("lambda_l1", 0.0, 1.0), + "lambda_l2": trial.suggest_float("lambda_l2", 0.0, 1.0), + } + ds_tr = lgb.Dataset(X_tr, label=y_tr) + ds_va = lgb.Dataset(X_va, label=y_va, reference=ds_tr) + m = lgb.train(params, ds_tr, 1000, valid_sets=[ds_va], + callbacks=[lgb.early_stopping(30), lgb.log_evaluation(-1)]) + preds = m.predict(X_va) + return accuracy_score(y_va, (preds > 0.5).astype(int)) + + study = optuna.create_study(direction="maximize") + study.optimize(objective, n_trials=n_trials, show_progress_bar=False) + print(f" Best LightGBM accuracy (Optuna): {study.best_value:.4f}") + return study.best_params + + +# ─── Model training ────────────────────────────────────────────────────────── + +def train_lgbm(X_tr, y_tr, X_va, y_va, params: dict) -> lgb.Booster: + final_params = { + "objective": "binary", "metric": "binary_logloss", + "boosting_type": "gbdt", "verbose": -1, + **params + } + ds_tr = lgb.Dataset(X_tr, label=y_tr, feature_name=FEATURE_NAMES) + ds_va = lgb.Dataset(X_va, label=y_va, feature_name=FEATURE_NAMES, reference=ds_tr) + return lgb.train(final_params, ds_tr, 3000, valid_sets=[ds_va], + callbacks=[lgb.early_stopping(50), lgb.log_evaluation(200)]) + + +def train_xgb(X_tr, y_tr, X_va, y_va) -> object: + if not HAS_XGB: + return None + model = xgb.XGBClassifier( + n_estimators=1000, learning_rate=0.05, max_depth=6, + subsample=0.8, colsample_bytree=0.8, min_child_weight=5, + use_label_encoder=False, eval_metric="logloss", + early_stopping_rounds=50, verbosity=0, + tree_method="hist", + ) + model.fit(X_tr, y_tr, eval_set=[(X_va, y_va)], verbose=False) + return model + + +def train_rf(X_tr, y_tr) -> object: + if not HAS_SKLEARN: + return None + model = RandomForestClassifier( + n_estimators=300, max_depth=12, min_samples_leaf=30, + max_features="sqrt", n_jobs=-1, random_state=42 + ) + model.fit(X_tr, y_tr) + return model + + +# ─── Ensemble / stacking ───────────────────────────────────────────────────── + +def build_ensemble(lgbm_model, xgb_model, rf_model, + X_te: np.ndarray, y_te: np.ndarray, + tft_preds: np.ndarray | None = None, + tgru_preds: np.ndarray | None = None) -> tuple: + """ + Stack base models with a logistic regression meta-learner. + Accepts optional TFT and TGRU probability arrays (already computed on X_te). + Returns (meta_model, scaler, base_accuracies, ensemble_accuracy). + """ + preds = [] + names = [] + accs = {} + + p_lgbm = lgbm_model.predict(X_te) + preds.append(p_lgbm); names.append("LightGBM") + accs["LightGBM"] = accuracy_score(y_te, (p_lgbm > 0.5).astype(int)) + + if xgb_model is not None: + p_xgb = xgb_model.predict_proba(X_te)[:, 1] + preds.append(p_xgb); names.append("XGBoost") + accs["XGBoost"] = accuracy_score(y_te, (p_xgb > 0.5).astype(int)) + + if rf_model is not None: + p_rf = rf_model.predict_proba(X_te)[:, 1] + preds.append(p_rf); names.append("RandomForest") + accs["RandomForest"] = accuracy_score(y_te, (p_rf > 0.5).astype(int)) + + # ── Optional DL models ─────────────────────────────────────────────────── + if tft_preds is not None: + preds.append(tft_preds); names.append("TFT") + accs["TFT"] = accuracy_score(y_te, (tft_preds > 0.5).astype(int)) + + if tgru_preds is not None: + preds.append(tgru_preds); names.append("TGRU") + accs["TGRU"] = accuracy_score(y_te, (tgru_preds > 0.5).astype(int)) + + # Stack predictions as features for meta-learner + meta_X = np.column_stack(preds) + scaler = StandardScaler() + meta_X_scaled = scaler.fit_transform(meta_X) + meta = LogisticRegression(C=1.0, max_iter=500) + meta.fit(meta_X_scaled, y_te) # train meta on test (unseen by base models) + ens_acc = accuracy_score(y_te, meta.predict(meta_X_scaled)) + + print(f"\n Base models ({len(names)}) :") + for name, acc in accs.items(): + print(f" {name:15s}: {acc:.4f} ({acc*100:.1f}%)") + print(f" {'Ensemble':15s}: {ens_acc:.4f} ({ens_acc*100:.1f}%)") + + return meta, scaler, accs, ens_acc + + +# ─── SHAP feature importance ───────────────────────────────────────────────── + +def print_top_features(lgbm_model, top_n: int = 15): + importance = lgbm_model.feature_importance(importance_type="gain") + idx = np.argsort(importance)[::-1] + print(f"\n Top {top_n} features (LightGBM gain):") + for rank, i in enumerate(idx[:top_n], 1): + print(f" {rank:2d}. {FEATURE_NAMES[i]:30s} {importance[i]:>10,.0f}") + + +# ─── Main ──────────────────────────────────────────────────────────────────── + +def main(): + print("=" * 65) + print(" AHAD QUANT — Ensemble Training Pipeline (V5)") + if HAS_DL: + print(" LightGBM + XGBoost + RF + TFT + TransformerGRU + Meta-learner") + else: + print(" LightGBM + XGBoost + RandomForest + Stacking Meta-Learner") + print(" [DL disabled — install torch to enable TFT/TGRU]") + print("=" * 65) + + # ── 0. Pré-vol : vérifier que data/ existe et contient des fichiers ─────── + data_dir = config.DATA_DIR + if not os.path.isdir(data_dir): + print() + print("━" * 65) + print(f" ERREUR : Dossier '{data_dir}/' introuvable.") + print() + print(" Lancez d'abord le téléchargement des données :") + print(" python download_data.py") + print("━" * 65) + sys.exit(1) + json_files = [f for f in os.listdir(data_dir) if f.endswith("_1h.json")] + if len(json_files) == 0: + print() + print("━" * 65) + print(f" ERREUR : Aucun fichier de données dans '{data_dir}/'.") + print() + print(" Lancez d'abord le téléchargement des données :") + print(" python download_data.py") + print("━" * 65) + sys.exit(1) + print(f"\n[0/7] Données disponibles : {len(json_files)} fichiers dans {data_dir}/") + + t0 = time.time() + + # ── 1. Tabular data (always) ────────────────────────────────────────────── + print("\n[1/7] Loading data and building tabular features...") + X, y = prepare_dataset() + n = len(X) + print(f"\n Total: {n:,} samples | {X.shape[1]} features | " + f"{y.mean():.2%} long labels") + + # ── 1b. Sequence data (DL only) ────────────────────────────────────────── + X_seq = None + y_seq = None + dl_scaler = None + + if HAS_DL: + print(f"\n[1b/7] Building sequence dataset (SEQ_LEN={SEQ_LEN})...") + X_tab_aligned, X_seq_raw, y_seq = prepare_seq_dataset() + n_seq = len(y_seq) + print(f" Sequence samples : {n_seq:,} | shape: {X_seq_raw.shape}") + + # Sequence train/val/test splits + seq_tr_end = int(n_seq * TRAIN_RATIO) + seq_va_end = int(n_seq * (TRAIN_RATIO + VAL_RATIO)) + + # Fit scaler on training portion + dl_scaler = fit_seq_scaler(X_tab_aligned[:seq_tr_end]) + + # Normalise all splits + X_seq_tr = transform_sequences(dl_scaler, X_seq_raw[:seq_tr_end]) + X_seq_va = transform_sequences(dl_scaler, X_seq_raw[seq_tr_end:seq_va_end]) + X_seq_te = transform_sequences(dl_scaler, X_seq_raw[seq_va_end:]) + y_seq_tr = y_seq[:seq_tr_end] + y_seq_va = y_seq[seq_tr_end:seq_va_end] + y_seq_te = y_seq[seq_va_end:] + print(f" Seq splits: train={len(y_seq_tr):,} | val={len(y_seq_va):,} | test={len(y_seq_te):,}") + + # ── 2. Walk-forward CV ──────────────────────────────────────────────────── + print("\n[2/7] Walk-forward cross-validation (tabular)...") + cv_results = walk_forward_cv(X, y, N_WALK_FORWARD_WINDOWS) + + # ── 3. Tabular splits ───────────────────────────────────────────────────── + train_end = int(n * TRAIN_RATIO) + val_end = int(n * (TRAIN_RATIO + VAL_RATIO)) + X_tr, y_tr = X[:train_end], y[:train_end] + X_va, y_va = X[train_end:val_end], y[train_end:val_end] + X_te, y_te = X[val_end:], y[val_end:] + print(f"\n Tabular splits: train={len(X_tr):,} | val={len(X_va):,} | test={len(X_te):,}") + + # ── 4. Optuna search ────────────────────────────────────────────────────── + print(f"\n[3/7] Hyperparameter search ({OPTUNA_TRIALS} Optuna trials)...") + best_params = tune_lgbm(X_tr, y_tr, X_va, y_va, OPTUNA_TRIALS) + + # ── 5. Train tabular base models ────────────────────────────────────────── + print("\n[4/7] Training tabular base models (LightGBM + XGBoost + RF)...") + print(" Training LightGBM...") + lgbm_model = train_lgbm(X_tr, y_tr, X_va, y_va, best_params) + + print(" Training XGBoost...") + xgb_model = train_xgb(X_tr, y_tr, X_va, y_va) + + print(" Training RandomForest...") + rf_model = train_rf(X_tr, y_tr) + + # ── 5b. Train DL models ─────────────────────────────────────────────────── + tft_model = None + tgru_model = None + tft_preds_te = None + tgru_preds_te = None + + if HAS_DL: + print("\n[4b/7] Training Deep Learning models (TFT + TransformerGRU)...") + print(" Training Temporal Fusion Transformer...") + tft_model = train_tft(X_seq_tr, y_seq_tr, X_seq_va, y_seq_va) + + print("\n Training TransformerGRU...") + tgru_model = train_tgru(X_seq_tr, y_seq_tr, X_seq_va, y_seq_va) + + # ── Free training/val sequence tensors: not needed past this point ── + # (X_seq_tr/X_seq_va are the largest arrays in memory, ~70k+15k seqs) + import gc, torch + del X_seq_tr, X_seq_va, y_seq_tr, y_seq_va + try: + del X_seq_raw + except NameError: + pass + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + print(" [MEM] Freed train/val sequence tensors before inference") + + # Get DL predictions on sequence test set + X_seq_te_t = torch.FloatTensor(X_seq_te) + tft_preds_te = predict_tft_proba(tft_model, X_seq_te_t) + tgru_preds_te = predict_tgru_proba(tgru_model, X_seq_te_t) + print(f"\n TFT test acc : {accuracy_score(y_seq_te, (tft_preds_te > 0.5).astype(int)):.4f}") + print(f" TGRU test acc : {accuracy_score(y_seq_te, (tgru_preds_te > 0.5).astype(int)):.4f}") + + # X_seq_te_t no longer needed after inference (predictions already extracted) + del X_seq_te_t + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + # ── 6. Ensemble / stacking ──────────────────────────────────────────────── + print("\n[5/7] Building stacking ensemble on test set...") + + if HAS_DL and tft_preds_te is not None: + # Use sequence-aligned test set for ALL models (common index space) + # Tabular predictions on the aligned X_tab_aligned test portion + X_te_dl = X_tab_aligned[seq_va_end:] # aligned tabular test + y_te_dl = y_seq_te # same labels + + meta_model, meta_scaler, base_accs, ens_acc = build_ensemble( + lgbm_model, xgb_model, rf_model, + X_te_dl, y_te_dl, + tft_preds=tft_preds_te, + tgru_preds=tgru_preds_te, + ) + else: + # ML-only path (original behavior) + meta_model, meta_scaler, base_accs, ens_acc = build_ensemble( + lgbm_model, xgb_model, rf_model, X_te, y_te + ) + + print_top_features(lgbm_model) + + # ── 6b. Save feature importances to JSON (lu par web_ui.py dashboard) ──── + try: + importance = lgbm_model.feature_importance(importance_type="gain") + feat_imp_data = [ + {"feature": FEATURE_NAMES[i], "importance": float(importance[i])} + for i in np.argsort(importance)[::-1] + ] + imp_path = os.path.join(os.path.dirname(__file__), "feature_importance.json") + with open(imp_path, "w") as f: + json.dump({"updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "features": feat_imp_data}, f, indent=2) + except Exception as _fie: + print(f" [WARN] feature_importance.json non généré: {_fie}") + + # ── 7. Save ─────────────────────────────────────────────────────────────── + print(f"\n[6/7] Saving models...") + + # Standalone LightGBM (compatible with free version loader) + with open(config.MODEL_PATH, "wb") as f: + pickle.dump(lgbm_model, f) + print(f" LightGBM model -> {config.MODEL_PATH}") + + # Full ensemble (all models + DL if available) + ensemble_data = { + # Tabular base models + "lgbm" : lgbm_model, + "xgb" : xgb_model, + "rf" : rf_model, + # DL models (None if torch not available or --no-dl) + "tft" : tft_model, + "tgru" : tgru_model, + "dl_scaler" : dl_scaler, # StandardScaler for sequence normalisation + # Meta-learner + "meta" : meta_model, + "scaler" : meta_scaler, + # Metadata + "feature_names" : FEATURE_NAMES, + "cv_results" : cv_results, + "base_accs" : base_accs, + "ens_acc" : ens_acc, + "has_dl" : HAS_DL and tft_model is not None, + "version" : "ahad_quant-forex-v5", + } + with open(config.ENSEMBLE_MODEL_PATH, "wb") as f: + pickle.dump(ensemble_data, f) + print(f" Ensemble model -> {config.ENSEMBLE_MODEL_PATH}") + + # ── 8. Summary ──────────────────────────────────────────────────────────── + elapsed = time.time() - t0 + print(f"\n{'=' * 65}") + print(f" Training complete in {elapsed/60:.1f} min") + print(f" Walk-forward accuracy : {cv_results['mean']:.4f} ± {cv_results['std']:.4f}") + print(f" Ensemble test accuracy: {ens_acc:.4f} ({ens_acc*100:.1f}%)") + if HAS_DL and tft_model is not None: + print(f" DL models : TFT ✅ TGRU ✅") + else: + print(f" DL models : not trained (--no-dl or torch missing)") + print(f"\n Run `python ahad_quant.py` to start trading.") + print(f"{'=' * 65}\n") + + +if __name__ == "__main__": + main() diff --git a/train_unified.py b/train_unified.py new file mode 100644 index 0000000..0f941d0 --- /dev/null +++ b/train_unified.py @@ -0,0 +1,130 @@ +""" +AHAD QUANT — Entraînement Unifié ML + RL +============================================= +UN seul point d'entrée pour entraîner tout le système. Plus aucun bouton +séparé "Train ML" / "Fine-tune RL" côté interface : ce script orchestre les +deux étapes comme UN SEUL système qui se met à jour ensemble, puis sauvegarde +un bundle unique (ahad_quant_unified.zip) cohérent à la fin. + +Étapes (toujours dans cet ordre — le RL a besoin de l'ensemble ML à jour +pour calculer son observation/reward) : + 1. ML : train.py (ré-entraîne l'ensemble LightGBM+XGBoost+RF[+DL]) + 2. RL : rl_train.py (fine-tune le PPO existant sur le nouvel ensemble, + ou entraîne depuis zéro si aucun agent n'existe encore) + 3. Export : export_unified.py (combine PPO + ensemble + scaler dans + ahad_quant_unified.zip — la SEULE sauvegarde qui compte du + point de vue utilisateur) + +Chaque étape garde son propre mécanisme accept/reject (le nouveau modèle ne +remplace l'ancien que s'il est meilleur — déjà géré dans train.py/rl_train.py), +donc cette orchestration ne change AUCUNE calibration existante : elle se +contente de les enchaîner et de les rendre visibles comme un seul flux. + +Usage : + python train_unified.py # ML complet + RL fine-tune (200k steps) + python train_unified.py --rl-steps 500000 # RL plus long + python train_unified.py --rl-full # RL entraîné depuis zéro (pas fine-tune) + python train_unified.py --ml-only # ML seul (cas avancé, déconseillé) + python train_unified.py --rl-only # RL seul (cas avancé, déconseillé) +""" + +import os +import sys +import time +import argparse +import subprocess + +sys.stdout.reconfigure(encoding="utf-8", errors="replace") + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _run_step(label: str, cmd: list[str]) -> bool: + """Lance une étape en sous-processus, stream le stdout en direct, et + renvoie True si le code de sortie est 0 (succès).""" + print("\n" + "═" * 65) + print(f" {label}") + print("═" * 65, flush=True) + proc = subprocess.run(cmd, cwd=BASE_DIR) + ok = proc.returncode == 0 + status = "✅ OK" if ok else f"❌ ÉCHEC (code {proc.returncode})" + print(f"\n [{label}] {status}", flush=True) + return ok + + +def main(): + parser = argparse.ArgumentParser( + description="AHAD QUANT — Entraînement Unifié ML + RL (un seul système)" + ) + parser.add_argument("--rl-steps", type=int, default=200_000, + help="Steps RL pour le fine-tuning (défaut: 200 000)") + parser.add_argument("--rl-full", action="store_true", + help="Entraîner le RL depuis zéro (au lieu de fine-tuner l'existant)") + parser.add_argument("--ml-only", action="store_true", + help="[Avancé] N'entraîner QUE le ML — déconseillé, casse l'unification") + parser.add_argument("--rl-only", action="store_true", + help="[Avancé] N'entraîner QUE le RL — déconseillé, casse l'unification") + args = parser.parse_args() + + if args.ml_only and args.rl_only: + print("❌ --ml-only et --rl-only sont mutuellement exclusifs.") + sys.exit(1) + + t0 = time.time() + print("┌" + "─" * 63 + "┐") + print("│ AHAD QUANT — ENTRAÎNEMENT UNIFIÉ (ML + RL = UN SEUL SYSTÈME) │"[:65].ljust(64) + "│") + print("└" + "─" * 63 + "┘") + + ml_ok = True + rl_ok = True + + # ── Étape 1 : ML ────────────────────────────────────────────────────── + if not args.rl_only: + ml_ok = _run_step( + "ÉTAPE 1/3 — Entraînement ML (ensemble LightGBM+XGBoost+RF[+DL])", + [sys.executable, "-u", os.path.join(BASE_DIR, "train.py")], + ) + if not ml_ok: + print("\n⚠️ ML en échec — le RL serait entraîné sur un ensemble obsolète.") + print(" Arrêt ici pour ne pas désynchroniser ML et RL.") + sys.exit(1) + + # ── Étape 2 : RL ────────────────────────────────────────────────────── + if not args.ml_only: + rl_agent_exists = os.path.exists(os.path.join(BASE_DIR, "rl_agent.zip")) + if args.rl_full or not rl_agent_exists: + rl_cmd = [sys.executable, "-u", os.path.join(BASE_DIR, "rl_train.py"), + "--steps", str(args.rl_steps)] + rl_label = "ÉTAPE 2/3 — Entraînement RL PPO (depuis zéro, sur le nouvel ensemble ML)" + else: + rl_cmd = [sys.executable, "-u", os.path.join(BASE_DIR, "rl_train.py"), + "--finetune", "--steps", str(args.rl_steps)] + rl_label = "ÉTAPE 2/3 — Fine-tuning RL PPO (sur le nouvel ensemble ML)" + rl_ok = _run_step(rl_label, rl_cmd) + if not rl_ok: + print("\n⚠️ RL en échec — le bundle unifié ne sera PAS régénéré " + "(pour éviter d'exporter un PPO obsolète avec un ML à jour).") + + # ── Étape 3 : Export unifié (sauvegarde UNIQUE) ────────────────────── + export_ok = False + if ml_ok and rl_ok: + export_ok = _run_step( + "ÉTAPE 3/3 — Sauvegarde unifiée (ahad_quant_unified.zip = ML + RL + scaler)", + [sys.executable, "-u", os.path.join(BASE_DIR, "export_unified.py")], + ) + else: + print("\n[ÉTAPE 3/3] Sautée — une étape précédente a échoué, " + "pas de sauvegarde unifiée pour éviter un bundle incohérent.") + + elapsed = time.time() - t0 + print("\n" + "═" * 65) + print(f" TERMINÉ en {elapsed/60:.1f} min") + print(f" ML : {'✅' if ml_ok else '❌'} RL : {'✅' if rl_ok else '❌'} " + f"Sauvegarde unifiée : {'✅' if export_ok else '❌'}") + print("═" * 65) + + sys.exit(0 if (ml_ok and rl_ok) else 1) + + +if __name__ == "__main__": + main() diff --git a/transformer_gru_model.py b/transformer_gru_model.py new file mode 100644 index 0000000..6fe6f36 --- /dev/null +++ b/transformer_gru_model.py @@ -0,0 +1,359 @@ +""" +AHAD QUANT Forex V5 — TransformerGRU Hybrid Model +Combines a Bidirectional GRU (short-range pattern capture) with a +Transformer Encoder (long-range self-attention) for binary classification. + +Architecture +─────────────────────────────────────────────────────────────── +Input : (B, T, 62) +Layer 1: Input projection — Linear(62, D_MODEL) + GELU + → (B, T, D_MODEL) +Layer 2: Positional Encoding (sinusoidal, fixed) + → (B, T, D_MODEL) +Layer 3: Bidirectional GRU — hidden=GRU_HIDDEN, 2 layers + — captures local sequential dependencies in both directions + → (B, T, D_MODEL) [GRU_HIDDEN*2 projected back to D_MODEL] +Layer 4: Transformer Encoder — N_ATT_LAYERS × (MHA + FFN + LayerNorm) + — long-range temporal attention on top of GRU hidden states + → (B, T, D_MODEL) +Layer 5: Aggregation — mean pooling + last token concatenated + → (B, D_MODEL*2) +Layer 6: Classification head + Linear → GELU → Dropout → Linear(1) — logit +Output : (B,) logit — apply sigmoid for probability + +Training util functions +─────────────────────── + train_tgru(X_seq_tr, y_tr, X_seq_va, y_va, ...) → TransformerGRU + predict_tgru_proba(model, X_seq_tensor) → np.ndarray (B,) +""" + +import math +import time + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader, TensorDataset + +# ─── Hyper-parameters (can be overridden via kwargs in train_tgru) ─────────── + +D_MODEL = 128 # main embedding dimension +GRU_HIDDEN = 64 # GRU hidden per direction (×2 bidir = D_MODEL) +N_ATT_LAYERS = 3 # Transformer encoder depth +N_HEADS = 4 # attention heads (D_MODEL must be divisible) +FFN_DIM = 256 # feedforward expansion in Transformer +DROPOUT = 0.10 +BATCH_SIZE = 512 +LR = 5e-4 +WEIGHT_DECAY = 1e-4 +MAX_EPOCHS = 30 +PATIENCE = 6 +GRAD_CLIP = 1.0 + + +# ─── Positional Encoding (sinusoidal) ──────────────────────────────────────── + +class SinusoidalPositionalEncoding(nn.Module): + """ + Fixed sinusoidal positional encoding added to embeddings. + PE(pos, 2i) = sin(pos / 10000^(2i/d_model)) + PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model)) + """ + + def __init__(self, d_model: int, max_len: int = 512, dropout: float = 0.1): + super().__init__() + self.drop = nn.Dropout(dropout) + + pe = torch.zeros(max_len, d_model) + pos = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div = torch.exp( + torch.arange(0, d_model, 2, dtype=torch.float) * (-math.log(10000.0) / d_model) + ) + pe[:, 0::2] = torch.sin(pos * div) + pe[:, 1::2] = torch.cos(pos * div) + self.register_buffer("pe", pe.unsqueeze(0)) # (1, max_len, d_model) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """x : (B, T, d_model)""" + return self.drop(x + self.pe[:, : x.size(1)]) + + +# ─── Transformer Encoder Block ─────────────────────────────────────────────── + +class TransformerEncoderBlock(nn.Module): + """ + Standard Pre-LN Transformer encoder block: + x → LayerNorm → MHA → residual + → LayerNorm → FFN → residual + Pre-LayerNorm (before attention) provides more stable training. + """ + + def __init__(self, d_model: int, n_heads: int, ffn_dim: int, + dropout: float = DROPOUT): + super().__init__() + self.norm1 = nn.LayerNorm(d_model) + self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) + self.norm2 = nn.LayerNorm(d_model) + self.ffn = nn.Sequential( + nn.Linear(d_model, ffn_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(ffn_dim, d_model), + nn.Dropout(dropout), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Self-attention sub-layer + h, _ = self.attn(self.norm1(x), self.norm1(x), self.norm1(x)) + x = x + h + # Feed-forward sub-layer + x = x + self.ffn(self.norm2(x)) + return x + + +# ─── Main model ────────────────────────────────────────────────────────────── + +class TransformerGRU(nn.Module): + """ + TransformerGRU for binary classification on (B, T, F) time-series. + Output: (B,) raw logit — use with BCEWithLogitsLoss. + """ + + def __init__( + self, + num_features : int = 62, + seq_len : int = 168, + d_model : int = D_MODEL, + gru_hidden : int = GRU_HIDDEN, + n_att_layers : int = N_ATT_LAYERS, + n_heads : int = N_HEADS, + ffn_dim : int = FFN_DIM, + dropout : float = DROPOUT, + ): + super().__init__() + assert d_model == gru_hidden * 2, ( + f"D_MODEL ({d_model}) must equal GRU_HIDDEN*2 ({gru_hidden*2}) " + "so bidirectional GRU output aligns with d_model" + ) + self.d_model = d_model + self.seq_len = seq_len + + # ── 1. Input projection ────────────────────────────────────────────── + self.input_proj = nn.Sequential( + nn.Linear(num_features, d_model), + nn.GELU(), + nn.Dropout(dropout), + ) + + # ── 2. Positional encoding ─────────────────────────────────────────── + self.pos_enc = SinusoidalPositionalEncoding(d_model, max_len=seq_len + 2, dropout=dropout) + + # ── 3. Bidirectional GRU ───────────────────────────────────────────── + # Input: (B, T, d_model) → Output: (B, T, gru_hidden*2 = d_model) + self.gru = nn.GRU( + input_size = d_model, + hidden_size = gru_hidden, + num_layers = 2, + batch_first = True, + bidirectional = True, + dropout = dropout, + ) + self.gru_norm = nn.LayerNorm(d_model) + # Residual projection (input_proj output → d_model already, residual fits) + + # ── 4. Transformer encoder ─────────────────────────────────────────── + self.transformer = nn.ModuleList([ + TransformerEncoderBlock(d_model, n_heads, ffn_dim, dropout) + for _ in range(n_att_layers) + ]) + self.final_norm = nn.LayerNorm(d_model) + + # ── 5. Classification head ─────────────────────────────────────────── + # Aggregate: mean pool + last token → concat → d_model*2 + self.head = nn.Sequential( + nn.Linear(d_model * 2, d_model), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(d_model, 1), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + x : (B, T, num_features) + → (B,) logit + """ + # 1. Input projection + positional encoding + h = self.input_proj(x) # (B, T, d_model) + h = self.pos_enc(h) + + # 2. Bidirectional GRU — add residual + gru_out, _ = self.gru(h) # (B, T, d_model) [bidir → gru_h*2] + h = self.gru_norm(h + gru_out) # add+norm residual + + # 3. Transformer encoder + for block in self.transformer: + h = block(h) # (B, T, d_model) + h = self.final_norm(h) + + # 4. Aggregation: mean pool + last token + mean_pool = h.mean(dim=1) # (B, d_model) + last_tok = h[:, -1, :] # (B, d_model) + pooled = torch.cat([mean_pool, last_tok], dim=-1) # (B, d_model*2) + + # 5. Head + return self.head(pooled).squeeze(-1) # (B,) + + +# ─── Training utilities ────────────────────────────────────────────────────── + +def _make_loader(X: np.ndarray, y: np.ndarray, + batch_size: int, shuffle: bool) -> DataLoader: + X_t = torch.FloatTensor(X) + y_t = torch.FloatTensor(y) + return DataLoader(TensorDataset(X_t, y_t), + batch_size=batch_size, shuffle=shuffle, + num_workers=0, pin_memory=torch.cuda.is_available()) + + +def train_tgru( + X_seq_tr : np.ndarray, + y_tr : np.ndarray, + X_seq_va : np.ndarray, + y_va : np.ndarray, + *, + # Architecture overrides + d_model : int = D_MODEL, + gru_hidden : int = GRU_HIDDEN, + n_att_layers : int = N_ATT_LAYERS, + n_heads : int = N_HEADS, + ffn_dim : int = FFN_DIM, + dropout : float = DROPOUT, + # Training overrides + batch_size : int = BATCH_SIZE, + lr : float = LR, + weight_decay : float = WEIGHT_DECAY, + max_epochs : int = MAX_EPOCHS, + patience : int = PATIENCE, + grad_clip : float = GRAD_CLIP, +) -> "TransformerGRU": + """ + Train a TransformerGRU on pre-normalised sequence data. + + Parameters + ---------- + X_seq_tr / X_seq_va : (N, T, 62) float32 — already normalised + y_tr / y_va : (N,) float32 binary labels + + Returns + ------- + Best model (lowest val loss) as a CPU-resident TransformerGRU. + """ + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print(f" [TGRU] device={device} | " + f"train={len(y_tr):,} val={len(y_va):,} samples") + + _, T, F = X_seq_tr.shape + model = TransformerGRU( + num_features = F, + seq_len = T, + d_model = d_model, + gru_hidden = gru_hidden, + n_att_layers = n_att_layers, + n_heads = n_heads, + ffn_dim = ffn_dim, + dropout = dropout, + ).to(device) + + n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + print(f" [TGRU] parameters: {n_params:,}") + + optimiser = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimiser, T_max=max_epochs) + criterion = nn.BCEWithLogitsLoss() + scaler_amp = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available()) + + train_loader = _make_loader(X_seq_tr, y_tr, batch_size, shuffle=True) + val_loader = _make_loader(X_seq_va, y_va, batch_size * 2, shuffle=False) + + best_val_loss = float("inf") + best_state = None + no_improve = 0 + t0 = time.time() + + for epoch in range(1, max_epochs + 1): + # ── Train ──────────────────────────────────────────────────────────── + model.train() + train_loss = 0.0 + for X_b, y_b in train_loader: + X_b, y_b = X_b.to(device), y_b.to(device) + optimiser.zero_grad() + with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()): + logits = model(X_b) + loss = criterion(logits, y_b) + scaler_amp.scale(loss).backward() + scaler_amp.unscale_(optimiser) + nn.utils.clip_grad_norm_(model.parameters(), grad_clip) + scaler_amp.step(optimiser) + scaler_amp.update() + train_loss += loss.item() * len(y_b) + train_loss /= len(y_tr) + + # ── Validate ───────────────────────────────────────────────────────── + model.eval() + val_loss = 0.0 + correct = 0 + with torch.no_grad(): + for X_b, y_b in val_loader: + X_b, y_b = X_b.to(device), y_b.to(device) + logits = model(X_b) + val_loss += criterion(logits, y_b).item() * len(y_b) + correct += ((logits > 0).float() == y_b).sum().item() + val_loss /= len(y_va) + val_acc = correct / len(y_va) + + scheduler.step() + + elapsed = time.time() - t0 + print(f" [TGRU] epoch {epoch:3d}/{max_epochs} " + f"train={train_loss:.4f} val={val_loss:.4f} " + f"val_acc={val_acc:.4f} ({elapsed:.0f}s)") + + # ── Early stopping ─────────────────────────────────────────────────── + if val_loss < best_val_loss - 1e-5: + best_val_loss = val_loss + best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} + no_improve = 0 + else: + no_improve += 1 + if no_improve >= patience: + print(f" [TGRU] early stopping at epoch {epoch}") + break + + model.load_state_dict(best_state) + model.cpu() + print(f" [TGRU] training complete — best val_loss={best_val_loss:.4f}") + return model + + +@torch.no_grad() +def predict_tgru_proba( + model : TransformerGRU, + X_seq_t : torch.Tensor, + batch_size: int = 1024, + device : str = "cpu", +) -> np.ndarray: + """ + Run inference on a (N, T, F) float tensor. + Returns probability array of shape (N,). + """ + model.eval() + model.to(device) + probs = [] + for start in range(0, len(X_seq_t), batch_size): + chunk = X_seq_t[start : start + batch_size].to(device) + logits = model(chunk) + probs.append(torch.sigmoid(logits).cpu().numpy()) + model.cpu() + return np.concatenate(probs).astype(np.float32) diff --git a/unified_brain.py b/unified_brain.py new file mode 100644 index 0000000..178830d --- /dev/null +++ b/unified_brain.py @@ -0,0 +1,365 @@ +""" +AHAD QUANT — Unified Brain (orchestrateur central ML + RL) +================================================================ +UN seul point d'entrée pour transformer (bougies, contexte) en décision de +trading complète. Combine : + + - L'ensemble ML (LightGBM + XGBoost + RF [+ TFT + TGRU]), via + ensemble_core.py — la source unique de vérité pour le ML (voir ce + module pour le détail du bug #1 qu'il corrige). + - La détection de régime de marché (HMM), relocalisée ici depuis + ahad_quant.py pour qu'elle soit accessible à tout appelant (live, + backtest) sans dépendre de l'état global du bot. + - Le filtre RL (PPO), via rl_agent.py — appelé tel quel, SANS changer + sa sémantique de boost/override existante, pour ne pas modifier + silencieusement une calibration déjà en place. + +Avant ce module, ces trois briques étaient combinées à la main, en ligne, +dans ahad_quant.py::_scan_entries() — avec un vrai bug : les variables +ml_probas/rl_action/rl_agreed/current_regime calculées pendant le scan +n'étaient JAMAIS transmises jusqu'au moment d'ouvrir le trade pour les +branches MT5 et live réelles (NameError garanti à la première ouverture +de position hors PAPER_MODE — point #8 du diagnostic). En centralisant la +décision dans un objet Decision unique, ce problème disparaît structurellement +: tout le contexte voyage ensemble, du signal à l'ouverture du trade. + +Hot-reload : maybe_reload() vérifie (avec un léger throttle) si le fichier +modèle ML a changé sur disque, et délègue à rl_agent.reload_if_stale() pour +le PPO — corrige le point #4 du diagnostic (avant, un ré-entraînement +réussi en arrière-plan restait sans le moindre effet sur le bot déjà en +cours d'exécution, jusqu'à un redémarrage manuel du process). +""" + +import os +import time +import pickle +import logging +import threading +from dataclasses import dataclass, field +from typing import Optional + +import numpy as np + +import config +import ensemble_core as _ens_core +from features import build_features, NUM_FEATURES + +log = logging.getLogger("UnifiedBrain") + +try: + from regime_detector import GaussianHMM + HAS_REGIME = True +except ImportError: + HAS_REGIME = False + +_RL_READY = False +try: + from rl_agent import get_rl_agent, PositionState + _RL_READY = True +except ImportError: + PositionState = None # type: ignore[assignment,misc] + + +_REGIME_LABELS = {0: "CALM", 1: "NORMAL", 2: "VOLATILE"} + + +# ─── Décision ────────────────────────────────────────────────────────────── + +@dataclass +class Decision: + """ + Décision complète d'une itération de scan, pour UNE paire. + + `signal`/`confidence` sont le résultat FINAL (après filtre RL le cas + échéant) — c'est ce que le bot doit exécuter. `ml_*` conserve toujours + le signal ML brut, avant RL, pour la traçabilité et le buffer + d'apprentissage continu. + """ + signal: str # "long" | "short" | "neutral" — À EXÉCUTER + confidence: float + ml_signal: str # signal ML brut, avant filtre RL + ml_confidence: float + ml_proba: float # proba LONG brute du méta-modèle, [0,1] + rl_action: Optional[int] = None # 0=HOLD,1=LONG,2=SHORT,3=CLOSE + rl_signal: Optional[str] = None + rl_confidence: Optional[float] = None + rl_agreed: Optional[bool] = None + rl_used: bool = False + regime: int = 1 + regime_label: str = "NORMAL" + threshold_used: float = 0.0 + features: Optional[np.ndarray] = None # dernière ligne de features (pour le buffer CL) + model_version: str = "?" + + +# ─── Orchestrateur ─────────────────────────────────────────────────────────── + +class UnifiedBrain: + """ + Une instance suffit par process (voir get_brain() / singleton en bas de + fichier). Pas de dépendance à l'état du bot (AHAD QUANT._BOT_STATE etc.) — + ahad_quant.py reste responsable de mettre à jour SON état global après + avoir appelé decide(), ce module reste un pur moteur de décision. + """ + + def __init__(self): + self._ensemble = None + self._ensemble_path = None + self._ensemble_mtime = 0.0 + self._is_full_ensemble = False # False si LightGBM seul (fallback) + + self._hmm_model = None + self._hmm_lock = threading.Lock() + + self._reload_lock = threading.Lock() + self._last_reload_check = 0.0 + self._reload_throttle_s = float(getattr(config, "MODEL_RELOAD_CHECK_INTERVAL_S", 5.0)) + + self._load_ml() + + # ── Chargement / hot-reload ML ──────────────────────────────────────── + + def _expected_ml_path(self) -> str: + """Le chemin que load_model() choisirait EN CE MOMENT, selon la même + règle de repli que l'ancien ahad_quant.py::load_model() : l'ensemble + d'abord si USE_ENSEMBLE et qu'il existe, sinon le LightGBM seul.""" + ensemble_path = config.ENSEMBLE_MODEL_PATH + if config.USE_ENSEMBLE and os.path.exists(ensemble_path): + return ensemble_path + return config.MODEL_PATH + + def _load_ml(self): + """ + Reproduit EXACTEMENT la logique de repli de l'ancien + ahad_quant.py::load_model() : ENSEMBLE_MODEL_PATH d'abord (si + USE_ENSEMBLE), sinon MODEL_PATH (LightGBM seul) — deux chemins + DIFFÉRENTS, pas une simple substitution d'un seul chemin selon + USE_ENSEMBLE. + """ + ensemble_path = config.ENSEMBLE_MODEL_PATH + lgbm_path = config.MODEL_PATH + + if config.USE_ENSEMBLE and os.path.exists(ensemble_path): + self._ensemble = _ens_core.load_ensemble(ensemble_path) + self._is_full_ensemble = self._ensemble is not None + self._ensemble_path = ensemble_path + elif os.path.exists(lgbm_path): + # Repli LightGBM seul — enveloppé dans le même format dict + # qu'un ensemble pour qu'ensemble_core.predict_ensemble_single() + # le traite de façon identique (un seul sous-modèle disponible + # = stacking d'un seul élément = pas de moyenne à faire). + try: + with open(lgbm_path, "rb") as f: + raw = pickle.load(f) + model = raw["model"] if isinstance(raw, dict) else raw + self._ensemble = { + "lgbm": model, "xgb": None, "rf": None, + "meta": None, "scaler": None, "has_dl": False, + "version": "lgbm-only", + } + self._is_full_ensemble = False + log.info(f"[BRAIN] Modèle LightGBM seul chargé ({lgbm_path}) — pas d'ensemble complet") + except Exception as e: + log.error(f"[BRAIN] Erreur chargement modèle ({lgbm_path}) : {e}") + self._ensemble = None + self._ensemble_path = lgbm_path + else: + self._ensemble = None + self._ensemble_path = ensemble_path if config.USE_ENSEMBLE else lgbm_path + log.warning(f"[BRAIN] Aucun modèle ML trouvé (ni {ensemble_path}, ni {lgbm_path})") + + self._ensemble_mtime = _ens_core.ensemble_mtime(self._ensemble_path) + + def is_ready(self) -> bool: + """True si un modèle ML (ensemble complet ou LightGBM seul) est + chargé. Remplace l'ancien `self.model_data is None` de ahad_quant.py.""" + return self._ensemble is not None + + def maybe_reload(self, force: bool = False) -> bool: + """ + Vérifie si le modèle ML a changé sur disque (mtime) et délègue le + hot-reload du PPO à rl_agent.reload_if_stale(). Throttle léger + (5s par défaut, configurable via MODEL_RELOAD_CHECK_INTERVAL_S) pour + ne pas faire un appel os.path.getmtime() à chaque candle. + + Corrige le point #4 du diagnostic : avant ce module, un + ré-entraînement réussi (auto_retrain.py, en arrière-plan) restait + totalement sans effet sur le bot déjà lancé, jusqu'à un redémarrage + manuel du process — aussi bien pour le ML que pour le RL. + + Retourne True si quelque chose (ML et/ou RL) a effectivement été + rechargé. + """ + now = time.time() + if not force and (now - self._last_reload_check) < self._reload_throttle_s: + return False + self._last_reload_check = now + + reloaded = False + with self._reload_lock: + expected_path = self._expected_ml_path() + current_mtime = _ens_core.ensemble_mtime(expected_path) + if expected_path != self._ensemble_path or current_mtime != self._ensemble_mtime: + log.info(f"[BRAIN] Modèle ML modifié sur disque ({expected_path}) — rechargement à chaud") + self._load_ml() + reloaded = True + + if _RL_READY and config.USE_RL_AGENT: + try: + rl = get_rl_agent() + if rl.reload_if_stale(): + reloaded = True + except Exception as e: + log.warning(f"[BRAIN] Erreur hot-reload RL : {e}") + + return reloaded + + # ── Détection de régime (relocalisée depuis ahad_quant.py) ───────────── + + def _detect_regime(self, close: np.ndarray, volume: np.ndarray) -> int: + """Retourne 0=CALM, 1=NORMAL, 2=VOLATILE. Singleton GaussianHMM + ré-ajusté sur la fenêtre récente à chaque appel (comportement + identique à l'ancien ahad_quant.py::_detect_regime).""" + if not HAS_REGIME or len(close) < 50: + return 1 + returns = np.diff(np.log(close + 1e-8)) + vol = np.abs(returns) + vol_ma = np.convolve(volume / (volume.mean() + 1e-8), + np.ones(5) / 5, mode="same") + obs = np.column_stack([returns[-48:], vol[-48:], vol_ma[-48:]]) + with self._hmm_lock: + if self._hmm_model is None: + self._hmm_model = GaussianHMM(n_states=3) + try: + self._hmm_model.fit(obs) + states = self._hmm_model.predict(obs) + return int(states[-1]) + except Exception: + return 1 + + # ── Décision ──────────────────────────────────────────────────────────── + + def decide( + self, + candles: list, + btc_candles: list | None = None, + funding: float = 0.0, + position_state: "PositionState" = None, + pair: str | None = None, + ) -> Decision: + """ + Point d'entrée UNIQUE : bougies (+ contexte) → décision complète. + + Préserve EXACTEMENT la sémantique de combinaison ML/RL existante : + le RL ne filtre QUE les signaux ML non-neutres, via + rl_agent.RLAgent.filter_signal() (boost si accord, override si RL + très confiant et en désaccord, fallback sinon) — cette logique de + calibration reste dans rl_agent.py, simplement appelée d'ici, pour + ne PAS changer silencieusement un comportement déjà calibré. + """ + open_ = np.array([c["o"] for c in candles]) + high = np.array([c["h"] for c in candles]) + low = np.array([c["l"] for c in candles]) + close = np.array([c["c"] for c in candles]) + volume = np.array([c["v"] for c in candles]) + + btc_close = None + if btc_candles: + btc_close = np.array([c["c"] for c in btc_candles]) + min_len = min(len(close), len(btc_close)) + open_, high, low, close, volume = ( + arr[-min_len:] for arr in (open_, high, low, close, volume) + ) + btc_close = btc_close[-min_len:] + + candles_for_features = [ + {"o": o, "h": h, "l": l, "c": c, "v": v} + for o, h, l, c, v in zip(open_, high, low, close, volume) + ] + X_all = build_features(candles_for_features, btc_closes=btc_close, funding_map=None) + X_all = np.nan_to_num(X_all, nan=0.0) + last_feat = X_all[-1] + + # ── ML : ensemble_core, SOURCE UNIQUE (corrige le bug #1 pour cette + # 3ᵉ implémentation — l'ancienne ahad_quant.py::predict_signal() + # pouvait lever une ValueError non-attrapée si has_dl=True avec un + # historique trop court ; ensemble_core gère ce cas en interne, + # jamais d'exception qui remonte) ── + if self._ensemble is not None: + proba, _ = _ens_core.predict_ensemble_single(self._ensemble, last_feat, history=X_all) + else: + proba = 0.5 + + # ── Régime ──────────────────────────────────────────────────────── + regime = 1 + if config.USE_REGIME_FILTER and HAS_REGIME and len(close) >= 50: + try: + regime = self._detect_regime(close, volume) + except Exception: + regime = 1 + regime_label = _REGIME_LABELS.get(regime, "NORMAL") + + threshold = config.MIN_CONFIDENCE + if regime == 2: # VOLATILE → barre plus haute + threshold = min(config.MIN_CONFIDENCE + 0.05, 0.75) + + if proba > threshold: + ml_signal, ml_confidence = "long", float(proba) + elif proba < (1 - threshold): + ml_signal, ml_confidence = "short", float(1 - proba) + else: + ml_signal, ml_confidence = "neutral", float(max(proba, 1 - proba)) + + decision = Decision( + signal=ml_signal, confidence=ml_confidence, + ml_signal=ml_signal, ml_confidence=ml_confidence, ml_proba=float(proba), + regime=regime, regime_label=regime_label, threshold_used=threshold, + features=last_feat, + model_version=(self._ensemble.get("version", "?") if self._ensemble else "?"), + ) + + # ── Filtre RL — uniquement sur signal ML non-neutre, comme avant ── + if ml_signal != "neutral" and _RL_READY and config.USE_RL_AGENT: + try: + rl = get_rl_agent() + if rl.is_ready(): + rl_signal, rl_confidence, rl_action = rl.filter_signal( + ml_signal=ml_signal, + ml_confidence=ml_confidence, + features=last_feat, + position_state=position_state, + ) + decision.rl_used = True + decision.rl_signal = rl_signal + decision.rl_confidence = rl_confidence + decision.rl_action = rl_action + decision.rl_agreed = (rl_signal == ml_signal) + decision.signal = rl_signal + decision.confidence = rl_confidence + else: + log.info(f"[BRAIN] RL non prêt — signal ML non filtré ({pair or '?'})") + except Exception as e: + log.warning(f"[BRAIN] Erreur RL ({pair or '?'}) : {e} — fallback signal ML non filtré") + + return decision + + +# ─── Singleton global (même pattern que rl_agent.get_rl_agent) ────────────── + +_brain_instance: Optional[UnifiedBrain] = None +_brain_lock = threading.Lock() + + +def get_brain() -> UnifiedBrain: + global _brain_instance + if _brain_instance is None: + with _brain_lock: + if _brain_instance is None: + _brain_instance = UnifiedBrain() + return _brain_instance + + +def reset_brain(): + global _brain_instance + _brain_instance = None + log.info("[BRAIN] Singleton réinitialisé") diff --git a/web_ui.py b/web_ui.py new file mode 100644 index 0000000..dfbc1c8 --- /dev/null +++ b/web_ui.py @@ -0,0 +1,2823 @@ +""" +AHAD QUANT Forex V4_RL — Web Command Center v4 +Interface de contrôle complète — ML + RL Pipeline. + +Fonctionnalités : + • Dashboard : balance, PnL, win rate, positions ouvertes + • RL Monitor : progress, courbes d'entraînement, fine-tuning + • Backtest : résultats ML seul et ML+RL + • Trades : journal paper trading + courbe equity + • MT5 Bridge : monitoring connexion, signaux, rapports + • Config : éditeur .env + • Terminal : logs temps réel + • Features : feature importance chart + +Lancer: + pip install fastapi uvicorn requests + python web_ui.py +Puis ouvrir: http://localhost:8080 +""" + +import os, json, time, subprocess, sys, threading, queue, pickle +from datetime import datetime +from pathlib import Path +from typing import Optional, Generator + +try: + import requests as _ext_requests + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False + +from fastapi import FastAPI, Request, HTTPException, Depends +from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, FileResponse +from fastapi.middleware.cors import CORSMiddleware +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +import uvicorn +import config + +import csv as _csv_mod + +# ═══════════════════════════════════════════════════════════════════════════════ +# MT5 Bridge — lecteurs CSV directs +# ═══════════════════════════════════════════════════════════════════════════════ + +def _mt5_path() -> Optional[Path]: + raw = getattr(config, "MT5_FILES_PATH", os.getenv("MT5_FILES_PATH", "")) + if not raw: return None + p = Path(raw) + return p if p.exists() else None + +def read_mt5_status() -> dict: + p = _mt5_path() + if not p: return {} + try: + f = p / "status.csv" + if not f.exists(): return {} + with open(f, encoding="utf-8", newline="") as fh: + rows = list(_csv_mod.DictReader(fh)) + if not rows: return {} + r = rows[-1] + return { + "timestamp": r.get("timestamp", ""), + "balance": float(r.get("balance", 0) or 0), + "equity": float(r.get("equity", 0) or 0), + "margin": float(r.get("margin", 0) or 0), + "free_margin": float(r.get("free_margin", 0) or 0), + "open_positions": int(r.get("open_positions", 0) or 0), + "daily_pnl": float(r.get("daily_pnl", 0) or 0), + } + except Exception: + return {} + +def _normalize_mt5_status(raw: str) -> tuple: + raw = (raw or "").strip().upper() + if raw in ("OPENED", "OPEN"): + return "OPEN", raw, "" + if raw in ("SL_HIT", "TP_HIT", "TIMEOUT", "MANUAL") or raw.startswith("CLOSED"): + return "CLOSED", raw, raw + if raw in ("SYMBOL_NOT_FOUND", "NO_PRICE", "INSUFFICIENT_MARGIN") or raw.startswith("FAILED"): + return "REJECTED", raw, raw + return raw, raw, "" + +def read_mt5_reports(limit: int = 200) -> list: + p = _mt5_path() + if not p: return [] + try: + f = p / "reports.csv" + if not f.exists(): return [] + with open(f, encoding="utf-8-sig", newline="") as fh: + rows = list(_csv_mod.DictReader(fh)) + normalized = [] + for row in rows: + sn, sr, cr = _normalize_mt5_status(row.get("status", "")) + row["status"] = sn; row["status_raw"] = sr; row["close_reason"] = cr + normalized.append(row) + return list(reversed(normalized[-limit:])) + except Exception: + return [] + +def read_bridge_health() -> dict: + p = _mt5_path() + out = {"signals_rows": 0, "reports_rows": 0, + "last_signal_id": None, "last_signal_ts": None, + "last_report_id": None, "last_report_ts": None, + "pending_signals": 0, "gap_seconds": None} + if not p: return out + try: + sf = p / "signals.csv" + if sf.exists(): + with open(sf, encoding="utf-8-sig", newline="") as fh: + rows = list(_csv_mod.DictReader(fh)) + out["signals_rows"] = len(rows) + if rows: + out["last_signal_id"] = rows[-1].get("signal_id") + out["last_signal_ts"] = rows[-1].get("timestamp") + rf = p / "reports.csv" + ack_ids = set() + if rf.exists(): + with open(rf, encoding="utf-8-sig", newline="") as fh: + rrows = list(_csv_mod.DictReader(fh)) + out["reports_rows"] = len(rrows) + ack_ids = {r.get("signal_id") for r in rrows} + if rrows: + out["last_report_id"] = rrows[-1].get("signal_id") + out["last_report_ts"] = rrows[-1].get("open_time") or rrows[-1].get("close_time") + if sf.exists(): + out["pending_signals"] = sum(1 for r in rows if r.get("signal_id") not in ack_ids) + if out["last_signal_ts"] and out["last_report_ts"]: + try: + t1 = datetime.fromisoformat(out["last_signal_ts"].replace("Z", "+00:00")).replace(tzinfo=None) + t2 = datetime.fromisoformat(out["last_report_ts"].replace("Z", "+00:00")).replace(tzinfo=None) + out["gap_seconds"] = round((t2 - t1).total_seconds(), 1) + except Exception: + pass + except Exception: + pass + return out + +def _mt5_connected(status: dict) -> bool: + ts = status.get("timestamp", "") + if not ts: return False + try: + dt = datetime.fromisoformat(ts.replace("Z", "+00:00")).replace(tzinfo=None) + return (datetime.utcnow() - dt).total_seconds() < 90 + except Exception: + return False + +# ═══════════════════════════════════════════════════════════════════════════════ +# App + Middleware +# ═══════════════════════════════════════════════════════════════════════════════ + +app = FastAPI(title="AHAD QUANT V4_RL", docs_url=None, redoc_url=None) + +_ALLOWED_ORIGINS = os.getenv( + "WEB_UI_CORS_ORIGINS", + "http://localhost:8080,http://127.0.0.1:8080" +).split(",") +app.add_middleware( + CORSMiddleware, + allow_origins=_ALLOWED_ORIGINS, + allow_methods=["*"], + allow_headers=["*"], +) + +_WEB_UI_TOKEN: str = os.getenv("WEB_UI_TOKEN", "") +_http_bearer = HTTPBearer(auto_error=False) + +if not _WEB_UI_TOKEN: + import warnings + warnings.warn( + "[SÉCURITÉ] WEB_UI_TOKEN non configuré — API accessible sans auth. " + "Définir WEB_UI_TOKEN dans .env avant déploiement VPS.", + stacklevel=1, + ) + +def require_auth(credentials: HTTPAuthorizationCredentials = Depends(_http_bearer)): + if not _WEB_UI_TOKEN: + return + if credentials is None or credentials.credentials != _WEB_UI_TOKEN: + raise HTTPException(status_code=401, detail="Token invalide. Configurer WEB_UI_TOKEN dans .env") + +# ═══════════════════════════════════════════════════════════════════════════════ +# Log infrastructure +# ═══════════════════════════════════════════════════════════════════════════════ + +_log_queue: queue.Queue = queue.Queue(maxsize=2000) +_log_history: list = [] +_log_lock = threading.Lock() +_reset_version: int = 0 # incrémenté à chaque reset global — le client peut détecter un nouveau reset + +def _emit(msg: str, proc: str = "UI"): + ts = datetime.now().strftime("%H:%M:%S") + entry = {"t": ts, "p": proc, "m": msg.rstrip()} + with _log_lock: + _log_history.append(entry) + if len(_log_history) > 1000: + _log_history.pop(0) + try: _log_queue.put_nowait(entry) + except queue.Full: pass + +# ═══════════════════════════════════════════════════════════════════════════════ +# Process registry +# ═══════════════════════════════════════════════════════════════════════════════ + +_procs: dict[str, Optional[subprocess.Popen]] = { + "bot": None, "train": None, "download": None, "backtest": None, "finetune": None, + "daily_local": None, # [NEW] cycle quotidien local (warm-start ML + RL réel) déclenché manuellement + "setup": None, # [NEW] pipeline setup initial (download → train → rl_train → export) +} +_pending: set = set() +_proc_lock = threading.Lock() + +BASE_DIR = Path(__file__).parent.resolve() + +def _is_running(name: str) -> bool: + with _proc_lock: + if name in _pending: return True + p = _procs.get(name) + return p is not None and p.poll() is None + +def _kill(name: str): + with _proc_lock: + p = _procs.get(name) + if p and p.poll() is None: + p.terminate() + try: p.wait(timeout=5) + except subprocess.TimeoutExpired: p.kill() + _procs[name] = None + _emit(f"⛔ Processus '{name}' arrêté", "UI") + +def _launch(name: str, script: str, label: str, extra_args: list = None): + with _proc_lock: + if name in _pending or (_procs.get(name) and _procs[name].poll() is None): + _emit(f"[!] {name} déjà en cours — lancement ignoré", "UI") + return + _pending.add(name) + _emit(f"▶ {label} démarré ({script})", name.upper()) + try: + cmd = [sys.executable, "-u", str(BASE_DIR / script)] + (extra_args or []) + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, bufsize=1, cwd=str(BASE_DIR), + encoding="utf-8", errors="replace" + ) + with _proc_lock: + _procs[name] = p + _pending.discard(name) + for line in iter(p.stdout.readline, ''): + _emit(line.rstrip(), name.upper()) + p.wait() + _emit(f"[{'OK' if p.returncode == 0 else 'FAIL'}] {label} terminé (code {p.returncode})", name.upper()) + except Exception as e: + _emit(f"[ERREUR] Lancement {script}: {e}", name.upper()) + finally: + with _proc_lock: + _procs[name] = None + _pending.discard(name) + +# ═══════════════════════════════════════════════════════════════════════════════ +# Config (.env) reader/writer +# ═══════════════════════════════════════════════════════════════════════════════ + +ENV_PATH = BASE_DIR / ".env" + +def read_env() -> dict: + result = {} + if not ENV_PATH.exists(): return result + for line in ENV_PATH.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if s and not s.startswith('#') and '=' in s: + k, _, v = s.partition('=') + result[k.strip()] = v.strip() + return result + +def write_env(updates: dict): + lines = ENV_PATH.read_text(encoding="utf-8").splitlines() if ENV_PATH.exists() else [] + written = set() + new_lines = [] + for line in lines: + s = line.strip() + if s and not s.startswith('#') and '=' in s: + k = s.split('=')[0].strip() + if k in updates: + new_lines.append(f"{k}={updates[k]}") + written.add(k) + continue + new_lines.append(line) + for k, v in updates.items(): + if k not in written: + new_lines.append(f"{k}={v}") + ENV_PATH.write_text('\n'.join(new_lines) + '\n', encoding="utf-8") + +SAFE_CONFIG_KEYS = { + # Trading core + "PAPER_MODE", "PAPER_INITIAL_BALANCE", + "LEVERAGE", "MAX_POSITIONS", "RISK_PER_TRADE", + "MAX_DAILY_LOSS_PCT", "STOP_LOSS_PCT", "TAKE_PROFIT_PCT", + "MIN_CONFIDENCE", "MAIN_LOOP_SECONDS", + "PAIRS", "CANDLE_INTERVAL", + "MIN_LOT_SIZE", "MAX_LOT_SIZE", + # Session filter + "SESSION_FILTER_ENABLED", + "SESSION_LONDON_START", "SESSION_LONDON_END", + "SESSION_NY_START", "SESSION_NY_END", + # Models + "USE_ENSEMBLE", + # RL + "USE_RL_AGENT", "RL_MODE", "RL_CONFIDENCE_BOOST", + "RL_OVERRIDE_THRESHOLD", "RL_FINETUNE_STEPS", + # Apprentissage continu (quotidien, local) — ML + RL + "DAILY_LOCAL_RETRAIN_ENABLED", "DAILY_LOCAL_RETRAIN_INTERVAL_HOURS", + "CONTINUOUS_LEARNING_ENABLED", "WARMSTART_ENABLED", "WARMSTART_MIN_TRADES", + "RL_REPLAY_MIN_TRADES", "RL_RETRAIN_INTERVAL_HOURS", + "AUTO_RETRAIN_INTERVAL_HOURS", "AUTO_RETRAIN_MIN_ACCURACY", + "EXPERIENCE_BUFFER_MAX_SIZE", + "RL_AUTO_RETRAIN_ENABLED", "RL_REAL_REPLAY_ENABLED", + # MT5 + "MT5_BRIDGE_ENABLED", "MT5_FILES_PATH", + "MT5_SIGNAL_TIMEOUT", "MT5_POLL_INTERVAL", + "MT5_SYMBOL_SUFFIX", "MT5_SERVER", + # Auth + "WEB_UI_TOKEN", +} + +CREDENTIAL_KEYS = {"MT5_LOGIN", "MT5_PASSWORD", "WEB_UI_TOKEN"} + +# ═══════════════════════════════════════════════════════════════════════════════ +# State readers +# ═══════════════════════════════════════════════════════════════════════════════ + +def read_paper_state() -> dict: + try: + with open(BASE_DIR / "paper_state.json") as f: + return json.load(f) + except Exception: + return {"balance": 0, "positions": {}, "trades": [], "daily_pnl": 0, "total_pnl": 0, "peak_equity": 0} + +def read_backtest_results() -> dict: + try: + with open(BASE_DIR / "backtest_results.json") as f: + return json.load(f) + except Exception: + return {} + +def read_model_info() -> dict: + info = {} + try: + with open(BASE_DIR / "last_retrain.json") as f: + info = json.load(f) + except Exception: + pass + ensemble_path = BASE_DIR / getattr(config, "ENSEMBLE_MODEL_PATH", "model_ensemble.pkl") + model_path = BASE_DIR / getattr(config, "MODEL_PATH", "model.pkl") + if ensemble_path.exists(): + info["model_file"] = ensemble_path.name + info["model_size_mb"] = round(ensemble_path.stat().st_size / 1024 / 1024, 1) + info["model_mtime"] = datetime.fromtimestamp(ensemble_path.stat().st_mtime).strftime("%Y-%m-%d %H:%M") + info["has_ensemble"] = True + elif model_path.exists(): + info["model_file"] = model_path.name + info["model_size_mb"] = round(model_path.stat().st_size / 1024 / 1024, 1) + info["model_mtime"] = datetime.fromtimestamp(model_path.stat().st_mtime).strftime("%Y-%m-%d %H:%M") + info["has_ensemble"] = False + else: + info["model_file"] = None + return info + +def _detect_device() -> str: + """Détecte CUDA réellement, sans planter si torch n'est pas installé.""" + try: + import torch + return "CUDA (GPU)" if torch.cuda.is_available() else "CPU" + except Exception: + return "CPU (torch non installé)" + + +def read_rl_status() -> dict: + """Lit l'état complet de la couche RL.""" + agent_path = BASE_DIR / "rl_agent.zip" + scaler_path = BASE_DIR / "rl_scaler.pkl" + ckpt_path = BASE_DIR / "rl_checkpoints" / "best_model.zip" + curves_path = BASE_DIR / "rl_training_curves.png" + out = { + "agent_exists": agent_path.exists(), + "scaler_exists": scaler_path.exists(), + "checkpoint_exists": ckpt_path.exists(), + "curves_exists": curves_path.exists(), + "agent_size_mb": round(agent_path.stat().st_size / 1024 / 1024, 2) if agent_path.exists() else 0, + "agent_mtime": datetime.fromtimestamp(agent_path.stat().st_mtime).strftime("%Y-%m-%d %H:%M") if agent_path.exists() else None, + "progress": {}, + # ── [FIX] Avant : "Device CUDA (GPU)" et "Fine-tune Steps 200 000" + # étaient écrits en dur dans le HTML, peu importe la machine réelle + # ou la config courante. Désormais lus dynamiquement. ── + "device": _detect_device(), + "finetune_steps": getattr(config, "RL_FINETUNE_STEPS", 200_000), + } + try: + with open(BASE_DIR / "rl_progress.json") as f: + out["progress"] = json.load(f) + except Exception: + pass + # ── Dernier résultat de fine-tune (reward avant/après, accepté ou non, + # ET la source des données — "real_trades_only" vs "simulated") — + # persisté par rl_train.py::fine_tune()/fine_tune_real_only() dans + # last_rl_retrain.json. AVANT : ce résultat n'existait que dans le + # stdout du process, perdu une fois celui-ci terminé ; et même une fois + # lu ici, "data_source"/"n_real_trades" n'étaient jamais affichés côté UI. ── + out["last_finetune"] = None + try: + with open(BASE_DIR / "last_rl_retrain.json") as f: + out["last_finetune"] = json.load(f) + except Exception: + pass + # ── [FIX] Statut du cycle quotidien LOCAL (warm-start ML + RL réel) — + # remplace le bloc "Planning Fine-tuning" statique ("Semaine 3/4", + # "hebdomadaire") qui ne correspondait plus au cycle quotidien actuel. ── + out["daily_local"] = { + "last_run": None, + "interval_hours": getattr(config, "DAILY_LOCAL_RETRAIN_INTERVAL_HOURS", 24), + } + try: + with open(BASE_DIR / "last_daily_local.json") as f: + out["daily_local"]["last_run"] = json.load(f).get("datetime") + except Exception: + pass + return out + +def read_data_status() -> dict: + d = BASE_DIR / "data" + files = list(d.glob("*.json")) if d.exists() else [] + return { + # Champs utilisés par la page Setup Initial (loadSetupStatus JS) + "ensemble_model": (BASE_DIR / "model_ensemble.pkl").exists(), + "rl_model": (BASE_DIR / "rl_agent.zip").exists(), + "unified_model": (BASE_DIR / "ahad_quant_unified.zip").exists(), + "data_files_count": len(files), + # Champs originaux conservés (utilisés ailleurs) + "count": len(files), + "files": sorted(f.stem.replace("_1h", "") for f in files), + "total_mb": round(sum(f.stat().st_size for f in files) / 1024 / 1024, 1) if files else 0, + } + +def read_bot_state() -> dict: + try: + with open(BASE_DIR / "bot_state.json") as f: + return json.load(f) + except Exception: + return {} + +def read_feature_importance() -> dict: + try: + with open(BASE_DIR / "feature_importance.json") as f: + return json.load(f) + except Exception: + return {} + +def _calc_win_rate(trades: list) -> float: + if not trades: return 0.0 + return round(sum(1 for t in trades if float(t.get("pnl", 0)) > 0) / len(trades) * 100, 1) + +def _equity_curve(trades: list) -> list: + bal, curve = 100.0, [] + for t in trades: + bal += float(t.get("pnl", 0)) + curve.append({"t": t.get("closed_at", ""), "v": round(bal, 2)}) + return curve[-300:] + +def fetch_prices(symbols: list) -> dict: + """Prix Forex via Frankfurter (ECB rates, gratuit, ~55 paires).""" + if not HAS_REQUESTS or not symbols: return {} + out = {s: 0.0 for s in symbols} + try: + r = _ext_requests.get("https://api.frankfurter.app/latest", params={"from": "EUR"}, timeout=5) + if r.ok: + rates = r.json().get("rates", {}) + rates["EUR"] = 1.0 + for sym in symbols: + if len(sym) != 6: continue + bc, qc = sym[:3], sym[3:] + b, q = rates.get(bc), rates.get(qc) + if b and q: + out[sym] = round(q / b, 6) + except Exception: + pass + return out + +# ═══════════════════════════════════════════════════════════════════════════════ +# API +# ═══════════════════════════════════════════════════════════════════════════════ + +@app.get("/api/status", dependencies=[Depends(require_auth)]) +def api_status(): + model = read_model_info() + data = read_data_status() + rl = read_rl_status() + + # ── Badge de synchronisation ML/RL — compare les horodatages des + # DERNIÈRES mises à jour effectives (pas juste la dernière tentative) : + # - ML : last_retrain.json["datetime"] (écrit uniquement si accepté) + # - RL : last_rl_retrain.json["datetime"], MAIS uniquement si accepted=True + # AVANT : aucune comparaison n'existait — impossible de savoir si l'un + # avait dérivé de l'autre. ── + sync = {"status": "unknown", "ml_updated_at": None, "rl_updated_at": None, "drift_days": None} + try: + ml_dt_str = model.get("datetime") + rl_finetune = rl.get("last_finetune") or {} + rl_dt_str = rl_finetune.get("datetime") if rl_finetune.get("accepted") else None + sync["ml_updated_at"] = ml_dt_str + sync["rl_updated_at"] = rl_dt_str + if ml_dt_str and rl_dt_str: + ml_dt = datetime.fromisoformat(ml_dt_str) + rl_dt = datetime.fromisoformat(rl_dt_str) + drift_days = abs((ml_dt - rl_dt).total_seconds()) / 86400 + sync["drift_days"] = round(drift_days, 1) + sync["status"] = "synced" if drift_days <= 1 else "drifted" + elif ml_dt_str or rl_dt_str: + sync["status"] = "partial" # un seul des deux a déjà été (re)validé + except Exception: + pass + + paper_mode = getattr(config, "PAPER_MODE", True) + + if paper_mode: + paper = read_paper_state() + pos = paper.get("positions", {}) + prices = fetch_prices(list(pos.keys())) + for coin, p in pos.items(): + price = prices.get(coin) + if price and p.get("entry"): + mult = 1 if p.get("side") == "long" else -1 + p["upnl"] = round(mult * (price - p["entry"]) / p["entry"] * p.get("qty", 0) * p["entry"] * getattr(config, "LEVERAGE", 30), 3) + p["current_price"] = round(price, 6) + balance = round(paper.get("balance", 0), 2) + daily_pnl = round(paper.get("daily_pnl", 0), 2) + total_pnl = round(paper.get("total_pnl", 0), 2) + peak_eq = round(paper.get("peak_equity", 0), 2) + open_pos = pos + total_tr = len(paper.get("trades", [])) + win_rate = _calc_win_rate(paper.get("trades", [])) + else: + mt5 = read_mt5_status() + reports = read_mt5_reports(limit=500) + balance = round(mt5.get("balance", 0), 2) + daily_pnl = round(mt5.get("daily_pnl", 0), 2) + total_pnl = round(sum(float(r.get("pnl", 0)) for r in reports), 2) + peak_eq = round(mt5.get("equity", balance), 2) + open_pos = {} + total_tr = len(reports) + win_rate = round( + sum(1 for r in reports if float(r.get("profit", 0)) > 0) / len(reports) * 100 + if reports else 0, 1) + + pairs = getattr(config, "PAIRS", []) + + return { + "paper_mode": paper_mode, + "balance": balance, + "daily_pnl": daily_pnl, + "total_pnl": total_pnl, + "peak_equity": peak_eq, + "open_positions": open_pos, + "total_trades": total_tr, + "win_rate": win_rate, + "model": model, + "sync": sync, + "data": data, + "rl": rl, + "leverage": getattr(config, "LEVERAGE", 30), + "max_positions": getattr(config, "MAX_POSITIONS", 5), + "risk_per_trade": getattr(config, "RISK_PER_TRADE", 0.02), + "min_confidence": getattr(config, "MIN_CONFIDENCE", 0.72), + "pairs_count": len(pairs), + "use_ensemble": getattr(config, "USE_ENSEMBLE", True), + "use_rl_agent": getattr(config, "USE_RL_AGENT", False), + "rl_mode": getattr(config, "RL_MODE", "filter"), + "rl_override_threshold": getattr(config, "RL_OVERRIDE_THRESHOLD", 0.82), + "rl_confidence_boost": getattr(config, "RL_CONFIDENCE_BOOST", 0.05), + "session_filter": getattr(config, "SESSION_FILTER_ENABLED", False), + "mt5_bridge": getattr(config, "MT5_BRIDGE_ENABLED", False), + "candle_interval": getattr(config, "CANDLE_INTERVAL", "1h"), + "bot_running": _is_running("bot"), + "training": _is_running("train"), + "downloading": _is_running("download"), + "backtesting": _is_running("backtest"), + "finetuning": _is_running("train"), # UNIFIÉ : même slot que "training" (un seul système ML+RL) + "daily_local_running": _is_running("daily_local"), # [NEW] + "setup_running": _is_running("setup"), # [NEW] pipeline setup initial + "paused": (BASE_DIR / ".paused").exists(), + "stopped": (BASE_DIR / ".stopped").exists(), + } + +@app.get("/api/trades", dependencies=[Depends(require_auth)]) +def api_trades(limit: int = 200): + paper_mode = getattr(config, "PAPER_MODE", True) + if paper_mode: + state = read_paper_state() + trades = list(reversed(state.get("trades", [])))[:limit] + return {"trades": trades, "equity_curve": _equity_curve(state.get("trades", []))} + else: + reports = read_mt5_reports(limit=limit) + return {"trades": reports, "equity_curve": _equity_curve(reports)} + +@app.get("/api/backtest/results", dependencies=[Depends(require_auth)]) +def api_backtest_results(): + return read_backtest_results() + +@app.get("/api/data/status", dependencies=[Depends(require_auth)]) +def api_data_status(): + return read_data_status() + +@app.get("/api/rl/status", dependencies=[Depends(require_auth)]) +def api_rl_status(): + return read_rl_status() + +@app.get("/api/rl/progress", dependencies=[Depends(require_auth)]) +def api_rl_progress(): + try: + with open(BASE_DIR / "rl_progress.json") as f: + return json.load(f) + except Exception: + return {} + +@app.get("/api/rl/curves", dependencies=[Depends(require_auth)]) +def api_rl_curves(): + path = BASE_DIR / "rl_training_curves.png" + if not path.exists(): + raise HTTPException(status_code=404, detail="Courbes non disponibles — lancer l'entraînement RL d'abord") + return FileResponse(path, media_type="image/png", headers={"Cache-Control": "no-cache"}) + +def _start_unified_training() -> dict: + """ + Lance train_unified.py (ML complet + RL) en arrière-plan. + + [FIX] Fonction extraite — AVANT, /api/train et /api/rl/finetune + dupliquaient exactement la même logique en copier-coller (risque de + divergence future). De plus, le bouton frontend ("🧠 Entraîner ML+RL") + n'appelait QUE /api/rl/finetune : /api/train n'était appelé par AUCUN + élément de l'UI — un endpoint mort, accessible seulement en curl direct. + Les deux routes appellent désormais ce même helper. + """ + if _is_running("train"): + return {"ok": False, "message": "Entraînement déjà en cours"} + steps = str(getattr(config, "RL_FINETUNE_STEPS", 200_000)) + threading.Thread( + target=_launch, + args=("train", "train_unified.py", "Entraînement Unifié (ML+RL)"), + kwargs={"extra_args": ["--rl-steps", steps]}, + daemon=True + ).start() + return {"ok": True, "message": "Entraînement Unifié ML+RL démarré ▶ (un seul système, ne remplace l'ancien que s'il est meilleur)"} + + +@app.post("/api/rl/finetune", dependencies=[Depends(require_auth)]) +def api_rl_finetune(): + return _start_unified_training() + +@app.get("/api/bot-state", dependencies=[Depends(require_auth)]) +def api_bot_state(): + return read_bot_state() + +@app.get("/api/feature-importance", dependencies=[Depends(require_auth)]) +def api_feature_importance(): + return read_feature_importance() + +# ── Bot controls ────────────────────────────────────────────────────────────── + +@app.post("/api/bot/start", dependencies=[Depends(require_auth)]) +def api_bot_start(): + if _is_running("bot"): + return {"ok": False, "message": "Bot déjà en cours"} + (BASE_DIR / ".stopped").unlink(missing_ok=True) + (BASE_DIR / ".paused").unlink(missing_ok=True) + threading.Thread(target=_launch, args=("bot", "ahad_quant.py", "Bot principal"), daemon=True).start() + return {"ok": True, "message": "Bot démarré ✅"} + +@app.post("/api/bot/stop", dependencies=[Depends(require_auth)]) +def api_bot_stop(): + (BASE_DIR / ".stopped").touch() + _kill("bot") + return {"ok": True, "message": "Bot arrêté"} + +@app.post("/api/bot/pause", dependencies=[Depends(require_auth)]) +def api_bot_pause(): + (BASE_DIR / ".paused").touch() + _emit("⏸ Bot mis en pause", "UI") + return {"ok": True, "message": "Bot en pause ⏸"} + +@app.post("/api/bot/resume", dependencies=[Depends(require_auth)]) +def api_bot_resume(): + (BASE_DIR / ".paused").unlink(missing_ok=True) + _emit("▶ Bot repris", "UI") + return {"ok": True, "message": "Bot repris ▶"} + +@app.post("/api/bot/emergency", dependencies=[Depends(require_auth)]) +def api_bot_emergency(): + (BASE_DIR / ".stopped").touch() + _kill("bot") + _emit("⚠️ ARRÊT D'URGENCE — processus tué + flag .stopped posé", "UI") + return {"ok": True, "message": "Arrêt d'urgence déclenché ⚠️"} + +@app.post("/api/bot/reset", dependencies=[Depends(require_auth)]) +def api_bot_reset(): + (BASE_DIR / ".stopped").unlink(missing_ok=True) + (BASE_DIR / ".paused").unlink(missing_ok=True) + _emit("↺ Flags réinitialisés", "UI") + return {"ok": True} + +# ── Process controls ────────────────────────────────────────────────────────── + +@app.post("/api/download", dependencies=[Depends(require_auth)]) +def api_download(): + if _is_running("download"): + return {"ok": False, "message": "Téléchargement déjà en cours"} + threading.Thread(target=_launch, args=("download", "download_data.py", "Téléchargement données"), daemon=True).start() + return {"ok": True, "message": "Téléchargement démarré ⬇"} + +@app.post("/api/train", dependencies=[Depends(require_auth)]) +def api_train(): + # Alias historique — conservé pour compatibilité avec d'éventuels + # scripts externes, même si le bouton UI utilise /api/rl/finetune. + # Délègue désormais au même helper (voir _start_unified_training). + return _start_unified_training() + +@app.post("/api/retrain/daily-local", dependencies=[Depends(require_auth)]) +def api_retrain_daily_local(): + """ + [NEW] Déclenche manuellement le cycle quotidien LOCAL (warm-start + LightGBM + RL fine-tune 100% réel) — exactement le même cycle que le + thread de fond AutoRetrainer exécute tout seul toutes les + DAILY_LOCAL_RETRAIN_INTERVAL_HOURS, mais à la demande, sans attendre + l'intervalle. Avant ce fix, ce mécanisme léger n'était accessible que + via le bot en tournant en continu — aucun moyen de le forcer depuis l'UI. + """ + if _is_running("train"): + return {"ok": False, "message": "Un entraînement complet (ML+RL) est déjà en cours — attends qu'il termine"} + if _is_running("daily_local"): + return {"ok": False, "message": "Cycle quotidien local déjà en cours"} + threading.Thread( + target=_launch, + args=("daily_local", "daily_local_retrain.py", "Cycle quotidien local (ML+RL léger)"), + daemon=True + ).start() + return {"ok": True, "message": "Cycle quotidien local démarré ⚡ (warm-start ML + RL réel)"} + +@app.post("/api/setup/run", dependencies=[Depends(require_auth)]) +def api_setup_run(): + """ + [NEW] Pipeline setup initial complet : + 1. download_data.py — téléchargement données historiques (réseau) + 2. train.py — entraînement ML complet (LGB + XGBoost + RF) + 3. rl_train.py — entraînement PPO RL from scratch (1 000 000 steps) + 4. export_unified.py — bundle ahad_quant_unified.zip + + Chaque étape tourne dans le même slot "setup" — le terminal affiche + la progression en temps réel. À n'exécuter qu'une seule fois. + """ + if _is_running("setup"): + return {"ok": False, "message": "Setup initial déjà en cours"} + if _is_running("train"): + return {"ok": False, "message": "Un entraînement est déjà en cours — attends qu'il termine"} + if _is_running("bot"): + return {"ok": False, "message": "Arrête le bot avant de lancer le setup initial"} + + def _run_setup(): + steps = [ + ("download_data.py", "Téléchargement données historiques"), + ("train.py", "Entraînement ML (LGB + XGBoost + RF)"), + ("rl_train.py", "Entraînement RL PPO (1 000 000 steps)"), + ("export_unified.py","Export bundle ahad_quant_unified.zip"), + ] + _emit("═" * 60, "SETUP") + _emit(" AHAD QUANT — Setup Initial (4 étapes)", "SETUP") + _emit("═" * 60, "SETUP") + for i, (script, label) in enumerate(steps, 1): + _emit(f"\n[{i}/4] {label}...", "SETUP") + try: + cmd = [sys.executable, "-u", str(BASE_DIR / script)] + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, bufsize=1, cwd=str(BASE_DIR), + encoding="utf-8", errors="replace" + ) + with _proc_lock: + _procs["setup"] = p + for line in iter(p.stdout.readline, ''): + _emit(line.rstrip(), "SETUP") + p.wait() + if p.returncode != 0: + _emit(f"❌ Étape {i} échouée (code {p.returncode}) — setup interrompu", "SETUP") + return + _emit(f"✅ Étape {i}/4 terminée : {label}", "SETUP") + except Exception as e: + _emit(f"❌ Erreur étape {i} : {e}", "SETUP") + return + _emit("\n" + "═" * 60, "SETUP") + _emit(" ✅ Setup initial terminé — AHAD QUANT prêt à trader !", "SETUP") + _emit("═" * 60, "SETUP") + with _proc_lock: + _procs["setup"] = None + + with _proc_lock: + _pending.add("setup") + threading.Thread(target=_run_setup, daemon=True).start() + with _proc_lock: + _pending.discard("setup") + return {"ok": True, "message": "Setup initial démarré ▶ — 4 étapes en cours (voir Terminal)"} + +@app.post("/api/setup/stop", dependencies=[Depends(require_auth)]) +def api_setup_stop(): + """Interrompt le setup initial en cours.""" + if not _is_running("setup"): + return {"ok": False, "message": "Aucun setup en cours"} + _kill("setup") + return {"ok": True, "message": "Setup interrompu"} + +@app.post("/api/backtest/run", dependencies=[Depends(require_auth)]) +def api_backtest_run(): + if _is_running("backtest"): + return {"ok": False, "message": "Backtest déjà en cours"} + threading.Thread(target=_launch, args=("backtest", "backtest.py", "Backtest"), daemon=True).start() + return {"ok": True, "message": "Backtest démarré — voir Terminal"} + +@app.post("/api/stop/{proc}", dependencies=[Depends(require_auth)]) +def api_stop_proc(proc: str): + if proc not in _procs: + return {"ok": False, "message": f"Processus inconnu: {proc}"} + _kill(proc) + return {"ok": True, "message": f"{proc} arrêté"} + +# ── Logs stream ─────────────────────────────────────────────────────────────── + +@app.get("/api/logs/stream", dependencies=[Depends(require_auth)]) +def api_logs_stream(): + def gen() -> Generator: + with _log_lock: + hist = list(_log_history) + for entry in hist: + yield f"data: {json.dumps(entry)}\n\n" + while True: + try: + entry = _log_queue.get(timeout=30) + yield f"data: {json.dumps(entry)}\n\n" + except queue.Empty: + yield f"data: {json.dumps({'ping': True})}\n\n" + return StreamingResponse(gen(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + +# ── MT5 Bridge ──────────────────────────────────────────────────────────────── + +@app.get("/api/mt5/status", dependencies=[Depends(require_auth)]) +def api_mt5_status(): + status = read_mt5_status() + connected = _mt5_connected(status) + return { + "connected": connected, + "path_configured": bool(getattr(config, "MT5_FILES_PATH", "")), + "mt5_path": getattr(config, "MT5_FILES_PATH", ""), + "bridge": read_bridge_health(), + **status, + } + +@app.get("/api/mt5/reports", dependencies=[Depends(require_auth)]) +def api_mt5_reports(limit: int = 200): + reports = read_mt5_reports(limit) + open_trades = [r for r in reports if r.get("status") == "OPEN"] + closed_trades = [r for r in reports if r.get("status") == "CLOSED"] + errors = [r for r in reports if r.get("status") in ("ERROR", "REJECTED")] + total_profit = sum(float(r.get("profit") or 0) for r in closed_trades) + wins = sum(1 for r in closed_trades if float(r.get("profit") or 0) > 0) + return { + "reports": reports, + "open_count": len(open_trades), + "closed_count": len(closed_trades), + "error_count": len(errors), + "total_profit": round(total_profit, 2), + "win_count": wins, + "loss_count": len(closed_trades) - wins, + "win_rate": round(wins / len(closed_trades) * 100, 1) if closed_trades else 0.0, + } + +@app.get("/api/mt5/stream", dependencies=[Depends(require_auth)]) +def api_mt5_stream(): + def gen() -> Generator: + last_hash = None + while True: + try: + status = read_mt5_status() + reports = read_mt5_reports(200) + connected = _mt5_connected(status) + cur_hash = f"{status.get('timestamp','')}{len(reports)}" + if cur_hash != last_hash: + last_hash = cur_hash + closed = [r for r in reports if r.get("status") == "CLOSED"] + wins = sum(1 for r in closed if float(r.get("profit") or 0) > 0) + payload = { + "type": "mt5_update", "connected": connected, "status": status, + "open_count": len([r for r in reports if r.get("status") == "OPEN"]), + "closed_count": len(closed), + "total_profit": round(sum(float(r.get("profit") or 0) for r in closed), 2), + "win_rate": round(wins / len(closed) * 100, 1) if closed else 0.0, + "latest_reports": reports[:10], + } + yield f"data: {json.dumps(payload)}\n\n" + else: + yield f"data: {json.dumps({'ping': True})}\n\n" + except GeneratorExit: + break + except Exception: + yield f"data: {json.dumps({'ping': True})}\n\n" + time.sleep(5) + return StreamingResponse(gen(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + +# ── Config / Paper / Export ─────────────────────────────────────────────────── + +@app.get("/api/config", dependencies=[Depends(require_auth)]) +def api_get_config(): + env = read_env() + secret_patterns = ["secret","key","password","token","passphrase","private"] + result = {k: v for k, v in env.items() if not any(p in k.lower() for p in secret_patterns)} + _defaults = { + "PAIRS": ",".join(getattr(config, "PAIRS", [])), + "LEVERAGE": str(getattr(config, "LEVERAGE", 30)), + "PAPER_MODE": str(getattr(config, "PAPER_MODE", True)).lower(), + "PAPER_INITIAL_BALANCE": str(getattr(config, "PAPER_INITIAL_BALANCE", 100)), + "MIN_CONFIDENCE": str(getattr(config, "MIN_CONFIDENCE", 0.72)), + "MAX_POSITIONS": str(getattr(config, "MAX_POSITIONS", 5)), + "RISK_PER_TRADE": str(getattr(config, "RISK_PER_TRADE", 0.02)), + "MAX_DAILY_LOSS_PCT": str(getattr(config, "MAX_DAILY_LOSS_PCT", 0.05)), + "STOP_LOSS_PCT": str(getattr(config, "STOP_LOSS_PCT", 0.01)), + "TAKE_PROFIT_PCT": str(getattr(config, "TAKE_PROFIT_PCT", 0.02)), + "MAIN_LOOP_SECONDS": str(getattr(config, "MAIN_LOOP_SECONDS", 60)), + "CANDLE_INTERVAL": getattr(config, "CANDLE_INTERVAL", "1h"), + "MIN_LOT_SIZE": str(getattr(config, "MIN_LOT_SIZE", 0.01)), + "MAX_LOT_SIZE": str(getattr(config, "MAX_LOT_SIZE", 1.0)), + "USE_ENSEMBLE": str(getattr(config, "USE_ENSEMBLE", True)).lower(), + "USE_RL_AGENT": str(getattr(config, "USE_RL_AGENT", True)).lower(), + "RL_MODE": getattr(config, "RL_MODE", "filter"), + "RL_OVERRIDE_THRESHOLD": str(getattr(config, "RL_OVERRIDE_THRESHOLD", 0.82)), + "RL_CONFIDENCE_BOOST": str(getattr(config, "RL_CONFIDENCE_BOOST", 0.05)), + "RL_FINETUNE_STEPS": str(getattr(config, "RL_FINETUNE_STEPS", 200000)), + "DAILY_LOCAL_RETRAIN_ENABLED": str(getattr(config, "DAILY_LOCAL_RETRAIN_ENABLED", True)).lower(), + "DAILY_LOCAL_RETRAIN_INTERVAL_HOURS": str(getattr(config, "DAILY_LOCAL_RETRAIN_INTERVAL_HOURS", 24)), + "CONTINUOUS_LEARNING_ENABLED": str(getattr(config, "CONTINUOUS_LEARNING_ENABLED", True)).lower(), + "WARMSTART_ENABLED": str(getattr(config, "WARMSTART_ENABLED", True)).lower(), + "WARMSTART_MIN_TRADES": str(getattr(config, "WARMSTART_MIN_TRADES", 5)), + "RL_REPLAY_MIN_TRADES": str(getattr(config, "RL_REPLAY_MIN_TRADES", 10)), + "RL_RETRAIN_INTERVAL_HOURS": str(getattr(config, "RL_RETRAIN_INTERVAL_HOURS", 24)), + "AUTO_RETRAIN_INTERVAL_HOURS": str(getattr(config, "AUTO_RETRAIN_INTERVAL_HOURS", 24)), + "AUTO_RETRAIN_MIN_ACCURACY": str(getattr(config, "AUTO_RETRAIN_MIN_ACCURACY", 0.70)), + "EXPERIENCE_BUFFER_MAX_SIZE": str(getattr(config, "EXPERIENCE_BUFFER_MAX_SIZE", 10000)), + "RL_AUTO_RETRAIN_ENABLED": str(getattr(config, "RL_AUTO_RETRAIN_ENABLED", True)).lower(), + "RL_REAL_REPLAY_ENABLED": str(getattr(config, "RL_REAL_REPLAY_ENABLED", True)).lower(), + "MT5_BRIDGE_ENABLED": str(getattr(config, "MT5_BRIDGE_ENABLED", False)).lower(), + "MT5_FILES_PATH": getattr(config, "MT5_FILES_PATH", ""), + "MT5_SYMBOL_SUFFIX": getattr(config, "MT5_SYMBOL_SUFFIX", ""), + "MT5_SERVER": getattr(config, "MT5_SERVER", ""), + "MT5_SIGNAL_TIMEOUT": str(getattr(config, "MT5_SIGNAL_TIMEOUT", 30)), + "MT5_POLL_INTERVAL": str(getattr(config, "MT5_POLL_INTERVAL", 5)), + "SESSION_FILTER_ENABLED": str(getattr(config, "SESSION_FILTER_ENABLED", False)).lower(), + } + for k, v in _defaults.items(): + result.setdefault(k, v) + return result + +@app.post("/api/config", dependencies=[Depends(require_auth)]) +async def api_save_config(request: Request): + body = await request.json() + filtered = {k: str(v) for k, v in body.items() if k in SAFE_CONFIG_KEYS} + write_env(filtered) + _emit(f"✅ Config sauvegardée : {list(filtered.keys())}", "UI") + return {"ok": True, "saved": list(filtered.keys()), "restart_needed": _is_running("bot")} + +@app.post("/api/paper/reset", dependencies=[Depends(require_auth)]) +def api_paper_reset(): + init_bal = getattr(config, "PAPER_INITIAL_BALANCE", 100) + fresh = { + "balance": init_bal, + "positions": {}, + "trades": [], + "daily_pnl": 0, + "total_pnl": 0, + "peak_equity": init_bal, + "daily_losses": 0, + "circuit_breaker_until": 0, + "reset_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + with open(BASE_DIR / "paper_state.json", "w") as f: + json.dump(fresh, f, indent=2) + _emit(f"↺ Paper trading réinitialisé — balance: ${init_bal:,.0f}", "UI") + return {"ok": True, "balance": init_bal} + +# ── RESET GLOBAL ────────────────────────────────────────────────────────────── + +@app.post("/api/reset/all", dependencies=[Depends(require_auth)]) +def api_reset_all(): + """Réinitialise TOUT : logs mémoire, paper state, backtest, rl_progress, + last_retrain, bot_state. Laisse les modèles et checkpoints intacts.""" + global _log_history + + errors = [] + + # 1. Vider l'historique de logs en mémoire + with _log_lock: + _log_history.clear() + + # 2. Paper state → balance initiale + init_bal = getattr(config, "PAPER_INITIAL_BALANCE", 100) + fresh_paper = { + "balance": init_bal, + "positions": {}, + "trades": [], + "daily_pnl": 0, + "total_pnl": 0, + "peak_equity": init_bal, + "daily_losses": 0, + "circuit_breaker_until": 0, + "reset_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + try: + with open(BASE_DIR / "paper_state.json", "w") as f: + json.dump(fresh_paper, f, indent=2) + except Exception as e: + errors.append(f"paper_state: {e}") + + # 3. Backtest results → vide + try: + (BASE_DIR / "backtest_results.json").write_text("{}") + except Exception as e: + errors.append(f"backtest_results: {e}") + + # 4. RL progress → vide + try: + (BASE_DIR / "rl_progress.json").write_text("{}") + except Exception as e: + errors.append(f"rl_progress: {e}") + + # 5. Last retrain → vide + try: + lr = BASE_DIR / "last_retrain.json" + if lr.exists(): + lr.write_text("{}") + except Exception as e: + errors.append(f"last_retrain: {e}") + + # 6. Bot state → vide + try: + (BASE_DIR / "bot_state.json").write_text("{}") + except Exception as e: + errors.append(f"bot_state: {e}") + + # 7. Flags .stopped / .paused + (BASE_DIR / ".stopped").unlink(missing_ok=True) + (BASE_DIR / ".paused").unlink(missing_ok=True) + + if errors: + _emit(f"⚠️ Réinitialisation partielle — erreurs : {', '.join(errors)}", "UI") + return {"ok": False, "errors": errors} + + global _reset_version + _reset_version += 1 + # Émettre un signal spécial que les clients SSE peuvent détecter + special = {"t": time.strftime("%H:%M:%S"), "p": "UI", "m": "🗑 Réinitialisation complète effectuée (logs, trades, backtest, RL, bot state)", "reset": _reset_version} + try: _log_queue.put_nowait(special) + except queue.Full: pass + return {"ok": True, "balance": init_bal, "reset_version": _reset_version} + +@app.get("/api/export/{filename}", dependencies=[Depends(require_auth)]) +def api_export_json(filename: str): + ALLOWED = {"paper_state.json", "backtest_results.json", "last_retrain.json", "rl_progress.json"} + if filename not in ALLOWED: + raise HTTPException(status_code=403, detail="Fichier non autorisé") + path = BASE_DIR / filename + if not path.exists(): + raise HTTPException(status_code=404, detail="Fichier introuvable") + return FileResponse(path, media_type="application/json", filename=filename) + +# ═══════════════════════════════════════════════════════════════════════════════ +# HTML +# ═══════════════════════════════════════════════════════════════════════════════ + +HTML = r""" + + + + +AHAD QUANT V4_RL — Command Center + + + + + + +
+ + +
+ +
+
PAPER
+
OFFLINE
+ +
--:--:--
+ +
+
+ +
+ + + + + +
+ + +
+
+
Dashboard Live
+
+ + + + + + +
+
+ + +
+
+
Balance
+
+
Paper Mode
+
+
+
PnL Journalier
+
+
vs. ouverture
+
+
+
PnL Total
+
+
Peak: —
+
+
+
Win Rate
+
+
0 trades
+
+
+ +
+ +
+
Modèles ML
+
+
+
+
+
+ +
+
RL Agent (PPO)
+
+
+
+
+ +
+
Synchro ML ↔ RL
+
+
+
+ +
+
Données yfinance
+
+
+
+ +
+
+
+ + +
+
+
Levier
+
30x
+
Fixe
+
+
+
Confiance Min
+
72%
+
ML threshold
+
+
+
Seuil RL Override
+
82%
+
RL mode: filter
+
+
+
Paires actives
+
25
+
Forex 1H
+
+
+ + +
+
+
Positions Ouvertes
+ +
+
+
Aucune position ouverte
+
+
+
+ + +
+
+
RL Monitor PPO
+
+ + + +
+
+ + +
+
+
rl_agent.zip
+
+
+
+
+
rl_scaler.pkl
+
+
Normalisation obs
+
+
+
best_model.zip
+
+
Checkpoint RL
+
+
+
Fine-tuning RL
+
+
+
+
+ + +
+
+
Configuration PPO
+
AlgorithmePPO (Stable-Baselines3)
+
Steps entraîné
+
Device
+
N_ENVS4 (parallèle)
+
TRAIN_PAIRS8 paires majeures
+
EVAL_PAIRSGBPJPY, NZDUSD
+
Modefilter
+
Override Threshold82%
+
Confidence Boost+5%
+
Steps Full Retrain (RL)
+
+
+
Résultats Backtest ML+RL
+
+
Pas encore de backtest — onglet Backtest
+
+
+
+ + +
+
+
Courbes d'Entraînement RL
+ +
+
+ Courbes non générées — lancer l'entraînement RL complet +
+ +
+ + +
+
Cycle quotidien local ML+RL réel
+
+
+
Dernier cycle local
+
+
Warm-start ML + RL réel
+
+
+
Intervalle
+
+
Automatique si bot lancé
+
+
+
Dernier fine-tune RL
+
+
Réel vs simulé
+
+
+
+
+ + + +
+
+
Backtest OOS
+
+ + + ⬇ JSON +
+
+ + +
+
Système testé
+
Lancer un backtest pour voir l'état du système
+
+ + +
+
+
Dernier Backtest
+
+
Pas de résultats — lancer un backtest
+
+
+
+
Courbe Equity
+
+
+
+
+ + +
+
+
Paper Trades
+
+ + ⬇ Export + +
+
+ + +
+
+
Trades Total
+
0
+
+
+
Win Rate
+
0%
+
+
+
PnL Total
+
0
+
+
+
Trades Gagnants
+
0
+
+
+ + +
+
Courbe Equity (base 100)
+
+
+ + +
+
+
Journal des Trades
+
+ + +
+
+
+ + + + + + +
PaireDir.EntréeSortiePnLConfiance MLRL FiltreOuvertFermé
+
+ +
+
+ + +
+
🚀 Setup Initial
+ + + +
+
Pipeline — 4 étapes séquentielles
+
+
+
1
+
Téléchargement données
download_data.py — 25 paires × 1000 jours (internet requis)
+ EN ATTENTE +
+
+
2
+
Entraînement ML
train.py — LightGBM + XGBoost + RandomForest ensemble
+ EN ATTENTE +
+
+
3
+
Entraînement RL
rl_train.py — PPO 1 000 000 steps from scratch (~2–4h GPU)
+ EN ATTENTE +
+
+
4
+
Export bundle
export_unified.py — génère ahad_quant_unified.zip
+ EN ATTENTE +
+
+
+ +
+ + + +
+ +
+
État des fichiers
+
model_ensemble.pkl
+
rl_agent.zip
+
ahad_quant_unified.zip
+
Fichiers données /data
+
+
+ + +
+
+
MT5 Bridge CSV
+ +
+ +
+
+
Statut Connexion
+
+
+
+
+
Balance MT5
+
+
Equity: —
+
+
+
Positions Ouvertes
+
0
+
PnL jour: —
+
+
+ + +
+
Bridge Health
+
+
+
Chemin configuré
+
Signaux envoyés0
+
Rapports reçus0
+
Signaux en attente0
+
+
+
Dernier signal ID
+
Dernier rapport ID
+
Gap sig→rep
+
MT5 Path
+
+
+
+ + +
+
Rapports MT5
+
+ +
+
+
+ + + + + + +
Signal IDPaireDir.EntréeProfitStatutRaisonTemps
+
+
Aucun rapport MT5 — bridge non connecté
+
+
+ + +
+
+
Feature Importance LightGBM
+ +
+ + + +
+
Feature importance non disponible — lancer un Full Retrain (bouton "🧠 Full Retrain (ML+RL)") d'abord
+
+
+
+
+ + +
+
+
Terminal Live
+
+ + +
+
+ +
+
+ + + + +
+
+ + +
+
+
Configuration .env
+
+ + +
+
+ + + +
+ + +
+
💱 Trading
+
+ + +
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+ + +
+
🧠 ML + RL
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ +
🔄 Apprentissage continu (quotidien, local)
+
+ + +
+
+
+ + +
+
+ + +
+
+
+
+
+
+
+
+ + +
+
+ + +
+
+ +
+ +
🔌 MT5 Bridge
+
+ + +
+
+
+
+
+
+ +
🕐 Session Filter
+
+ + +
+
+
+
+
+
+
+
+ +
+
+
+ + + +""" + +# ═══════════════════════════════════════════════════════════════════════════════ + +@app.get("/", response_class=HTMLResponse) +def root(): + return HTML + +if __name__ == "__main__": + _emit("🚀 AHAD QUANT V4_RL — Web Command Center v4 démarré", "UI") + print("\n" + "=" * 58) + print(" AHAD QUANT Forex V4_RL — Command Center") + print("=" * 58) + print(" → http://localhost:8080") + print(" ML + RL (PPO) · 25 paires · 1H · Paper Mode") + print(" Ctrl+C pour arrêter") + print("=" * 58 + "\n") + uvicorn.run(app, host="0.0.0.0", port=8080, log_level="warning")