gavindiaz/profitable-expert-advisor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
zhutoutoutousan
2026-04-01 05:40:17 +02:00
parent 28e7daf1e3
commit 842a2f8fac
53 changed files with 9222 additions and 962 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai/btcusd1min/BTCUSD_M1_EA.mq5
+620
View File
@@ -0,0 +1,620 @@
//+------------------------------------------------------------------+
//| BTCUSD_M1_EA.mq5 |
//| Copyright 2025, MetaQuotes Ltd. |
//| https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2025, MetaQuotes Ltd."
#property link "https://www.mql5.com"
#property version "1.00"
#property description "Expert Advisor using ONNX model for BTCUSD 1-minute price prediction"
#property description "Based on: https://www.mql5.com/en/docs/onnx/onnx_test"
#include <Trade\Trade.mqh>
//--- Resource: Embed ONNX model in EA
// Based on: https://www.mql5.com/en/docs/onnx/onnx_test
// Path is relative to MQL5 directory (not starting with \\Files\\)
#resource "Files\\BTCUSD_M1_model.onnx" as uchar ExtModel[]
//--- Input parameters
input group "ONNX Model Settings"
input string InpModelPath = ""; // ONNX Model Path (leave empty to use embedded resource)
input int InpLookback = 60; // Lookback Period (bars)
input bool InpUsePrediction = true; // Use Model Prediction
input group "Trading Settings"
input double InpLotSize = 0.01; // Lot Size
input int InpMagicNumber = 123456; // Magic Number
input int InpSlippage = 3; // Slippage (points)
input bool InpUsePredictedSLTP = true; // Use Predicted SL/TP (based on prediction & volatility)
input int InpStopLoss = 50; // Stop Loss (pips) - used if InpUsePredictedSLTP=false
input int InpTakeProfit = 100; // Take Profit (pips) - used if InpUsePredictedSLTP=false
input double InpSLMultiplier = 1.5; // SL Multiplier (ATR-based, e.g., 1.5 = 1.5x ATR)
input double InpTPMultiplier = 2.0; // TP Multiplier (ATR-based, e.g., 2.0 = 2x ATR)
input double InpMinSLATR = 0.5; // Minimum SL (ATR multiplier)
input double InpMinTPATR = 1.0; // Minimum TP (ATR multiplier)
input group "Prediction Settings"
input double InpPredictionThreshold = 0.00005; // Min Prediction Change (0.005% as decimal, e.g., 0.00005 = 0.005%)
input bool InpUseConfidence = true; // Use Confidence Filter
input double InpMinConfidence = 0.1; // Minimum Confidence (0.1 = 10%)
//--- Global variables
CTrade trade;
long onnx_handle = INVALID_HANDLE;
datetime last_bar_time = 0;
double last_prediction = 0.0;
double last_confidence = 0.0;
//+------------------------------------------------------------------+
//| Expert initialization function |
//+------------------------------------------------------------------+
int OnInit()
{
// Set trade parameters
trade.SetExpertMagicNumber(InpMagicNumber);
trade.SetDeviationInPoints(InpSlippage);
trade.SetTypeFilling(ORDER_FILLING_FOK);
// Check symbol
if(_Symbol != "BTCUSD" && _Symbol != "BTCUSD#")
{
Print("WARNING: This EA is designed for BTCUSD. Current symbol: ", _Symbol);
}
// Check timeframe
if(_Period != PERIOD_M1)
{
Print("WARNING: This EA is designed for M1 timeframe. Current timeframe: ", EnumToString(_Period));
}
// Load ONNX model
// Based on: https://www.mql5.com/en/docs/onnx/onnx_test
Print("Loading ONNX model from embedded resource...");
// Create model from resource buffer
onnx_handle = OnnxCreateFromBuffer(ExtModel, ONNX_DEBUG_LOGS);
if(onnx_handle == INVALID_HANDLE)
{
int error = GetLastError();
Print("ERROR: Failed to create ONNX model from resource. Error: ", error);
Print("Make sure the model file exists at: MQL5\\Files\\BTCUSD_M1_model.onnx");
Print("Then recompile the EA to embed it as a resource.");
return(INIT_FAILED);
}
// Set input shape - per MQL5 documentation
const long ExtInputShape[] = {1, InpLookback, 13}; // batch=1, lookback bars, 13 features
if(!OnnxSetInputShape(onnx_handle, 0, ExtInputShape))
{
Print("OnnxSetInputShape failed, error ", GetLastError());
OnnxRelease(onnx_handle);
return(INIT_FAILED);
}
// Set output shape - per MQL5 documentation
const long ExtOutputShape[] = {1, 1}; // batch=1, single output value
if(!OnnxSetOutputShape(onnx_handle, 0, ExtOutputShape))
{
Print("OnnxSetOutputShape failed, error ", GetLastError());
OnnxRelease(onnx_handle);
return(INIT_FAILED);
}
// Get model info
long input_count = OnnxGetInputCount(onnx_handle);
long output_count = OnnxGetOutputCount(onnx_handle);
Print("ONNX Model loaded successfully");
Print(" Inputs: ", input_count);
Print(" Outputs: ", output_count);
if(input_count > 0)
{
string input_name = OnnxGetInputName(onnx_handle, 0);
Print(" Input name: ", input_name);
}
if(output_count > 0)
{
string output_name = OnnxGetOutputName(onnx_handle, 0);
Print(" Output name: ", output_name);
}
return(INIT_SUCCEEDED);
}
//+------------------------------------------------------------------+
//| Expert deinitialization function |
//+------------------------------------------------------------------+
void OnDeinit(const int reason)
{
// Release ONNX model
if(onnx_handle != INVALID_HANDLE)
{
OnnxRelease(onnx_handle);
Print("ONNX model released");
}
}
//+------------------------------------------------------------------+
//| Expert tick function |
//+------------------------------------------------------------------+
void OnTick()
{
// Check if new bar
datetime current_bar_time = iTime(_Symbol, PERIOD_CURRENT, 0);
if(current_bar_time == last_bar_time)
{
return; // Still the same bar
}
last_bar_time = current_bar_time;
// Check if we should use prediction
if(!InpUsePrediction)
{
return;
}
// Prepare input data
float input_data[];
if(!PrepareInputData(input_data))
{
Print("ERROR: Failed to prepare input data");
return;
}
// Check if input data is valid
if(ArraySize(input_data) != InpLookback * 13)
{
Print("ERROR: Input data size mismatch. Expected: ", InpLookback * 13, ", Got: ", ArraySize(input_data));
return;
}
// Convert flat array to matrixf for OnnxRun
// Shape: [lookback, features] = [60, 13] - batch dimension is added automatically
matrixf input_matrix;
input_matrix.Resize(InpLookback, 13);
// Fill matrix from flat array
int idx = 0;
for(int i = 0; i < InpLookback; i++)
{
for(int j = 0; j < 13; j++)
{
if(idx >= ArraySize(input_data))
{
Print("ERROR: Index out of bounds when filling matrix. idx=", idx, ", array size=", ArraySize(input_data));
return;
}
input_matrix[i][j] = input_data[idx++];
}
}
// Verify matrix is not empty
if(input_matrix.Rows() == 0 || input_matrix.Cols() == 0)
{
Print("ERROR: Input matrix is empty. Rows: ", input_matrix.Rows(), ", Cols: ", input_matrix.Cols());
return;
}
// Run ONNX model - use matrixf and vectorf per MQL5 documentation
vectorf output_vector(1);
if(!RunONNXModel(input_matrix, output_vector))
{
Print("ERROR: Failed to run ONNX model");
return;
}
// Get prediction
if(output_vector.Size() == 0)
{
Print("ERROR: Empty output from ONNX model");
return;
}
// Model now predicts price change percentage directly (e.g., -0.003 = -0.3%)
double predicted_change_pct = output_vector[0];
double current_price = SymbolInfoDouble(_Symbol, SYMBOL_BID);
// Check if prediction is percentage (between -1 and 1) or absolute price (old format)
double price_change_pct;
double predicted_price;
if(MathAbs(predicted_change_pct) < 1.0)
{
// New format: percentage (e.g., -0.003 = -0.3%)
price_change_pct = predicted_change_pct * 100.0; // Convert to percentage
predicted_price = current_price * (1.0 + predicted_change_pct); // Calculate predicted price
}
else
{
// Old format: absolute price
predicted_price = predicted_change_pct;
double price_change = predicted_price - current_price;
price_change_pct = (price_change / current_price) * 100.0;
}
// Calculate confidence (for percentage predictions: 0.001 = 0.1% = 10% confidence)
double confidence;
if(MathAbs(price_change_pct) < 1.0)
{
// It's a decimal percentage (e.g., 0.001 = 0.1%)
confidence = MathMin(MathAbs(predicted_change_pct) / 0.01, 1.0); // 0.01 = 1% = 100% confidence
}
else
{
// It's already in percentage form
confidence = MathMin(MathAbs(price_change_pct) / 1.0, 1.0);
}
last_prediction = predicted_price;
last_confidence = confidence;
// Calculate ATR for dynamic SL/TP
double atr_value = 0.0;
double atr_array[];
ArraySetAsSeries(atr_array, true);
int atr_handle = iATR(_Symbol, PERIOD_CURRENT, 14);
if(atr_handle != INVALID_HANDLE)
{
if(CopyBuffer(atr_handle, 0, 0, 1, atr_array) > 0)
{
atr_value = atr_array[0];
}
IndicatorRelease(atr_handle);
}
// Calculate predicted SL/TP based on prediction, confidence, and volatility
double predicted_sl = 0.0;
double predicted_tp = 0.0;
if(InpUsePredictedSLTP && atr_value > 0)
{
// Calculate SL/TP based on ATR, prediction, and confidence
double predicted_move = MathAbs(predicted_price - current_price);
// SL: Based on ATR and confidence
// Higher confidence = tighter SL, lower confidence = wider SL
double sl_atr_mult = InpSLMultiplier / MathMax(confidence, 0.1);
sl_atr_mult = MathMax(sl_atr_mult, InpMinSLATR);
predicted_sl = atr_value * sl_atr_mult;
// TP: Use a fraction of predicted move (not the full move)
// Take 30-50% of predicted move as TP, but ensure minimum
double tp_fraction = 0.3 + (confidence * 0.2); // 30-50% based on confidence
double tp_from_prediction = predicted_move * tp_fraction;
// Also calculate TP from ATR multiplier
double tp_from_atr = atr_value * InpTPMultiplier;
// Use the smaller of the two (more conservative)
predicted_tp = MathMin(tp_from_prediction, tp_from_atr);
predicted_tp = MathMax(predicted_tp, atr_value * InpMinTPATR); // Minimum TP
// Ensure TP is at least 1.5x SL for risk/reward
if(predicted_tp < predicted_sl * 1.5)
{
predicted_tp = predicted_sl * 1.5;
}
// Cap TP at maximum 80% of predicted move (don't be too greedy)
double max_tp = predicted_move * 0.8;
if(predicted_tp > max_tp)
{
predicted_tp = max_tp;
}
}
// Log prediction
Print("Prediction: Current=", current_price,
" Predicted Change=", price_change_pct, "%",
" Predicted Price=", predicted_price,
" Confidence=", confidence,
" ATR=", atr_value);
if(InpUsePredictedSLTP && predicted_sl > 0 && predicted_tp > 0)
{
Print(" Predicted SL=", predicted_sl, " (", predicted_sl/current_price*100, "%)",
" Predicted TP=", predicted_tp, " (", predicted_tp/current_price*100, "%)");
}
// Check if we should trade
if(!InpUseConfidence || confidence >= InpMinConfidence)
{
// Check if prediction is significant
// price_change_pct is in percentage (e.g., 5.72 = 5.72%)
// InpPredictionThreshold is in decimal (e.g., 0.00005 = 0.005%)
// Convert threshold to percentage for comparison
double threshold_pct = InpPredictionThreshold * 100.0;
double abs_change_pct = MathAbs(price_change_pct); // Already in percentage
Print("Trade Check: Change=", price_change_pct, "% Threshold=", threshold_pct, "% Confidence=", confidence);
if(abs_change_pct >= threshold_pct)
{
// Check existing position
if(PositionSelect(_Symbol))
{
// Manage existing position
ManagePosition(predicted_price, price_change_pct);
}
else
{
// Open new position based on prediction
if(price_change_pct > threshold_pct)
{
Print(">>> Opening BUY position: Change=", price_change_pct, "% Threshold=", threshold_pct, "%");
OpenBuyPosition(predicted_sl, predicted_tp);
}
else if(price_change_pct < -threshold_pct)
{
Print(">>> Opening SELL position: Change=", price_change_pct, "% Threshold=", threshold_pct, "%");
OpenSellPosition(predicted_sl, predicted_tp);
}
}
}
else
{
Print("Prediction below threshold: Change=", price_change_pct, "% < Threshold=", threshold_pct, "%");
}
}
else
{
Print("Confidence too low: ", confidence, " < ", InpMinConfidence);
}
}
//+------------------------------------------------------------------+
//| Prepare input data for ONNX model |
//+------------------------------------------------------------------+
bool PrepareInputData(float &input_array[])
{
int lookback = InpLookback;
int features = 13; // OHLC(4) + volume(1) + RSI(1) + EMA20(1) + EMA50(1) + ATR(1) + price_change(1) + high_low_ratio(1) + volume_ma(1) + volume_ratio(1) = 13
ArrayResize(input_array, lookback * features);
ArrayInitialize(input_array, 0.0);
// Get historical data
double open[], high[], low[], close[];
long volume[]; // CopyTickVolume requires long[] not double[]
ArraySetAsSeries(open, true);
ArraySetAsSeries(high, true);
ArraySetAsSeries(low, true);
ArraySetAsSeries(close, true);
ArraySetAsSeries(volume, true);
int copied_open = CopyOpen(_Symbol, PERIOD_CURRENT, 0, lookback + 50, open);
if(copied_open < lookback)
{
Print("ERROR: CopyOpen failed. Got ", copied_open, " bars, need ", lookback);
return false;
}
int copied_high = CopyHigh(_Symbol, PERIOD_CURRENT, 0, lookback + 50, high);
if(copied_high < lookback)
{
Print("ERROR: CopyHigh failed. Got ", copied_high, " bars, need ", lookback);
return false;
}
int copied_low = CopyLow(_Symbol, PERIOD_CURRENT, 0, lookback + 50, low);
if(copied_low < lookback)
{
Print("ERROR: CopyLow failed. Got ", copied_low, " bars, need ", lookback);
return false;
}
int copied_close = CopyClose(_Symbol, PERIOD_CURRENT, 0, lookback + 50, close);
if(copied_close < lookback)
{
Print("ERROR: CopyClose failed. Got ", copied_close, " bars, need ", lookback);
return false;
}
int copied_volume = CopyTickVolume(_Symbol, PERIOD_CURRENT, 0, lookback + 50, volume);
if(copied_volume < lookback)
{
Print("ERROR: CopyTickVolume failed. Got ", copied_volume, " bars, need ", lookback);
return false;
}
// Calculate indicators
double rsi[], ema20[], ema50[], atr[];
ArraySetAsSeries(rsi, true);
ArraySetAsSeries(ema20, true);
ArraySetAsSeries(ema50, true);
ArraySetAsSeries(atr, true);
// Calculate RSI
int rsi_handle = iRSI(_Symbol, PERIOD_CURRENT, 14, PRICE_CLOSE);
if(rsi_handle == INVALID_HANDLE) return false;
if(CopyBuffer(rsi_handle, 0, 0, lookback + 50, rsi) < lookback)
{
IndicatorRelease(rsi_handle);
return false;
}
IndicatorRelease(rsi_handle);
// Calculate EMAs
int ema20_handle = iMA(_Symbol, PERIOD_CURRENT, 20, 0, MODE_EMA, PRICE_CLOSE);
int ema50_handle = iMA(_Symbol, PERIOD_CURRENT, 50, 0, MODE_EMA, PRICE_CLOSE);
if(ema20_handle == INVALID_HANDLE || ema50_handle == INVALID_HANDLE) return false;
if(CopyBuffer(ema20_handle, 0, 0, lookback + 50, ema20) < lookback ||
CopyBuffer(ema50_handle, 0, 0, lookback + 50, ema50) < lookback)
{
IndicatorRelease(ema20_handle);
IndicatorRelease(ema50_handle);
return false;
}
IndicatorRelease(ema20_handle);
IndicatorRelease(ema50_handle);
// Calculate ATR
int atr_handle = iATR(_Symbol, PERIOD_CURRENT, 14);
if(atr_handle == INVALID_HANDLE) return false;
if(CopyBuffer(atr_handle, 0, 0, lookback + 50, atr) < lookback)
{
IndicatorRelease(atr_handle);
return false;
}
IndicatorRelease(atr_handle);
// Calculate volume MA for normalization
double volume_ma[];
ArraySetAsSeries(volume_ma, true);
ArrayResize(volume_ma, lookback);
ArrayInitialize(volume_ma, 0.0);
// Calculate volume MA (20-period rolling average)
for(int j = 0; j < lookback; j++)
{
double sum = 0.0;
int count = 0;
for(int k = j; k < j + 20 && k < ArraySize(volume); k++)
{
sum += (double)volume[k];
count++;
}
volume_ma[j] = count > 0 ? sum / count : (double)volume[j];
}
// Prepare features - MUST match Python training exactly (13 features)
int idx = 0;
for(int i = 0; i < lookback; i++)
{
// Feature 1-4: OHLC
input_array[idx++] = (float)open[i];
input_array[idx++] = (float)high[i];
input_array[idx++] = (float)low[i];
input_array[idx++] = (float)close[i];
// Feature 5: Volume (normalized by 1,000,000)
input_array[idx++] = (float)((double)volume[i] / 1000000.0);
// Feature 6: RSI (normalized by 100)
input_array[idx++] = (float)(rsi[i] / 100.0);
// Feature 7: EMA20 normalized difference
input_array[idx++] = (float)((ema20[i] - close[i]) / close[i]);
// Feature 8: EMA50 normalized difference
input_array[idx++] = (float)((ema50[i] - close[i]) / close[i]);
// Feature 9: ATR normalized
input_array[idx++] = (float)(atr[i] / close[i]);
// Feature 10: Price change (percentage)
double price_change = i > 0 ? (close[i] - close[i+1]) / close[i+1] : 0.0;
input_array[idx++] = (float)price_change;
// Feature 11: High/Low ratio
input_array[idx++] = (float)(high[i] / low[i]);
// Feature 12: Volume MA (normalized by 1,000,000)
input_array[idx++] = (float)(volume_ma[i] / 1000000.0);
// Feature 13: Volume ratio
double vol_ratio = volume_ma[i] > 0 ? (double)volume[i] / volume_ma[i] : 1.0;
input_array[idx++] = (float)vol_ratio;
}
return true;
}
//+------------------------------------------------------------------+
//| Run ONNX model |
//+------------------------------------------------------------------+
bool RunONNXModel(matrixf &input_matrix, vectorf &output_vector)
{
if(onnx_handle == INVALID_HANDLE)
return false;
// Run model - shapes are already set in OnInit per MQL5 documentation
// Based on: https://www.mql5.com/en/docs/onnx/onnx_test
// OnnxRun expects matrixf and vectorf, not flat arrays
if(!OnnxRun(onnx_handle, ONNX_DEBUG_LOGS | ONNX_NO_CONVERSION, input_matrix, output_vector))
{
Print("ERROR: Failed to run ONNX model. Error: ", GetLastError());
return false;
}
return true;
}
//+------------------------------------------------------------------+
//| Open buy position |
//+------------------------------------------------------------------+
void OpenBuyPosition(double predicted_sl = 0.0, double predicted_tp = 0.0)
{
double price = SymbolInfoDouble(_Symbol, SYMBOL_ASK);
double sl = 0.0;
double tp = 0.0;
if(InpUsePredictedSLTP && predicted_sl > 0 && predicted_tp > 0)
{
// Use predicted SL/TP
sl = price - predicted_sl;
tp = price + predicted_tp;
Print("Using predicted SL/TP: SL=", sl, " TP=", tp);
}
else
{
// Use fixed SL/TP from input parameters
sl = InpStopLoss > 0 ? price - InpStopLoss * _Point * 10 : 0;
tp = InpTakeProfit > 0 ? price + InpTakeProfit * _Point * 10 : 0;
}
if(trade.Buy(InpLotSize, _Symbol, price, sl, tp, "ONNX Buy Signal"))
{
Print("Buy order opened. Ticket: ", trade.ResultOrder(), " Price: ", price, " SL: ", sl, " TP: ", tp);
}
else
{
Print("Failed to open buy order. Error: ", trade.ResultRetcodeDescription());
}
}
//+------------------------------------------------------------------+
//| Open sell position |
//+------------------------------------------------------------------+
void OpenSellPosition(double predicted_sl = 0.0, double predicted_tp = 0.0)
{
double price = SymbolInfoDouble(_Symbol, SYMBOL_BID);
double sl = 0.0;
double tp = 0.0;
if(InpUsePredictedSLTP && predicted_sl > 0 && predicted_tp > 0)
{
// Use predicted SL/TP
sl = price + predicted_sl;
tp = price - predicted_tp;
Print("Using predicted SL/TP: SL=", sl, " TP=", tp);
}
else
{
// Use fixed SL/TP from input parameters
sl = InpStopLoss > 0 ? price + InpStopLoss * _Point * 10 : 0;
tp = InpTakeProfit > 0 ? price - InpTakeProfit * _Point * 10 : 0;
}
if(trade.Sell(InpLotSize, _Symbol, price, sl, tp, "ONNX Sell Signal"))
{
Print("Sell order opened. Ticket: ", trade.ResultOrder(), " Price: ", price, " SL: ", sl, " TP: ", tp);
}
else
{
Print("Failed to open sell order. Error: ", trade.ResultRetcodeDescription());
}
}
//+------------------------------------------------------------------+
//| Manage existing position |
//+------------------------------------------------------------------+
void ManagePosition(double predicted_price, double price_change_pct)
{
if(!PositionSelect(_Symbol))
return;
// Simple position management - can be enhanced
// For now, just log the position status
double position_profit = PositionGetDouble(POSITION_PROFIT);
Print("Position exists. Profit: ", position_profit, " Predicted change: ", price_change_pct, "%");
}
ai/btcusd1min/README.md
+68
View File
@@ -0,0 +1,68 @@
# BTCUSD 1-Minute ONNX Model Training
This directory contains the training script for a BTCUSD price prediction model using 1-minute timeframe data from 2017 to 2026.
## Requirements
```bash
pip install yfinance tensorflow scikit-learn pandas numpy tf2onnx onnx tqdm
```
## Usage
1. **Run the training script:**
```bash
cd ai/btcusd1min
python main.py
```
**Note:** yfinance 1-minute data is limited to the last 7 days. For longer historical training, the script will use the most recent available data.
## Configuration
The script is configured with:
- **Symbol**: BTCUSD
- **Timeframe**: M1 (1 minute)
- **Lookback**: 60 bars (60 minutes of history)
- **Date Range**: 2017-01-01 to 2026-01-01
- **Model Architecture**: LSTM with 3 layers (128, 64, 32 units)
- **Epochs**: 50 (with early stopping)
- **Batch Size**: 64
## Output
The script will create:
- `models/BTCUSD_M1_model.onnx` - The trained ONNX model
- `models/BTCUSD_M1_model_scaler.pkl` - The MinMaxScaler used for normalization
## Model Features
The model uses 13 features:
1. Open
2. High
3. Low
4. Close
5. Tick Volume
6. RSI (14 period)
7. EMA 20
8. EMA 50
9. ATR (14 period)
10. Price Change (percentage)
11. High/Low Ratio
12. Volume MA (20 period)
13. Volume Ratio
## Model Output
The model predicts the **price change percentage** for the next bar (1 minute ahead).
## Notes
- Training on 9 years of 1-minute data will take significant time and memory
- The script fetches data in 3-month chunks to manage memory
- Early stopping and learning rate reduction are enabled to prevent overfitting
- The model uses dropout (0.3) for regularization
## Using the Model in MQL5
After training, copy the ONNX model to your MT5 `MQL5/Files/` directory and use it in an Expert Advisor similar to the XAUUSD H1 EA.
ai/btcusd1min/main.py
+455
View File
@@ -0,0 +1,455 @@
"""
ONNX Model Training Script for BTCUSD 1-Minute Data
Uses yfinance (Yahoo Finance) for historical data
This script trains a neural network model for BTCUSD price prediction on 1-minute timeframe
and exports it to ONNX format.
Usage:
python main.py
"""
import os
import sys
from datetime import datetime, timedelta
import numpy as np
import pandas as pd
import yfinance as yf
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import tf2onnx
import onnx
from tqdm import tqdm
import pickle
class BTCUSD1MinTrainer:
"""
Trainer class for creating ONNX models from BTCUSD 1-minute MT5 data.
"""
def __init__(self, symbol: str = "BTC-USD", lookback: int = 60,
prediction_horizon: int = 1):
"""
Initialize the trainer.
Args:
symbol: Trading symbol (default: 'BTC-USD' for Yahoo Finance)
lookback: Number of bars to look back for prediction (default: 60)
prediction_horizon: Number of bars ahead to predict (default: 1)
"""
self.symbol = symbol
self.lookback = lookback
self.prediction_horizon = prediction_horizon
self.scaler = MinMaxScaler()
self.model = None
print(f"Using yfinance for data source. Symbol: {self.symbol}")
def fetch_data(self, start_date: datetime, end_date: datetime) -> pd.DataFrame:
"""
Fetch historical data from Yahoo Finance using yfinance.
Args:
start_date: Start date for data
end_date: End date for data
Returns:
DataFrame with OHLCV data
"""
print(f"\nFetching {self.symbol} 1-minute data from {start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}...")
# yfinance can only fetch 7 days of 1-minute data at a time
# For longer periods, we need to fetch in chunks
all_data = []
current_start = start_date
# For 1-minute data, yfinance limits to last 7 days
# So we'll fetch the most recent 7 days available
print("Note: yfinance 1-minute data is limited to last 7 days")
print("Fetching most recent available 1-minute data...")
# Get ticker
ticker = yf.Ticker(self.symbol)
# Try to fetch 1-minute data (limited to 7 days)
# If we need more data, we'll use daily data and resample
try:
# Fetch 1-minute data (max 7 days)
df = ticker.history(start=start_date, end=end_date, interval='1m')
if df is None or len(df) == 0:
print("Warning: No 1-minute data available, trying daily data...")
# Fall back to daily data
df = ticker.history(start=start_date, end=end_date, interval='1d')
if df is None or len(df) == 0:
raise ValueError(f"No data available for {self.symbol}")
print(f"Using daily data instead (will resample to 1-minute for training)")
except Exception as e:
print(f"Error fetching 1-minute data: {e}")
print("Falling back to daily data...")
df = ticker.history(start=start_date, end=end_date, interval='1d')
if df is None or len(df) == 0:
raise ValueError(f"No data available for {self.symbol}: {e}")
# Rename columns to match expected format
df.columns = [col.lower().replace(' ', '_') for col in df.columns]
# Ensure we have the required columns
required_cols = ['open', 'high', 'low', 'close', 'volume']
missing_cols = [col for col in required_cols if col not in df.columns]
if missing_cols:
raise ValueError(f"Missing required columns: {missing_cols}")
# Rename 'volume' to 'tick_volume' for consistency
if 'volume' in df.columns:
df['tick_volume'] = df['volume']
df = df.drop('volume', axis=1)
# Remove duplicates and sort
df = df[~df.index.duplicated(keep='first')]
df = df.sort_index()
print(f"Total fetched: {len(df)} bars")
if len(df) > 0:
print(f"Date range: {df.index[0]} to {df.index[-1]}")
print(f"Timeframe: {df.index[1] - df.index[0] if len(df) > 1 else 'N/A'}")
else:
raise ValueError("DataFrame is empty after processing")
return df
def prepare_features(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Prepare features for training.
Args:
df: Raw OHLCV data
Returns:
DataFrame with features
"""
print("\nPreparing features...")
feature_df = df[['open', 'high', 'low', 'close', 'tick_volume']].copy()
# Add technical indicators as features
print(" Calculating RSI...")
feature_df['rsi'] = self._calculate_rsi(df['close'], period=14)
print(" Calculating EMAs...")
feature_df['ema_20'] = df['close'].ewm(span=20, adjust=False).mean()
feature_df['ema_50'] = df['close'].ewm(span=50, adjust=False).mean()
print(" Calculating ATR...")
feature_df['atr'] = self._calculate_atr(df, period=14)
# Price changes
feature_df['price_change'] = df['close'].pct_change()
feature_df['high_low_ratio'] = df['high'] / df['low']
# Volume features
feature_df['volume_ma'] = df['tick_volume'].rolling(window=20).mean()
feature_df['volume_ratio'] = df['tick_volume'] / feature_df['volume_ma']
# Drop NaN values
feature_df = feature_df.dropna()
print(f" Features prepared: {len(feature_df)} samples, {len(feature_df.columns)} features")
print(f" Features: {list(feature_df.columns)}")
return feature_df
def _calculate_rsi(self, prices: pd.Series, period: int = 14) -> pd.Series:
"""Calculate RSI indicator."""
delta = prices.diff()
gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()
loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()
rs = gain / loss
rsi = 100 - (100 / (1 + rs))
return rsi
def _calculate_atr(self, df: pd.DataFrame, period: int = 14) -> pd.Series:
"""Calculate ATR indicator."""
high_low = df['high'] - df['low']
high_close = np.abs(df['high'] - df['close'].shift())
low_close = np.abs(df['low'] - df['close'].shift())
tr = pd.concat([high_low, high_close, low_close], axis=1).max(axis=1)
atr = tr.rolling(window=period).mean()
return atr
def create_sequences(self, data: np.ndarray, target: np.ndarray) -> tuple:
"""
Create sequences for LSTM/RNN training.
Args:
data: Feature data
target: Target values (price change percentages)
Returns:
Tuple of (X, y) sequences
"""
print("\nCreating sequences...")
X, y = [], []
for i in tqdm(range(self.lookback, len(data) - self.prediction_horizon + 1), desc="Creating sequences"):
X.append(data[i - self.lookback:i])
y.append(target[i])
X = np.array(X)
y = np.array(y)
print(f" Sequences created: X shape {X.shape}, y shape {y.shape}")
return X, y
def build_model(self, input_shape: tuple) -> keras.Model:
"""
Build the neural network model.
Args:
input_shape: Shape of input data (lookback, features)
Returns:
Compiled Keras model
"""
print(f"\nBuilding model with input shape: {input_shape}")
model = keras.Sequential([
layers.LSTM(128, return_sequences=True, input_shape=input_shape),
layers.Dropout(0.3),
layers.LSTM(64, return_sequences=True),
layers.Dropout(0.3),
layers.LSTM(32),
layers.Dropout(0.3),
layers.Dense(32, activation='relu'),
layers.Dense(16, activation='relu'),
layers.Dense(1) # Predict price change percentage
])
model.compile(
optimizer=keras.optimizers.Adam(learning_rate=0.0005),
loss='mse',
metrics=['mae']
)
print(f" Model parameters: {model.count_params():,}")
model.summary()
return model
def train(self, start_date: datetime, end_date: datetime,
epochs: int = 50, batch_size: int = 32,
validation_split: float = 0.2, verbose: int = 1):
"""
Train the model.
Args:
start_date: Start date for training data
end_date: End date for training data
epochs: Number of training epochs
batch_size: Batch size for training
validation_split: Fraction of data to use for validation
verbose: Verbosity level
"""
# Fetch data
df = self.fetch_data(start_date, end_date)
feature_df = self.prepare_features(df)
# Prepare target: price change percentage for next bar
# Calculate future price change: (next_close - current_close) / current_close
close_prices = feature_df['close'].values
target = []
for i in range(len(close_prices)):
if i + self.prediction_horizon < len(close_prices):
current_price = close_prices[i]
future_price = close_prices[i + self.prediction_horizon]
price_change_pct = (future_price - current_price) / current_price if current_price > 0 else 0.0
target.append(price_change_pct)
else:
target.append(0.0)
target = pd.Series(target, index=feature_df.index)
# Keep 'close' in features - it's needed for the model
# Align target with features
valid_idx = ~(target.isna() | feature_df.isna().any(axis=1))
feature_df = feature_df[valid_idx]
target = target[valid_idx]
print(f"\nValid samples after alignment: {len(feature_df)}")
# Normalize features
print("\nNormalizing features...")
feature_array = self.scaler.fit_transform(feature_df.values)
# Create sequences
X, y = self.create_sequences(feature_array, target.values)
# Split into train and validation
split_idx = int(len(X) * (1 - validation_split))
X_train, X_val = X[:split_idx], X[split_idx:]
y_train, y_val = y[:split_idx], y[split_idx:]
print(f"\nTrain set: {len(X_train)} samples")
print(f"Validation set: {len(X_val)} samples")
# Build model
input_shape = (self.lookback, feature_array.shape[1])
self.model = self.build_model(input_shape)
# Train model
print(f"\nTraining model for {epochs} epochs...")
history = self.model.fit(
X_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_val, y_val),
verbose=verbose,
callbacks=[
keras.callbacks.EarlyStopping(
monitor='val_loss',
patience=10,
restore_best_weights=True
),
keras.callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.5,
patience=5,
min_lr=1e-7
)
]
)
# Evaluate
print("\nEvaluating model...")
train_loss = self.model.evaluate(X_train, y_train, verbose=0)
val_loss = self.model.evaluate(X_val, y_val, verbose=0)
print(f"Train Loss: {train_loss[0]:.6f}, MAE: {train_loss[1]:.6f}")
print(f"Val Loss: {val_loss[0]:.6f}, MAE: {val_loss[1]:.6f}")
return history
def export_to_onnx(self, output_path: str):
"""
Export the trained model to ONNX format.
Args:
output_path: Path to save ONNX model
"""
if self.model is None:
raise ValueError("Model must be trained before exporting")
print(f"\nExporting model to ONNX format: {output_path}")
# Get number of features
num_features = self.model.input_shape[2] if len(self.model.input_shape) > 2 else self.model.input_shape[1]
print(f"Using {num_features} features for ONNX export")
# Create input signature
input_shape = (None, self.lookback, num_features)
spec = (tf.TensorSpec(input_shape, tf.float32, name="input"),)
# Fix output_names for Sequential model
if not hasattr(self.model, 'output_names'):
if hasattr(self.model, 'outputs') and self.model.outputs:
self.model.output_names = [f'output_{i}' for i in range(len(self.model.outputs))]
else:
self.model.output_names = ['output']
# Convert to ONNX
onnx_model, _ = tf2onnx.convert.from_keras(
self.model,
input_signature=spec,
opset=13
)
# Save ONNX model
os.makedirs(os.path.dirname(output_path), exist_ok=True)
onnx.save_model(onnx_model, output_path)
print(f"ONNX model saved to: {output_path}")
# Save scaler
scaler_path = output_path.replace('.onnx', '_scaler.pkl')
with open(scaler_path, 'wb') as f:
pickle.dump(self.scaler, f)
print(f"Scaler saved to: {scaler_path}")
def cleanup(self):
"""Clean up (no-op for yfinance)."""
pass
def main():
"""Main function."""
print("="*60)
print("BTCUSD 1-Minute ONNX Model Training")
print("="*60)
# Training parameters
symbol = "BTC-USD" # Yahoo Finance symbol
lookback = 60 # 60 minutes of history
epochs = 50
batch_size = 64 # Larger batch for 1-minute data
# Date range: Use recent data (yfinance 1m data limited to 7 days)
# For longer training, we'll use the most recent available data
end_date = datetime.now()
start_date = end_date - timedelta(days=7) # Last 7 days for 1-minute data
print(f"Note: yfinance 1-minute data is limited to last 7 days")
print(f"Using date range: {start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}")
# Output paths
output_dir = "models"
os.makedirs(output_dir, exist_ok=True)
model_path = os.path.join(output_dir, f"{symbol}_M1_model.onnx")
trainer = None
try:
# Create trainer
trainer = BTCUSD1MinTrainer(
symbol=symbol,
lookback=lookback
)
# Train model
history = trainer.train(
start_date=start_date,
end_date=end_date,
epochs=epochs,
batch_size=batch_size,
validation_split=0.2,
verbose=1
)
# Export to ONNX
trainer.export_to_onnx(model_path)
print("\n" + "="*60)
print("Training completed successfully!")
print("="*60)
print(f"Model saved to: {model_path}")
except Exception as e:
print(f"\nERROR: Training failed: {e}")
import traceback
traceback.print_exc()
return 1
finally:
if trainer:
trainer.cleanup()
return 0
if __name__ == '__main__':
sys.exit(main())
ai/btcusd1min/models/BTC-USD_M1_model.onnx
BIN
View File
Binary file not shown.
ai/btcusd1min/models/BTC-USD_M1_model_scaler.pkl
BIN
View File
Binary file not shown.