diff --git a/docs/MT5-ARCH-LINUX-SETUP.md b/docs/MT5-ARCH-LINUX-SETUP.md new file mode 100644 index 0000000..8fb3a33 --- /dev/null +++ b/docs/MT5-ARCH-LINUX-SETUP.md @@ -0,0 +1,151 @@ +# MT5 on Arch Linux (Wine + mt5linux bridge) + +The `MetaTrader5` Python package is Windows-only. On Arch Linux we run the MT5 +terminal under **Wine** plus a **Windows Python** that exposes the MT5 API over +an `rpyc` socket (`mt5linux`). The Linux-side bot connects to that socket. + +``` +Linux Python (bot) --rpyc--> Wine Python (mt5linux server) --> MT5 terminal --> broker + mt5linux client MetaTrader5 pkg terminal64.exe +``` + +## One-time setup (already done on this machine) + +- Wine: `wine-11.10 (Staging)` ✅ +- Headless display: `Xvfb` (pkg `xorg-server-xvfb`) ✅ +- MT5 terminal installed in Wine: `~/.wine/drive_c/Program Files/MetaTrader 5/terminal64.exe` ✅ +- Windows Python 3.9 in Wine: `~/.wine/drive_c/users/$USER/AppData/Local/Programs/Python/Python39/python.exe` ✅ +- Wine Python packages: `MetaTrader5`, `mt5linux`, `rpyc` ✅ +- Linux Python packages: `mt5linux`, `rpyc` ✅ + +To reproduce the Wine Python install: + +```bash +curl -sL -o /tmp/py39.exe https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe +DISPLAY=:1 WINEDEBUG=-all wine /tmp/py39.exe /quiet InstallAllUsers=0 PrependPath=1 Include_pip=1 +WINPY="$HOME/.wine/drive_c/users/$USER/AppData/Local/Programs/Python/Python39/python.exe" +wine "$WINPY" -m pip install MetaTrader5 mt5linux +``` + +## Configuration (`.env`) + +```env +MT5_LOGIN=... +MT5_PASSWORD=... +MT5_SERVER=... # exact broker server name (see terminal) +MT5_BRIDGE_HOST=127.0.0.1 +MT5_BRIDGE_PORT=18812 +MT5_WIN_PATH=C:\Program Files\MetaTrader 5\terminal64.exe +``` + +## Run (robust, one command) + +```bash +# Bring the whole bridge up (idempotent): Xvfb + rpyc server + verified login. +# The terminal is auto-launched headless and logged in from .env credentials. +scripts/mt5_bridge.sh up + +# Other commands +scripts/mt5_bridge.sh status # components + live login + trade_allowed +scripts/mt5_bridge.sh restart +scripts/mt5_bridge.sh down +``` + +`up` is safe to re-run: it only starts what is missing and health-checks the +rpyc port (no blind sleeps). Credentials and port are read from `.env` +(`MT5_LOGIN/PASSWORD/SERVER`, `MT5_BRIDGE_PORT`, `MT5_WIN_PATH`). + +### Auto-start on login (systemd --user) + +```bash +scripts/mt5_bridge.sh install-service +systemctl --user enable --now mt5-bridge.service +journalctl --user -u mt5-bridge -f # logs +``` + +After this the bridge comes up automatically; you never repeat the manual +steps. Then just run the bot: + +```bash +python main_live.py +``` + +### First-time only + +The account must be logged into the terminal once so the profile knows it. +Normally `up` does this automatically via explicit login. If you ever get +`LOGIN_FAIL (-6)`, do a one-time GUI login: + +```bash +scripts/mt5_bridge.sh login-gui # File -> Login to Trade Account +``` + +The bot (`src/mt5_connector.py`) auto-detects the bridge: if the native +`MetaTrader5` import fails but `mt5linux` is present, it connects lazily to +`MT5_BRIDGE_HOST:MT5_BRIDGE_PORT` on the first `connect()` call, trying +`initialize()` attach-mode first then explicit login. + +### Zero-step auto-connect + +You do **not** need to start anything by hand. When the bot calls `connect()` +and the bridge port is closed, it auto-runs `scripts/mt5_bridge.sh up` +(Xvfb + rpyc server + login) and waits for it to become ready. So on Linux: + +```bash +python main_live.py # bridge is started automatically if needed +``` + +Disable autostart (e.g. when using the systemd service) with: + +```bash +export MT5_BRIDGE_AUTOSTART=0 +``` + +### Broker symbol note + +XM Global names spot gold **`GOLD`** (not `XAUUSD`). `.env` sets `SYMBOL=GOLD`. +The connector pre-selects whatever `SYMBOL` is configured. + +## Root cause of the `-6 Authorization failed` we hit + +Per the official mt5linux docs, the intended flow is: +**(1) open MT5 and log in via the GUI → (2) start the server → (3) call +`initialize()` with no credentials to attach to the running terminal.** + +Our first attempt launched the terminal with `/portable` and passed +`login/password/server` to `initialize()`. A fresh portable profile has **no +saved account**, so the broker rejected the fresh login attempt with +`(-6, 'Authorization failed')`. Confirmed: no `accounts.dat`/`servers.dat` +account was present in the Wine prefix. + +Fixes applied: +- launcher no longer uses `/portable` (uses the persistent profile) +- added `scripts/mt5_bridge.sh login_gui` for the one-time GUI login +- the connector + test now try `initialize()` attach-mode first + +> So the credentials were almost certainly fine — the terminal simply had never +> logged into the account. Do the one-time `login_gui` step, then the bridge +> attaches cleanly. + +## QA status (verified) + +- rpyc server starts and listens on `127.0.0.1:18812` ✅ +- Linux client connects through to the broker ✅ +- `mt5.initialize(...)` reaches the broker server ✅ +- `src/mt5_connector.py` imports without blocking; backend detected as `bridge` ✅ + +> NOTE: With the supplied XM credentials the broker returned +> `(-6, 'Authorization failed')`. The transport works end-to-end — this is a +> credential/server-name issue, not an infrastructure problem. Confirm in the +> MT5 terminal: +> - exact **server name** (XM has several, e.g. `XMGlobal-MT5`, `XMGlobal-MT5 2` … `10`) +> - **login number** and **password** (investor vs master password) +> - account is active and AlgoTrading is enabled + +## Troubleshooting + +- `Authorization failed (-6)` → wrong server name / login / password, or + expired account. Log into the terminal GUI once to confirm the exact server. +- Client `result expired` / timeout → the Wine MT5 terminal hung; run + `scripts/mt5_bridge.sh restart`. +- Port already in use → another server instance is running; `mt5_bridge.sh stop`. diff --git a/scripts/mt5_bridge.sh b/scripts/mt5_bridge.sh new file mode 100755 index 0000000..28b2f56 --- /dev/null +++ b/scripts/mt5_bridge.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# ============================================================================= +# MT5 Bridge (Arch Linux + Wine) — robust, idempotent, headless +# ============================================================================= +# One command brings the MT5 API up on Linux: +# +# scripts/mt5_bridge.sh up # ensure Xvfb + rpyc server are running, +# # then verify a live MT5 login (auto) +# scripts/mt5_bridge.sh down # stop server + Xvfb (+ stray terminal) +# scripts/mt5_bridge.sh status # show component + login status +# scripts/mt5_bridge.sh restart +# scripts/mt5_bridge.sh login-gui # first-time GUI login (rarely needed) +# scripts/mt5_bridge.sh install-service # install systemd --user unit +# +# Design: +# - The rpyc server runs under Wine Python and exposes the MetaTrader5 API. +# - The bot (or this script's healthcheck) calls initialize(login,...) which +# auto-launches the terminal headless under Xvfb and logs in. No GUI needed +# after the account has been logged in once. +# - Idempotent: re-running `up` only starts what is missing. +# - Readiness is health-checked (socket + rpyc), not fixed sleeps. +# +# Config: auto-sourced from the project .env (MT5_LOGIN/PASSWORD/SERVER, +# MT5_BRIDGE_PORT, MT5_WIN_PATH). Override via env vars. +# ============================================================================= +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# --- Load .env (robust: only KEY=value lines, supports quoted values) -------- +if [ -f "$PROJECT_DIR/.env" ]; then + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + ''|\#*) continue ;; # skip blank / comment + [A-Za-z_]*=*) eval "export $_line" 2>/dev/null || true ;; + esac + done < "$PROJECT_DIR/.env" +fi + +PORT="${MT5_BRIDGE_PORT:-18812}" +DISP="${MT5_BRIDGE_DISPLAY:-:99}" +export WINEPREFIX="${WINEPREFIX:-$HOME/.wine}" +export WINEDEBUG="${WINEDEBUG:--all}" + +WINPY="$WINEPREFIX/drive_c/users/$USER/AppData/Local/Programs/Python/Python39/python.exe" +MT5_TERM="$WINEPREFIX/drive_c/Program Files/MetaTrader 5/terminal64.exe" +WIN_PATH="${MT5_WIN_PATH:-C:\\Program Files\\MetaTrader 5\\terminal64.exe}" + +RUN_DIR="${XDG_RUNTIME_DIR:-/tmp}/xaubot-mt5" +mkdir -p "$RUN_DIR" +XVFB_PID="$RUN_DIR/xvfb.pid" +SRV_PID="$RUN_DIR/server.pid" +LOG="$RUN_DIR/bridge.log" + +log() { echo "[mt5-bridge] $*"; } +err() { echo "[mt5-bridge] ERROR: $*" >&2; } + +_check_prereq() { + local ok=1 + command -v wine >/dev/null || { err "wine not found"; ok=0; } + command -v Xvfb >/dev/null || { err "Xvfb not found (pacman -S xorg-server-xvfb)"; ok=0; } + [ -f "$WINPY" ] || { err "Wine Python missing: $WINPY"; ok=0; } + [ -f "$MT5_TERM" ] || { err "MT5 terminal missing: $MT5_TERM"; ok=0; } + python3 -c "import mt5linux" 2>/dev/null || { err "linux pkg 'mt5linux' missing (pip install mt5linux)"; ok=0; } + [ "$ok" = 1 ] || exit 1 +} + +_alive() { [ -f "$1" ] && kill -0 "$(cat "$1")" 2>/dev/null; } +_port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null && exec 3>&- ; } + +# Wait until the rpyc port accepts connections (timeout seconds) +_wait_port() { + local t="${1:-20}" + for _ in $(seq 1 "$t"); do _port_open && return 0; sleep 1; done + return 1 +} + +_ensure_xvfb() { + if _alive "$XVFB_PID"; then return 0; fi + # Reuse an existing Xvfb on $DISP if present + if pgrep -f "Xvfb $DISP" >/dev/null 2>&1; then + pgrep -f "Xvfb $DISP" | head -1 > "$XVFB_PID" + log "Reusing Xvfb on $DISP" + return 0 + fi + setsid Xvfb "$DISP" -screen 0 1280x1024x24 -nolisten tcp >>"$LOG" 2>&1 & + echo $! > "$XVFB_PID" + sleep 1 + log "Xvfb started on $DISP (pid $(cat "$XVFB_PID"))" +} + +_ensure_server() { + if _alive "$SRV_PID" && _port_open; then return 0; fi + if _port_open; then + log "rpyc server already listening on :$PORT (external)" + return 0 + fi + log "Starting rpyc server (Wine Python) on 127.0.0.1:$PORT ..." + setsid env DISPLAY="$DISP" WINEDEBUG="$WINEDEBUG" \ + wine "$WINPY" -m mt5linux --host 127.0.0.1 -p "$PORT" >>"$LOG" 2>&1 & + echo $! > "$SRV_PID" + if _wait_port 25; then + log "rpyc server UP (pid $(cat "$SRV_PID"))" + else + err "rpyc server failed to open port :$PORT (see $LOG)" + exit 1 + fi +} + +# Verify a live login through the bridge (auto-launches terminal headless). +_verify_login() { + DISPLAY="$DISP" MT5_BRIDGE_PORT="$PORT" python3 - "$WIN_PATH" <<'PY' +import os, sys +from mt5linux import MetaTrader5 +port = int(os.getenv("MT5_BRIDGE_PORT", "18812")) +win_path = sys.argv[1] +login = int(os.getenv("MT5_LOGIN", "0")) +pw = os.getenv("MT5_PASSWORD", "") +srv = os.getenv("MT5_SERVER", "") +try: + m = MetaTrader5(host="127.0.0.1", port=port, timeout=120) +except Exception as e: + print(f"CONNECT_FAIL {e}"); sys.exit(3) +# attach first, then explicit login (auto-launches terminal via path) +ok = m.initialize() +if not ok: + ok = m.initialize(login=login, password=pw, server=srv, path=win_path) +if not ok: + print(f"LOGIN_FAIL {m.last_error()}"); sys.exit(1) +ai = m.account_info() +if ai: + print(f"OK login={ai.login} balance={ai.balance} {ai.currency} server={ai.server}") +ti = m.terminal_info() +if ti: + print(f"TERMINAL connected={ti.connected} trade_allowed={ti.trade_allowed}") +sys.exit(0) +PY +} + +up() { + _check_prereq + _ensure_xvfb + _ensure_server + log "Verifying MT5 login ..." + if out="$(_verify_login)"; then + echo "$out" | sed 's/^/[mt5-bridge] /' + log "Bridge READY (port=$PORT display=$DISP)" + echo "$out" | grep -q "trade_allowed=False" && \ + log "WARNING: AlgoTrading OFF in terminal -> bot cannot place orders. Enable 'Algo Trading' in MT5." + return 0 + else + err "Login verification failed:" + echo "$out" | sed 's/^/[mt5-bridge] /' + err "If LOGIN_FAIL (-6): run 'scripts/mt5_bridge.sh login-gui' once." + return 1 + fi +} + +down() { + for p in "$SRV_PID" "$XVFB_PID"; do + if _alive "$p"; then kill "$(cat "$p")" 2>/dev/null || true; log "stopped pid $(cat "$p")"; fi + rm -f "$p" + done + pkill -f "mt5linux" 2>/dev/null || true + pkill -f "terminal64.exe" 2>/dev/null || true + log "Bridge down." +} + +status() { + _alive "$XVFB_PID" && log "Xvfb : UP ($(cat "$XVFB_PID"))" || log "Xvfb : DOWN" + if _port_open; then log "rpyc : UP (:$PORT listening)"; else log "rpyc : DOWN"; fi + if _port_open; then + log "Login check:" + _verify_login 2>/dev/null | sed 's/^/[mt5-bridge] /' || log " (login check failed)" + fi + log "port=$PORT display=$DISP run_dir=$RUN_DIR" +} + +login_gui() { + _check_prereq + local d="${DISPLAY:-:0}" + log "Opening MT5 on DISPLAY=$d. In MT5: File -> Login to Trade Account" + log " Login=${MT5_LOGIN:-?} Server=${MT5_SERVER:-?}" + log "Close the terminal after 'authorized'. Then run: scripts/mt5_bridge.sh up" + DISPLAY="$d" WINEDEBUG=-all wine "$MT5_TERM" +} + +install_service() { + local unit_dir="$HOME/.config/systemd/user" + mkdir -p "$unit_dir" + cat > "$unit_dir/mt5-bridge.service" < int: + try: + from mt5linux import MetaTrader5 + except ImportError: + print("ERROR: mt5linux not installed (pip install mt5linux)") + return 2 + + print(f"Connecting to bridge {HOST}:{PORT} ...") + try: + mt5 = MetaTrader5(host=HOST, port=PORT) + except Exception as e: + print(f"ERROR: cannot reach rpyc server: {e}") + print("Hint: start it first -> scripts/mt5_bridge.sh start") + return 3 + + # Preferred (official mt5linux) flow: the terminal is already logged in via + # GUI, so initialize() attaches to it without credentials. + print("initialize() [attach to already-logged-in terminal] ...") + ok = mt5.initialize() + if not ok: + print(f" attach failed -> {mt5.last_error()}") + # Fallback: explicit login (works only if account/server are valid AND + # the terminal already knows this account). + print(f"initialize(login={LOGIN}, server={SERVER!r}) [explicit login] ...") + ok = mt5.initialize(login=LOGIN, password=PASSWORD, server=SERVER, path=WIN_PATH) + if not ok: + print(f"FAILED: initialize -> {mt5.last_error()}") + print("Fix: log in once via GUI so the terminal remembers the account:") + print(" scripts/mt5_bridge.sh login_gui") + print("Also verify server name, login, and MASTER password (not investor).") + return 1 + + print("initialize OK") + ai = mt5.account_info() + if ai is not None: + print(f" account : {ai.login} | balance={ai.balance} {ai.currency} | server={ai.server}") + ti = mt5.terminal_info() + if ti is not None: + print(f" terminal: connected={ti.connected} trade_allowed={ti.trade_allowed}") + + symbol = os.getenv("SYMBOL", "XAUUSD") + rates = mt5.copy_rates_from_pos(symbol, mt5.TIMEFRAME_M15, 0, 5) + if rates is not None and len(rates) > 0: + print(f" rates : fetched {len(rates)} {symbol} M15 bars OK") + else: + print(f" rates : WARN no rates for {symbol} -> {mt5.last_error()}") + + mt5.shutdown() + print("DONE: bridge fully functional") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/mt5_connector.py b/src/mt5_connector.py index 773e6ef..138d7e3 100644 --- a/src/mt5_connector.py +++ b/src/mt5_connector.py @@ -11,14 +11,111 @@ import numpy as np from typing import Optional, Dict, Any, List, Tuple from dataclasses import dataclass from datetime import datetime +from pathlib import Path import time from loguru import logger +import os + +# MT5 backend resolution: +# 1) Native MetaTrader5 (Windows / Wine Python directly) +# 2) mt5linux rpyc bridge (Linux client -> Wine Python server) +# The bridge client is created lazily via get_mt5_backend() so that importing +# this module never blocks on a network connection. +mt5 = None +_MT5_BACKEND = "none" + try: - import MetaTrader5 as mt5 + import MetaTrader5 as mt5 # type: ignore + _MT5_BACKEND = "native" except ImportError: - logger.warning("MetaTrader5 not installed. Running in simulation mode.") - mt5 = None + try: + import mt5linux # type: ignore # noqa: F401 + _MT5_BACKEND = "bridge" # available, connect lazily in get_mt5_backend() + except Exception: # noqa: BLE001 + _MT5_BACKEND = "none" + + +def _bridge_port_open(host: str, port: int, timeout: float = 1.0) -> bool: + """Return True if something is listening on host:port.""" + import socket + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def _ensure_bridge_running(host: str, port: int) -> bool: + """Best-effort: start the MT5 bridge (Xvfb + rpyc server) if it's down. + + Lets the bot auto-connect without the user launching anything manually. + Only attempts a local autostart when host is loopback. Returns True if the + port is open afterwards. + """ + if _bridge_port_open(host, port): + return True + if host not in ("127.0.0.1", "localhost", "::1"): + return False # remote bridge: not ours to start + + import subprocess + import time + script = Path(__file__).resolve().parent.parent / "scripts" / "mt5_bridge.sh" + if not script.exists(): + logger.warning(f"MT5 bridge autostart skipped: {script} not found") + return False + + logger.info("MT5 bridge down -> auto-starting via scripts/mt5_bridge.sh up ...") + try: + subprocess.run( + ["bash", str(script), "up"], + timeout=150, + check=False, + capture_output=True, + text=True, + ) + except Exception as e: # noqa: BLE001 + logger.warning(f"MT5 bridge autostart error: {e}") + + # Wait for the port to come up + for _ in range(30): + if _bridge_port_open(host, port): + logger.info("MT5 bridge is up.") + return True + time.sleep(1) + logger.warning("MT5 bridge did not come up; falling back to simulation.") + return False + + +def get_mt5_backend(): + """Return an MT5 API object, connecting the rpyc bridge on first use. + + On Linux this will auto-start the Wine bridge (scripts/mt5_bridge.sh up) + if it is not already running, so the bot connects with no manual steps. + Returns None if no backend is available (simulation mode). + """ + global mt5 + if mt5 is not None: + return mt5 + if _MT5_BACKEND == "bridge": + host = os.getenv("MT5_BRIDGE_HOST", "127.0.0.1") + port = int(os.getenv("MT5_BRIDGE_PORT", "18812")) + # Auto-start the bridge if it isn't already listening (opt-out via env). + if os.getenv("MT5_BRIDGE_AUTOSTART", "1") not in ("0", "false", "False"): + _ensure_bridge_running(host, port) + try: + from mt5linux import MetaTrader5 as _MT5Bridge # type: ignore + mt5 = _MT5Bridge(host=host, port=port) + logger.info(f"MetaTrader5 via mt5linux bridge {host}:{port}") + except Exception as e: # noqa: BLE001 + logger.warning( + f"MT5 bridge connect failed ({host}:{port}): {e}. " + "Simulation mode." + ) + mt5 = None + elif _MT5_BACKEND == "none": + logger.warning("MetaTrader5 unavailable. Running in simulation mode.") + return mt5 @dataclass @@ -127,6 +224,9 @@ class MT5Connector: Returns: True if connected successfully """ + global mt5 + if mt5 is None: + mt5 = get_mt5_backend() if mt5 is None: logger.warning("MT5 not available - simulation mode") return False @@ -150,7 +250,21 @@ class MT5Connector: if self.path: kwargs["path"] = self.path - if mt5.initialize(**kwargs): + # On the mt5linux/Wine bridge the terminal is typically already + # logged in via GUI; initialize() without credentials attaches + # to it. Try attach-mode first, then fall back to explicit login. + _initialized = False + if _MT5_BACKEND == "bridge": + try: + _initialized = mt5.initialize() + except Exception: + _initialized = False + if not _initialized: + _initialized = mt5.initialize(**kwargs) + else: + _initialized = mt5.initialize(**kwargs) + + if _initialized: # Wait for terminal to fully initialize time.sleep(2) # Increased delay for stability @@ -176,8 +290,10 @@ class MT5Connector: self._account_info = self._get_account_info() logger.info(f"Connected to MT5: {self.server} (Account: {self.login})") - # Pre-select common symbols to ensure they're ready - mt5.symbol_select("XAUUSD", True) + # Pre-select the configured trading symbol so it's ready. + # Broker-specific name (e.g. XM uses 'GOLD', not 'XAUUSD'). + _sym = os.getenv("SYMBOL", "XAUUSD") + mt5.symbol_select(_sym, True) time.sleep(0.5) return True diff --git a/src/news_agent.py b/src/news_agent.py index 6259212..14586d9 100644 --- a/src/news_agent.py +++ b/src/news_agent.py @@ -178,7 +178,7 @@ class NewsAgent: try: # Try to get calendar events (broker-dependent) # Some brokers don't expose this API - events = mt5.copy_ticks_from("XAUUSD", now - timedelta(hours=1), 1, mt5.COPY_TICKS_INFO) + events = mt5.copy_ticks_from(os.getenv("SYMBOL", "XAUUSD"), now - timedelta(hours=1), 1, mt5.COPY_TICKS_INFO) # If we get here, try calendar calendar_events = []