Initial commit: orderflow analysis system with 5 pattern detectors
Real-time orderflow trading system with absorption, initiative, sweep, exhaustion, and divergence detection. Features volume profile framing, state machine trade lifecycle, MT5 + Bybit feeds, FastAPI dashboard, and Telegram alerts for 30+ instruments. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
BlackboxAI
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
Bybit WebSocket data feed — connects to public aggTrade + orderbook depth streams.
|
||||
Free, no API key needed for public data.
|
||||
Provides tick-by-tick trades with aggressor side and L2 orderbook updates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Callable, Optional
|
||||
|
||||
import websockets
|
||||
|
||||
from orderflow_system.data.models import Tick, Side, OrderbookSnapshot, OrderbookLevel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BYBIT_WS_URL = "wss://stream.bybit.com/v5/public/linear"
|
||||
|
||||
|
||||
class BybitFeed:
|
||||
"""
|
||||
Real-time data feed from Bybit perpetual futures.
|
||||
Subscribes to:
|
||||
- publicTrade.<symbol> → Tick data with aggressor side
|
||||
- orderbook.50.<symbol> → 50-level L2 orderbook snapshots + deltas
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
symbols: list[str],
|
||||
on_tick: Optional[Callable] = None,
|
||||
on_orderbook: Optional[Callable] = None,
|
||||
):
|
||||
self.symbols = symbols
|
||||
self.on_tick = on_tick
|
||||
self.on_orderbook = on_orderbook
|
||||
self._ws = None
|
||||
self._running = False
|
||||
self._orderbooks: dict[str, OrderbookSnapshot] = {}
|
||||
self._tick_buffer: dict[str, list[Tick]] = {s: [] for s in symbols}
|
||||
self._reconnect_delay = 1.0
|
||||
|
||||
async def start(self):
|
||||
"""Connect and begin receiving data."""
|
||||
self._running = True
|
||||
while self._running:
|
||||
try:
|
||||
await self._connect_and_listen()
|
||||
except (
|
||||
websockets.ConnectionClosed,
|
||||
ConnectionRefusedError,
|
||||
OSError,
|
||||
) as e:
|
||||
logger.warning(f"WebSocket disconnected: {e}. Reconnecting in {self._reconnect_delay}s...")
|
||||
await asyncio.sleep(self._reconnect_delay)
|
||||
self._reconnect_delay = min(self._reconnect_delay * 2, 30.0)
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error in feed: {e}", exc_info=True)
|
||||
await asyncio.sleep(5.0)
|
||||
|
||||
async def stop(self):
|
||||
self._running = False
|
||||
if self._ws:
|
||||
await self._ws.close()
|
||||
|
||||
async def _connect_and_listen(self):
|
||||
async with websockets.connect(BYBIT_WS_URL, ping_interval=20) as ws:
|
||||
self._ws = ws
|
||||
self._reconnect_delay = 1.0
|
||||
logger.info(f"Connected to Bybit WebSocket")
|
||||
|
||||
# Subscribe to trades + orderbook for each symbol
|
||||
subscribe_args = []
|
||||
for sym in self.symbols:
|
||||
subscribe_args.append(f"publicTrade.{sym}")
|
||||
subscribe_args.append(f"orderbook.50.{sym}")
|
||||
|
||||
subscribe_msg = {
|
||||
"op": "subscribe",
|
||||
"args": subscribe_args,
|
||||
}
|
||||
await ws.send(json.dumps(subscribe_msg))
|
||||
logger.info(f"Subscribed to: {subscribe_args}")
|
||||
|
||||
async for raw_msg in ws:
|
||||
if not self._running:
|
||||
break
|
||||
try:
|
||||
msg = json.loads(raw_msg)
|
||||
await self._handle_message(msg)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"Invalid JSON: {raw_msg[:100]}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error handling message: {e}", exc_info=True)
|
||||
|
||||
async def _handle_message(self, msg: dict):
|
||||
topic = msg.get("topic", "")
|
||||
|
||||
if topic.startswith("publicTrade."):
|
||||
await self._handle_trades(msg)
|
||||
elif topic.startswith("orderbook."):
|
||||
await self._handle_orderbook(msg)
|
||||
|
||||
async def _handle_trades(self, msg: dict):
|
||||
"""
|
||||
Parse Bybit public trade messages.
|
||||
Each trade has: price, size, side (Buy/Sell), timestamp.
|
||||
The 'side' from Bybit = the TAKER side = the aggressor.
|
||||
"""
|
||||
data_list = msg.get("data", [])
|
||||
symbol = msg.get("topic", "").replace("publicTrade.", "")
|
||||
|
||||
for trade in data_list:
|
||||
side_str = trade.get("S", "")
|
||||
tick = Tick(
|
||||
timestamp_ms=trade.get("T", int(time.time() * 1000)),
|
||||
price=float(trade.get("p", 0)),
|
||||
size=float(trade.get("v", 0)),
|
||||
side=Side.BUY if side_str == "Buy" else Side.SELL,
|
||||
trade_id=trade.get("i", ""),
|
||||
)
|
||||
|
||||
self._tick_buffer[symbol].append(tick)
|
||||
|
||||
if self.on_tick:
|
||||
await self.on_tick(symbol, tick)
|
||||
|
||||
async def _handle_orderbook(self, msg: dict):
|
||||
"""
|
||||
Parse Bybit orderbook messages.
|
||||
Type 'snapshot' = full book replacement.
|
||||
Type 'delta' = incremental update.
|
||||
"""
|
||||
data = msg.get("data", {})
|
||||
msg_type = msg.get("type", "")
|
||||
topic = msg.get("topic", "")
|
||||
symbol = topic.split(".")[-1] if "." in topic else ""
|
||||
ts = data.get("u", int(time.time() * 1000))
|
||||
|
||||
if msg_type == "snapshot":
|
||||
bids = [
|
||||
OrderbookLevel(price=float(b[0]), quantity=float(b[1]))
|
||||
for b in data.get("b", [])
|
||||
]
|
||||
asks = [
|
||||
OrderbookLevel(price=float(a[0]), quantity=float(a[1]))
|
||||
for a in data.get("a", [])
|
||||
]
|
||||
self._orderbooks[symbol] = OrderbookSnapshot(
|
||||
timestamp_ms=ts,
|
||||
bids=sorted(bids, key=lambda x: -x.price),
|
||||
asks=sorted(asks, key=lambda x: x.price),
|
||||
)
|
||||
elif msg_type == "delta":
|
||||
book = self._orderbooks.get(symbol)
|
||||
if book is None:
|
||||
return
|
||||
self._apply_delta(book, data)
|
||||
book.timestamp_ms = ts
|
||||
|
||||
if symbol in self._orderbooks and self.on_orderbook:
|
||||
await self.on_orderbook(symbol, self._orderbooks[symbol])
|
||||
|
||||
def _apply_delta(self, book: OrderbookSnapshot, data: dict):
|
||||
"""Apply incremental orderbook updates."""
|
||||
# Update bids
|
||||
for b in data.get("b", []):
|
||||
price, qty = float(b[0]), float(b[1])
|
||||
if qty == 0:
|
||||
book.bids = [lv for lv in book.bids if lv.price != price]
|
||||
else:
|
||||
found = False
|
||||
for lv in book.bids:
|
||||
if lv.price == price:
|
||||
lv.quantity = qty
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
book.bids.append(OrderbookLevel(price=price, quantity=qty))
|
||||
book.bids.sort(key=lambda x: -x.price)
|
||||
|
||||
# Update asks
|
||||
for a in data.get("a", []):
|
||||
price, qty = float(a[0]), float(a[1])
|
||||
if qty == 0:
|
||||
book.asks = [lv for lv in book.asks if lv.price != price]
|
||||
else:
|
||||
found = False
|
||||
for lv in book.asks:
|
||||
if lv.price == price:
|
||||
lv.quantity = qty
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
book.asks.append(OrderbookLevel(price=price, quantity=qty))
|
||||
book.asks.sort(key=lambda x: x.price)
|
||||
|
||||
def get_orderbook(self, symbol: str) -> Optional[OrderbookSnapshot]:
|
||||
return self._orderbooks.get(symbol)
|
||||
|
||||
def flush_tick_buffer(self, symbol: str) -> list[Tick]:
|
||||
"""Return and clear buffered ticks for batch DB insert."""
|
||||
ticks = self._tick_buffer.get(symbol, [])
|
||||
self._tick_buffer[symbol] = []
|
||||
return ticks
|
||||
@@ -0,0 +1,133 @@
|
||||
"""
|
||||
Candle Builder — aggregates ticks into OHLCV candles with footprint data.
|
||||
Builds candles in real-time from the tick stream.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Callable, Optional
|
||||
|
||||
from orderflow_system.data.models import Tick, Candle, FootprintLevel, Side
|
||||
|
||||
|
||||
class CandleBuilder:
|
||||
"""
|
||||
Builds time-based candles from a tick stream.
|
||||
Each candle includes full footprint data (bid/ask volume at each price level).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
interval_seconds: int = 60,
|
||||
tick_size: float = 0.1,
|
||||
on_candle_close: Optional[Callable] = None,
|
||||
):
|
||||
self.interval_ms = interval_seconds * 1000
|
||||
self.tick_size = tick_size
|
||||
self.on_candle_close = on_candle_close
|
||||
self._current_candle: Optional[Candle] = None
|
||||
self._candle_history: list[Candle] = []
|
||||
self._max_history = 5000
|
||||
|
||||
def _round_price(self, price: float) -> float:
|
||||
"""Round price to tick size for footprint grouping."""
|
||||
return round(round(price / self.tick_size) * self.tick_size, 10)
|
||||
|
||||
def _candle_start_ms(self, timestamp_ms: int) -> int:
|
||||
"""Align timestamp to candle interval boundary."""
|
||||
return (timestamp_ms // self.interval_ms) * self.interval_ms
|
||||
|
||||
async def process_tick(self, tick: Tick) -> Optional[Candle]:
|
||||
"""
|
||||
Feed a tick into the builder. Returns a closed candle if interval completed.
|
||||
"""
|
||||
candle_start = self._candle_start_ms(tick.timestamp_ms)
|
||||
closed_candle = None
|
||||
|
||||
# Check if we need to close current candle and start new one
|
||||
if self._current_candle is not None:
|
||||
if candle_start > self._current_candle.timestamp_ms:
|
||||
closed_candle = self._current_candle
|
||||
# Deduplicate: replace last entry if same timestamp
|
||||
if (self._candle_history
|
||||
and self._candle_history[-1].timestamp_ms
|
||||
== closed_candle.timestamp_ms):
|
||||
self._candle_history[-1] = closed_candle
|
||||
else:
|
||||
self._candle_history.append(closed_candle)
|
||||
if len(self._candle_history) > self._max_history:
|
||||
self._candle_history = self._candle_history[-self._max_history:]
|
||||
|
||||
if self.on_candle_close:
|
||||
await self.on_candle_close(closed_candle)
|
||||
|
||||
self._current_candle = None
|
||||
|
||||
# Start new candle if needed
|
||||
if self._current_candle is None:
|
||||
self._current_candle = Candle(
|
||||
timestamp_ms=candle_start,
|
||||
open=tick.price,
|
||||
high=tick.price,
|
||||
low=tick.price,
|
||||
close=tick.price,
|
||||
)
|
||||
|
||||
# Update OHLCV
|
||||
c = self._current_candle
|
||||
c.high = max(c.high, tick.price)
|
||||
c.low = min(c.low, tick.price)
|
||||
c.close = tick.price
|
||||
c.volume += tick.size
|
||||
c.tick_count += 1
|
||||
|
||||
if tick.is_buy:
|
||||
c.buy_volume += tick.size
|
||||
else:
|
||||
c.sell_volume += tick.size
|
||||
|
||||
# Update footprint at this price level
|
||||
fp_price = self._round_price(tick.price)
|
||||
if fp_price not in c.footprint:
|
||||
c.footprint[fp_price] = FootprintLevel(price=fp_price)
|
||||
|
||||
if tick.is_buy:
|
||||
c.footprint[fp_price].ask_volume += tick.size
|
||||
else:
|
||||
c.footprint[fp_price].bid_volume += tick.size
|
||||
|
||||
return closed_candle
|
||||
|
||||
@property
|
||||
def current_candle(self) -> Optional[Candle]:
|
||||
return self._current_candle
|
||||
|
||||
@property
|
||||
def history(self) -> list[Candle]:
|
||||
return self._candle_history
|
||||
|
||||
def get_recent_candles(self, n: int) -> list[Candle]:
|
||||
"""Return last N closed candles."""
|
||||
return self._candle_history[-n:]
|
||||
|
||||
def load_historical_candles(self, candles: list[Candle]) -> int:
|
||||
"""
|
||||
Bulk-load historical candles (e.g. from MT5 bars).
|
||||
Prepends them before any real-time candles, deduplicating by timestamp.
|
||||
Returns the number of candles actually added.
|
||||
"""
|
||||
if not candles:
|
||||
return 0
|
||||
|
||||
# Existing timestamps for dedup
|
||||
existing_ts = {c.timestamp_ms for c in self._candle_history}
|
||||
new_candles = [c for c in candles if c.timestamp_ms not in existing_ts]
|
||||
|
||||
if not new_candles:
|
||||
return 0
|
||||
|
||||
# Merge: historical first, then real-time, sorted by time
|
||||
merged = sorted(new_candles + self._candle_history, key=lambda c: c.timestamp_ms)
|
||||
self._candle_history = merged[-self._max_history:]
|
||||
return len(new_candles)
|
||||
@@ -0,0 +1,278 @@
|
||||
"""
|
||||
SQLite storage for ticks, candles, volume profiles, and signals.
|
||||
Lightweight, zero-cost, zero-config alternative to TimescaleDB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import aiosqlite
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from orderflow_system.data.models import Tick, Side, Candle, Signal, SignalType, VolumeProfileResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Database:
|
||||
"""Async SQLite database for orderflow data storage."""
|
||||
|
||||
def __init__(self, db_path: str = "orderflow_data.db"):
|
||||
self.db_path = db_path
|
||||
self._db: Optional[aiosqlite.Connection] = None
|
||||
|
||||
async def connect(self):
|
||||
self._db = await aiosqlite.connect(self.db_path)
|
||||
await self._db.execute("PRAGMA journal_mode=WAL")
|
||||
await self._db.execute("PRAGMA synchronous=NORMAL")
|
||||
await self._create_tables()
|
||||
logger.info(f"Database connected: {self.db_path}")
|
||||
|
||||
async def close(self):
|
||||
if self._db:
|
||||
await self._db.close()
|
||||
logger.info("Database closed")
|
||||
|
||||
async def _create_tables(self):
|
||||
await self._db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS ticks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
instrument TEXT NOT NULL,
|
||||
timestamp_ms INTEGER NOT NULL,
|
||||
price REAL NOT NULL,
|
||||
size REAL NOT NULL,
|
||||
side TEXT NOT NULL,
|
||||
trade_id TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ticks_instrument_ts
|
||||
ON ticks(instrument, timestamp_ms);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS candles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
instrument TEXT NOT NULL,
|
||||
timestamp_ms INTEGER NOT NULL,
|
||||
timeframe TEXT NOT NULL,
|
||||
open REAL, high REAL, low REAL, close REAL,
|
||||
volume REAL,
|
||||
buy_volume REAL,
|
||||
sell_volume REAL,
|
||||
delta REAL,
|
||||
tick_count INTEGER,
|
||||
footprint_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_candles_instrument_ts
|
||||
ON candles(instrument, timestamp_ms, timeframe);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS volume_profiles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
instrument TEXT NOT NULL,
|
||||
session_date TEXT NOT NULL,
|
||||
poc REAL, vah REAL, val REAL,
|
||||
total_volume REAL,
|
||||
shape TEXT,
|
||||
poc_position_pct REAL,
|
||||
lvn_json TEXT,
|
||||
volume_at_price_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_vp_instrument_date
|
||||
ON volume_profiles(instrument, session_date);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
instrument TEXT NOT NULL,
|
||||
timestamp_ms INTEGER NOT NULL,
|
||||
signal_type TEXT NOT NULL,
|
||||
direction TEXT NOT NULL,
|
||||
price_level REAL,
|
||||
strength REAL,
|
||||
details_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_signals_instrument_ts
|
||||
ON signals(instrument, timestamp_ms);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trade_journal (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
instrument TEXT NOT NULL,
|
||||
direction TEXT NOT NULL,
|
||||
entry_time_ms INTEGER,
|
||||
exit_time_ms INTEGER,
|
||||
entry_price REAL,
|
||||
exit_price REAL,
|
||||
stop_loss REAL,
|
||||
take_profit REAL,
|
||||
pnl_ticks REAL,
|
||||
rr_ratio REAL,
|
||||
signals_json TEXT,
|
||||
notes TEXT
|
||||
);
|
||||
""")
|
||||
await self._db.commit()
|
||||
|
||||
# ── Ticks ──
|
||||
|
||||
async def insert_tick(self, instrument: str, tick: Tick):
|
||||
await self._db.execute(
|
||||
"INSERT INTO ticks (instrument, timestamp_ms, price, size, side, trade_id) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(instrument, tick.timestamp_ms, tick.price, tick.size,
|
||||
tick.side.value, tick.trade_id),
|
||||
)
|
||||
|
||||
async def insert_ticks_batch(self, instrument: str, ticks: list[Tick]):
|
||||
data = [
|
||||
(instrument, t.timestamp_ms, t.price, t.size, t.side.value, t.trade_id)
|
||||
for t in ticks
|
||||
]
|
||||
await self._db.executemany(
|
||||
"INSERT INTO ticks (instrument, timestamp_ms, price, size, side, trade_id) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
data,
|
||||
)
|
||||
await self._db.commit()
|
||||
|
||||
async def get_ticks(
|
||||
self, instrument: str, start_ms: int, end_ms: int
|
||||
) -> list[Tick]:
|
||||
cursor = await self._db.execute(
|
||||
"SELECT timestamp_ms, price, size, side, trade_id FROM ticks "
|
||||
"WHERE instrument = ? AND timestamp_ms >= ? AND timestamp_ms <= ? "
|
||||
"ORDER BY timestamp_ms",
|
||||
(instrument, start_ms, end_ms),
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
return [
|
||||
Tick(
|
||||
timestamp_ms=r[0], price=r[1], size=r[2],
|
||||
side=Side(r[3]), trade_id=r[4] or ""
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
# ── Candles ──
|
||||
|
||||
async def insert_candle(self, instrument: str, timeframe: str, candle: Candle):
|
||||
fp_json = json.dumps({
|
||||
str(price): {"bid": lvl.bid_volume, "ask": lvl.ask_volume}
|
||||
for price, lvl in candle.footprint.items()
|
||||
}) if candle.footprint else "{}"
|
||||
|
||||
await self._db.execute(
|
||||
"INSERT INTO candles "
|
||||
"(instrument, timestamp_ms, timeframe, open, high, low, close, "
|
||||
"volume, buy_volume, sell_volume, delta, tick_count, footprint_json) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(instrument, candle.timestamp_ms, timeframe,
|
||||
candle.open, candle.high, candle.low, candle.close,
|
||||
candle.volume, candle.buy_volume, candle.sell_volume,
|
||||
candle.delta, candle.tick_count, fp_json),
|
||||
)
|
||||
await self._db.commit()
|
||||
|
||||
async def get_candles(
|
||||
self, instrument: str, timeframe: str, start_ms: int, end_ms: int
|
||||
) -> list[Candle]:
|
||||
cursor = await self._db.execute(
|
||||
"SELECT timestamp_ms, open, high, low, close, volume, "
|
||||
"buy_volume, sell_volume, tick_count FROM candles "
|
||||
"WHERE instrument = ? AND timeframe = ? "
|
||||
"AND timestamp_ms >= ? AND timestamp_ms <= ? "
|
||||
"ORDER BY timestamp_ms",
|
||||
(instrument, timeframe, start_ms, end_ms),
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
return [
|
||||
Candle(
|
||||
timestamp_ms=r[0], open=r[1], high=r[2], low=r[3], close=r[4],
|
||||
volume=r[5], buy_volume=r[6], sell_volume=r[7], tick_count=r[8],
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
# ── Volume Profiles ──
|
||||
|
||||
async def insert_volume_profile(self, instrument: str, vp: VolumeProfileResult):
|
||||
await self._db.execute(
|
||||
"INSERT INTO volume_profiles "
|
||||
"(instrument, session_date, poc, vah, val, total_volume, shape, "
|
||||
"poc_position_pct, lvn_json, volume_at_price_json) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(instrument, vp.session_date, vp.poc, vp.vah, vp.val,
|
||||
vp.total_volume, vp.shape, vp.poc_position_pct,
|
||||
json.dumps(vp.lvn_levels),
|
||||
json.dumps({str(k): v for k, v in vp.volume_at_price.items()})),
|
||||
)
|
||||
await self._db.commit()
|
||||
|
||||
async def get_volume_profiles(
|
||||
self, instrument: str, days: int = 5
|
||||
) -> list[VolumeProfileResult]:
|
||||
cursor = await self._db.execute(
|
||||
"SELECT session_date, poc, vah, val, total_volume, shape, "
|
||||
"poc_position_pct, lvn_json, volume_at_price_json "
|
||||
"FROM volume_profiles WHERE instrument = ? "
|
||||
"ORDER BY session_date DESC LIMIT ?",
|
||||
(instrument, days),
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
results = []
|
||||
for r in rows:
|
||||
vap_raw = json.loads(r[8]) if r[8] else {}
|
||||
results.append(VolumeProfileResult(
|
||||
session_date=r[0], poc=r[1], vah=r[2], val=r[3],
|
||||
total_volume=r[4], shape=r[5], poc_position_pct=r[6],
|
||||
lvn_levels=json.loads(r[7]) if r[7] else [],
|
||||
volume_at_price={float(k): v for k, v in vap_raw.items()},
|
||||
))
|
||||
return list(reversed(results)) # Oldest first
|
||||
|
||||
# ── Signals ──
|
||||
|
||||
async def insert_signal(self, instrument: str, signal: Signal):
|
||||
await self._db.execute(
|
||||
"INSERT INTO signals "
|
||||
"(instrument, timestamp_ms, signal_type, direction, price_level, "
|
||||
"strength, details_json) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(instrument, signal.timestamp_ms, signal.signal_type.value,
|
||||
signal.direction.value, signal.price_level, signal.strength,
|
||||
json.dumps(signal.details)),
|
||||
)
|
||||
await self._db.commit()
|
||||
|
||||
# ── Trade Journal ──
|
||||
|
||||
async def log_trade(
|
||||
self,
|
||||
instrument: str,
|
||||
direction: str,
|
||||
entry_price: float,
|
||||
exit_price: float,
|
||||
stop_loss: float,
|
||||
take_profit: float,
|
||||
pnl_ticks: float,
|
||||
rr_ratio: float,
|
||||
signals: list[Signal],
|
||||
notes: str = "",
|
||||
entry_time_ms: int = 0,
|
||||
exit_time_ms: int = 0,
|
||||
):
|
||||
signals_json = json.dumps([
|
||||
{"type": s.signal_type.value, "strength": s.strength,
|
||||
"price": s.price_level, "ts": s.timestamp_ms}
|
||||
for s in signals
|
||||
])
|
||||
await self._db.execute(
|
||||
"INSERT INTO trade_journal "
|
||||
"(instrument, direction, entry_time_ms, exit_time_ms, entry_price, "
|
||||
"exit_price, stop_loss, take_profit, pnl_ticks, rr_ratio, "
|
||||
"signals_json, notes) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(instrument, direction, entry_time_ms, exit_time_ms,
|
||||
entry_price, exit_price, stop_loss, take_profit,
|
||||
pnl_ticks, rr_ratio, signals_json, notes),
|
||||
)
|
||||
await self._db.commit()
|
||||
@@ -0,0 +1,290 @@
|
||||
"""
|
||||
Core data models used across the entire system.
|
||||
Tick, OrderbookSnapshot, Candle, Signal, TradeState — the common language.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Enums
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
class Side(Enum):
|
||||
BUY = "buy"
|
||||
SELL = "sell"
|
||||
|
||||
|
||||
class SignalType(Enum):
|
||||
ABSORPTION = "absorption"
|
||||
INITIATIVE = "initiative_auction"
|
||||
SWEEP = "book_sweep"
|
||||
EXHAUSTION = "exhaustion"
|
||||
DIVERGENCE = "delta_divergence"
|
||||
|
||||
|
||||
class TradePhase(Enum):
|
||||
"""State machine phases for the execution model."""
|
||||
WATCHING = "watching" # Monitoring a qualified level
|
||||
ABSORPTION_DETECTED = "absorption" # Entry signal seen
|
||||
POSITION_OPEN = "position_open" # Trade entered
|
||||
BREAK_EVEN = "break_even" # SL moved to BE after initiative
|
||||
TRAILING = "trailing" # Trailing on initiative prints
|
||||
CLOSED = "closed" # Trade finished
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Raw Market Data
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Tick:
|
||||
"""Single executed trade from the exchange."""
|
||||
timestamp_ms: int # Unix ms
|
||||
price: float
|
||||
size: float # Contracts / quantity
|
||||
side: Side # Aggressor side (taker)
|
||||
trade_id: str = ""
|
||||
|
||||
@property
|
||||
def timestamp(self) -> float:
|
||||
return self.timestamp_ms / 1000.0
|
||||
|
||||
@property
|
||||
def is_buy(self) -> bool:
|
||||
return self.side == Side.BUY
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class OrderbookLevel:
|
||||
"""Single price level in the orderbook."""
|
||||
price: float
|
||||
quantity: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrderbookSnapshot:
|
||||
"""L2 orderbook state at a point in time."""
|
||||
timestamp_ms: int
|
||||
bids: list[OrderbookLevel] = field(default_factory=list) # Sorted desc by price
|
||||
asks: list[OrderbookLevel] = field(default_factory=list) # Sorted asc by price
|
||||
|
||||
@property
|
||||
def best_bid(self) -> Optional[float]:
|
||||
return self.bids[0].price if self.bids else None
|
||||
|
||||
@property
|
||||
def best_ask(self) -> Optional[float]:
|
||||
return self.asks[0].price if self.asks else None
|
||||
|
||||
@property
|
||||
def mid_price(self) -> Optional[float]:
|
||||
if self.best_bid and self.best_ask:
|
||||
return (self.best_bid + self.best_ask) / 2.0
|
||||
return None
|
||||
|
||||
@property
|
||||
def spread(self) -> Optional[float]:
|
||||
if self.best_bid and self.best_ask:
|
||||
return self.best_ask - self.best_bid
|
||||
return None
|
||||
|
||||
def bid_depth(self, levels: int = 5) -> float:
|
||||
"""Total bid quantity in top N levels."""
|
||||
return sum(b.quantity for b in self.bids[:levels])
|
||||
|
||||
def ask_depth(self, levels: int = 5) -> float:
|
||||
"""Total ask quantity in top N levels."""
|
||||
return sum(a.quantity for a in self.asks[:levels])
|
||||
|
||||
def imbalance_ratio(self, levels: int = 5) -> float:
|
||||
"""Book imbalance: +1 = all bids, -1 = all asks."""
|
||||
bd = self.bid_depth(levels)
|
||||
ad = self.ask_depth(levels)
|
||||
total = bd + ad
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return (bd - ad) / total
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Aggregated Structures
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class FootprintLevel:
|
||||
"""Bid/Ask volume at a single price level within a candle."""
|
||||
price: float
|
||||
bid_volume: float = 0.0 # Aggressive sell volume hitting this bid
|
||||
ask_volume: float = 0.0 # Aggressive buy volume hitting this ask
|
||||
|
||||
@property
|
||||
def delta(self) -> float:
|
||||
"""Horizontal delta at this level."""
|
||||
return self.ask_volume - self.bid_volume
|
||||
|
||||
@property
|
||||
def total_volume(self) -> float:
|
||||
return self.bid_volume + self.ask_volume
|
||||
|
||||
@property
|
||||
def imbalance_ratio(self) -> float:
|
||||
"""Buy/sell ratio. >3 = strong buy imbalance."""
|
||||
if self.bid_volume == 0:
|
||||
return float("inf") if self.ask_volume > 0 else 0.0
|
||||
return self.ask_volume / self.bid_volume
|
||||
|
||||
|
||||
@dataclass
|
||||
class Candle:
|
||||
"""OHLCV candle enriched with orderflow data."""
|
||||
timestamp_ms: int
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: float = 0.0
|
||||
buy_volume: float = 0.0 # Aggressive buy volume
|
||||
sell_volume: float = 0.0 # Aggressive sell volume
|
||||
tick_count: int = 0
|
||||
footprint: dict[float, FootprintLevel] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def delta(self) -> float:
|
||||
"""Vertical delta for this candle."""
|
||||
return self.buy_volume - self.sell_volume
|
||||
|
||||
@property
|
||||
def is_green(self) -> bool:
|
||||
return self.close >= self.open
|
||||
|
||||
@property
|
||||
def body_size(self) -> float:
|
||||
return abs(self.close - self.open)
|
||||
|
||||
@property
|
||||
def range_size(self) -> float:
|
||||
return self.high - self.low
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Volume Profile
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class VolumeProfileResult:
|
||||
"""Output of the volume profile engine for a session."""
|
||||
session_date: str = "" # YYYY-MM-DD
|
||||
poc: float = 0.0 # Point of Control
|
||||
vah: float = 0.0 # Value Area High
|
||||
val: float = 0.0 # Value Area Low
|
||||
volume_at_price: dict[float, float] = field(default_factory=dict)
|
||||
total_volume: float = 0.0
|
||||
lvn_levels: list[float] = field(default_factory=list)
|
||||
shape: str = "unknown" # p_shape, b_shape, d_shape, double_dist
|
||||
poc_position_pct: float = 0.5 # POC position within range (0=bottom, 1=top)
|
||||
|
||||
@property
|
||||
def value_area_range(self) -> float:
|
||||
return self.vah - self.val
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Signals
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class Signal:
|
||||
"""Output from a pattern detector."""
|
||||
timestamp_ms: int
|
||||
signal_type: SignalType
|
||||
direction: Side # Suggested direction
|
||||
price_level: float # Key price
|
||||
strength: float = 0.0 # 0-100 confidence score
|
||||
details: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def is_bullish(self) -> bool:
|
||||
return self.direction == Side.BUY
|
||||
|
||||
def __repr__(self) -> str:
|
||||
dir_str = "LONG" if self.is_bullish else "SHORT"
|
||||
return (
|
||||
f"Signal({self.signal_type.value} {dir_str} "
|
||||
f"@ {self.price_level:.2f}, strength={self.strength:.0f})"
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Trade State (State Machine)
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class TradeState:
|
||||
"""
|
||||
Tracks the state machine for a single trade idea.
|
||||
Qualified Level → Absorption → Position → Break-Even → Trail → Closed
|
||||
"""
|
||||
instrument: str
|
||||
direction: Side
|
||||
phase: TradePhase = TradePhase.WATCHING
|
||||
qualified_level: float = 0.0 # Level from profile framing
|
||||
entry_price: float = 0.0
|
||||
stop_loss: float = 0.0
|
||||
take_profit: float = 0.0
|
||||
break_even_price: float = 0.0
|
||||
trail_stop: float = 0.0
|
||||
absorption_signals: list[Signal] = field(default_factory=list)
|
||||
initiative_signals: list[Signal] = field(default_factory=list)
|
||||
entry_time_ms: int = 0
|
||||
rr_ratio: float = 0.0
|
||||
pnl_ticks: float = 0.0
|
||||
notes: str = ""
|
||||
|
||||
def advance_to_absorption(self, signal: Signal):
|
||||
"""Absorption detected at qualified level — entry signal."""
|
||||
self.phase = TradePhase.ABSORPTION_DETECTED
|
||||
self.absorption_signals.append(signal)
|
||||
|
||||
def advance_to_position(self, entry_price: float, stop_loss: float, take_profit: float):
|
||||
"""Trade entered."""
|
||||
self.phase = TradePhase.POSITION_OPEN
|
||||
self.entry_price = entry_price
|
||||
self.stop_loss = stop_loss
|
||||
self.take_profit = take_profit
|
||||
self.break_even_price = entry_price
|
||||
self.entry_time_ms = int(time.time() * 1000)
|
||||
risk = abs(entry_price - stop_loss)
|
||||
if risk > 0:
|
||||
self.rr_ratio = abs(take_profit - entry_price) / risk
|
||||
|
||||
def advance_to_break_even(self, signal: Signal):
|
||||
"""Initiative auction confirmed — move SL to break even."""
|
||||
self.phase = TradePhase.BREAK_EVEN
|
||||
self.stop_loss = self.break_even_price
|
||||
self.trail_stop = self.break_even_price
|
||||
self.initiative_signals.append(signal)
|
||||
|
||||
def update_trail(self, new_trail_level: float, signal: Signal):
|
||||
"""New initiative print — trail stop to the candle's extreme."""
|
||||
self.phase = TradePhase.TRAILING
|
||||
if self.direction == Side.BUY:
|
||||
self.trail_stop = max(self.trail_stop, new_trail_level)
|
||||
else:
|
||||
self.trail_stop = min(self.trail_stop, new_trail_level)
|
||||
self.stop_loss = self.trail_stop
|
||||
self.initiative_signals.append(signal)
|
||||
|
||||
def close_trade(self, exit_price: float, reason: str = ""):
|
||||
"""Trade finished."""
|
||||
self.phase = TradePhase.CLOSED
|
||||
if self.direction == Side.BUY:
|
||||
self.pnl_ticks = exit_price - self.entry_price
|
||||
else:
|
||||
self.pnl_ticks = self.entry_price - exit_price
|
||||
self.notes = reason
|
||||
@@ -0,0 +1,495 @@
|
||||
"""
|
||||
MetaTrader 5 Data Feed — connects to MT5 terminal for real NAS100 & Gold tick data.
|
||||
|
||||
Provides:
|
||||
- Real-time tick polling from MT5 terminal (bid/ask/last, volume, buy/sell flags)
|
||||
- Historical tick/bar download for backtesting & VP computation
|
||||
- Book of market (DOM) data for orderbook analysis
|
||||
- Symbol info (tick size, contract size, session times)
|
||||
|
||||
Requirements:
|
||||
- MetaTrader 5 terminal installed and running on Windows
|
||||
- pip install MetaTrader5
|
||||
- Broker account connected in MT5
|
||||
|
||||
MT5 Symbol Mapping (broker-dependent, adjust in config):
|
||||
- NAS100: "NAS100", "USTEC", "US100", "NAS100.cash", "USTEC.cash"
|
||||
- Gold: "XAUUSD", "GOLD", "XAUUSD.cash"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Callable, Optional
|
||||
|
||||
from orderflow_system.data.models import (
|
||||
Tick, Side, OrderbookSnapshot, OrderbookLevel, Candle,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# MT5 tick flags (from MetaTrader5 module constants)
|
||||
TICK_FLAG_BID = 0x02
|
||||
TICK_FLAG_ASK = 0x04
|
||||
TICK_FLAG_LAST = 0x08
|
||||
TICK_FLAG_VOLUME = 0x10
|
||||
TICK_FLAG_BUY = 0x20
|
||||
TICK_FLAG_SELL = 0x40
|
||||
|
||||
|
||||
class MT5Feed:
|
||||
"""
|
||||
Real-time and historical data feed from MetaTrader 5 terminal.
|
||||
|
||||
Polling-based: MT5 Python API is synchronous, so we poll ticks in an
|
||||
async loop with configurable interval. For orderflow, we need the
|
||||
LAST price + BUY/SELL flags, not just bid/ask.
|
||||
|
||||
Usage:
|
||||
feed = MT5Feed(
|
||||
symbols={"NAS100USDT": "USTEC", "XAUUSDT": "XAUUSD"},
|
||||
on_tick=my_tick_handler,
|
||||
on_orderbook=my_book_handler,
|
||||
)
|
||||
await feed.start()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
symbols: dict[str, str], # {internal_name: mt5_symbol}
|
||||
on_tick: Optional[Callable] = None,
|
||||
on_orderbook: Optional[Callable] = None,
|
||||
poll_interval_ms: int = 100,
|
||||
enable_book: bool = True,
|
||||
):
|
||||
self.symbols = symbols # e.g., {"NAS100USDT": "USTEC", "XAUUSDT": "XAUUSD"}
|
||||
self.on_tick = on_tick
|
||||
self.on_orderbook = on_orderbook
|
||||
self.poll_interval_ms = poll_interval_ms
|
||||
self.enable_book = enable_book
|
||||
self._running = False
|
||||
self._mt5 = None
|
||||
self._last_tick_time: dict[str, int] = {} # Track last seen tick per symbol
|
||||
self._initialized = False
|
||||
|
||||
def connect(self) -> bool:
|
||||
"""Initialize MT5 connection (call before download_historical_ticks)."""
|
||||
if self._initialized:
|
||||
return True
|
||||
return self._initialize_mt5()
|
||||
|
||||
async def start(self):
|
||||
"""Initialize MT5 connection and begin polling."""
|
||||
if not self._initialized and not self._initialize_mt5():
|
||||
logger.error("Failed to initialize MT5. Make sure MT5 terminal is running.")
|
||||
return
|
||||
|
||||
self._running = True
|
||||
|
||||
# Enable market book for each symbol (DOM data)
|
||||
if self.enable_book:
|
||||
for internal, mt5_sym in self.symbols.items():
|
||||
self._mt5.market_book_add(mt5_sym)
|
||||
logger.info(f"Market book enabled for {mt5_sym}")
|
||||
|
||||
logger.info(
|
||||
f"MT5 feed started. Polling {len(self.symbols)} symbols "
|
||||
f"every {self.poll_interval_ms}ms"
|
||||
)
|
||||
|
||||
try:
|
||||
while self._running:
|
||||
await self._poll_cycle()
|
||||
await asyncio.sleep(self.poll_interval_ms / 1000.0)
|
||||
finally:
|
||||
await self.stop()
|
||||
|
||||
async def stop(self):
|
||||
"""Disconnect from MT5."""
|
||||
self._running = False
|
||||
if self._mt5 and self._initialized:
|
||||
if self.enable_book:
|
||||
for mt5_sym in self.symbols.values():
|
||||
try:
|
||||
self._mt5.market_book_release(mt5_sym)
|
||||
except Exception:
|
||||
pass
|
||||
self._mt5.shutdown()
|
||||
self._initialized = False
|
||||
logger.info("MT5 disconnected")
|
||||
|
||||
def _initialize_mt5(self) -> bool:
|
||||
"""Initialize MT5 connection."""
|
||||
try:
|
||||
import MetaTrader5 as mt5
|
||||
self._mt5 = mt5
|
||||
except ImportError:
|
||||
logger.error(
|
||||
"MetaTrader5 package not installed. Install with: "
|
||||
"pip install MetaTrader5"
|
||||
)
|
||||
return False
|
||||
|
||||
if not mt5.initialize():
|
||||
error = mt5.last_error()
|
||||
logger.error(f"MT5 initialize() failed: {error}")
|
||||
return False
|
||||
|
||||
self._initialized = True
|
||||
|
||||
# Log account info
|
||||
account = mt5.account_info()
|
||||
if account:
|
||||
logger.info(
|
||||
f"MT5 connected: {account.server} | "
|
||||
f"Account: {account.login} | Balance: {account.balance}"
|
||||
)
|
||||
|
||||
# Validate symbols exist
|
||||
for internal, mt5_sym in list(self.symbols.items()):
|
||||
info = mt5.symbol_info(mt5_sym)
|
||||
if info is None:
|
||||
logger.warning(
|
||||
f"Symbol '{mt5_sym}' not found in MT5. "
|
||||
f"Trying alternatives..."
|
||||
)
|
||||
# Try common alternatives
|
||||
found = self._find_symbol_alternative(mt5_sym, internal)
|
||||
if not found:
|
||||
logger.error(
|
||||
f"Could not find any matching symbol for {internal}. "
|
||||
f"Available symbols can be listed with mt5.symbols_get()"
|
||||
)
|
||||
else:
|
||||
if not info.visible:
|
||||
mt5.symbol_select(mt5_sym, True)
|
||||
logger.info(
|
||||
f"Symbol {mt5_sym} ({internal}): "
|
||||
f"tick_size={info.trade_tick_size}, "
|
||||
f"digits={info.digits}, "
|
||||
f"spread={info.spread}"
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
# Known alternative symbol names per asset class (broker-dependent)
|
||||
_SYMBOL_ALTERNATIVES: dict[str, list[str]] = {
|
||||
# Indices
|
||||
"USTEC": ["USTEC", "USTECm", "NAS100", "US100", "USTEC.cash", "NAS100.cash", "USTECH100", "#NAS100", "NASDAQ"],
|
||||
"US500": ["US500", "US500m", "SP500", "SPX500", "US500.cash", "#SP500", "SP500m"],
|
||||
"US30": ["US30", "US30m", "DJ30", "DJI30", "US30.cash", "#DJ30", "DJ30m"],
|
||||
"UK100": ["UK100", "UK100m", "FTSE100", "UK100.cash", "#UK100"],
|
||||
"DE30": ["DE30", "DE30m", "DE40", "DE40m", "DAX40", "GER40", "GER30", "DE30.cash"],
|
||||
"JP225": ["JP225", "JP225m", "NI225", "NIKKEI225", "JP225.cash"],
|
||||
"FR40": ["FR40", "FR40m", "CAC40", "FRA40", "FR40.cash"],
|
||||
"AUS200":["AUS200", "AUS200m", "AU200", "ASX200", "AUS200.cash"],
|
||||
"HK50": ["HK50", "HK50m", "HSI50", "HK50.cash"],
|
||||
# Metals
|
||||
"XAUUSD":["XAUUSD", "XAUUSDm", "GOLD", "GOLDm", "XAUUSD.cash", "#XAUUSD"],
|
||||
"XAGUSD":["XAGUSD", "XAGUSDm", "SILVER", "SILVERm", "XAGUSD.cash"],
|
||||
# Energy
|
||||
"USOIL": ["USOIL", "USOILm", "WTI", "XTIUSD", "XTIUSDm", "CrudeOIL", "USCrude"],
|
||||
"UKOIL": ["UKOIL", "UKOILm", "BRENT", "XBRUSD", "XBRUSDm", "BrentOIL"],
|
||||
# Forex Majors
|
||||
"EURUSD":["EURUSD", "EURUSDm", "EURUSD.cash"],
|
||||
"GBPUSD":["GBPUSD", "GBPUSDm"],
|
||||
"USDJPY":["USDJPY", "USDJPYm"],
|
||||
"AUDUSD":["AUDUSD", "AUDUSDm"],
|
||||
"USDCAD":["USDCAD", "USDCADm"],
|
||||
"USDCHF":["USDCHF", "USDCHFm"],
|
||||
"NZDUSD":["NZDUSD", "NZDUSDm"],
|
||||
# Forex Crosses
|
||||
"EURGBP":["EURGBP", "EURGBPm"],
|
||||
"EURJPY":["EURJPY", "EURJPYm"],
|
||||
"GBPJPY":["GBPJPY", "GBPJPYm"],
|
||||
# Stocks
|
||||
"AAPL": ["AAPL", "AAPLm", "#AAPL", "AAPL.US"],
|
||||
"TSLA": ["TSLA", "TSLAm", "#TSLA", "TSLA.US"],
|
||||
"AMZN": ["AMZN", "AMZNm", "#AMZN", "AMZN.US"],
|
||||
"MSFT": ["MSFT", "MSFTm", "#MSFT", "MSFT.US"],
|
||||
"NVDA": ["NVDA", "NVDAm", "#NVDA", "NVDA.US"],
|
||||
"META": ["META", "METAm", "#META", "META.US"],
|
||||
"GOOGL": ["GOOGL", "GOOGLm", "#GOOGL", "GOOGL.US", "GOOG", "GOOGm"],
|
||||
# Crypto
|
||||
"BTCUSD":["BTCUSD", "BTCUSDm", "BTCUSDT"],
|
||||
}
|
||||
|
||||
def _find_symbol_alternative(self, mt5_sym: str, internal: str) -> bool:
|
||||
"""Try to find alternative symbol names for common instruments."""
|
||||
mt5 = self._mt5
|
||||
base = mt5_sym.upper().replace(".CASH", "").replace(".", "").rstrip("M")
|
||||
|
||||
# Find matching alternatives list
|
||||
alternatives = []
|
||||
for key, alts in self._SYMBOL_ALTERNATIVES.items():
|
||||
if base == key.upper() or mt5_sym.upper().rstrip("M") == key.upper():
|
||||
alternatives = alts
|
||||
break
|
||||
|
||||
# Fallback: try plain name with/without 'm' suffix
|
||||
if not alternatives:
|
||||
alternatives = [mt5_sym, mt5_sym.rstrip('m'), mt5_sym + 'm']
|
||||
|
||||
for alt in alternatives:
|
||||
info = mt5.symbol_info(alt)
|
||||
if info is not None:
|
||||
if not info.visible:
|
||||
mt5.symbol_select(alt, True)
|
||||
self.symbols[internal] = alt
|
||||
logger.info(f"Found alternative symbol: {alt} for {internal}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
async def _poll_cycle(self):
|
||||
"""Poll MT5 for new ticks and book data for all symbols."""
|
||||
mt5 = self._mt5
|
||||
|
||||
for internal, mt5_sym in self.symbols.items():
|
||||
try:
|
||||
# ── Poll ticks ──
|
||||
await self._poll_ticks(internal, mt5_sym)
|
||||
|
||||
# ── Poll orderbook (DOM) ──
|
||||
if self.enable_book and self.on_orderbook:
|
||||
await self._poll_book(internal, mt5_sym)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error polling {mt5_sym}: {e}", exc_info=True)
|
||||
|
||||
async def _poll_ticks(self, internal: str, mt5_sym: str):
|
||||
"""
|
||||
Poll new ticks since last check.
|
||||
MT5 ticks have flags indicating BUY or SELL direction.
|
||||
"""
|
||||
mt5 = self._mt5
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
if internal not in self._last_tick_time:
|
||||
# First poll — get ticks from last 2 seconds
|
||||
from_dt = now - timedelta(seconds=2)
|
||||
else:
|
||||
# Get ticks since last poll
|
||||
from_dt = datetime.fromtimestamp(
|
||||
self._last_tick_time[internal] / 1000.0, tz=timezone.utc
|
||||
)
|
||||
|
||||
# copy_ticks_from returns numpy array of ticks
|
||||
ticks_data = mt5.copy_ticks_from(mt5_sym, from_dt, 1000, mt5.COPY_TICKS_ALL)
|
||||
|
||||
if ticks_data is None or len(ticks_data) == 0:
|
||||
return
|
||||
|
||||
for t in ticks_data:
|
||||
# Skip if we already processed this tick
|
||||
tick_time_ms = int(t['time_msc'])
|
||||
if internal in self._last_tick_time and tick_time_ms <= self._last_tick_time[internal]:
|
||||
continue
|
||||
|
||||
# Determine aggressor side from tick flags
|
||||
flags = int(t['flags'])
|
||||
if flags & TICK_FLAG_BUY:
|
||||
side = Side.BUY
|
||||
elif flags & TICK_FLAG_SELL:
|
||||
side = Side.SELL
|
||||
else:
|
||||
# No buy/sell flag — use price vs previous bid/ask heuristic
|
||||
last_price = float(t['last'])
|
||||
bid = float(t['bid'])
|
||||
ask = float(t['ask'])
|
||||
if last_price >= ask:
|
||||
side = Side.BUY
|
||||
elif last_price <= bid:
|
||||
side = Side.SELL
|
||||
else:
|
||||
side = Side.BUY # Default to buy if ambiguous
|
||||
|
||||
# Use 'last' price (actual trade price) when available,
|
||||
# fall back to mid of bid/ask
|
||||
last_price = float(t['last'])
|
||||
if last_price == 0:
|
||||
last_price = (float(t['bid']) + float(t['ask'])) / 2.0
|
||||
|
||||
volume = float(t['volume_real']) if t['volume_real'] > 0 else float(t['volume'])
|
||||
if volume == 0:
|
||||
volume = 1.0 # Some brokers don't provide real volume
|
||||
|
||||
tick = Tick(
|
||||
timestamp_ms=tick_time_ms,
|
||||
price=last_price,
|
||||
size=volume,
|
||||
side=side,
|
||||
trade_id=f"mt5_{tick_time_ms}",
|
||||
)
|
||||
|
||||
if self.on_tick:
|
||||
await self.on_tick(internal, tick)
|
||||
|
||||
# Update last tick time
|
||||
self._last_tick_time[internal] = int(ticks_data[-1]['time_msc'])
|
||||
|
||||
async def _poll_book(self, internal: str, mt5_sym: str):
|
||||
"""
|
||||
Poll the order book (DOM / Market Depth) from MT5.
|
||||
Converts MT5 book entries to our OrderbookSnapshot model.
|
||||
"""
|
||||
mt5 = self._mt5
|
||||
book = mt5.market_book_get(mt5_sym)
|
||||
|
||||
if book is None or len(book) == 0:
|
||||
return
|
||||
|
||||
bids = []
|
||||
asks = []
|
||||
now_ms = int(time.time() * 1000)
|
||||
|
||||
for entry in book:
|
||||
level = OrderbookLevel(
|
||||
price=entry.price,
|
||||
quantity=float(entry.volume_real if entry.volume_real > 0 else entry.volume),
|
||||
)
|
||||
# MT5 book type: 1 = SELL (ask side), 2 = BUY (bid side)
|
||||
if entry.type == 1: # BOOK_TYPE_SELL
|
||||
asks.append(level)
|
||||
elif entry.type == 2: # BOOK_TYPE_BUY
|
||||
bids.append(level)
|
||||
|
||||
snapshot = OrderbookSnapshot(
|
||||
timestamp_ms=now_ms,
|
||||
bids=sorted(bids, key=lambda x: -x.price),
|
||||
asks=sorted(asks, key=lambda x: x.price),
|
||||
)
|
||||
|
||||
if self.on_orderbook:
|
||||
await self.on_orderbook(internal, snapshot)
|
||||
|
||||
# ── Historical Data Methods ──
|
||||
|
||||
async def download_historical_ticks(
|
||||
self,
|
||||
mt5_sym: str,
|
||||
from_date: datetime,
|
||||
to_date: datetime,
|
||||
) -> list[Tick]:
|
||||
"""
|
||||
Download historical ticks from MT5 for backtesting.
|
||||
Uses copy_ticks_range() which can return millions of ticks.
|
||||
"""
|
||||
mt5 = self._mt5
|
||||
if not self._initialized:
|
||||
self._initialize_mt5()
|
||||
|
||||
logger.info(f"Downloading ticks for {mt5_sym} from {from_date} to {to_date}")
|
||||
|
||||
ticks_data = mt5.copy_ticks_range(
|
||||
mt5_sym, from_date, to_date, mt5.COPY_TICKS_ALL
|
||||
)
|
||||
|
||||
if ticks_data is None or len(ticks_data) == 0:
|
||||
logger.warning(f"No ticks returned for {mt5_sym}")
|
||||
return []
|
||||
|
||||
ticks = []
|
||||
for t in ticks_data:
|
||||
flags = int(t['flags'])
|
||||
if flags & TICK_FLAG_BUY:
|
||||
side = Side.BUY
|
||||
elif flags & TICK_FLAG_SELL:
|
||||
side = Side.SELL
|
||||
else:
|
||||
last_price = float(t['last'])
|
||||
bid = float(t['bid'])
|
||||
ask = float(t['ask'])
|
||||
side = Side.BUY if last_price >= ask else Side.SELL
|
||||
|
||||
last_price = float(t['last'])
|
||||
if last_price == 0:
|
||||
last_price = (float(t['bid']) + float(t['ask'])) / 2.0
|
||||
|
||||
volume = float(t['volume_real']) if t['volume_real'] > 0 else float(t['volume'])
|
||||
if volume == 0:
|
||||
volume = 1.0
|
||||
|
||||
ticks.append(Tick(
|
||||
timestamp_ms=int(t['time_msc']),
|
||||
price=last_price,
|
||||
size=volume,
|
||||
side=side,
|
||||
trade_id=f"mt5_{t['time_msc']}",
|
||||
))
|
||||
|
||||
logger.info(f"Downloaded {len(ticks)} ticks for {mt5_sym}")
|
||||
return ticks
|
||||
|
||||
async def download_historical_candles(
|
||||
self,
|
||||
mt5_sym: str,
|
||||
timeframe: int, # MT5 timeframe constant (e.g., mt5.TIMEFRAME_M1)
|
||||
from_date: datetime,
|
||||
to_date: datetime,
|
||||
) -> list[Candle]:
|
||||
"""
|
||||
Download historical OHLCV bars from MT5.
|
||||
Note: MT5 bars don't have buy/sell volume split — only total.
|
||||
"""
|
||||
mt5 = self._mt5
|
||||
if not self._initialized:
|
||||
self._initialize_mt5()
|
||||
|
||||
rates = mt5.copy_rates_range(mt5_sym, timeframe, from_date, to_date)
|
||||
|
||||
if rates is None or len(rates) == 0:
|
||||
logger.warning(f"No bars returned for {mt5_sym}")
|
||||
return []
|
||||
|
||||
candles = []
|
||||
for r in rates:
|
||||
candles.append(Candle(
|
||||
timestamp_ms=int(r['time']) * 1000,
|
||||
open=float(r['open']),
|
||||
high=float(r['high']),
|
||||
low=float(r['low']),
|
||||
close=float(r['close']),
|
||||
volume=float(r['real_volume'] if r['real_volume'] > 0 else r['tick_volume']),
|
||||
buy_volume=0.0, # MT5 bars don't split buy/sell
|
||||
sell_volume=0.0,
|
||||
tick_count=int(r['tick_volume']),
|
||||
))
|
||||
|
||||
logger.info(f"Downloaded {len(candles)} bars for {mt5_sym}")
|
||||
return candles
|
||||
|
||||
def get_symbol_info(self, mt5_sym: str) -> Optional[dict]:
|
||||
"""Get symbol properties from MT5."""
|
||||
mt5 = self._mt5
|
||||
info = mt5.symbol_info(mt5_sym)
|
||||
if info is None:
|
||||
return None
|
||||
return {
|
||||
"name": info.name,
|
||||
"description": info.description,
|
||||
"tick_size": info.trade_tick_size,
|
||||
"tick_value": info.trade_tick_value,
|
||||
"digits": info.digits,
|
||||
"spread": info.spread,
|
||||
"contract_size": info.trade_contract_size,
|
||||
"volume_min": info.volume_min,
|
||||
"volume_max": info.volume_max,
|
||||
"volume_step": info.volume_step,
|
||||
"currency_base": info.currency_base,
|
||||
"currency_profit": info.currency_profit,
|
||||
}
|
||||
|
||||
def list_available_symbols(self, filter_text: str = "") -> list[str]:
|
||||
"""List available symbols in MT5 matching a filter."""
|
||||
mt5 = self._mt5
|
||||
if filter_text:
|
||||
symbols = mt5.symbols_get(filter_text)
|
||||
else:
|
||||
symbols = mt5.symbols_get()
|
||||
if symbols is None:
|
||||
return []
|
||||
return [s.name for s in symbols]
|
||||
Reference in New Issue
Block a user