""" Trading Environment for RL Agents. Gym-compatible environment for training RL trading agents. Supports single-asset (EUR/USD) trading with technical indicators and portfolio state as observations. Inspired by common RL trading environment patterns, implemented from scratch for NexQuant. """ import gymnasium as gym import numpy as np import pandas as pd from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple @dataclass class TradingState: """Current state of the trading environment.""" position: float = 0.0 # Current position (-1 to 1 for short/long) cash: float = 100000.0 # Available cash equity: float = 100000.0 # Total equity (cash + position value) entry_price: float = 0.0 # Entry price of current position step: int = 0 # Current time step holdings_history: List[float] = field(default_factory=list) # Historical holdings class TradingEnv(gym.Env): """ Trading environment for RL agents. State: price history, technical indicators, portfolio state Action: position size (-1 to 1, short to long) Reward: risk-adjusted return with transaction costs Parameters ---------- prices : np.ndarray Array of asset prices (1D) indicators : np.ndarray, optional Array of technical indicators (n_steps x n_features) initial_balance : float Starting cash balance transaction_cost : float Cost per unit of position change (e.g. 0.0001 = 1 basis point) window_size : int Lookback window for observations max_steps : int Maximum steps per episode Examples -------- >>> prices = np.random.randn(1000) + 100 >>> env = TradingEnv(prices, window_size=30) >>> obs, info = env.reset() >>> action = np.array([0.5]) >>> obs, reward, terminated, truncated, info = env.step(action) """ metadata = {"render_modes": ["human"]} def __init__( self, prices: np.ndarray, indicators: Optional[np.ndarray] = None, initial_balance: float = 100000.0, transaction_cost: float = 0.0001, window_size: int = 60, max_steps: int = 10000, ) -> None: super().__init__() self.prices = np.asarray(prices, dtype=np.float64) self.indicators = np.asarray(indicators, dtype=np.float32) if indicators is not None else None self.initial_balance = initial_balance self.transaction_cost = transaction_cost self.window_size = window_size self.max_steps = max_steps # Environment state (reset each episode) self.current_step: int = 0 self.balance: float = initial_balance self.position: float = 0.0 self.entry_price: float = 0.0 self.equity_history: List[float] = [initial_balance] self.trades: List[Dict] = [] # Observation space: window_size x (1 + n_indicators + 3) n_indicators = self.indicators.shape[1] if self.indicators is not None else 0 obs_dim = window_size * (1 + n_indicators + 3) # +3 for position, pnl, step self.observation_space = gym.spaces.Box( low=-np.inf, high=np.inf, shape=(obs_dim,), dtype=np.float32 ) # Action space: continuous position from -1 (short) to 1 (long) self.action_space = gym.spaces.Box(low=-1.0, high=1.0, shape=(1,), dtype=np.float32) # ------------------------------------------------------------------ # # Observation # ------------------------------------------------------------------ # def _get_observation(self) -> np.ndarray: """Build observation vector from current state. Returns ------- np.ndarray Flattened observation vector of shape (window_size * (1 + n_ind + 3),) """ start_idx = self.current_step end_idx = start_idx + self.window_size # Price window price_window = self.prices[start_idx:end_idx] # Normalize prices relative to first value if len(price_window) == self.window_size and price_window[0] != 0: price_norm = price_window / price_window[0] - 1.0 else: price_norm = np.zeros(self.window_size) if len(price_window) > 0 and price_window[0] != 0: valid_len = len(price_window) price_norm[-valid_len:] = price_window / price_window[0] - 1.0 # Indicators window if self.indicators is not None: indicators_window = self.indicators[start_idx:end_idx] if len(indicators_window) < self.window_size: padded = np.zeros((self.window_size, self.indicators.shape[1]), dtype=np.float32) padded[-len(indicators_window):] = indicators_window indicators_window = padded else: indicators_window = np.zeros((self.window_size, 0), dtype=np.float32) # Portfolio state features (repeated for each timestep in window) current_price = float(self.prices[min(self.current_step, len(self.prices) - 1)]) position_feature = np.full(self.window_size, self.position, dtype=np.float32) if self.entry_price > 0 and self.position != 0: pnl = (current_price - self.entry_price) / self.entry_price * np.sign(self.position) else: pnl = 0.0 pnl_feature = np.full(self.window_size, pnl, dtype=np.float32) step_feature = np.full(self.window_size, self.current_step / max(self.max_steps, 1), dtype=np.float32) # Combine all features observation = np.column_stack( [ price_norm.astype(np.float32), indicators_window, position_feature.reshape(-1, 1), pnl_feature.reshape(-1, 1), step_feature.reshape(-1, 1), ] ) return observation.flatten().astype(np.float32) # ------------------------------------------------------------------ # # Reward # ------------------------------------------------------------------ # def _calculate_reward(self, new_equity: float, old_equity: float) -> float: """Calculate risk-adjusted reward with penalties. Parameters ---------- new_equity : float Equity after the step old_equity : float Equity before the step Returns ------- float Reward value """ # Simple return if old_equity > 0: simple_return = (new_equity - old_equity) / old_equity else: simple_return = 0.0 # Transaction cost penalty position_change = abs(self.position) if len(self.trades) > 0 else 0.0 cost_penalty = -position_change * self.transaction_cost # Drawdown penalty if len(self.equity_history) > 1: max_equity = max(self.equity_history) if max_equity > 0: drawdown = (max_equity - new_equity) / max_equity else: drawdown = 0.0 drawdown_penalty = -drawdown * 2.0 # Heavy penalty for drawdowns else: drawdown_penalty = 0.0 return float(simple_return + cost_penalty + drawdown_penalty) # ------------------------------------------------------------------ # # Gym API: reset / step # ------------------------------------------------------------------ # def reset( self, seed: Optional[int] = None, options: Optional[dict] = None ) -> Tuple[np.ndarray, dict]: """Reset environment to initial state. Parameters ---------- seed : int, optional Random seed for reproducibility options : dict, optional Additional reset options (unused) Returns ------- observation : np.ndarray Initial observation info : dict Additional info """ super().reset(seed=seed) self.current_step = 0 self.balance = self.initial_balance self.position = 0.0 self.entry_price = 0.0 self.equity_history = [self.initial_balance] self.trades = [] return self._get_observation(), {} def step( self, action: np.ndarray ) -> Tuple[np.ndarray, float, bool, bool, dict]: """Execute one time step in the environment. Parameters ---------- action : np.ndarray Target position size in [-1, 1] Returns ------- observation : np.ndarray Next observation reward : float Step reward terminated : bool Whether episode ended due to terminal condition truncated : bool Whether episode ended due to time limit info : dict Additional info """ current_price = float(self.prices[min(self.current_step, len(self.prices) - 1)]) old_equity = self.balance + self.position * current_price # Execute action target_position = float(np.clip(action[0], -1.0, 1.0)) # Calculate transaction costs when position changes significantly if abs(target_position - self.position) > 0.01: cost = ( abs(target_position - self.position) * current_price * self.transaction_cost ) self.balance -= cost self.trades.append( {"step": self.current_step, "action": target_position, "cost": cost} ) # Update position self.position = target_position if abs(self.position) > 0.01: self.entry_price = current_price # Advance time self.current_step += 1 # Calculate new equity if self.current_step < len(self.prices): next_price = float(self.prices[self.current_step]) new_equity = self.balance + self.position * next_price else: new_equity = self.balance self.equity_history.append(new_equity) # Reward reward = self._calculate_reward(new_equity, old_equity) # Termination conditions terminated = new_equity < self.initial_balance * 0.5 # Liquidation truncated = self.current_step >= min(self.max_steps, len(self.prices) - 1) observation = self._get_observation() info = { "equity": new_equity, "balance": self.balance, "position": self.position, "trades_count": len(self.trades), "return": (new_equity - self.initial_balance) / self.initial_balance if self.initial_balance > 0 else 0.0, } return observation, reward, terminated, truncated, info # ------------------------------------------------------------------ # # Utility # ------------------------------------------------------------------ # def get_equity_curve(self) -> np.ndarray: """Return equity curve for the current episode. Returns ------- np.ndarray Equity values over time """ return np.array(self.equity_history, dtype=np.float64) def get_trade_log(self) -> List[Dict]: """Return trade log for the current episode. Returns ------- list[dict] List of trade records """ return list(self.trades)