XauBot/backtests/ml_v2/ml_v2_target.py

"""
ML V2 Target Builder
====================
Better target variables to reduce noise and improve ML predictive power.

Problem with current target (src/feature_eng.py::create_target):
- Predicts 1-bar ahead price movement with threshold=0 → too noisy
- Captures noise, not tradeable moves

Solutions:
A. Multi-bar + ATR threshold (primary)
B. 3-class target (BUY/SELL/HOLD)
C. Baseline (current method for comparison)
"""

import polars as pl
import numpy as np
from typing import Tuple, Optional
from loguru import logger


class TargetBuilder:
    """
    Builder for improved target variables.

    Key improvements:
    1. Multi-bar lookahead (reduces noise)
    2. ATR-based threshold (filters small moves)
    3. 3-class option (explicit HOLD class)
    """

    def __init__(self):
        """Initialize target builder."""
        pass

    def create_multi_bar_target(
        self,
        df: pl.DataFrame,
        lookahead: int = 3,
        threshold_atr_mult: float = 0.3,
    ) -> pl.DataFrame:
        """
        Create multi-bar binary target with ATR-based threshold.

        Logic:
        - Look at next `lookahead` bars (default 3 = 45 min on M15)
        - Find max close in that window
        - UP (1) if: max_future_close - current_close > threshold * ATR
        - DOWN (0) if: current_close - min_future_close > threshold * ATR
        - Filtered out: moves smaller than threshold (noise)

        Why this works:
        - Multi-bar: reduces bar-to-bar noise
        - ATR threshold: filters moves too small to trade profitably
        - For XAUUSD @ ATR ~$12, threshold=0.3 means $3.6 minimum move

        Args:
            df: DataFrame with OHLCV and ATR
            lookahead: Number of bars to look ahead (default 3)
            threshold_atr_mult: ATR multiplier for minimum move (default 0.3)

        Returns:
            DataFrame with multi_bar_target column (1=UP, 0=DOWN, null=HOLD/filtered)
        """
        # Ensure ATR exists
        if "atr" not in df.columns:
            logger.error("ATR column required for multi-bar target")
            return df

        # Calculate future max/min close in lookahead window
        df = df.with_columns([
            # Rolling max of future closes (reverse window)
            pl.col("close").shift(-lookahead).alias("_future_start_close"),
            pl.col("close").shift(-1).alias("_future_1"),
            pl.col("close").shift(-2).alias("_future_2") if lookahead >= 2 else pl.col("close").alias("_future_2"),
            pl.col("close").shift(-3).alias("_future_3") if lookahead >= 3 else pl.col("close").alias("_future_3"),
        ])

        # Get max and min across future window
        if lookahead == 1:
            df = df.with_columns([
                pl.col("_future_1").alias("_max_future_close"),
                pl.col("_future_1").alias("_min_future_close"),
            ])
        elif lookahead == 2:
            df = df.with_columns([
                pl.max_horizontal("_future_1", "_future_2").alias("_max_future_close"),
                pl.min_horizontal("_future_1", "_future_2").alias("_min_future_close"),
            ])
        else:  # lookahead >= 3
            df = df.with_columns([
                pl.max_horizontal("_future_1", "_future_2", "_future_3").alias("_max_future_close"),
                pl.min_horizontal("_future_1", "_future_2", "_future_3").alias("_min_future_close"),
            ])

        # Calculate move sizes
        df = df.with_columns([
            (pl.col("_max_future_close") - pl.col("close")).alias("_up_move"),
            (pl.col("close") - pl.col("_min_future_close")).alias("_down_move"),
        ])

        # Calculate threshold (ATR * multiplier)
        df = df.with_columns([
            (pl.col("atr") * threshold_atr_mult).alias("_threshold"),
        ])

        # Create target:
        # - UP (1): if up_move > threshold AND up_move > down_move
        # - DOWN (0): if down_move > threshold AND down_move > up_move
        # - null: otherwise (filtered as noise)
        df = df.with_columns([
            pl.when(
                (pl.col("_up_move") > pl.col("_threshold")) &
                (pl.col("_up_move") > pl.col("_down_move"))
            )
                .then(1)
                .when(
                    (pl.col("_down_move") > pl.col("_threshold")) &
                    (pl.col("_down_move") > pl.col("_up_move"))
                )
                .then(0)
                .otherwise(None)  # Filter out noise
                .alias("multi_bar_target")
                .cast(pl.Int32),
        ])

        # Drop temporary columns
        df = df.drop([
            "_future_start_close", "_future_1", "_future_2", "_future_3",
            "_max_future_close", "_min_future_close",
            "_up_move", "_down_move", "_threshold"
        ])

        # Log statistics
        total = len(df)
        ups = df.filter(pl.col("multi_bar_target") == 1).height
        downs = df.filter(pl.col("multi_bar_target") == 0).height
        filtered = total - ups - downs

        logger.info(
            f"Multi-bar target (lookahead={lookahead}, threshold={threshold_atr_mult}*ATR): "
            f"{ups} UP ({ups/total*100:.1f}%), "
            f"{downs} DOWN ({downs/total*100:.1f}%), "
            f"{filtered} filtered ({filtered/total*100:.1f}%)"
        )

        return df

    def create_3class_target(
        self,
        df: pl.DataFrame,
        lookahead: int = 3,
        threshold_atr_mult: float = 0.3,
    ) -> pl.DataFrame:
        """
        Create 3-class target: BUY (0), SELL (1), HOLD (2).

        Same logic as multi_bar_target but keeps HOLD as explicit class
        instead of filtering it out.

        Use with XGBoost multi:softprob objective.

        Args:
            df: DataFrame with OHLCV and ATR
            lookahead: Number of bars to look ahead
            threshold_atr_mult: ATR multiplier for threshold

        Returns:
            DataFrame with target_3class column (0=BUY, 1=SELL, 2=HOLD)
        """
        # Reuse multi_bar logic but map null to HOLD (2)
        df = self.create_multi_bar_target(df, lookahead, threshold_atr_mult)

        # Convert to 3-class: 0=BUY, 1=SELL, 2=HOLD
        df = df.with_columns([
            pl.when(pl.col("multi_bar_target") == 1)
                .then(0)  # UP → BUY
                .when(pl.col("multi_bar_target") == 0)
                .then(1)  # DOWN → SELL
                .otherwise(2)  # null → HOLD
                .alias("target_3class")
                .cast(pl.Int32),
        ])

        # Log distribution
        total = len(df)
        buys = df.filter(pl.col("target_3class") == 0).height
        sells = df.filter(pl.col("target_3class") == 1).height
        holds = df.filter(pl.col("target_3class") == 2).height

        logger.info(
            f"3-class target: "
            f"{buys} BUY ({buys/total*100:.1f}%), "
            f"{sells} SELL ({sells/total*100:.1f}%), "
            f"{holds} HOLD ({holds/total*100:.1f}%)"
        )

        return df

    def create_baseline_target(
        self,
        df: pl.DataFrame,
        lookahead: int = 1,
        threshold: float = 0.0,
    ) -> pl.DataFrame:
        """
        Create baseline target (mirrors current FeatureEngineer.create_target()).

        For comparison with V1 model.

        Args:
            df: DataFrame with price data
            lookahead: Bars to look ahead (default 1)
            threshold: Minimum return threshold (default 0.0)

        Returns:
            DataFrame with baseline_target column
        """
        df = df.with_columns([
            pl.col("close").shift(-lookahead).alias("_future_close"),
        ])

        df = df.with_columns([
            ((pl.col("_future_close") / pl.col("close") - 1) > threshold)
                .cast(pl.Int32)
                .alias("baseline_target"),
        ])

        df = df.drop(["_future_close"])

        ups = df.filter(pl.col("baseline_target") == 1).height
        total = len(df)
        logger.info(
            f"Baseline target (lookahead={lookahead}, threshold={threshold}): "
            f"{ups} UP ({ups/total*100:.1f}%), {total-ups} DOWN ({(total-ups)/total*100:.1f}%)"
        )

        return df

    def create_all_targets(
        self,
        df: pl.DataFrame,
        lookahead: int = 3,
        threshold_atr_mult: float = 0.3,
    ) -> pl.DataFrame:
        """
        Create all target variants for comparison.

        Args:
            df: DataFrame with OHLCV and ATR
            lookahead: Lookahead for multi-bar targets
            threshold_atr_mult: ATR threshold multiplier

        Returns:
            DataFrame with all target columns added
        """
        logger.info(f"Creating all target variants (lookahead={lookahead}, threshold={threshold_atr_mult}*ATR)...")

        # Baseline (V1)
        df = self.create_baseline_target(df, lookahead=1, threshold=0.0)

        # Multi-bar binary
        df = self.create_multi_bar_target(df, lookahead=lookahead, threshold_atr_mult=threshold_atr_mult)

        # 3-class
        df = self.create_3class_target(df, lookahead=lookahead, threshold_atr_mult=threshold_atr_mult)

        return df


if __name__ == "__main__":
    # Test target builder
    import numpy as np
    from datetime import datetime, timedelta

    # Create synthetic OHLCV data with trend
    np.random.seed(42)
    n = 500

    base_price = 2000.0
    # Add uptrend
    trend = np.linspace(0, 50, n)
    noise = np.random.randn(n) * 5
    prices = base_price + trend + noise

    # Create ATR (realistic for XAUUSD)
    atr_values = np.random.uniform(10, 14, n)

    df = pl.DataFrame({
        "time": [datetime.now() - timedelta(minutes=15*i) for i in range(n-1, -1, -1)],
        "open": prices,
        "high": prices + np.abs(np.random.randn(n)) * 2,
        "low": prices - np.abs(np.random.randn(n)) * 2,
        "close": prices + np.random.randn(n) * 1,
        "volume": np.random.randint(1000, 10000, n),
        "atr": atr_values,
    })

    # Build targets
    builder = TargetBuilder()
    df = builder.create_all_targets(df, lookahead=3, threshold_atr_mult=0.3)

    # Show comparison
    print("\n=== Target Builder Test ===")
    print(f"Total bars: {len(df)}")
    print(f"\nTarget columns created:")
    print(f"  - baseline_target (1-bar, threshold=0)")
    print(f"  - multi_bar_target (3-bar, 0.3*ATR threshold)")
    print(f"  - target_3class (3-class version)")

    # Sample
    print("\n=== Sample Data (Last 10 Rows) ===")
    cols = ["time", "close", "atr", "baseline_target", "multi_bar_target", "target_3class"]
    print(df.select([c for c in cols if c in df.columns]).tail(10))

    # Class distribution comparison
    print("\n=== Class Distribution ===")

    baseline_up = df.filter(pl.col("baseline_target") == 1).height
    baseline_down = len(df) - baseline_up
    print(f"Baseline: {baseline_up} UP, {baseline_down} DOWN")

    multi_up = df.filter(pl.col("multi_bar_target") == 1).height
    multi_down = df.filter(pl.col("multi_bar_target") == 0).height
    multi_filtered = len(df) - multi_up - multi_down
    print(f"Multi-bar: {multi_up} UP, {multi_down} DOWN, {multi_filtered} filtered")

    class3_buy = df.filter(pl.col("target_3class") == 0).height
    class3_sell = df.filter(pl.col("target_3class") == 1).height
    class3_hold = df.filter(pl.col("target_3class") == 2).height
    print(f"3-class: {class3_buy} BUY, {class3_sell} SELL, {class3_hold} HOLD")