From b1d911c3a988cdad7699ac586e950a6f62930ea5 Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Thu, 2 Apr 2026 21:08:18 +0200 Subject: [PATCH] chore: Remove unused scripts - Removed apply_config.py (not used, .env is manual) - Removed start_trading.sh (too complex, interactive prompts) - Removed setup_predix_eurusd.sh (one-time setup, already done) Kept: - data_config.yaml (central configuration, German comments OK) - start_loop.sh (useful for 24/7 trading) Co-authored-by: Qwen-Coder --- apply_config.py | 91 ------------- setup_predix_eurusd.sh | 301 ----------------------------------------- start_trading.sh | 73 ---------- 3 files changed, 465 deletions(-) delete mode 100755 apply_config.py delete mode 100755 setup_predix_eurusd.sh delete mode 100755 start_trading.sh diff --git a/apply_config.py b/apply_config.py deleted file mode 100755 index b70c0f9f..00000000 --- a/apply_config.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -""" -Liest data_config.yaml und schreibt alle Werte in: -- .env (Zeiträume, Pfade) -- generate.py (Qlib Datengenerierung) -""" -import yaml -import re -from pathlib import Path - -CONFIG = Path(__file__).parent / "data_config.yaml" -ENV = Path(__file__).parent / ".env" -GENERATE = Path("/home/nico/miniconda3/envs/rdagent/lib/python3.10/site-packages/rdagent/scenarios/qlib/experiment/factor_data_template/generate.py") - -with open(CONFIG) as f: - cfg = yaml.safe_load(f) - -# --- .env updaten --- -env_text = ENV.read_text() - -replacements = { - r"QLIB_DATA_DIR=.*": f"QLIB_DATA_DIR={cfg['data_path'].replace('~', str(Path.home()))}", - r"QLIB_FREQ=.*": f"QLIB_FREQ={cfg['frequency']}", - r"QLIB_FACTOR_TRAIN_START=.*": f"QLIB_FACTOR_TRAIN_START={cfg['train_start']}", - r"QLIB_FACTOR_TRAIN_END=.*": f"QLIB_FACTOR_TRAIN_END={cfg['train_end']}", - r"QLIB_FACTOR_VALID_START=.*": f"QLIB_FACTOR_VALID_START={cfg['valid_start']}", - r"QLIB_FACTOR_VALID_END=.*": f"QLIB_FACTOR_VALID_END={cfg['valid_end']}", - r"QLIB_FACTOR_TEST_START=.*": f"QLIB_FACTOR_TEST_START={cfg['test_start']}", - r"QLIB_FACTOR_TEST_END=.*": f"QLIB_FACTOR_TEST_END={cfg['test_end']}", - r"QLIB_MODEL_TRAIN_START=.*": f"QLIB_MODEL_TRAIN_START={cfg['train_start']}", - r"QLIB_MODEL_TRAIN_END=.*": f"QLIB_MODEL_TRAIN_END={cfg['train_end']}", - r"QLIB_MODEL_VALID_START=.*": f"QLIB_MODEL_VALID_START={cfg['valid_start']}", - r"QLIB_MODEL_VALID_END=.*": f"QLIB_MODEL_VALID_END={cfg['valid_end']}", - r"QLIB_MODEL_TEST_START=.*": f"QLIB_MODEL_TEST_START={cfg['test_start']}", - r"QLIB_MODEL_TEST_END=.*": f"QLIB_MODEL_TEST_END={cfg['test_end']}", - r"QLIB_QUANT_TRAIN_START=.*": f"QLIB_QUANT_TRAIN_START={cfg['train_start']}", - r"QLIB_QUANT_TRAIN_END=.*": f"QLIB_QUANT_TRAIN_END={cfg['train_end']}", - r"QLIB_QUANT_VALID_START=.*": f"QLIB_QUANT_VALID_START={cfg['valid_start']}", - r"QLIB_QUANT_VALID_END=.*": f"QLIB_QUANT_VALID_END={cfg['valid_end']}", - r"QLIB_QUANT_TEST_START=.*": f"QLIB_QUANT_TEST_START={cfg['test_start']}", - r"QLIB_QUANT_TEST_END=.*": f"QLIB_QUANT_TEST_END={cfg['test_end']}", -} - -for pattern, replacement in replacements.items(): - env_text = re.sub(pattern, replacement, env_text) - -ENV.write_text(env_text) -print("✓ .env aktualisiert") - -# --- generate.py updaten --- -data_path = cfg['data_path'] -freq = cfg['frequency'] -train_start = cfg['train_start'] -test_end = cfg['test_end'] -valid_start = cfg['valid_start'] -cols = str(cfg['columns']) - -generate_text = f'''import qlib -import pandas as pd - -qlib.init(provider_uri="{data_path}", freq="{freq}") - -from qlib.data import D - -instruments = D.instruments(market="all") -fields = {cols} - -data = ( - D.features(instruments, fields, freq="{freq}") - .swaplevel() - .sort_index() - .loc["{train_start}":] - .sort_index() -) -data.to_hdf("./daily_pv_all.h5", key="data") - -data_debug = ( - D.features(instruments, fields, start_time="{valid_start}", end_time="{test_end}", freq="{freq}") - .swaplevel() - .sort_index() -) -data_debug.to_hdf("./daily_pv_debug.h5", key="data") -''' - -GENERATE.write_text(generate_text) -print("✓ generate.py aktualisiert") -print(f"\nKonfiguration angewendet:") -print(f" Instrument: {cfg['instrument']}") -print(f" Frequenz: {cfg['frequency']}") -print(f" Train: {cfg['train_start']} → {cfg['train_end']}") -print(f" Test: {cfg['test_start']} → {cfg['test_end']}") diff --git a/setup_predix_eurusd.sh b/setup_predix_eurusd.sh deleted file mode 100755 index c2ec7178..00000000 --- a/setup_predix_eurusd.sh +++ /dev/null @@ -1,301 +0,0 @@ -#!/bin/bash -# ============================================================================= -# setup_predix_eurusd.sh -# Sets up Predix for EURUSD 15min trading -# Usage: bash setup_predix_eurusd.sh -# ============================================================================= - -set -e # Exit on error - -PREDIX_DIR="$HOME/Predix" -CSV_SOURCE="$HOME/Downloads/eurusd_data.csv" -DATA_DIR="$PREDIX_DIR/git_ignore_folder/eurusd_data" -QLIB_DIR="$HOME/.qlib/qlib_data/eurusd_data" - -echo "========================================" -echo " Predix EURUSD Setup" -echo "========================================" - -# ─── 1. Check prerequisites ─────────────────────────────────────────────── -echo "" -echo "[1/7] Checking prerequisites..." - -if [ ! -d "$PREDIX_DIR" ]; then - echo "ERROR: $PREDIX_DIR not found!" - exit 1 -fi - -if [ ! -f "$CSV_SOURCE" ]; then - echo "ERROR: $CSV_SOURCE not found!" - echo "Please place eurusd_data.csv in ~/Downloads/" - exit 1 -fi - -echo "✓ Predix found: $PREDIX_DIR" -echo "✓ CSV found: $CSV_SOURCE" - -# ─── 2. Create directory structure ───────────────────────────────────────── -echo "" -echo "[2/7] Creating directory structure..." - -mkdir -p "$DATA_DIR" -mkdir -p "$QLIB_DIR/calendars" -mkdir -p "$QLIB_DIR/instruments" -mkdir -p "$QLIB_DIR/features/eurusd" -mkdir -p "$PREDIX_DIR/git_ignore_folder/log" - -cp "$CSV_SOURCE" "$DATA_DIR/eurusd_data.csv" -echo "✓ CSV copied to $DATA_DIR" - -# ─── 3. Convert CSV to Qlib format ───────────────────────────────────────── -echo "" -echo "[3/7] Converting CSV to Qlib format..." - -python3 << 'PYEOF' -import pandas as pd -import numpy as np -from pathlib import Path -import os - -QLIB_DIR = Path(os.path.expanduser("~/.qlib/qlib_data/eurusd_data")) -CSV_PATH = Path(os.path.expanduser("~/Downloads/eurusd_data.csv")) - -# Load + sort -df = pd.read_csv(CSV_PATH, parse_dates=["datetime"]) -df = df.sort_values("datetime").reset_index(drop=True) -df.columns = [c.lower() for c in df.columns] - -print(f" Rows: {len(df):,} | Range: {df['datetime'].min().date()} -> {df['datetime'].max().date()}") - -# ── Calendar (all 15min timestamps) ──────────────────────────────────────── -cal = df["datetime"].dt.strftime("%Y-%m-%d %H:%M:%S") -cal_path = QLIB_DIR / "calendars" / "15min.txt" -cal.to_csv(cal_path, index=False, header=False) -print(f" ✓ Calendar: {len(cal)} entries -> {cal_path}") - -# ── Instruments (only EURUSD) ────────────────────────────────────────────── -inst_path = QLIB_DIR / "instruments" / "all.txt" -start = df["datetime"].min().strftime("%Y-%m-%d") -end = df["datetime"].max().strftime("%Y-%m-%d") -with open(inst_path, "w") as f: - f.write(f"EURUSD\t{start}\t{end}\n") -print(f" ✓ Instruments -> {inst_path}") - -# ── Features (Qlib binary format via CSV) ────────────────────────────────── -feat_dir = QLIB_DIR / "features" / "eurusd" -feat_dir.mkdir(parents=True, exist_ok=True) - -# Qlib expects: $open, $high, $low, $close, $volume -for col in ["open", "high", "low", "close", "volume"]: - out = feat_dir / f"{col}.day.bin" - # Qlib binary: float32 array - arr = df[col].astype("float32").values - arr.tofile(str(out).replace(".day.bin", f"_15min.bin")) - -# Also as simple CSV for direct access -df.to_csv(QLIB_DIR / "eurusd_15min.csv", index=False) -print(f" ✓ Features + CSV -> {feat_dir}") - -# ── Returns + technical features precomputation ──────────────────────────── -def ema(s, p): return s.ewm(span=p, adjust=False).mean() -def rsi(c, p=14): - d = c.diff() - g = d.clip(lower=0).ewm(span=p, adjust=False).mean() - l = (-d.clip(upper=0)).ewm(span=p, adjust=False).mean() - return 100 - 100/(1 + g/(l+1e-9)) - -feat = pd.DataFrame() -feat["datetime"] = df["datetime"] -feat["close"] = df["close"] -for n in [1,4,8,16,96]: - feat[f"ret_{n}"] = df["close"].pct_change(n) -feat["rsi_14"] = rsi(df["close"], 14) -feat["macd_hist"] = ema(df["close"],12) - ema(df["close"],26) -feat["hour"] = df["datetime"].dt.hour -feat["is_london"] = feat["hour"].isin([8,9,10,11]).astype(int) -feat["is_ny"] = feat["hour"].isin([13,14,15,16]).astype(int) -feat["adx_proxy"] = df["close"].rolling(14).std() / df["close"].rolling(96).std() - -feat.dropna(inplace=True) -feat.to_csv(QLIB_DIR / "eurusd_features.csv", index=False) -print(f" ✓ Features CSV: {len(feat):,} rows, {len(feat.columns)} columns") -print(" Done!") -PYEOF - -echo "✓ Qlib data converted" - -# ─── 4. Update .env ──────────────────────────────────────────────────────── -echo "" -echo "[4/7] Updating .env..." - -ENV_FILE="$PREDIX_DIR/.env" - -# Backup -cp "$ENV_FILE" "$ENV_FILE.backup_$(date +%Y%m%d_%H%M%S)" - -# Update QLIB_DATA_DIR to EURUSD -sed -i "s|QLIB_DATA_DIR=.*|QLIB_DATA_DIR=$QLIB_DIR|" "$ENV_FILE" - -# Fix LOG_PATH (was /home/nico/RD-Agent-Local/log) -sed -i "s|LOG_PATH=.*|LOG_PATH=$PREDIX_DIR/git_ignore_folder/log|" "$ENV_FILE" - -# Add EURUSD-specific vars (if not already present) -grep -q "EURUSD_DATA_PATH" "$ENV_FILE" || cat >> "$ENV_FILE" << 'ENVEOF' - -# ---------- EURUSD ---------- -EURUSD_DATA_PATH=/home/nico/.qlib/qlib_data/eurusd_data/eurusd_15min.csv -QLIB_FREQ=15min -QLIB_MARKET=eurusd -BACKTEST_START_TIME=2024-08-09 -BACKTEST_END_TIME=2026-03-20 -COST_RATE=0.00015 -ENVEOF - -echo "✓ .env updated (backup created)" - -# ─── 5. Adjust prompts for EURUSD ────────────────────────────────────────── -echo "" -echo "[5/7] Adjusting Qlib prompts for EURUSD..." - -PROMPT_FILE="$PREDIX_DIR/rdagent/app/qlib_rd_loop/prompts.yaml" -cp "$PROMPT_FILE" "${PROMPT_FILE}.backup_$(date +%Y%m%d_%H%M%S)" - -cat > "$PROMPT_FILE" << 'YAMLEOF' -hypothesis_generation: - system: |- - You are an expert quantitative researcher specialized in FX (foreign exchange) trading, - specifically EURUSD intraday strategies on 15-minute bars. - - EURUSD domain knowledge you must apply: - - London session (08:00-12:00 UTC): highest volatility, trending behavior — favor momentum strategies - - NY session (13:00-17:00 UTC): second volatility peak, also trending - - Asian session (00:00-07:00 UTC): low volatility, mean-reverting behavior - - London/NY overlap (13:00-17:00 UTC): strongest directional moves of the day - - Weekend gap risk: avoid holding positions after Friday 20:00 UTC - - Spread cost: ~1.5 bps per trade — strategies must minimize unnecessary entries - - EURUSD is mean-reverting on short windows (<1h), trending on longer (>4h) - - Key macro drivers: ECB/Fed rate decisions, NFP (first Friday of month), CPI releases - - Available model types you can propose: - - TimeSeries: LSTM, GRU, TCN (Temporal Convolutional Network), Transformer, PatchTST - - Tabular: XGBoost, LightGBM, RandomForest (on engineered features) - - Hybrid: CNN+LSTM, XGBoost+LSTM ensemble - - Statistical: Regime-switching (HMM), Kalman filter - - Available features in the dataset: - - OHLCV: open, high, low, close, volume (15min bars) - - Returns: ret_1, ret_4, ret_8, ret_16, ret_96 - - Technical: rsi_14, macd_hist, adx_14, atr_14, bb_pct, stoch_k, cci_14 - - Volatility: vol_real_4, vol_real_16, vol_ratio, zscore_ret_96 - - Time/Session: hour, is_london, is_ny, is_overlap, hour_sin, hour_cos - - Lags: rsi_14_lag1-8, macd_hist_lag1-8, bb_pct_lag1-8 - - Your hypothesis must: - 1. Specify which session(s) the strategy targets - 2. Name which model type to use and why it fits EURUSD - 3. Include a session filter (is_london / is_ny) - 4. Include a spread filter (only trade when expected |return| > 0.0003) - 5. Specify target: classification (fwd_sign_4) or regression (fwd_ret_4) - - Please ensure your response is in JSON format: - { - "hypothesis": "A clear and concise trading hypothesis for EURUSD 15min.", - "reason": "Detailed explanation including session, model choice, and expected edge.", - "model_type": "One of: TimeSeries / Tabular / XGBoost", - "target_session": "london / ny / asian / all", - "expected_arr_range": "e.g. 8-12%" - } - - user: |- - Previously tried approaches and their results: - {{ factor_descriptions }} - - Additional context: - {{ report_content }} - - Generate a NEW hypothesis that is meaningfully different from what has been tried. - Focus on approaches that have NOT been tested yet. - Target: beat current best ARR of 9.62%. -YAMLEOF - -echo "✓ prompts.yaml updated (backup created)" - -# ─── 6. Extend Model Coder Prompt ────────────────────────────────────────── -echo "" -echo "[6/7] Extending Model Coder prompts..." - -MODEL_PROMPT="$PREDIX_DIR/rdagent/components/coder/model_coder/prompts.yaml" -cp "$MODEL_PROMPT" "${MODEL_PROMPT}.backup_$(date +%Y%m%d_%H%M%S)" - -# Inject EURUSD session filter as comment in evolving_strategy block -python3 << 'PYEOF' -import re -from pathlib import Path - -path = Path("/home/nico/Predix/rdagent/components/coder/model_coder/prompts.yaml") -content = path.read_text() - -eurusd_note = """ - EURUSD-specific rules (ALWAYS apply these in generated code): - 1. Session filter: use is_london and is_ny columns — weight/filter signals to active sessions - 2. Spread filter: only generate signal when abs(predicted_return) > 0.0003 - 3. ADX regime: if adx_proxy > 1.2 use trend model; if adx_proxy < 0.8 use mean-reversion - 4. Weekend filter: zero out signals when dayofweek==4 and hour>=20 - 5. Max trade frequency: target <15 trades per day (avoid spread cost death) - 6. Supported model_type values: "Tabular", "TimeSeries", "XGBoost" -""" - -# Inject after the scenario line in evolving_strategy_model_coder -content = content.replace( - " Your code is expected to align the scenario in any form", - eurusd_note + "\n Your code is expected to align the scenario in any form" -) -path.write_text(content) -print(" ✓ Model coder prompt extended") -PYEOF - -echo "✓ Model coder prompt adjusted" - -# ─── 7. Git Commits ──────────────────────────────────────────────────────── -echo "" -echo "[7/7] Git Commits..." - -cd "$PREDIX_DIR" - -git add rdagent/app/qlib_rd_loop/prompts.yaml -git commit -m "feat: EURUSD 15min prompts - session filter, FX domain knowledge - -- Add London/NY/Asian session awareness to hypothesis generation -- Add model type suggestions: LSTM, GRU, TCN, Transformer, XGBoost, LightGBM -- Add spread filter (1.5 bps) and ADX regime detection -- Target: beat current best ARR of 9.62%" - -git add rdagent/components/coder/model_coder/prompts.yaml -git commit -m "feat: inject EURUSD trading rules into model coder - -- Session filter (is_london, is_ny) -- Spread filter (|return| > 0.0003) -- Weekend position close -- Max trade frequency guidance" - -git add .env 2>/dev/null || true -echo " (Note: .env not committed - contains API keys)" - -echo "" -echo "========================================" -echo " Setup completed!" -echo "========================================" -echo "" -echo "Next steps:" -echo "" -echo " 1. Start:" -echo " cd ~/Predix && rdagent fin_quant" -echo "" -echo " 2. Dashboard (in second terminal):" -echo " cd ~/Predix && rdagent server_ui --port 19899" -echo " → http://localhost:19899" -echo "" -echo " 3. Logs:" -echo " tail -f $PREDIX_DIR/git_ignore_folder/log/*.log" -echo "" -echo "Backup files (.backup_*) can be deleted after successful testing." diff --git a/start_trading.sh b/start_trading.sh deleted file mode 100755 index 99826325..00000000 --- a/start_trading.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Start EURUSD Trading mit automatischem Dashboard -# Verwendung: ./start_trading.sh - -set -e - -echo "============================================================" -echo " Predix EURUSD Trading - Start mit Dashboard" -echo "============================================================" -echo "" - -# Conda Environment aktivieren -if [ -f ~/miniconda3/etc/profile.d/conda.sh ]; then - source ~/miniconda3/etc/profile.d/conda.sh - conda activate rdagent - echo "✓ Conda Environment 'rdagent' aktiviert" -else - echo "⚠️ Conda nicht gefunden, versuche mit system Python..." -fi - -echo "" -echo "Verwendung:" -echo " Option 1: Web Dashboard (empfohlen)" -echo " rdagent fin_quant --with-dashboard" -echo "" -echo " Option 2: CLI Dashboard (Terminal UI)" -echo " rdagent fin_quant --cli-dashboard" -echo "" -echo " Option 3: Beide Dashboards" -echo " rdagent fin_quant -d -c" -echo "" -echo " Option 4: Endlosschleife mit Auto-Restart" -echo " ./start_loop.sh" -echo "" -echo "============================================================" - -# Dashboard API im Hintergrund starten (optional) -read -p "Dashboard im Hintergrund starten? (y/n): " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]]; then - echo "" - echo "🚀 Starte Dashboard API..." - cd /home/nico/Predix - nohup python web/dashboard_api.py > /tmp/dashboard.log 2>&1 & - DASHBOARD_PID=$! - echo "✓ Dashboard API gestartet (PID: $DASHBOARD_PID)" - echo "" - echo "📊 Dashboard URL: http://localhost:5000/dashboard.html" - echo " Dashboard Log: /tmp/dashboard.log" - echo "" - - # Cleanup Funktion - cleanup() { - echo "" - echo "⏹️ Stoppe Dashboard (PID: $DASHBOARD_PID)..." - kill $DASHBOARD_PID 2>/dev/null || true - echo "✓ Gestoppt" - exit 0 - } - - # Trap für Ctrl+C - trap cleanup SIGINT SIGTERM -fi - -# RD-Agent fin_quant starten -echo "🔄 Starte EURUSD Trading-Agent..." -echo "" -dotenv run -- rdagent fin_quant - -# Cleanup wenn fertig -if [[ $REPLY =~ ^[Yy]$ ]]; then - cleanup -fi