fix: mt5_parser, trade analysis charts and batch backtest

mt5_parser:
- Remove deprecated infer_datetime_format (removed in pandas 2.0)
- Add ISO datetime format fallback for CSV imports (YYYY-MM-DD)
- Fix missing comment column in _enrich causing KeyError on plain CSVs

Trade Analysis:
- Add daily P&L bar chart below equity curve
- Add drawdown panel below daily P&L
- Fix NaT in date filter when open_time has null values

Batch Backtest:
- Fix read_utf16/write_utf16 to detect and preserve original encoding
- Fix _clear_use_default_flags to clear bit 2 on all parameters
- Fix snapshot approach for report finding — before/after file scan
- Fix cleanup to remove all new htm files after each run
- Fix report selection to prefer named file over newest fallback
- Apply all encoding and flag fixes to CLI script mt5_batch_backtest.py

mt5_parser.py

@@ -67,6 +67,8 @@ def _enrich(df):
         )
     df['win']      = df['net_profit'] > 0
     df['type']     = df['type'].str.lower().str.strip()
+    if 'comment' not in df.columns:
+        df['comment'] = ''
     df['strategy'] = df['comment'].apply(extract_strategy)
     # Normalise symbol — strip .a suffix for display matching
     df['symbol_base'] = df['symbol'].str.replace(r'\.[a-z]+$', '', regex=True).str.upper()
@@ -256,12 +258,15 @@ def parse_quant_csv(file_bytes):
 
     df = pd.DataFrame(result)
 
-    # Parse datetimes — QA uses DD.MM.YYYY HH:MM:SS
+    # Parse datetimes — try QA format first (DD.MM.YYYY), then ISO (YYYY-MM-DD)
     for col in ['open_time', 'close_time']:
         if col in df.columns:
-            df[col] = pd.to_datetime(df[col], format='%d.%m.%Y %H:%M:%S', errors='coerce')
+            parsed = pd.to_datetime(df[col], format='%d.%m.%Y %H:%M:%S', errors='coerce')
-            if df[col].isna().all():
+            if parsed.isna().all():
-                df[col] = pd.to_datetime(df[col], infer_datetime_format=True, errors='coerce')
+                parsed = pd.to_datetime(df[col], format='%Y-%m-%d %H:%M:%S', errors='coerce')
+            if parsed.isna().all():
+                parsed = pd.to_datetime(df[col], errors='coerce')
+            df[col] = parsed
 
     # QA comm_swap is combined — split evenly as approximation if no separate swap
     if 'commission' in df.columns and 'swap' not in df.columns:

trade_analysis.py

@@ -1,333 +0,0 @@
-"""
-pages/trade_analysis.py
-=======================
-MT5 Trade Analysis page — migrated from main dashboard.
-"""
-
-import streamlit as st
-import plotly.graph_objects as go
-import sys, os
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from mt5_parser import detect_and_parse, calc_stats
-
-
-def render():
-    st.title("📊 Trade Analysis")
-
-    # ── Session state ─────────────────────────────────────────────────────────
-    if 'ta_df' not in st.session_state:
-        st.session_state['ta_df']     = None
-        st.session_state['ta_format'] = None
-
-    # ── File upload ───────────────────────────────────────────────────────────
-    col1, col2 = st.columns([4, 1])
-    with col1:
-        uploaded = st.file_uploader(
-            "Upload MT5 Report (HTM/HTML) or Quant Analyzer CSV",
-            type=['html', 'htm', 'csv'],
-            key='ta_upload'
-        )
-    with col2:
-        st.markdown("<br>", unsafe_allow_html=True)
-        if st.button("🗑 Clear", key='ta_clear'):
-            st.session_state['ta_df']     = None
-            st.session_state['ta_format'] = None
-            st.rerun()
-
-    if uploaded:
-        df, fmt = detect_and_parse(uploaded.read(), uploaded.name)
-        if df is not None:
-            st.session_state['ta_df']     = df
-            st.session_state['ta_format'] = fmt
-            st.success(f"✓ Loaded {len(df)} trades — {fmt}")
-        else:
-            st.error("Could not parse report — check file format")
-
-    df_all = st.session_state['ta_df']
-    fmt    = st.session_state['ta_format']
-
-    if df_all is None or len(df_all) == 0:
-        st.markdown("""
-        <div class="info-card">
-            Upload an MT5 account history report (.htm/.html), MT5 backtest report,
-            or a Quant Analyzer CSV export to begin analysis.
-        </div>
-        """, unsafe_allow_html=True)
-        return
-
-    if fmt:
-        st.caption(f"Format detected: **{fmt}** · {len(df_all)} total trades")
-
-    # ── Filters ───────────────────────────────────────────────────────────────
-    st.divider()
-    fc1, fc2, fc3, fc4 = st.columns(4)
-
-    with fc1:
-        date_min  = df_all['open_time'].min().date()
-        date_max  = df_all['open_time'].max().date()
-        date_from = st.date_input("From", value=date_min, min_value=date_min,
-                                  max_value=date_max, key='ta_from')
-        date_to   = st.date_input("To",   value=date_max, min_value=date_min,
-                                  max_value=date_max, key='ta_to')
-
-    with fc2:
-        symbols    = sorted(df_all['symbol'].dropna().unique().tolist())
-        sel_symbol = st.multiselect("Symbol", symbols, key='ta_sym')
-
-    with fc3:
-        strategies    = sorted(df_all['strategy'].dropna().unique().tolist())
-        sel_strategy  = st.multiselect("Strategy / EA", strategies, key='ta_strat')
-
-    with fc4:
-        days     = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday']
-        sel_days = st.multiselect("Day of week", days, key='ta_days')
-        sel_type = st.multiselect("Type", ['buy', 'sell'], key='ta_type')
-
-    # Apply filters
-    df = df_all.copy()
-    df = df[(df['open_time'].dt.date >= date_from) &
-            (df['open_time'].dt.date <= date_to)]
-    if sel_symbol:
-        df = df[df['symbol'].isin(sel_symbol)]
-    if sel_strategy:
-        df = df[df['strategy'].isin(sel_strategy)]
-    if sel_days:
-        df = df[df['day_of_week'].isin(sel_days)]
-    if sel_type:
-        df = df[df['type'].isin(sel_type)]
-
-    st.caption(f"Showing **{len(df)}** trades after filters")
-
-    # ── Analysis mode ─────────────────────────────────────────────────────────
-    mode = st.radio(
-        "Analysis mode",
-        ["Overall", "By Strategy", "By Symbol", "By Day of Week"],
-        horizontal=True, key='ta_mode'
-    )
-    st.divider()
-
-    # ── Helpers ───────────────────────────────────────────────────────────────
-    def render_stats(stats, label=""):
-        if label:
-            st.markdown(f"**{label}**")
-
-        c1, c2, c3, c4, c5 = st.columns(5)
-        c1.metric("Net Profit",    f"${stats['net_profit']:,.2f}")
-        c2.metric("Win Rate",      f"{stats['win_rate']}%")
-        c3.metric("Profit Factor", f"{stats['profit_factor']}")
-        c4.metric("R:R Ratio",     f"{stats['rr_ratio']}")
-        c5.metric("Expectancy",    f"${stats['expectancy']:,.2f}")
-
-        c1, c2, c3, c4, c5 = st.columns(5)
-        c1.metric("Total Trades",  stats['total_trades'])
-        c2.metric("Avg Win",       f"${stats['avg_win']:,.2f}")
-        c3.metric("Avg Loss",      f"${stats['avg_loss']:,.2f}")
-        c4.metric("Max DD",        f"${stats['max_drawdown']:,.2f}")
-        c5.metric("Best Trade",    f"${stats['best_trade']:,.2f}")
-
-        c1, c2, c3, c4, c5 = st.columns(5)
-        c1.metric("Max Consec Wins",   stats['max_consec_wins'])
-        c2.metric("Max Consec Losses", stats['max_consec_losses'])
-        c3.metric("Avg Win Dur",       f"{stats['avg_win_duration']}m")
-        c4.metric("Avg Loss Dur",      f"{stats['avg_loss_duration']}m")
-        c5.metric("Worst Trade",       f"${stats['worst_trade']:,.2f}")
-
-        c1, c2, c3, c4 = st.columns(4)
-        c1.metric("Long Trades",   stats['long_trades'])
-        c2.metric("Long Win Rate", f"{stats['long_win_rate']}%")
-        c3.metric("Short Trades",  stats['short_trades'])
-        c4.metric("Short Win Rate",f"{stats['short_win_rate']}%")
-
-    def render_equity_curve(df_plot, label="Equity Curve"):
-        df_s = df_plot.sort_values('close_time').copy()
-        df_s['cumulative'] = df_s['net_profit'].cumsum()
-        fig = go.Figure()
-        fig.add_trace(go.Scatter(
-            x=df_s['close_time'], y=df_s['cumulative'],
-            mode='lines',
-            line=dict(color='#7c6af7', width=2),
-            fill='tozeroy',
-            fillcolor='rgba(124,106,247,0.08)',
-            name='Equity'
-        ))
-        fig.update_layout(
-            title=label, height=300,
-            plot_bgcolor='rgba(10,10,15,1)',
-            paper_bgcolor='rgba(10,10,15,1)',
-            font=dict(color='#aaa', family='JetBrains Mono'),
-            xaxis=dict(gridcolor='rgba(255,255,255,0.04)'),
-            yaxis=dict(gridcolor='rgba(255,255,255,0.04)', tickprefix='$'),
-            margin=dict(l=60, r=20, t=40, b=40)
-        )
-        st.plotly_chart(fig, use_container_width=True)
-
-    def render_dow_chart(df_plot):
-        dow_order = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
-        dow = df_plot.groupby('day_of_week').agg(
-            trades     = ('net_profit', 'count'),
-            net_profit = ('net_profit', 'sum'),
-            win_rate   = ('win', lambda x: round(x.mean()*100, 1))
-        ).reindex([d for d in dow_order if d in df_plot['day_of_week'].unique()])
-
-        wins_dow   = df_plot[df_plot['win']].groupby('day_of_week')['net_profit'].sum().reindex(dow.index, fill_value=0)
-        losses_dow = df_plot[~df_plot['win']].groupby('day_of_week')['net_profit'].sum().reindex(dow.index, fill_value=0)
-
-        fig = go.Figure()
-        fig.add_trace(go.Bar(x=dow.index, y=wins_dow,   name='Profit', marker_color='rgba(45,198,83,0.8)'))
-        fig.add_trace(go.Bar(x=dow.index, y=losses_dow, name='Loss',   marker_color='rgba(230,57,70,0.8)'))
-        fig.update_layout(
-            title='P&L by Day of Week', height=280, barmode='relative',
-            plot_bgcolor='rgba(10,10,15,1)', paper_bgcolor='rgba(10,10,15,1)',
-            font=dict(color='#aaa'), margin=dict(l=60, r=20, t=40, b=40),
-            xaxis=dict(gridcolor='rgba(255,255,255,0.04)'),
-            yaxis=dict(gridcolor='rgba(255,255,255,0.04)', tickprefix='$'),
-            legend=dict(bgcolor='rgba(0,0,0,0.3)')
-        )
-        st.plotly_chart(fig, use_container_width=True)
-
-        dt = dow.reset_index()
-        dt.columns = ['Day', 'Trades', 'Net Profit', 'Win Rate %']
-        dt['Net Profit'] = dt['Net Profit'].round(2)
-        st.dataframe(dt, use_container_width=True, hide_index=True)
-
-    def render_hour_chart(df_plot):
-        hourly = df_plot.groupby('hour').agg(
-            trades     = ('net_profit', 'count'),
-            net_profit = ('net_profit', 'sum'),
-        )
-        wins_h   = df_plot[df_plot['win']].groupby('hour')['net_profit'].sum().reindex(hourly.index, fill_value=0)
-        losses_h = df_plot[~df_plot['win']].groupby('hour')['net_profit'].sum().reindex(hourly.index, fill_value=0)
-
-        fig = go.Figure()
-        fig.add_trace(go.Bar(x=wins_h.index,   y=wins_h,   name='Profit', marker_color='rgba(45,198,83,0.8)'))
-        fig.add_trace(go.Bar(x=losses_h.index, y=losses_h, name='Loss',   marker_color='rgba(230,57,70,0.8)'))
-        fig.update_layout(
-            title='P&L by Hour of Day', height=280, barmode='relative',
-            plot_bgcolor='rgba(10,10,15,1)', paper_bgcolor='rgba(10,10,15,1)',
-            font=dict(color='#aaa'), margin=dict(l=60, r=20, t=40, b=40),
-            xaxis=dict(gridcolor='rgba(255,255,255,0.04)', title='Hour (UTC)'),
-            yaxis=dict(gridcolor='rgba(255,255,255,0.04)', tickprefix='$'),
-            legend=dict(bgcolor='rgba(0,0,0,0.3)')
-        )
-        st.plotly_chart(fig, use_container_width=True)
-
-    def colour_profit(val):
-        try:
-            v = float(str(val).replace(',', ''))
-            if v > 0: return 'background-color: rgba(0,180,0,0.12)'
-            if v < 0: return 'background-color: rgba(180,0,0,0.12)'
-        except:
-            pass
-        return ''
-
-    # ── Render mode ───────────────────────────────────────────────────────────
-    if mode == "Overall":
-        stats = calc_stats(df)
-        render_stats(stats, "Overall Statistics")
-        render_equity_curve(df)
-        col1, col2 = st.columns(2)
-        with col1:
-            render_dow_chart(df)
-        with col2:
-            render_hour_chart(df)
-
-    elif mode == "By Strategy":
-        strats = sorted(df['strategy'].dropna().unique().tolist())
-        if not strats:
-            st.info("No strategies found")
-        else:
-            st.subheader("Strategy Comparison")
-            rows = []
-            for s in strats:
-                sdf  = df[df['strategy'] == s]
-                stat = calc_stats(sdf)
-                rows.append({
-                    'Strategy'      : s,
-                    'Trades'        : stat['total_trades'],
-                    'Net Profit'    : stat['net_profit'],
-                    'Win Rate %'    : stat['win_rate'],
-                    'Profit Factor' : stat['profit_factor'],
-                    'R:R'           : stat['rr_ratio'],
-                    'Expectancy'    : stat['expectancy'],
-                    'Max DD'        : stat['max_drawdown'],
-                    'Max Consec W'  : stat['max_consec_wins'],
-                    'Max Consec L'  : stat['max_consec_losses'],
-                })
-            sdf_sum = __import__('pandas').DataFrame(rows).sort_values('Net Profit', ascending=False)
-            st.dataframe(
-                sdf_sum.style.map(colour_profit, subset=['Net Profit', 'Expectancy', 'Max DD']),
-                use_container_width=True, hide_index=True
-            )
-            st.divider()
-            sel = st.selectbox("Select strategy for detail", strats)
-            if sel:
-                sdf  = df[df['strategy'] == sel]
-                stat = calc_stats(sdf)
-                render_stats(stat, sel)
-                render_equity_curve(sdf, f"{sel} — Equity Curve")
-                col1, col2 = st.columns(2)
-                with col1: render_dow_chart(sdf)
-                with col2: render_hour_chart(sdf)
-
-    elif mode == "By Symbol":
-        syms = sorted(df['symbol'].dropna().unique().tolist())
-        rows = []
-        for s in syms:
-            sdf  = df[df['symbol'] == s]
-            stat = calc_stats(sdf)
-            rows.append({
-                'Symbol'        : s,
-                'Trades'        : stat['total_trades'],
-                'Net Profit'    : stat['net_profit'],
-                'Win Rate %'    : stat['win_rate'],
-                'Profit Factor' : stat['profit_factor'],
-                'R:R'           : stat['rr_ratio'],
-                'Expectancy'    : stat['expectancy'],
-                'Max DD'        : stat['max_drawdown'],
-            })
-        sdf_sum = __import__('pandas').DataFrame(rows).sort_values('Net Profit', ascending=False)
-        st.dataframe(
-            sdf_sum.style.map(colour_profit, subset=['Net Profit', 'Expectancy', 'Max DD']),
-            use_container_width=True, hide_index=True
-        )
-        sel = st.selectbox("Select symbol for detail", syms)
-        if sel:
-            sdf  = df[df['symbol'] == sel]
-            stat = calc_stats(sdf)
-            render_stats(stat, sel)
-            render_equity_curve(sdf, f"{sel} — Equity Curve")
-            col1, col2 = st.columns(2)
-            with col1: render_dow_chart(sdf)
-            with col2: render_hour_chart(sdf)
-
-    elif mode == "By Day of Week":
-        render_dow_chart(df)
-        render_hour_chart(df)
-
-    # ── Raw trade log ─────────────────────────────────────────────────────────
-    st.divider()
-    with st.expander("Raw Trade Log"):
-        show_cols = ['open_time', 'close_time', 'symbol', 'type', 'strategy',
-                     'volume', 'open_price', 'close_price', 'sl', 'tp',
-                     'commission', 'swap', 'profit', 'net_profit', 'duration_min']
-        show_cols = [c for c in show_cols if c in df.columns]
-
-        def colour_net(val):
-            try:
-                v = float(val)
-                if v > 0: return 'background-color: rgba(0,180,0,0.12)'
-                if v < 0: return 'background-color: rgba(180,0,0,0.12)'
-            except:
-                pass
-            return ''
-
-        st.dataframe(
-            df[show_cols].style.map(colour_net, subset=['net_profit', 'profit']),
-            use_container_width=True, hide_index=True, height=400
-        )
-        st.download_button(
-            "⬇ Download filtered trades CSV",
-            data      = df[show_cols].to_csv(index=False),
-            file_name = f"mt5_trades_{date_from}_{date_to}.csv",
-            mime      = 'text/csv'
-        )

view_trade_analysis.py

@@ -161,8 +161,9 @@ def render():
     fc1, fc2, fc3, fc4 = st.columns(4)
 
     with fc1:
-        date_min  = df_all['open_time'].min().date()
+        valid_times = df_all['open_time'].dropna()
-        date_max  = df_all['open_time'].max().date()
+        date_min  = valid_times.min().date()
+        date_max  = valid_times.max().date()
         date_from = st.date_input("From", value=date_min, min_value=date_min,
                                   max_value=date_max, key='ta_from')
         date_to   = st.date_input("To",   value=date_max, min_value=date_min,
@@ -301,6 +302,54 @@ def render():
         )
         st.plotly_chart(fig, use_container_width=True, key=f"eq_fig_{safe_key}")
 
+        # ── Daily P&L bars ────────────────────────────────────────────────
+        st.markdown("**Daily P&L**")
+        daily = (df_s.groupby(df_s['close_time'].dt.date)['net_profit']
+                 .sum().reset_index())
+        daily.columns = ['date','pnl']
+        daily['color'] = daily['pnl'].apply(
+            lambda v: 'rgba(52,194,122,0.75)' if v >= 0 else 'rgba(220,80,80,0.75)')
+        fig_d = go.Figure(go.Bar(
+            x=daily['date'], y=daily['pnl'],
+            marker_color=daily['color'], name='Daily P&L',
+        ))
+        fig_d.update_layout(
+            height=160,
+            plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)',
+            font=dict(family='sans-serif'),
+            xaxis=dict(gridcolor='rgba(128,128,128,0.15)', showgrid=False,
+                       showticklabels=False),
+            yaxis=dict(gridcolor='rgba(128,128,128,0.15)', tickprefix='$',
+                       showgrid=True, zeroline=True,
+                       zerolinecolor='rgba(128,128,128,0.3)'),
+            margin=dict(l=60, r=20, t=8, b=20),
+            showlegend=False,
+        )
+        st.plotly_chart(fig_d, use_container_width=True, key=f"eq_daily_{safe_key}")
+
+        # ── Drawdown panel ────────────────────────────────────────────────
+        st.markdown("**Drawdown**")
+        df_s['_cum2']  = df_s['net_profit'].cumsum()
+        df_s['_peak']  = df_s['_cum2'].cummax()
+        df_s['_dd']    = df_s['_cum2'] - df_s['_peak']
+        fig_dd = go.Figure(go.Scatter(
+            x=df_s['close_time'], y=df_s['_dd'],
+            mode='lines', fill='tozeroy',
+            line=dict(color='rgba(220,80,80,0.6)', width=1),
+            fillcolor='rgba(220,80,80,0.12)', name='Drawdown',
+        ))
+        fig_dd.update_layout(
+            height=120,
+            plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)',
+            font=dict(family='sans-serif'),
+            xaxis=dict(gridcolor='rgba(128,128,128,0.15)', showgrid=True),
+            yaxis=dict(gridcolor='rgba(128,128,128,0.15)', tickprefix='$',
+                       showgrid=True),
+            margin=dict(l=60, r=20, t=8, b=40),
+            showlegend=False,
+        )
+        st.plotly_chart(fig_dd, use_container_width=True, key=f"eq_dd_{safe_key}")
+
     def render_dow_chart(df_plot):
         dow_order = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
         dow = df_plot.groupby('day_of_week').agg(