Files
All-in-one-Financial-Analysis/atlas-terminal/server/services/research_dashboard.py
T

512 lines
19 KiB
Python

"""Research deep-dive dashboard: F-Score history, DuPont tree, Sankey, waterfall, anomalies.
All quantitative; no LLM. See claude.md hybrid separation principle.
"""
from __future__ import annotations
import threading
import time
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
from server.utils.safe_float import _safe_float
from server.services.market_fetcher import _get_annual_financials_balance_cashflow, _get_row_series
from server.models.schemas import (
DuPontTreeNode,
DuPontTreePayload,
FScoreCriterionSeries,
FScoreYearPoint,
FinancialAnomalyItem,
ResearchDashboardResponse,
SankeyGraphPayload,
SankeyNivoLink,
SankeyNivoNode,
WaterfallStep,
)
try:
import yfinance as yf
except ImportError:
yf = None # type: ignore[assignment]
_DASHBOARD_CACHE_TTL_SECONDS = 300
_dashboard_cache_lock = threading.RLock()
_dashboard_cache: Dict[str, tuple[float, ResearchDashboardResponse]] = {}
_dashboard_inflight: Dict[str, threading.Event] = {}
_FSCORE_KEYS = [
("profitable", "Net income > 0"),
("ocf_pos", "Operating cash flow > 0"),
("roa_up", "ROA vs prior year"),
("ocf_gt_ni", "OCF > net income"),
("leverage_down", "Lower LT debt / assets"),
("cr_up", "Current ratio improved"),
("no_dilution", "Shares flat or down"),
("gm_up", "Gross margin improved"),
("at_up", "Asset turnover improved"),
]
def _col_year(fin: pd.DataFrame, idx: int) -> int:
if fin is None or fin.empty or idx >= len(fin.columns):
return 0
c = fin.columns[idx]
s = str(c)[:4]
return int(s) if s.isdigit() else 2024 - idx
def _vx(s: Optional[pd.Series], i: int) -> Optional[float]:
if s is None or len(s) <= i:
return None
x = _safe_float(s.iloc[i])
return x if x is not None and x == x and not pd.isna(x) else None
def _fscore_passes_for_pair(
fin: pd.DataFrame,
bal: pd.DataFrame,
cf: pd.DataFrame,
ticker: str,
i: int,
) -> Tuple[bool, ...]:
"""Piotroski 9 booleans comparing fiscal column i vs i+1 (i is more recent)."""
ncol = len(fin.columns)
if i + 1 >= ncol:
return tuple([False] * 9)
rev = _get_row_series(fin, "Total Revenue", "Revenue")
ni = _get_row_series(fin, "Net Income", "Net Income Common Stockholders")
gross = _get_row_series(fin, "Gross Profit")
ta = _get_row_series(bal, "Total Assets")
lt_debt = _get_row_series(bal, "Long Term Debt")
ca = _get_row_series(bal, "Current Assets")
cl = _get_row_series(bal, "Current Liabilities")
ocf = _get_row_series(cf, "Operating Cash Flow", "Cash From Operating Activities") if not cf.empty else None
shares = _get_row_series(bal, "Share Issued")
if shares is None or (hasattr(shares, "empty") and shares.empty):
shares = _get_row_series(bal, "Ordinary Shares Number")
ni0, ni1 = _vx(ni, i), _vx(ni, i + 1)
ocf0 = _vx(ocf, i) if ocf is not None else None
ta0, ta1 = _vx(ta, i), _vx(ta, i + 1)
roa0 = (ni0 / ta0 * 100) if (ni0 is not None and ta0 and ta0 != 0) else None
roa1 = (ni1 / ta1 * 100) if (ni1 is not None and ta1 and ta1 != 0) else None
lt0 = _vx(lt_debt, i) or 0
lt1 = _vx(lt_debt, i + 1) or 0
cl0, cl1 = _vx(cl, i), _vx(cl, i + 1)
ca0, ca1 = _vx(ca, i), _vx(ca, i + 1)
cr0 = (ca0 / cl0) if (ca0 is not None and cl0 and cl0 != 0) else None
cr1 = (ca1 / cl1) if (ca1 is not None and cl1 and cl1 != 0) else None
sh0, sh1 = _vx(shares, i), _vx(shares, i + 1)
rev0, rev1 = _vx(rev, i), _vx(rev, i + 1)
gm0 = (_vx(gross, i) / rev0 * 100) if (gross is not None and rev0 and rev0 != 0) else None
gm1 = (_vx(gross, i + 1) / rev1 * 100) if (gross is not None and rev1 and rev1 != 0) else None
at0 = (rev0 / ta0) if (rev0 and ta0 and ta0 != 0) else None
at1 = (rev1 / ta1) if (rev1 and ta1 and ta1 != 0) else None
if shares is None or (sh0 is None and sh1 is None) and yf:
try:
info = (yf.Ticker(ticker.upper()).info) or {}
sh_out = info.get("sharesOutstanding")
if sh_out is not None:
sh0 = sh0 or _safe_float(sh_out)
except Exception:
pass
p1 = ni0 is not None and ni0 > 0
p2 = ocf0 is not None and ocf0 > 0
p3 = roa0 is not None and roa1 is not None and roa0 > roa1
p4 = ocf0 is not None and ni0 is not None and ocf0 > ni0
p5 = bool(ta0 and ta0 != 0 and ta1 and ta1 != 0 and (lt0 / ta0) < (lt1 / ta1))
p6 = cr0 is not None and cr1 is not None and cr0 > cr1
p7 = (sh0 is not None and sh1 is not None and sh0 <= sh1) if (sh0 is not None and sh1 is not None) else True
p8 = gm0 is not None and gm1 is not None and gm0 > gm1
p9 = at0 is not None and at1 is not None and at0 > at1
return (p1, p2, p3, p4, p5, p6, p7, p8, p9)
def _build_fscore_series(ticker: str, fin: pd.DataFrame, bal: pd.DataFrame, cf: pd.DataFrame) -> Tuple[int, List[FScoreCriterionSeries]]:
criteria_out: List[FScoreCriterionSeries] = []
ncol = len(fin.columns)
if ncol < 2:
return 0, []
max_pairs = min(3, ncol - 1)
latest_passes = _fscore_passes_for_pair(fin, bal, cf, ticker, 0)
total = sum(1 for p in latest_passes if p)
for ci, (key, label) in enumerate(_FSCORE_KEYS):
hist: List[FScoreYearPoint] = []
for k in range(max_pairs):
passes = _fscore_passes_for_pair(fin, bal, cf, ticker, k)
yr = _col_year(fin, k)
hist.append(FScoreYearPoint(year=yr, pass_flag=bool(passes[ci])))
criteria_out.append(FScoreCriterionSeries(key=key, label=label, history=list(reversed(hist))))
return total, criteria_out
def _dupont_tree_from_df(dupont_df: pd.DataFrame) -> Optional[DuPontTreePayload]:
if dupont_df is None or dupont_df.empty:
return None
row0 = dupont_df.iloc[0]
npm = float(row0.get("NPM %") or 0)
at = float(row0.get("Asset Turnover") or 0)
em = float(row0.get("Equity Mult.") or 0)
roe = float(row0.get("ROE %") or 0)
tail = dupont_df.iloc[1:6]
def avg(col: str) -> Optional[float]:
if col not in tail.columns:
return None
s = tail[col].dropna()
if s.empty:
return None
return float(s.mean())
def node(
nid: str,
lbl: str,
val: float,
unit: str,
col: str,
) -> DuPontTreeNode:
a = avg(col)
vs = ((val - a) / abs(a) * 100) if (a is not None and a != 0) else None
if vs is None:
tr = "flat"
elif vs > 2:
tr = "up"
elif vs < -2:
tr = "down"
else:
tr = "flat"
return DuPontTreeNode(
id=nid,
label=lbl,
value=round(val, 4),
unit=unit,
avg_5y=round(a, 4) if a is not None else None,
vs_5y_avg_pct=round(vs, 2) if vs is not None else None,
trend=tr,
)
root = node("roe", "ROE", roe, "pct", "ROE %")
return DuPontTreePayload(
root=root,
npm=node("npm", "Net margin", npm, "pct", "NPM %"),
asset_turnover=node("at", "Asset turnover", at, "x", "Asset Turnover"),
equity_mult=node("em", "Equity multiplier", em, "x", "Equity Mult."),
)
def _build_dupont_tree_from_statements(fin: pd.DataFrame, bal: pd.DataFrame) -> Optional[DuPontTreePayload]:
if fin is None or fin.empty or bal is None or bal.empty:
return None
rev = _get_row_series(fin, "Total Revenue", "Revenue", "Net Revenue")
ni = _get_row_series(fin, "Net Income", "Net Income Common Stockholders")
total_assets = _get_row_series(bal, "Total Assets")
total_equity = _get_row_series(
bal,
"Total Stockholder Equity",
"Stockholders Equity",
"Total Equity Gross Minority Interest",
)
if rev is None or ni is None or total_assets is None or total_equity is None:
return None
rows: List[Dict[str, Any]] = []
for i, d in enumerate(fin.columns[:6]):
revenue = _safe_float(rev.get(d))
net_income = _safe_float(ni.get(d))
assets = _safe_float(total_assets.get(d))
equity = _safe_float(total_equity.get(d))
if not revenue or not assets or not equity:
continue
npm = (net_income / revenue * 100) if net_income is not None else None
at = revenue / assets if assets else None
em = assets / equity if equity else None
roe = (net_income / equity * 100) if (net_income is not None and equity) else None
if npm is None or at is None or em is None or roe is None:
continue
yr = int(str(d)[:4]) if str(d)[:4].isdigit() else (d.year if hasattr(d, "year") else (2024 - i))
rows.append(
{
"Year": yr,
"Revenue": revenue,
"Net Income": net_income,
"NPM %": round(npm, 2),
"Asset Turnover": round(at, 4),
"Equity Mult.": round(em, 2),
"ROE %": round(roe, 2),
}
)
if not rows:
return None
return _dupont_tree_from_df(pd.DataFrame(rows))
def _build_sankey_nivo_from_fin(fin: pd.DataFrame) -> SankeyGraphPayload:
if fin is None or fin.empty:
return SankeyGraphPayload(nodes=[], links=[])
rev = _get_row_series(fin, "Total Revenue", "Revenue", "Net Revenue")
cogs = _get_row_series(fin, "Cost Of Revenue", "Cost Of Goods Sold")
gross = _get_row_series(fin, "Gross Profit")
op_inc = _get_row_series(fin, "Operating Income", "EBIT")
ni = _get_row_series(fin, "Net Income", "Net Income Common Stockholders")
if rev is None or len(rev) == 0:
return SankeyGraphPayload(nodes=[], links=[])
d = rev.index[0]
revenue = abs(_safe_float(rev.get(d)) or 0)
cogs_val = abs(_safe_float(cogs.get(d)) or 0) if cogs is not None and d in cogs.index else 0
gross_val = _safe_float(gross.get(d)) if gross is not None and d in gross.index else None
if gross_val is None:
gross_val = (revenue - cogs_val) if revenue else 0
gross_val = abs(gross_val or 0)
operating_income = _safe_float(op_inc.get(d)) if op_inc is not None and d in op_inc.index else 0
net_income = _safe_float(ni.get(d)) if ni is not None and d in ni.index else 0
opex = max(0, gross_val - operating_income) if gross_val >= operating_income else 0
tax_interest_other = max(0, operating_income - net_income) if (operating_income - net_income) > 0 else abs(min(0, operating_income - net_income))
nodes = [
SankeyNivoNode(id="revenue", label="Revenue"),
SankeyNivoNode(id="cogs", label="COGS"),
SankeyNivoNode(id="gross_profit", label="Gross profit"),
SankeyNivoNode(id="opex", label="Operating expenses"),
SankeyNivoNode(id="operating_income", label="Operating income"),
SankeyNivoNode(id="tax_other", label="Tax & other"),
SankeyNivoNode(id="net_income", label="Net income"),
]
revenue_safe = max(revenue, 1)
gross_safe = max(gross_val, revenue_safe - min(cogs_val, revenue_safe * 0.999))
links: List[SankeyNivoLink] = [
SankeyNivoLink(source="revenue", target="cogs", value=float(min(cogs_val, revenue_safe * 0.999))),
SankeyNivoLink(source="revenue", target="gross_profit", value=float(max(gross_safe, revenue_safe - cogs_val))),
]
gp_v = links[-1].value
links.append(SankeyNivoLink(source="gross_profit", target="opex", value=float(min(opex, gp_v))))
links.append(SankeyNivoLink(source="gross_profit", target="operating_income", value=float(max(operating_income, gp_v - opex))))
oi_v = links[-1].value
tax_v = min(tax_interest_other, max(oi_v, 0))
links.append(SankeyNivoLink(source="operating_income", target="tax_other", value=float(tax_v)))
links.append(SankeyNivoLink(source="operating_income", target="net_income", value=float(max(abs(net_income), 0))))
return SankeyGraphPayload(nodes=nodes, links=links)
def _build_sankey_nivo(ticker: str) -> SankeyGraphPayload:
fin, _, _ = _get_annual_financials_balance_cashflow(ticker)
return _build_sankey_nivo_from_fin(fin)
def _build_waterfall(fin: pd.DataFrame) -> List[WaterfallStep]:
if fin is None or fin.empty or len(fin.columns) < 2:
return []
c0, c1 = fin.columns[0], fin.columns[1]
def cell(row_names: Tuple[str, ...]) -> Tuple[float, float]:
v0 = v1 = 0.0
for name in row_names:
if name in fin.index:
v0 += float(_safe_float(fin.loc[name, c0]) or 0)
v1 += float(_safe_float(fin.loc[name, c1]) or 0)
return v0, v1
gp0, gp1 = cell(("Gross Profit",))
if gp0 == 0 and gp1 == 0:
r0, r1 = cell(("Total Revenue", "Revenue"))
cg0, cg1 = cell(("Cost Of Revenue", "Cost of Revenue"))
gp0, gp1 = r0 - cg0, r1 - cg1
oi0, oi1 = cell(("Operating Income", "EBIT"))
opex0 = max(gp0 - oi0, 0)
opex1 = max(gp1 - oi1, 0)
d_gp = gp0 - gp1
d_opex = opex0 - opex1
y0 = _col_year(fin, 0)
y1 = _col_year(fin, 1)
cum = oi1
steps: List[WaterfallStep] = [
WaterfallStep(id="prior_oi", label=f"Operating income {y1}", value=oi1, cumulative=cum, step_type="total"),
WaterfallStep(id="d_gp", label="Gross profit change", value=d_gp, cumulative=cum + d_gp, step_type="relative"),
WaterfallStep(id="d_opex", label="Operating expense change", value=-d_opex, cumulative=cum + d_gp - d_opex, step_type="relative"),
]
final = cum + d_gp - d_opex
diff = oi0 - final
if abs(diff) > max(abs(oi0), 1) * 0.02:
steps.append(WaterfallStep(id="other", label="Other / rounding", value=diff, cumulative=oi0, step_type="relative"))
steps.append(WaterfallStep(id="current_oi", label=f"Operating income {y0}", value=oi0, cumulative=oi0, step_type="total"))
return steps
_ANOMALY_ROWS = [
("Total Revenue", "Revenue"),
("Cost Of Revenue", "COGS"),
("Research And Development", "R&D"),
("Selling General And Administration", "SG&A"),
("Operating Expense", "Operating expenses"),
("Operating Income", "Operating income"),
("Net Income", "Net income"),
("Total Debt", "Total debt"),
("Long Term Debt", "Long-term debt"),
("Current Debt", "Short-term debt"),
]
def _detect_anomalies(fin: pd.DataFrame, bal: pd.DataFrame, threshold: float = 0.30) -> List[FinancialAnomalyItem]:
out: List[FinancialAnomalyItem] = []
if fin is None or fin.empty or len(fin.columns) < 2:
return out
c0, c1 = fin.columns[0], fin.columns[1]
def check_row(idx_name: str, display: str) -> None:
if idx_name not in fin.index:
return
cur = _safe_float(fin.loc[idx_name, c0])
prev = _safe_float(fin.loc[idx_name, c1])
if cur is None or prev is None or abs(prev) < 1e-6:
return
chg = (cur - prev) / abs(prev)
if abs(chg) < threshold:
return
out.append(
FinancialAnomalyItem(
account_key=idx_name,
display_name=display,
prior_value=prev,
current_value=cur,
change_pct=round(chg * 100, 2),
direction="up" if chg > 0 else "down",
)
)
for key, disp in _ANOMALY_ROWS:
check_row(key, disp)
if bal is not None and not bal.empty and len(bal.columns) >= 2:
b0, b1 = bal.columns[0], bal.columns[1]
for idx_name, display in [
("Current Debt", "Short-term borrowings"),
("Long Term Debt", "Long-term debt"),
("Total Debt", "Total debt"),
]:
if idx_name not in bal.index:
continue
cur = _safe_float(bal.loc[idx_name, b0])
prev = _safe_float(bal.loc[idx_name, b1])
if cur is None or prev is None or abs(prev) < 1e-6:
continue
chg = (cur - prev) / abs(prev)
if abs(chg) < threshold:
continue
out.append(
FinancialAnomalyItem(
account_key=f"BS:{idx_name}",
display_name=display,
prior_value=prev,
current_value=cur,
change_pct=round(chg * 100, 2),
direction="up" if chg > 0 else "down",
)
)
out.sort(key=lambda x: abs(x.change_pct or 0), reverse=True)
return out[:12]
def sankey_nivo_for_ticker(ticker: str) -> Dict[str, Any]:
"""Public helper for /api/market/sankey extended payload."""
return _build_sankey_nivo(ticker).model_dump()
def _get_cached_dashboard(sym: str) -> Optional[ResearchDashboardResponse]:
now = time.time()
with _dashboard_cache_lock:
cached = _dashboard_cache.get(sym)
if not cached:
return None
ts, payload = cached
if now - ts >= _DASHBOARD_CACHE_TTL_SECONDS:
_dashboard_cache.pop(sym, None)
return None
return payload.model_copy(deep=True)
def _store_cached_dashboard(sym: str, payload: ResearchDashboardResponse) -> None:
if payload.error:
return
with _dashboard_cache_lock:
_dashboard_cache[sym] = (time.time(), payload.model_copy(deep=True))
def _build_research_dashboard_uncached(sym: str) -> ResearchDashboardResponse:
fin, bal, cf = _get_annual_financials_balance_cashflow(sym)
if fin is None or fin.empty or bal is None or bal.empty:
return ResearchDashboardResponse(ticker=sym, error="Insufficient financial statements")
if cf is None or cf.empty:
cf = pd.DataFrame()
total, fseries = _build_fscore_series(sym, fin, bal, cf)
tree = _build_dupont_tree_from_statements(fin, bal)
sankey = _build_sankey_nivo_from_fin(fin)
waterfall = _build_waterfall(fin)
anomalies = _detect_anomalies(fin, bal)
return ResearchDashboardResponse(
ticker=sym,
fscore_total=total,
fscore_criteria=fseries,
dupont_tree=tree,
sankey=sankey,
waterfall=waterfall,
anomalies=anomalies,
)
def build_research_dashboard(ticker: str) -> ResearchDashboardResponse:
sym = ticker.upper().strip()
cached = _get_cached_dashboard(sym)
if cached is not None:
return cached
wait_event: Optional[threading.Event] = None
with _dashboard_cache_lock:
cached = _get_cached_dashboard(sym)
if cached is not None:
return cached
wait_event = _dashboard_inflight.get(sym)
if wait_event is None:
wait_event = threading.Event()
_dashboard_inflight[sym] = wait_event
leader = True
else:
leader = False
if not leader:
wait_event.wait(timeout=15)
cached = _get_cached_dashboard(sym)
if cached is not None:
return cached
return _build_research_dashboard_uncached(sym)
try:
payload = _build_research_dashboard_uncached(sym)
_store_cached_dashboard(sym, payload)
return payload.model_copy(deep=True)
finally:
with _dashboard_cache_lock:
event = _dashboard_inflight.pop(sym, None)
if event is not None:
event.set()