phase 5: harden earnings call delta

atlas-terminal/README.md

@@ -130,7 +130,7 @@ Credential API:
 
 ## Recent Work
 
-- Phase 5 earnings-call delta MVP: FMP transcript pair lookup, deterministic new/faded/emphasis phrase analysis, tone shift scoring, and best-effort AI narrative on the Earnings page
+- Phase 5 earnings-call delta: FMP transcript pair lookup, rule-based lemmatisation, bigram/trigram TF-IDF phrase ranking, finance-topic shift detection, tone shift scoring, and best-effort Claude/Gemini narrative on the Earnings page
 - Phase 4 peer comparison: gateway-backed peer discovery, parallel fundamentals matrix, percentile-colored valuation/quality cells, and backward-compatible `/api/market/peers/{ticker}` responses for overview/report flows
 - Phase 3 security hardening: AES-GCM envelope encryption, credential tables, credential access audit logs, and `ATLAS_MASTER_KEY` documentation for future KIS/IBKR key storage
 - v2 refactor foundation: baseline measurements in `docs/baseline-2026-04.md`, CI workflow, pytest smoke tests, and Playwright route smoke tests

atlas-terminal/apps/web/src/app/earnings/page.tsx

@@ -38,8 +38,15 @@ interface TranscriptDeltaData {
   previous?: { year: number; quarter: number };
   new_phrases?: { phrase: string; count: number }[];
   removed_phrases?: { phrase: string; previous_count: number }[];
-  emphasis_shift?: { phrase: string; current_count: number; previous_count: number; delta: number }[];
-  tone_shift?: { current_score: number; previous_score: number };
+  emphasis_shift?: { phrase: string; current_count: number; previous_count: number; delta: number; score?: number }[];
+  tone_shift?: {
+    current_score: number;
+    previous_score: number;
+    delta?: number;
+    current_label?: string;
+    previous_label?: string;
+  };
+  topic_shift?: { topic: string; current_count: number; previous_count: number; delta: number }[];
   narrative?: {
     key_shifts?: string[];
     what_it_means?: string;
@@ -280,6 +287,11 @@ function TranscriptDeltaPanel({ data }: { data: TranscriptDeltaData }) {
             <div className="font-mono text-sm text-brand-navy">
               {data.tone_shift.previous_score.toFixed(1)} → {data.tone_shift.current_score.toFixed(1)}
             </div>
+            {data.tone_shift.current_label && (
+              <div className="mt-0.5 text-[10px] uppercase tracking-[0.12em] text-text-muted">
+                {data.tone_shift.previous_label} → {data.tone_shift.current_label}
+              </div>
+            )}
           </div>
         )}
       </div>
@@ -321,6 +333,24 @@ function TranscriptDeltaPanel({ data }: { data: TranscriptDeltaData }) {
         </div>
       </div>
 
+      {data.topic_shift && data.topic_shift.length > 0 && (
+        <div className="mt-4 rounded border border-border bg-surface-raised p-4">
+          <div className="text-[11px] uppercase tracking-[0.12em] text-brand-navy font-semibold mb-3">Topic Shift</div>
+          <div className="grid gap-2 md:grid-cols-2 lg:grid-cols-3">
+            {data.topic_shift.slice(0, 6).map((row) => (
+              <div key={row.topic} className="rounded border border-border bg-surface-sunken px-3 py-2">
+                <div className="flex items-center justify-between gap-3">
+                  <span className="truncate text-sm text-text-secondary">{row.topic}</span>
+                  <span className={`font-mono text-xs ${row.delta >= 0 ? "text-fin-positive" : "text-fin-negative"}`}>
+                    {row.previous_count} → {row.current_count}
+                  </span>
+                </div>
+              </div>
+            ))}
+          </div>
+        </div>
+      )}
+
       {data.narrative && (
         <div className="mt-4 border-l-4 border-brand-gold bg-brand-gold/10 p-4">
           <div className="text-[11px] uppercase tracking-[0.12em] text-brand-navy font-semibold mb-2">AI Interpretation</div>
@@ -334,6 +364,16 @@ function TranscriptDeltaPanel({ data }: { data: TranscriptDeltaData }) {
             </div>
           )}
           <p className="text-sm leading-relaxed text-text-primary">{data.narrative.what_it_means}</p>
+          {data.narrative.questions_to_ask && data.narrative.questions_to_ask.length > 0 && (
+            <div className="mt-3">
+              <div className="mb-1 text-[11px] uppercase tracking-[0.12em] text-brand-navy font-semibold">Questions for next call</div>
+              <ul className="space-y-1 text-sm text-text-secondary">
+                {data.narrative.questions_to_ask.slice(0, 3).map((question) => (
+                  <li key={question}>› {question}</li>
+                ))}
+              </ul>
+            </div>
+          )}
           {data.narrative.variant_view && <p className="mt-2 text-sm text-text-secondary">Variant view: {data.narrative.variant_view}</p>}
         </div>
       )}

atlas-terminal/server/services/earnings_transcripts.py

@@ -1,12 +1,15 @@
 """Earnings-call transcript delta analysis.
 
-This intentionally starts lightweight: deterministic phrase deltas are computed
-locally, and the LLM narrative is best-effort so the feature still works without
-an AI key.
+Deterministic phrase deltas are computed locally so the feature remains useful
+without an LLM key.  A best-effort Claude/Gemini narrative is layered on top
+when a server-side key is configured.
 """
 
 from __future__ import annotations
 
+import json
+import math
+import os
 import re
 from collections import Counter
 from dataclasses import dataclass
@@ -26,6 +29,15 @@ _STOPWORDS = {
 _POSITIVE = {"growth", "accelerate", "strong", "record", "improve", "expansion", "demand", "margin", "profitable"}
 _NEGATIVE = {"decline", "pressure", "risk", "weak", "slower", "headwind", "inventory", "cost", "uncertain"}
 
+_TOPIC_LEXICON: dict[str, set[str]] = {
+    "AI / Data Centre": {"ai", "artificial intelligence", "data center", "data centre", "accelerated computing", "inference", "training"},
+    "Capex / Supply": {"capex", "capital expenditure", "supply", "capacity", "manufacturing", "inventory", "lead time"},
+    "Margins / Pricing": {"margin", "gross margin", "pricing", "cost", "mix", "profitability", "operating leverage"},
+    "Demand / Customers": {"demand", "customer", "enterprise", "cloud", "hyperscaler", "consumer", "orders"},
+    "Risk / Regulation": {"risk", "regulation", "export", "competition", "uncertain", "headwind", "restriction"},
+    "Product Mix": {"gaming", "automotive", "software", "services", "networking", "platform", "segment"},
+}
+
 
 @dataclass(frozen=True)
 class Transcript:
@@ -36,6 +48,13 @@ class Transcript:
     source: str = "fmp"
 
 
+@dataclass(frozen=True)
+class Token:
+    text: str
+    lemma: str
+    position: int
+
+
 def default_quarter_pair(today: date | None = None) -> tuple[tuple[int, int], tuple[int, int]]:
     """Return a reasonable current/previous quarter pair for transcript lookup."""
 
@@ -74,22 +93,62 @@ async def fetch_transcript(ticker: str, year: int, quarter: int) -> Optional[Tra
     return Transcript(ticker=ticker.upper(), year=year, quarter=quarter, content=content)
 
 
-def tokenize_and_normalize(text: str) -> list[str]:
+def _lemma(word: str) -> str:
+    irregular = {
+        "centres": "centre",
+        "centers": "center",
+        "margins": "margin",
+        "revenues": "revenue",
+        "customers": "customer",
+        "orders": "order",
+        "risks": "risk",
+        "costs": "cost",
+        "services": "service",
+    }
+    if word in irregular:
+        return irregular[word]
+    if word in {"ai", "data", "capex", "cloud"}:
+        return word
+    if len(word) > 5 and word.endswith("ies"):
+        return word[:-3] + "y"
+    if len(word) > 6 and word.endswith("ing"):
+        base = word[:-3]
+        return base[:-1] if len(base) > 3 and base[-1] == base[-2] else base
+    if len(word) > 5 and word.endswith("ed"):
+        return word[:-2]
+    if len(word) > 4 and word.endswith("s") and not word.endswith("ss"):
+        return word[:-1]
+    return word
+
+
+def tokenize_and_normalize(text: str) -> list[Token]:
     words = re.findall(r"[a-zA-Z][a-zA-Z\-']{1,}", text.lower())
     normalized = [word.strip("-'") for word in words]
-    return [word for word in normalized if (len(word) > 2 or word == "ai") and word not in _STOPWORDS]
+    tokens: list[Token] = []
+    for position, word in enumerate(normalized):
+        if (len(word) <= 2 and word != "ai") or word in _STOPWORDS:
+            continue
+        tokens.append(Token(text=word, lemma=_lemma(word), position=position))
+    return tokens
 
 
 def _phrase_counts(text: str) -> Counter[str]:
     tokens = tokenize_and_normalize(text)
-    phrases: Counter[str] = Counter(tokens)
+    lemmas = [token.lemma for token in tokens]
+    phrases: Counter[str] = Counter(lemmas)
     for size in (2, 3):
-        for idx in range(0, max(0, len(tokens) - size + 1)):
-            phrase = " ".join(tokens[idx : idx + size])
+        for idx in range(0, max(0, len(lemmas) - size + 1)):
+            phrase = " ".join(lemmas[idx : idx + size])
             phrases[phrase] += 1
     return phrases
 
 
+def _tfidf_score(phrase: str, count: int, curr: Counter[str], prev: Counter[str]) -> float:
+    doc_freq = int(curr.get(phrase, 0) > 0) + int(prev.get(phrase, 0) > 0)
+    idf = math.log((1 + 2) / (1 + doc_freq)) + 1
+    return round(count * idf, 3)
+
+
 def _sentiment_score(counts: Counter[str]) -> float:
     total = sum(counts.values()) or 1
     pos = sum(counts[word] for word in _POSITIVE)
@@ -97,26 +156,36 @@ def _sentiment_score(counts: Counter[str]) -> float:
     return round((pos - neg) / total * 100, 2)
 
 
+def _tone_label(score: float) -> str:
+    if score >= 0.12:
+        return "bullish"
+    if score <= -0.12:
+        return "bearish"
+    return "neutral"
+
+
 def _top_new(curr: Counter[str], prev: Counter[str], limit: int = 10) -> list[dict[str, Any]]:
     rows = [
-        {"phrase": phrase, "count": count}
+        {"phrase": phrase, "count": count, "score": _tfidf_score(phrase, count, curr, prev)}
         for phrase, count in curr.items()
         if count >= 2 and prev.get(phrase, 0) == 0 and " " in phrase
     ]
-    return sorted(rows, key=lambda row: row["count"], reverse=True)[:limit]
+    return sorted(rows, key=lambda row: row["score"], reverse=True)[:limit]
 
 
 def _top_removed(curr: Counter[str], prev: Counter[str], limit: int = 10) -> list[dict[str, Any]]:
     rows = [
-        {"phrase": phrase, "previous_count": count}
+        {"phrase": phrase, "previous_count": count, "score": _tfidf_score(phrase, count, curr, prev)}
         for phrase, count in prev.items()
         if count >= 2 and curr.get(phrase, 0) == 0 and " " in phrase
     ]
-    return sorted(rows, key=lambda row: row["previous_count"], reverse=True)[:limit]
+    return sorted(rows, key=lambda row: row["score"], reverse=True)[:limit]
 
 
 def _emphasis_shift(curr: Counter[str], prev: Counter[str], limit: int = 12) -> list[dict[str, Any]]:
     rows: list[dict[str, Any]] = []
+    curr_total = sum(curr.values()) or 1
+    prev_total = sum(prev.values()) or 1
     for phrase in set(curr) | set(prev):
         if " " not in phrase:
             continue
@@ -125,13 +194,41 @@ def _emphasis_shift(curr: Counter[str], prev: Counter[str], limit: int = 12) ->
         delta = curr_count - prev_count
         if abs(delta) < 2:
             continue
-        rows.append({"phrase": phrase, "current_count": curr_count, "previous_count": prev_count, "delta": delta})
-    return sorted(rows, key=lambda row: abs(row["delta"]), reverse=True)[:limit]
+        curr_rate = curr_count / curr_total
+        prev_rate = prev_count / prev_total
+        score = abs(curr_rate - prev_rate) * math.log(curr_count + prev_count + 2)
+        rows.append({
+            "phrase": phrase,
+            "current_count": curr_count,
+            "previous_count": prev_count,
+            "delta": delta,
+            "score": round(score, 5),
+        })
+    return sorted(rows, key=lambda row: row["score"], reverse=True)[:limit]
+
+
+def _topic_shift(curr: Counter[str], prev: Counter[str]) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for topic, keywords in _TOPIC_LEXICON.items():
+        curr_count = sum(curr.get(keyword, 0) for keyword in keywords)
+        prev_count = sum(prev.get(keyword, 0) for keyword in keywords)
+        delta = curr_count - prev_count
+        if curr_count == 0 and prev_count == 0:
+            continue
+        rows.append({
+            "topic": topic,
+            "current_count": curr_count,
+            "previous_count": prev_count,
+            "delta": delta,
+        })
+    return sorted(rows, key=lambda row: abs(row["delta"]), reverse=True)
 
 
 def compute_delta(curr: Transcript, prev: Transcript) -> dict[str, Any]:
     curr_counts = _phrase_counts(curr.content)
     prev_counts = _phrase_counts(prev.content)
+    current_tone = _sentiment_score(curr_counts)
+    previous_tone = _sentiment_score(prev_counts)
     return {
         "ticker": curr.ticker,
         "available": True,
@@ -139,11 +236,15 @@ def compute_delta(curr: Transcript, prev: Transcript) -> dict[str, Any]:
         "previous": {"year": prev.year, "quarter": prev.quarter},
         "new_phrases": _top_new(curr_counts, prev_counts),
         "removed_phrases": _top_removed(curr_counts, prev_counts),
-        "emphasis_shift": _emphasis_shift(curr_counts, prev_counts),
+        "emphasis_shift": _emphasis_shift(curr_counts, prev_counts, limit=20),
         "tone_shift": {
-            "current_score": _sentiment_score(curr_counts),
-            "previous_score": _sentiment_score(prev_counts),
+            "current_score": current_tone,
+            "previous_score": previous_tone,
+            "delta": round(current_tone - previous_tone, 2),
+            "current_label": _tone_label(current_tone),
+            "previous_label": _tone_label(previous_tone),
         },
+        "topic_shift": _topic_shift(curr_counts, prev_counts),
     }
 
 
@@ -154,20 +255,59 @@ async def generate_delta_narrative(delta: dict[str, Any], ticker: str) -> dict[s
         "questions_to_ask": ["Which new phrases are one-off comments versus strategy?", "Are margin or capex terms increasing?"],
         "variant_view": "Use phrase shifts as a prompt for deeper research, not as standalone evidence.",
     }
+    prompt = (
+        "You are a skeptical institutional equity analyst reviewing earnings-call language drift.\n"
+        "Return ONLY valid JSON with this exact shape:\n"
+        '{"key_shifts":["..."],"what_it_means":"...","questions_to_ask":["..."],"variant_view":"..."}\n'
+        "Keep what_it_means to five concise analyst-style lines or fewer. Do not invent numbers.\n\n"
+        f"TICKER: {ticker.upper()}\n"
+        f"DELTA_DATA: {json.dumps(delta, ensure_ascii=False)[:12000]}"
+    )
+
+    anthropic_key = (os.getenv("ANTHROPIC_API_KEY") or os.getenv("CLAUDE_API_KEY") or "").strip()
+    if anthropic_key:
+        try:
+            from server.ai.llm_router import LLMConfig, LLMProvider, llm_router
+
+            text = await llm_router.generate(
+                prompt,
+                config=LLMConfig(
+                    provider=LLMProvider.CLAUDE,
+                    model="claude-sonnet-4-20250514",
+                    api_key=anthropic_key,
+                    temperature=0.2,
+                    max_tokens=900,
+                ),
+                system_prompt="You return strict JSON for equity research workflows.",
+            )
+            parsed = _parse_json_object(text)
+            if parsed:
+                return {**fallback, **parsed}
+        except Exception:
+            pass
+
     try:
         from server.services.gemini_service import generate_text
 
-        prompt = (
-            f"Analyze {ticker.upper()} earnings call transcript delta. Return concise JSON with keys "
-            "key_shifts, what_it_means, questions_to_ask, variant_view. Data:\n"
-            f"{delta}"
-        )
         text = await generate_text(prompt)
-        import json
-
-        parsed = json.loads(text.strip().removeprefix("```json").removesuffix("```").strip())
-        if isinstance(parsed, dict):
+        parsed = _parse_json_object(text)
+        if parsed:
             return {**fallback, **parsed}
     except Exception:
         return fallback
     return fallback
+
+
+def _parse_json_object(text: str) -> dict[str, Any] | None:
+    cleaned = text.strip()
+    if cleaned.startswith("```"):
+        cleaned = re.sub(r"^```(?:json)?", "", cleaned).strip()
+        cleaned = re.sub(r"```$", "", cleaned).strip()
+    match = re.search(r"\{.*\}", cleaned, flags=re.S)
+    if match:
+        cleaned = match.group(0)
+    try:
+        parsed = json.loads(cleaned)
+    except json.JSONDecodeError:
+        return None
+    return parsed if isinstance(parsed, dict) else None

atlas-terminal/server/services/gemini_service.py

@@ -6,8 +6,10 @@ No Streamlit dependencies.
 """
 
 import json
+import os
 import re
 import time
+import asyncio
 from typing import Any, Dict, Generator, List, Optional
 
 from server.utils.safe_float import _safe_float
@@ -38,6 +40,30 @@ def get_gemini_model(api_key: str) -> Any:
     return genai.GenerativeModel(GEMINI_MODEL)
 
 
+async def generate_text(prompt: str, temperature: float = 0.3, max_tokens: int = 1200) -> str:
+    """Async convenience wrapper used by lightweight best-effort AI features.
+
+    It reads a server-side Gemini key from the environment.  Browser-local keys
+    are intentionally not pulled in here because routers should not receive API
+    secrets implicitly from localStorage.
+    """
+
+    api_key = (os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY") or "").strip()
+    if not api_key:
+        raise RuntimeError("GOOGLE_API_KEY or GEMINI_API_KEY is not configured")
+
+    def _run() -> str:
+        model = get_gemini_model(api_key)
+        response = _generate_with_retry(
+            model,
+            prompt,
+            {"temperature": temperature, "max_output_tokens": max_tokens},
+        )
+        return (response.text or "").strip()
+
+    return await asyncio.to_thread(_run)
+
+
 # ---------------------------------------------------------------------------
 # Retry / streaming helpers
 # ---------------------------------------------------------------------------

atlas-terminal/tests/test_earnings_transcripts.py

@@ -14,11 +14,12 @@ def test_default_quarter_pair_uses_completed_quarter() -> None:
 
 def test_tokenize_and_normalize_removes_common_call_words() -> None:
     tokens = tokenize_and_normalize("Thank you operator. Sovereign AI demand was strong, strong, strong.")
+    lemmas = [token.lemma for token in tokens]
 
-    assert "thank" not in tokens
-    assert "operator" not in tokens
-    assert "sovereign" in tokens
-    assert tokens.count("strong") == 3
+    assert "thank" not in lemmas
+    assert "operator" not in lemmas
+    assert "sovereign" in lemmas
+    assert lemmas.count("strong") == 3
 
 
 def test_compute_delta_surfaces_new_removed_and_emphasis_phrases() -> None:
@@ -41,3 +42,5 @@ def test_compute_delta_surfaces_new_removed_and_emphasis_phrases() -> None:
     assert any(row["phrase"] == "sovereign ai" for row in delta["new_phrases"])
     assert any(row["phrase"] == "inventory correction" for row in delta["removed_phrases"])
     assert any(row["phrase"] == "data center" for row in delta["emphasis_shift"])
+    assert any(row["topic"] == "AI / Data Centre" for row in delta["topic_shift"])
+    assert "delta" in delta["tone_shift"]