phase 5: harden earnings call delta

atlas-terminal/server/services/earnings_transcripts.py

@@ -1,12 +1,15 @@
 """Earnings-call transcript delta analysis.
 
-This intentionally starts lightweight: deterministic phrase deltas are computed
-locally, and the LLM narrative is best-effort so the feature still works without
-an AI key.
+Deterministic phrase deltas are computed locally so the feature remains useful
+without an LLM key.  A best-effort Claude/Gemini narrative is layered on top
+when a server-side key is configured.
 """
 
 from __future__ import annotations
 
+import json
+import math
+import os
 import re
 from collections import Counter
 from dataclasses import dataclass
@@ -26,6 +29,15 @@ _STOPWORDS = {
 _POSITIVE = {"growth", "accelerate", "strong", "record", "improve", "expansion", "demand", "margin", "profitable"}
 _NEGATIVE = {"decline", "pressure", "risk", "weak", "slower", "headwind", "inventory", "cost", "uncertain"}
 
+_TOPIC_LEXICON: dict[str, set[str]] = {
+    "AI / Data Centre": {"ai", "artificial intelligence", "data center", "data centre", "accelerated computing", "inference", "training"},
+    "Capex / Supply": {"capex", "capital expenditure", "supply", "capacity", "manufacturing", "inventory", "lead time"},
+    "Margins / Pricing": {"margin", "gross margin", "pricing", "cost", "mix", "profitability", "operating leverage"},
+    "Demand / Customers": {"demand", "customer", "enterprise", "cloud", "hyperscaler", "consumer", "orders"},
+    "Risk / Regulation": {"risk", "regulation", "export", "competition", "uncertain", "headwind", "restriction"},
+    "Product Mix": {"gaming", "automotive", "software", "services", "networking", "platform", "segment"},
+}
+
 
 @dataclass(frozen=True)
 class Transcript:
@@ -36,6 +48,13 @@ class Transcript:
     source: str = "fmp"
 
 
+@dataclass(frozen=True)
+class Token:
+    text: str
+    lemma: str
+    position: int
+
+
 def default_quarter_pair(today: date | None = None) -> tuple[tuple[int, int], tuple[int, int]]:
     """Return a reasonable current/previous quarter pair for transcript lookup."""
 
@@ -74,22 +93,62 @@ async def fetch_transcript(ticker: str, year: int, quarter: int) -> Optional[Tra
     return Transcript(ticker=ticker.upper(), year=year, quarter=quarter, content=content)
 
 
-def tokenize_and_normalize(text: str) -> list[str]:
+def _lemma(word: str) -> str:
+    irregular = {
+        "centres": "centre",
+        "centers": "center",
+        "margins": "margin",
+        "revenues": "revenue",
+        "customers": "customer",
+        "orders": "order",
+        "risks": "risk",
+        "costs": "cost",
+        "services": "service",
+    }
+    if word in irregular:
+        return irregular[word]
+    if word in {"ai", "data", "capex", "cloud"}:
+        return word
+    if len(word) > 5 and word.endswith("ies"):
+        return word[:-3] + "y"
+    if len(word) > 6 and word.endswith("ing"):
+        base = word[:-3]
+        return base[:-1] if len(base) > 3 and base[-1] == base[-2] else base
+    if len(word) > 5 and word.endswith("ed"):
+        return word[:-2]
+    if len(word) > 4 and word.endswith("s") and not word.endswith("ss"):
+        return word[:-1]
+    return word
+
+
+def tokenize_and_normalize(text: str) -> list[Token]:
     words = re.findall(r"[a-zA-Z][a-zA-Z\-']{1,}", text.lower())
     normalized = [word.strip("-'") for word in words]
-    return [word for word in normalized if (len(word) > 2 or word == "ai") and word not in _STOPWORDS]
+    tokens: list[Token] = []
+    for position, word in enumerate(normalized):
+        if (len(word) <= 2 and word != "ai") or word in _STOPWORDS:
+            continue
+        tokens.append(Token(text=word, lemma=_lemma(word), position=position))
+    return tokens
 
 
 def _phrase_counts(text: str) -> Counter[str]:
     tokens = tokenize_and_normalize(text)
-    phrases: Counter[str] = Counter(tokens)
+    lemmas = [token.lemma for token in tokens]
+    phrases: Counter[str] = Counter(lemmas)
     for size in (2, 3):
-        for idx in range(0, max(0, len(tokens) - size + 1)):
-            phrase = " ".join(tokens[idx : idx + size])
+        for idx in range(0, max(0, len(lemmas) - size + 1)):
+            phrase = " ".join(lemmas[idx : idx + size])
             phrases[phrase] += 1
     return phrases
 
 
+def _tfidf_score(phrase: str, count: int, curr: Counter[str], prev: Counter[str]) -> float:
+    doc_freq = int(curr.get(phrase, 0) > 0) + int(prev.get(phrase, 0) > 0)
+    idf = math.log((1 + 2) / (1 + doc_freq)) + 1
+    return round(count * idf, 3)
+
+
 def _sentiment_score(counts: Counter[str]) -> float:
     total = sum(counts.values()) or 1
     pos = sum(counts[word] for word in _POSITIVE)
@@ -97,26 +156,36 @@ def _sentiment_score(counts: Counter[str]) -> float:
     return round((pos - neg) / total * 100, 2)
 
 
+def _tone_label(score: float) -> str:
+    if score >= 0.12:
+        return "bullish"
+    if score <= -0.12:
+        return "bearish"
+    return "neutral"
+
+
 def _top_new(curr: Counter[str], prev: Counter[str], limit: int = 10) -> list[dict[str, Any]]:
     rows = [
-        {"phrase": phrase, "count": count}
+        {"phrase": phrase, "count": count, "score": _tfidf_score(phrase, count, curr, prev)}
         for phrase, count in curr.items()
         if count >= 2 and prev.get(phrase, 0) == 0 and " " in phrase
     ]
-    return sorted(rows, key=lambda row: row["count"], reverse=True)[:limit]
+    return sorted(rows, key=lambda row: row["score"], reverse=True)[:limit]
 
 
 def _top_removed(curr: Counter[str], prev: Counter[str], limit: int = 10) -> list[dict[str, Any]]:
     rows = [
-        {"phrase": phrase, "previous_count": count}
+        {"phrase": phrase, "previous_count": count, "score": _tfidf_score(phrase, count, curr, prev)}
         for phrase, count in prev.items()
         if count >= 2 and curr.get(phrase, 0) == 0 and " " in phrase
     ]
-    return sorted(rows, key=lambda row: row["previous_count"], reverse=True)[:limit]
+    return sorted(rows, key=lambda row: row["score"], reverse=True)[:limit]
 
 
 def _emphasis_shift(curr: Counter[str], prev: Counter[str], limit: int = 12) -> list[dict[str, Any]]:
     rows: list[dict[str, Any]] = []
+    curr_total = sum(curr.values()) or 1
+    prev_total = sum(prev.values()) or 1
     for phrase in set(curr) | set(prev):
         if " " not in phrase:
             continue
@@ -125,13 +194,41 @@ def _emphasis_shift(curr: Counter[str], prev: Counter[str], limit: int = 12) ->
         delta = curr_count - prev_count
         if abs(delta) < 2:
             continue
-        rows.append({"phrase": phrase, "current_count": curr_count, "previous_count": prev_count, "delta": delta})
-    return sorted(rows, key=lambda row: abs(row["delta"]), reverse=True)[:limit]
+        curr_rate = curr_count / curr_total
+        prev_rate = prev_count / prev_total
+        score = abs(curr_rate - prev_rate) * math.log(curr_count + prev_count + 2)
+        rows.append({
+            "phrase": phrase,
+            "current_count": curr_count,
+            "previous_count": prev_count,
+            "delta": delta,
+            "score": round(score, 5),
+        })
+    return sorted(rows, key=lambda row: row["score"], reverse=True)[:limit]
+
+
+def _topic_shift(curr: Counter[str], prev: Counter[str]) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for topic, keywords in _TOPIC_LEXICON.items():
+        curr_count = sum(curr.get(keyword, 0) for keyword in keywords)
+        prev_count = sum(prev.get(keyword, 0) for keyword in keywords)
+        delta = curr_count - prev_count
+        if curr_count == 0 and prev_count == 0:
+            continue
+        rows.append({
+            "topic": topic,
+            "current_count": curr_count,
+            "previous_count": prev_count,
+            "delta": delta,
+        })
+    return sorted(rows, key=lambda row: abs(row["delta"]), reverse=True)
 
 
 def compute_delta(curr: Transcript, prev: Transcript) -> dict[str, Any]:
     curr_counts = _phrase_counts(curr.content)
     prev_counts = _phrase_counts(prev.content)
+    current_tone = _sentiment_score(curr_counts)
+    previous_tone = _sentiment_score(prev_counts)
     return {
         "ticker": curr.ticker,
         "available": True,
@@ -139,11 +236,15 @@ def compute_delta(curr: Transcript, prev: Transcript) -> dict[str, Any]:
         "previous": {"year": prev.year, "quarter": prev.quarter},
         "new_phrases": _top_new(curr_counts, prev_counts),
         "removed_phrases": _top_removed(curr_counts, prev_counts),
-        "emphasis_shift": _emphasis_shift(curr_counts, prev_counts),
+        "emphasis_shift": _emphasis_shift(curr_counts, prev_counts, limit=20),
         "tone_shift": {
-            "current_score": _sentiment_score(curr_counts),
-            "previous_score": _sentiment_score(prev_counts),
+            "current_score": current_tone,
+            "previous_score": previous_tone,
+            "delta": round(current_tone - previous_tone, 2),
+            "current_label": _tone_label(current_tone),
+            "previous_label": _tone_label(previous_tone),
         },
+        "topic_shift": _topic_shift(curr_counts, prev_counts),
     }
 
 
@@ -154,20 +255,59 @@ async def generate_delta_narrative(delta: dict[str, Any], ticker: str) -> dict[s
         "questions_to_ask": ["Which new phrases are one-off comments versus strategy?", "Are margin or capex terms increasing?"],
         "variant_view": "Use phrase shifts as a prompt for deeper research, not as standalone evidence.",
     }
+    prompt = (
+        "You are a skeptical institutional equity analyst reviewing earnings-call language drift.\n"
+        "Return ONLY valid JSON with this exact shape:\n"
+        '{"key_shifts":["..."],"what_it_means":"...","questions_to_ask":["..."],"variant_view":"..."}\n'
+        "Keep what_it_means to five concise analyst-style lines or fewer. Do not invent numbers.\n\n"
+        f"TICKER: {ticker.upper()}\n"
+        f"DELTA_DATA: {json.dumps(delta, ensure_ascii=False)[:12000]}"
+    )
+
+    anthropic_key = (os.getenv("ANTHROPIC_API_KEY") or os.getenv("CLAUDE_API_KEY") or "").strip()
+    if anthropic_key:
+        try:
+            from server.ai.llm_router import LLMConfig, LLMProvider, llm_router
+
+            text = await llm_router.generate(
+                prompt,
+                config=LLMConfig(
+                    provider=LLMProvider.CLAUDE,
+                    model="claude-sonnet-4-20250514",
+                    api_key=anthropic_key,
+                    temperature=0.2,
+                    max_tokens=900,
+                ),
+                system_prompt="You return strict JSON for equity research workflows.",
+            )
+            parsed = _parse_json_object(text)
+            if parsed:
+                return {**fallback, **parsed}
+        except Exception:
+            pass
+
     try:
         from server.services.gemini_service import generate_text
 
-        prompt = (
-            f"Analyze {ticker.upper()} earnings call transcript delta. Return concise JSON with keys "
-            "key_shifts, what_it_means, questions_to_ask, variant_view. Data:\n"
-            f"{delta}"
-        )
         text = await generate_text(prompt)
-        import json
-
-        parsed = json.loads(text.strip().removeprefix("```json").removesuffix("```").strip())
-        if isinstance(parsed, dict):
+        parsed = _parse_json_object(text)
+        if parsed:
             return {**fallback, **parsed}
     except Exception:
         return fallback
     return fallback
+
+
+def _parse_json_object(text: str) -> dict[str, Any] | None:
+    cleaned = text.strip()
+    if cleaned.startswith("```"):
+        cleaned = re.sub(r"^```(?:json)?", "", cleaned).strip()
+        cleaned = re.sub(r"```$", "", cleaned).strip()
+    match = re.search(r"\{.*\}", cleaned, flags=re.S)
+    if match:
+        cleaned = match.group(0)
+    try:
+        parsed = json.loads(cleaned)
+    except json.JSONDecodeError:
+        return None
+    return parsed if isinstance(parsed, dict) else None