fix: connection pool saturation causing dashboard all-zeros

- Increase httpx pool from 50 to 200 connections (140+ concurrent requests)
- Add semaphore(25) to RSS feed fetcher (119 feeds no longer flood the pool)
- Give slow sources (news, alerts, posture) 90s timeout instead of 45s
- Result: 56/58 sources OK, 0 timeouts (was: everything timing out)

src/world_intel_mcp/dashboard/app.py

@@ -157,14 +157,22 @@ async def _fetch_overview() -> dict:
     }
 
     # Per-coro timeout so no single slow source blocks the entire dashboard.
-    # Without this, 80+ RSS feeds timing out sequentially can delay the
+    # news_feed and trending_keywords get extra time (119 RSS feeds with semaphore).
-    # first SSE frame for minutes, leaving the dashboard stuck at all-zeros.
+    _SLOW_SOURCES = {
+        "news_feed",
+        "trending_keywords",
+        "alert_digest",
+        "weekly_trends",
+        "strategic_posture",
+    }
+
     async def _with_timeout(name: str, coro, timeout: float = 45.0):
+        effective = 90.0 if name in _SLOW_SOURCES else timeout
         try:
-            return await asyncio.wait_for(coro, timeout=timeout)
+            return await asyncio.wait_for(coro, timeout=effective)
         except asyncio.TimeoutError:
-            logger.warning("Dashboard fetch %s timed out after %.0fs", name, timeout)
+            logger.warning("Dashboard fetch %s timed out after %.0fs", name, effective)
-            return {"error": f"timeout after {timeout}s", "_timeout": True}
+            return {"error": f"timeout after {effective}s", "_timeout": True}
 
     gathered = await asyncio.gather(
         *[_with_timeout(name, c) for name, c in zip(coros.keys(), coros.values())],

src/world_intel_mcp/fetcher.py

@@ -67,7 +67,7 @@ class Fetcher:
             self._client = httpx.AsyncClient(
                 timeout=httpx.Timeout(self.default_timeout),
                 follow_redirects=True,
-                limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
+                limits=httpx.Limits(max_connections=200, max_keepalive_connections=40),
                 headers={"User-Agent": "PhoenixAGI-WorldIntel/0.1"},
                 proxy=None,  # never inherit system SOCKS proxy
             )

src/world_intel_mcp/sources/news.py

@@ -30,7 +30,10 @@ _RSS_FEEDS: dict[str, list[tuple[str, str]]] = {
         ("BBC World", "https://feeds.bbci.co.uk/news/world/rss.xml"),
         ("Al Jazeera", "https://www.aljazeera.com/xml/rss/all.xml"),
         ("AP Top News", "https://feedx.net/rss/ap.xml"),
-        ("Reuters World", "https://news.google.com/rss/search?q=source:Reuters&hl=en-US&gl=US&ceid=US:en"),
+        (
+            "Reuters World",
+            "https://news.google.com/rss/search?q=source:Reuters&hl=en-US&gl=US&ceid=US:en",
+        ),
         ("The Guardian World", "https://www.theguardian.com/world/rss"),
         ("DW News", "https://rss.dw.com/rss/en/top_news/rss-en-top"),
         ("France24", "https://www.france24.com/en/rss"),
@@ -56,7 +59,10 @@ _RSS_FEEDS: dict[str, list[tuple[str, str]]] = {
         ("The Register", "https://www.theregister.com/headlines.atom"),
     ],
     "finance": [
-        ("CNBC", "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114"),
+        (
+            "CNBC",
+            "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114",
+        ),
         ("MarketWatch", "https://feeds.content.dowjones.io/public/rss/mw_topstories"),
         ("FT World", "https://www.ft.com/rss/home/uk"),
         ("Bloomberg", "https://feeds.bloomberg.com/markets/news.rss"),
@@ -68,12 +74,18 @@ _RSS_FEEDS: dict[str, list[tuple[str, str]]] = {
         ("War on the Rocks", "https://warontherocks.com/feed/"),
         ("The War Zone", "https://www.twz.com/feed"),
         ("Breaking Defense", "https://breakingdefense.com/feed/"),
-        ("Military Times", "https://www.militarytimes.com/arc/outboundfeeds/rss/?outputType=xml"),
+        (
+            "Military Times",
+            "https://www.militarytimes.com/arc/outboundfeeds/rss/?outputType=xml",
+        ),
         ("USNI News", "https://news.usni.org/feed"),
     ],
     "science": [
         ("Nature", "https://www.nature.com/nature.rss"),
-        ("Science", "https://www.science.org/action/showFeed?type=etoc&feed=rss&jc=science"),
+        (
+            "Science",
+            "https://www.science.org/action/showFeed?type=etoc&feed=rss&jc=science",
+        ),
         ("Phys.org", "https://phys.org/rss-feed/"),
         ("New Scientist", "https://www.newscientist.com/feed/home/"),
         ("Scientific American", "https://rss.sciam.com/ScientificAmerican-Global"),
@@ -90,7 +102,10 @@ _RSS_FEEDS: dict[str, list[tuple[str, str]]] = {
     ],
     "middle_east": [
         ("Middle East Eye", "https://www.middleeasteye.net/rss"),
-        ("The National UAE", "https://www.thenationalnews.com/arc/outboundfeeds/rss/?outputType=xml"),
+        (
+            "The National UAE",
+            "https://www.thenationalnews.com/arc/outboundfeeds/rss/?outputType=xml",
+        ),
         ("Times of Israel", "https://www.timesofisrael.com/feed/"),
         ("Iran Intl", "https://www.iranintl.com/en/feed"),
     ],
@@ -98,7 +113,10 @@ _RSS_FEEDS: dict[str, list[tuple[str, str]]] = {
         ("SCMP", "https://www.scmp.com/rss/91/feed"),
         ("Nikkei Asia", "https://asia.nikkei.com/rss/feed/nar"),
         ("The Diplomat", "https://thediplomat.com/feed/"),
-        ("Channel News Asia", "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml"),
+        (
+            "Channel News Asia",
+            "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml",
+        ),
         ("Lowy Interpreter", "https://www.lowyinstitute.org/the-interpreter/rss.xml"),
     ],
     "africa": [
@@ -136,7 +154,10 @@ _RSS_FEEDS: dict[str, list[tuple[str, str]]] = {
     ],
     "government": [
         ("State Dept", "https://www.state.gov/rss-feed/press-releases/feed/"),
-        ("DoD News", "https://www.defense.gov/DesktopModules/ArticleCS/RSS.ashx?ContentType=1&Site=945&max=10"),
+        (
+            "DoD News",
+            "https://www.defense.gov/DesktopModules/ArticleCS/RSS.ashx?ContentType=1&Site=945&max=10",
+        ),
         ("UN News", "https://news.un.org/feed/subscribe/en/news/all/rss.xml"),
         ("White House", "https://www.whitehouse.gov/feed/"),
     ],
@@ -297,23 +318,149 @@ SOURCE_TIERS: dict[str, str] = {
 }
 
 _STOPWORDS: set[str] = {
-    "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+    "the",
-    "have", "has", "had", "do", "does", "did", "will", "would", "shall",
+    "a",
-    "should", "may", "might", "must", "can", "could", "in", "on", "at",
+    "an",
-    "to", "for", "of", "and", "or", "but", "nor", "not", "no", "so",
+    "is",
-    "yet", "both", "either", "neither", "with", "from", "by", "as",
+    "are",
-    "into", "through", "during", "before", "after", "above", "below",
+    "was",
-    "between", "under", "over", "about", "against", "out", "off", "up",
+    "were",
-    "down", "then", "once", "here", "there", "when", "where", "why",
+    "be",
-    "how", "all", "each", "every", "any", "few", "more", "most", "some",
+    "been",
-    "such", "only", "own", "same", "than", "too", "very", "just", "also",
+    "being",
-    "now", "it", "its", "he", "she", "they", "them", "their", "his",
+    "have",
-    "her", "we", "you", "your", "our", "my", "me", "him", "us",
+    "has",
-    "that", "this", "these", "those", "which", "who", "whom", "what",
+    "had",
-    "if", "while", "because", "until", "although", "since", "whether",
+    "do",
-    "new", "says", "said", "one", "two", "first", "last", "many",
+    "does",
-    "much", "get", "got", "back", "even", "still", "well", "way",
+    "did",
-    "s", "t", "re", "ve", "d", "ll", "m",
+    "will",
+    "would",
+    "shall",
+    "should",
+    "may",
+    "might",
+    "must",
+    "can",
+    "could",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "of",
+    "and",
+    "or",
+    "but",
+    "nor",
+    "not",
+    "no",
+    "so",
+    "yet",
+    "both",
+    "either",
+    "neither",
+    "with",
+    "from",
+    "by",
+    "as",
+    "into",
+    "through",
+    "during",
+    "before",
+    "after",
+    "above",
+    "below",
+    "between",
+    "under",
+    "over",
+    "about",
+    "against",
+    "out",
+    "off",
+    "up",
+    "down",
+    "then",
+    "once",
+    "here",
+    "there",
+    "when",
+    "where",
+    "why",
+    "how",
+    "all",
+    "each",
+    "every",
+    "any",
+    "few",
+    "more",
+    "most",
+    "some",
+    "such",
+    "only",
+    "own",
+    "same",
+    "than",
+    "too",
+    "very",
+    "just",
+    "also",
+    "now",
+    "it",
+    "its",
+    "he",
+    "she",
+    "they",
+    "them",
+    "their",
+    "his",
+    "her",
+    "we",
+    "you",
+    "your",
+    "our",
+    "my",
+    "me",
+    "him",
+    "us",
+    "that",
+    "this",
+    "these",
+    "those",
+    "which",
+    "who",
+    "whom",
+    "what",
+    "if",
+    "while",
+    "because",
+    "until",
+    "although",
+    "since",
+    "whether",
+    "new",
+    "says",
+    "said",
+    "one",
+    "two",
+    "first",
+    "last",
+    "many",
+    "much",
+    "get",
+    "got",
+    "back",
+    "even",
+    "still",
+    "well",
+    "way",
+    "s",
+    "t",
+    "re",
+    "ve",
+    "d",
+    "ll",
+    "m",
 }
 
 _GDELT_DOC_URL = "https://api.gdeltproject.org/api/v2/doc/doc"
@@ -326,6 +473,7 @@ _PUNCT_RE = re.compile(f"[{re.escape(string.punctuation)}]")
 # Helpers
 # ---------------------------------------------------------------------------
 
+
 def _parse_published(entry: dict) -> str | None:
     """Parse an RSS entry's published date to ISO 8601 UTC string.
 
@@ -370,6 +518,7 @@ def _truncate(text: str | None, max_len: int = 200) -> str:
 # Function 1: RSS News Feed Aggregation
 # ---------------------------------------------------------------------------
 
+
 async def fetch_news_feed(
     fetcher: Fetcher,
     category: str | None = None,
@@ -440,15 +589,17 @@ async def fetch_news_feed(
         for entry in parsed.get("entries", []):
             published = _parse_published(entry)
             summary_raw = entry.get("summary") or entry.get("description") or ""
-            items.append({
+            items.append(
-                "title": entry.get("title", ""),
+                {
-                "link": entry.get("link", ""),
+                    "title": entry.get("title", ""),
-                "published": published,
+                    "link": entry.get("link", ""),
-                "summary": _truncate(summary_raw, 200),
+                    "published": published,
-                "feed_name": feed_name,
+                    "summary": _truncate(summary_raw, 200),
-                "category": cat,
+                    "feed_name": feed_name,
-                "source_tier": SOURCE_TIERS.get(feed_name, "unknown"),
+                    "category": cat,
-            })
+                    "source_tier": SOURCE_TIERS.get(feed_name, "unknown"),
+                }
+            )
 
         return items
 
@@ -456,15 +607,24 @@ async def fetch_news_feed(
         """Wrap single feed fetch with a hard timeout."""
         try:
             return await asyncio.wait_for(
-                _fetch_single_feed(feed_name, url, cat), timeout=12.0,
+                _fetch_single_feed(feed_name, url, cat),
+                timeout=12.0,
             )
         except asyncio.TimeoutError:
             logger.debug("RSS feed %s timed out", feed_name)
             return []
 
-    # Fetch ALL feeds across ALL categories in one parallel batch
+    # Fetch feeds with concurrency cap to avoid saturating the httpx pool.
+    # Without this, 119 RSS feeds + other source calls exceed the 50-connection
+    # pool limit, causing cascade timeouts across the entire dashboard.
+    sem = asyncio.Semaphore(25)
+
+    async def _throttled(feed_name: str, url: str, cat: str) -> list[dict]:
+        async with sem:
+            return await _safe_fetch(feed_name, url, cat)
+
     all_tasks = [
-        _safe_fetch(feed_name, url, cat)
+        _throttled(feed_name, url, cat)
         for cat, feeds in categories_to_fetch.items()
         for feed_name, url in feeds
     ]
@@ -494,6 +654,7 @@ async def fetch_news_feed(
 # Function 2: Keyword Trending / Spike Detection
 # ---------------------------------------------------------------------------
 
+
 async def fetch_trending_keywords(
     fetcher: Fetcher,
     hours: int = 6,
@@ -559,6 +720,7 @@ async def fetch_trending_keywords(
 # Function 3: GDELT 2.0 Doc API Search
 # ---------------------------------------------------------------------------
 
+
 async def fetch_gdelt_search(
     fetcher: Fetcher,
     query: str = "conflict",
@@ -613,15 +775,17 @@ async def fetch_gdelt_search(
     if mode == "artlist":
         articles = []
         for article in data.get("articles", []):
-            articles.append({
+            articles.append(
-                "title": article.get("title"),
+                {
-                "url": article.get("url"),
+                    "title": article.get("title"),
-                "seendate": article.get("seendate"),
+                    "url": article.get("url"),
-                "socialimage": article.get("socialimage"),
+                    "seendate": article.get("seendate"),
-                "domain": article.get("domain"),
+                    "socialimage": article.get("socialimage"),
-                "language": article.get("language"),
+                    "domain": article.get("domain"),
-                "sourcecountry": article.get("sourcecountry"),
+                    "language": article.get("language"),
-            })
+                    "sourcecountry": article.get("sourcecountry"),
+                }
+            )
 
         return {
             "articles": articles,