commit 01cb4255ef492bce995dcba7b2868cbc99e13cd7 Author: Marc Shade Date: Mon Feb 23 10:33:57 2026 -0500 feat: add world-intel-mcp server — 36 tools across 14 intelligence domains New MCP server providing real-time global intelligence: financial markets (Yahoo Finance, CoinGecko), economic indicators (FRED, EIA, World Bank), conflict tracking (ACLED, UCDP, HDX), military flights (OpenSky), infrastructure monitoring (Cloudflare Radar, NGA cable health), maritime warnings (NGA), climate anomalies (Open-Meteo), news aggregation (RSS, GDELT), prediction markets (Polymarket), displacement data (UNHCR), aviation delays (FAA), cyber threats (Feodo, CISA KEV, SANS, URLhaus), country intelligence briefs (Ollama LLM), and HTML report generation (Jinja2 + Chart.js). Includes: SQLite TTL cache, per-source circuit breakers, async HTTP fetcher with retry/rate-limiting, Click CLI with 27 commands, 4 analysis modules (instability scoring, geo-convergence, signal aggregation, news clustering), 4 HTML report templates, and 28 unit tests. Co-Authored-By: Claude Opus 4.6 diff --git a/README.md b/README.md new file mode 100644 index 0000000..da395db --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +# world-intel-mcp + +Real-time global intelligence MCP server for Phoenix AGI. Covers financial markets, economic indicators, earthquakes, wildfires, conflict, military flights, infrastructure health, and more. + +## Quick Start + +```bash +pip install -e . +world-intel-mcp # Run as MCP server +intel markets # CLI: stock indices +intel earthquakes # CLI: recent quakes +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..105ac16 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[project] +name = "world-intel-mcp" +version = "0.1.0" +description = "World Intelligence MCP Server - real-time global intelligence across 17 domains" +readme = "README.md" +requires-python = ">=3.11" +license = {text = "MIT"} +authors = [ + {name = "Phoenix AGI", email = "phoenix@2acrestudios.com"} +] +keywords = ["mcp", "intelligence", "osint", "markets", "geopolitical", "phoenix"] + +dependencies = [ + "mcp>=1.0.0", + "httpx>=0.27.0", + "pydantic>=2.0.0", + "feedparser>=6.0.0", + "click>=8.1.0", + "jinja2>=3.1.0", + "rich>=13.0.0", +] + +[project.optional-dependencies] +pdf = ["weasyprint>=62.0"] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "pytest-cov>=4.0.0", + "respx>=0.21.0", +] + +[project.scripts] +world-intel-mcp = "world_intel_mcp.server:run" +intel = "world_intel_mcp.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/world_intel_mcp"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["src/world_intel_mcp/tests"] +addopts = "-v --tb=short" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a59f354 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +mcp==1.8.1 +httpx[socks]==0.28.1 +pydantic==2.10.6 +feedparser==6.0.11 +click==8.1.8 +jinja2==3.1.5 +rich==13.9.4 diff --git a/src/world_intel_mcp/__init__.py b/src/world_intel_mcp/__init__.py new file mode 100644 index 0000000..5734cf8 --- /dev/null +++ b/src/world_intel_mcp/__init__.py @@ -0,0 +1,3 @@ +"""World Intelligence MCP Server — real-time global intelligence across 17 domains.""" + +__version__ = "0.1.0" diff --git a/src/world_intel_mcp/analysis/__init__.py b/src/world_intel_mcp/analysis/__init__.py new file mode 100644 index 0000000..ed04749 --- /dev/null +++ b/src/world_intel_mcp/analysis/__init__.py @@ -0,0 +1 @@ +"""Analysis modules for world-intel-mcp.""" diff --git a/src/world_intel_mcp/analysis/clustering.py b/src/world_intel_mcp/analysis/clustering.py new file mode 100644 index 0000000..30b0e62 --- /dev/null +++ b/src/world_intel_mcp/analysis/clustering.py @@ -0,0 +1,98 @@ +"""News article clustering using Jaccard similarity. + +Groups news articles by content similarity to reduce noise +and identify story clusters. +""" + +import logging +import re + +logger = logging.getLogger("world-intel-mcp.analysis.clustering") + +_STOPWORDS = frozenset({ + "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", + "have", "has", "had", "do", "does", "did", "will", "would", "could", + "should", "may", "might", "can", "shall", "in", "on", "at", "to", + "for", "of", "and", "or", "but", "not", "no", "nor", "with", "from", + "by", "it", "its", "that", "this", "these", "those", "he", "she", + "they", "we", "you", "i", "me", "my", "his", "her", "our", "your", + "their", "what", "which", "who", "whom", "how", "when", "where", + "why", "if", "then", "than", "so", "as", "up", "out", "about", + "into", "over", "after", "before", "between", "under", "again", + "also", "just", "more", "most", "other", "some", "such", "very", + "new", "said", "says", "one", "two", "first", +}) + + +def _tokenize(text: str) -> set[str]: + """Extract meaningful word tokens from text.""" + words = re.findall(r'[a-z]{3,}', text.lower()) + return {w for w in words if w not in _STOPWORDS} + + +def jaccard_similarity(set_a: set, set_b: set) -> float: + """Compute Jaccard similarity between two sets.""" + if not set_a or not set_b: + return 0.0 + intersection = len(set_a & set_b) + union = len(set_a | set_b) + return intersection / union if union > 0 else 0.0 + + +def cluster_articles( + articles: list[dict], + similarity_threshold: float = 0.3, + title_field: str = "title", +) -> list[dict]: + """Cluster articles by title similarity. + + Args: + articles: List of article dicts. + similarity_threshold: Minimum Jaccard similarity to merge. + title_field: Key containing the article title. + + Returns: + List of cluster dicts with representative article and member count. + """ + if not articles: + return [] + + # Tokenize all titles + tokenized = [] + for article in articles: + title = article.get(title_field, "") or "" + tokens = _tokenize(title) + tokenized.append(tokens) + + # Greedy single-linkage clustering + assigned = [False] * len(articles) + clusters: list[dict] = [] + + for i in range(len(articles)): + if assigned[i]: + continue + + cluster_members = [i] + assigned[i] = True + + for j in range(i + 1, len(articles)): + if assigned[j]: + continue + + # Check similarity against cluster representative (first member) + sim = jaccard_similarity(tokenized[i], tokenized[j]) + if sim >= similarity_threshold: + cluster_members.append(j) + assigned[j] = True + + # Build cluster output + representative = articles[cluster_members[0]] + clusters.append({ + "representative": representative, + "member_count": len(cluster_members), + "member_indices": cluster_members, + }) + + # Sort by member count descending + clusters.sort(key=lambda c: c["member_count"], reverse=True) + return clusters diff --git a/src/world_intel_mcp/analysis/convergence.py b/src/world_intel_mcp/analysis/convergence.py new file mode 100644 index 0000000..8b41762 --- /dev/null +++ b/src/world_intel_mcp/analysis/convergence.py @@ -0,0 +1,84 @@ +"""Geo-convergence detection for multi-domain signal overlap. + +Detects when signals from different domains (conflict, natural disasters, +military activity) converge geographically, suggesting elevated risk. +""" + +import logging +from collections import defaultdict +from math import floor + +logger = logging.getLogger("world-intel-mcp.analysis.convergence") + + +def _grid_key(lat: float, lon: float, resolution: float = 1.0) -> tuple[int, int]: + """Convert lat/lon to grid cell key.""" + return (int(floor(lat / resolution)), int(floor(lon / resolution))) + + +def detect_convergence( + events: list[dict], + resolution: float = 1.0, + min_types: int = 2, + min_total: int = 3, +) -> list[dict]: + """Detect geographic convergence of multi-domain signals. + + Args: + events: List of dicts with 'lat', 'lon', 'type' (domain), and optional 'weight'. + resolution: Grid cell size in degrees. + min_types: Minimum number of different signal types for convergence. + min_total: Minimum total events in cell for convergence. + + Returns: + List of convergence hotspots sorted by score descending. + """ + grid: dict[tuple[int, int], list[dict]] = defaultdict(list) + + for event in events: + lat = event.get("lat") + lon = event.get("lon") + if lat is None or lon is None: + continue + try: + lat_f = float(lat) + lon_f = float(lon) + except (ValueError, TypeError): + continue + + key = _grid_key(lat_f, lon_f, resolution) + grid[key].append(event) + + hotspots = [] + for (grid_lat, grid_lon), cell_events in grid.items(): + if len(cell_events) < min_total: + continue + + types = set() + total_weight = 0.0 + for e in cell_events: + types.add(e.get("type", "unknown")) + total_weight += e.get("weight", 1.0) + + if len(types) < min_types: + continue + + # Center of grid cell + center_lat = (grid_lat + 0.5) * resolution + center_lon = (grid_lon + 0.5) * resolution + + # Score: number of types * log of total events * weight + score = len(types) * (1 + len(cell_events) ** 0.5) * (total_weight / len(cell_events)) + + hotspots.append({ + "lat": round(center_lat, 2), + "lon": round(center_lon, 2), + "event_count": len(cell_events), + "signal_types": sorted(types), + "type_count": len(types), + "total_weight": round(total_weight, 1), + "convergence_score": round(score, 2), + }) + + hotspots.sort(key=lambda h: h["convergence_score"], reverse=True) + return hotspots diff --git a/src/world_intel_mcp/analysis/instability.py b/src/world_intel_mcp/analysis/instability.py new file mode 100644 index 0000000..d384033 --- /dev/null +++ b/src/world_intel_mcp/analysis/instability.py @@ -0,0 +1,110 @@ +"""Country Instability Index (CII) computation. + +Pure scoring functions that transform raw data into instability scores. +""" + +import logging + +logger = logging.getLogger("world-intel-mcp.analysis.instability") + + +def score_conflict_intensity(event_count: int, days: int = 30) -> float: + """Score conflict intensity 0-20 based on event count per period.""" + daily_rate = event_count / max(days, 1) + # 0 events = 0, 10+/day = 20 + return min(20.0, daily_rate * 2.0) + + +def score_economic_stress(inflation_rate: float | None, gdp_growth: float | None) -> float: + """Score economic stress 0-20 from inflation and GDP growth.""" + score = 0.0 + if inflation_rate is not None: + # High inflation (>20%) = 10, moderate (>10%) = 5 + if inflation_rate > 20: + score += 10.0 + elif inflation_rate > 10: + score += 5.0 + elif inflation_rate > 5: + score += 2.0 + if gdp_growth is not None: + # Negative GDP = 10, stagnant = 5 + if gdp_growth < -5: + score += 10.0 + elif gdp_growth < 0: + score += 7.0 + elif gdp_growth < 1: + score += 3.0 + return min(20.0, score) + + +def score_humanitarian_crisis(dataset_count: int, displacement_total: int = 0) -> float: + """Score humanitarian crisis 0-20 based on HDX datasets and displacement.""" + score = 0.0 + # Many crisis datasets = high concern + score += min(10.0, dataset_count * 0.5) + # Displacement + if displacement_total > 1_000_000: + score += 10.0 + elif displacement_total > 100_000: + score += 5.0 + return min(20.0, score) + + +def score_infrastructure_disruption(outage_count: int, cable_warnings: int = 0) -> float: + """Score infrastructure disruption 0-20.""" + score = 0.0 + score += min(10.0, outage_count * 2.0) + score += min(10.0, cable_warnings * 2.5) + return min(20.0, score) + + +def score_military_activity(aircraft_count: int, flight_density: float = 0.0) -> float: + """Score military activity 0-20 based on aircraft presence.""" + score = 0.0 + if aircraft_count > 50: + score = 20.0 + elif aircraft_count > 20: + score = 15.0 + elif aircraft_count > 10: + score = 10.0 + elif aircraft_count > 5: + score = 5.0 + elif aircraft_count > 0: + score = 2.0 + return score + + +def compute_cii( + conflict: float = 0.0, + economic: float = 0.0, + humanitarian: float = 0.0, + infrastructure: float = 0.0, + military: float = 0.0, +) -> dict: + """Compute Country Instability Index (0-100) from component scores. + + Returns dict with index, components, and risk_level. + """ + total = conflict + economic + humanitarian + infrastructure + military + total = min(100.0, max(0.0, total)) + + if total >= 75: + risk_level = "critical" + elif total >= 50: + risk_level = "high" + elif total >= 25: + risk_level = "medium" + else: + risk_level = "low" + + return { + "instability_index": round(total, 1), + "components": { + "conflict_intensity": round(conflict, 1), + "economic_stress": round(economic, 1), + "humanitarian_crisis": round(humanitarian, 1), + "infrastructure_disruption": round(infrastructure, 1), + "military_activity": round(military, 1), + }, + "risk_level": risk_level, + } diff --git a/src/world_intel_mcp/analysis/signals.py b/src/world_intel_mcp/analysis/signals.py new file mode 100644 index 0000000..db161b2 --- /dev/null +++ b/src/world_intel_mcp/analysis/signals.py @@ -0,0 +1,98 @@ +"""Signal aggregation per country. + +Collects and normalizes signals from multiple domains into +a per-country summary for dashboard and reporting. +""" + +import logging +from collections import defaultdict + +logger = logging.getLogger("world-intel-mcp.analysis.signals") + + +def aggregate_country_signals( + conflict_events: list[dict] | None = None, + displacement_data: list[dict] | None = None, + earthquake_data: list[dict] | None = None, + fire_data: dict | None = None, +) -> dict[str, dict]: + """Aggregate multi-domain signals by country. + + Args: + conflict_events: ACLED/UCDP events with 'country' field. + displacement_data: UNHCR data with 'country' field. + earthquake_data: USGS earthquakes (uses reverse geocoding heuristic). + fire_data: Wildfire data by region (maps to countries approximately). + + Returns: + Dict mapping country name to signal summary. + """ + countries: dict[str, dict] = defaultdict(lambda: { + "conflict_events": 0, + "fatalities": 0, + "displaced_persons": 0, + "earthquakes": 0, + "max_earthquake_mag": 0.0, + "fires": 0, + "signal_count": 0, + "domains": set(), + }) + + # Conflict events + if conflict_events: + for event in conflict_events: + country = event.get("country") + if not country: + continue + c = countries[country] + c["conflict_events"] += 1 + fat = event.get("fatalities", 0) + if isinstance(fat, (int, float)): + c["fatalities"] += int(fat) + c["domains"].add("conflict") + + # Displacement + if displacement_data: + for record in displacement_data: + country = record.get("country") + if not country: + continue + c = countries[country] + total = record.get("total_displaced", 0) + if isinstance(total, (int, float)): + c["displaced_persons"] += int(total) + c["domains"].add("displacement") + + # Earthquakes (approximate country from place string) + if earthquake_data: + for quake in earthquake_data: + place = quake.get("place", "") or "" + # USGS format: "123km SSE of City, Country" + parts = place.rsplit(", ", 1) + country = parts[-1] if len(parts) > 1 else "Unknown" + c = countries[country] + c["earthquakes"] += 1 + mag = quake.get("magnitude", 0) or 0 + if isinstance(mag, (int, float)) and mag > c["max_earthquake_mag"]: + c["max_earthquake_mag"] = float(mag) + c["domains"].add("seismology") + + # TODO: fire_data integration (requires region-to-country mapping) + _ = fire_data + + # Compute signal counts + result = {} + for country, data in countries.items(): + domains = data.pop("domains") + data["signal_count"] = len(domains) + data["active_domains"] = sorted(domains) + result[country] = data + + # Sort by signal count, then fatalities + result = dict(sorted( + result.items(), + key=lambda x: (x[1]["signal_count"], x[1]["fatalities"]), + reverse=True, + )) + + return result diff --git a/src/world_intel_mcp/cache.py b/src/world_intel_mcp/cache.py new file mode 100644 index 0000000..54d2bbb --- /dev/null +++ b/src/world_intel_mcp/cache.py @@ -0,0 +1,105 @@ +"""SQLite TTL cache for world-intel-mcp. + +Simple key-value cache with per-key TTL, WAL mode, and automatic eviction. +No external deps — just stdlib sqlite3. +""" + +import json +import logging +import sqlite3 +import time +from pathlib import Path +from typing import Any + +logger = logging.getLogger("world-intel-mcp.cache") + +_DEFAULT_DB = Path.home() / ".cache" / "world-intel-mcp" / "cache.db" + + +class Cache: + """SQLite-backed TTL cache.""" + + def __init__(self, db_path: Path | None = None): + self.db_path = db_path or _DEFAULT_DB + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._conn: sqlite3.Connection | None = None + self._init_db() + + def _init_db(self) -> None: + conn = self._get_conn() + conn.execute(""" + CREATE TABLE IF NOT EXISTS cache ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + expires_at REAL NOT NULL, + created_at REAL NOT NULL + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_cache_expires ON cache(expires_at)") + conn.commit() + + def _get_conn(self) -> sqlite3.Connection: + if self._conn is None: + self._conn = sqlite3.connect(str(self.db_path), timeout=10) + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA busy_timeout=5000") + self._conn.execute("PRAGMA synchronous=NORMAL") + return self._conn + + def get(self, key: str) -> Any | None: + """Get a cached value. Returns None if missing or expired.""" + conn = self._get_conn() + row = conn.execute( + "SELECT value, expires_at FROM cache WHERE key = ?", (key,) + ).fetchone() + if row is None: + return None + value, expires_at = row + if time.time() > expires_at: + conn.execute("DELETE FROM cache WHERE key = ?", (key,)) + conn.commit() + return None + return json.loads(value) + + def set(self, key: str, value: Any, ttl_seconds: int) -> None: + """Store a value with TTL in seconds.""" + conn = self._get_conn() + now = time.time() + conn.execute( + "INSERT OR REPLACE INTO cache (key, value, expires_at, created_at) VALUES (?, ?, ?, ?)", + (key, json.dumps(value, default=str), now + ttl_seconds, now), + ) + conn.commit() + + def delete(self, key: str) -> None: + """Delete a specific key.""" + conn = self._get_conn() + conn.execute("DELETE FROM cache WHERE key = ?", (key,)) + conn.commit() + + def evict_expired(self) -> int: + """Remove all expired entries. Returns count removed.""" + conn = self._get_conn() + cursor = conn.execute("DELETE FROM cache WHERE expires_at < ?", (time.time(),)) + conn.commit() + return cursor.rowcount + + def stats(self) -> dict[str, Any]: + """Cache statistics.""" + conn = self._get_conn() + now = time.time() + total = conn.execute("SELECT COUNT(*) FROM cache").fetchone()[0] + expired = conn.execute( + "SELECT COUNT(*) FROM cache WHERE expires_at < ?", (now,) + ).fetchone()[0] + return { + "total_entries": total, + "active_entries": total - expired, + "expired_entries": expired, + "db_path": str(self.db_path), + } + + def close(self) -> None: + if self._conn: + self._conn.close() + self._conn = None diff --git a/src/world_intel_mcp/circuit_breaker.py b/src/world_intel_mcp/circuit_breaker.py new file mode 100644 index 0000000..b72d80a --- /dev/null +++ b/src/world_intel_mcp/circuit_breaker.py @@ -0,0 +1,92 @@ +"""Per-source circuit breaker for world-intel-mcp. + +Tracks failures per data source. Trips after N consecutive failures, +blocks calls for a cooldown period, then allows a single probe. +""" + +import logging +import time +from dataclasses import dataclass, field + +logger = logging.getLogger("world-intel-mcp.circuit_breaker") + + +@dataclass +class _State: + failures: int = 0 + last_failure: float = 0.0 + tripped_at: float = 0.0 + is_open: bool = False + total_trips: int = 0 + total_successes: int = 0 + total_failures: int = 0 + + +class CircuitBreaker: + """Circuit breaker with configurable thresholds per source.""" + + def __init__( + self, + failure_threshold: int = 3, + cooldown_seconds: float = 300.0, + ): + self.failure_threshold = failure_threshold + self.cooldown_seconds = cooldown_seconds + self._states: dict[str, _State] = {} + + def _get(self, source: str) -> _State: + if source not in self._states: + self._states[source] = _State() + return self._states[source] + + def is_available(self, source: str) -> bool: + """Check if source is available (circuit closed or cooldown elapsed).""" + state = self._get(source) + if not state.is_open: + return True + if time.time() - state.tripped_at >= self.cooldown_seconds: + return True # allow probe + return False + + def record_success(self, source: str) -> None: + """Record successful call — resets failure count, closes circuit.""" + state = self._get(source) + state.failures = 0 + state.is_open = False + state.total_successes += 1 + + def record_failure(self, source: str) -> None: + """Record failed call — increments counter, may trip breaker.""" + state = self._get(source) + state.failures += 1 + state.last_failure = time.time() + state.total_failures += 1 + if state.failures >= self.failure_threshold and not state.is_open: + state.is_open = True + state.tripped_at = time.time() + state.total_trips += 1 + logger.warning( + "Circuit breaker TRIPPED for %s (failures=%d, cooldown=%.0fs)", + source, state.failures, self.cooldown_seconds, + ) + + def status(self) -> dict[str, dict]: + """Return status of all tracked sources.""" + now = time.time() + result = {} + for source, state in self._states.items(): + if state.is_open: + remaining = max(0, self.cooldown_seconds - (now - state.tripped_at)) + status = "open" if remaining > 0 else "half-open" + else: + remaining = 0 + status = "closed" + result[source] = { + "status": status, + "failures": state.failures, + "cooldown_remaining_s": round(remaining, 1), + "total_trips": state.total_trips, + "total_successes": state.total_successes, + "total_failures": state.total_failures, + } + return result diff --git a/src/world_intel_mcp/cli.py b/src/world_intel_mcp/cli.py new file mode 100644 index 0000000..ef6b96d --- /dev/null +++ b/src/world_intel_mcp/cli.py @@ -0,0 +1,978 @@ +#!/usr/bin/env python3 +""" +intel — CLI for World Intelligence MCP. + +Calls source functions directly (no MCP protocol overhead). +""" + +import asyncio +import json +from typing import Any + +import click +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich import box + +from .cache import Cache +from .circuit_breaker import CircuitBreaker +from .fetcher import Fetcher +from .sources import markets, economic, seismology, wildfire, conflict, military, infrastructure, maritime, climate, news, intelligence, prediction, displacement, aviation, cyber +from .reports import generator as report_gen + +console = Console() + +# Shared infrastructure — lazily initialized +_fetcher: Fetcher | None = None + + +def _get_fetcher() -> Fetcher: + global _fetcher + if _fetcher is None: + cache = Cache() + breaker = CircuitBreaker() + _fetcher = Fetcher(cache=cache, breaker=breaker) + return _fetcher + + +def _run(coro: Any) -> Any: + """Run an async coroutine from sync CLI context.""" + return asyncio.run(coro) + + +def _print_json(data: dict) -> None: + """Print raw JSON if --json flag, otherwise formatted.""" + console.print_json(json.dumps(data, default=str)) + + +# --------------------------------------------------------------------------- +# Root group +# --------------------------------------------------------------------------- + +@click.group() +@click.option("--json-output", is_flag=True, help="Output raw JSON") +@click.pass_context +def main(ctx: click.Context, json_output: bool) -> None: + """World Intelligence CLI — real-time global intelligence.""" + ctx.ensure_object(dict) + ctx.obj["json"] = json_output + + +# --------------------------------------------------------------------------- +# Markets +# --------------------------------------------------------------------------- + +@main.command(name="markets") +@click.option("--symbols", "-s", multiple=True, help="Ticker symbols") +@click.pass_context +def markets_cmd(ctx: click.Context, symbols: tuple[str, ...]) -> None: + """Stock market index quotes.""" + f = _get_fetcher() + sym_list = list(symbols) if symbols else None + data = _run(markets.fetch_market_quotes(f, symbols=sym_list)) + + if ctx.obj.get("json"): + _print_json(data) + return + + quotes = data.get("quotes", []) + if not quotes: + console.print("[yellow]No market data available[/yellow]") + return + + table = Table(title="Market Indices", box=box.SIMPLE_HEAVY) + table.add_column("Symbol", style="bold") + table.add_column("Price", justify="right") + table.add_column("Change %", justify="right") + table.add_column("Currency") + + for q in quotes: + chg = q.get("change_pct") or 0 + price = q.get("price") or 0 + style = "green" if chg >= 0 else "red" + table.add_row( + q.get("symbol", "?"), + f"{price:,.2f}", + f"[{style}]{chg:+.2f}%[/{style}]", + q.get("currency", ""), + ) + console.print(table) + + +@main.command() +@click.option("--limit", "-n", default=20, help="Number of coins") +@click.pass_context +def crypto(ctx: click.Context, limit: int) -> None: + """Top cryptocurrency prices.""" + f = _get_fetcher() + data = _run(markets.fetch_crypto_quotes(f, limit=limit)) + + if ctx.obj.get("json"): + _print_json(data) + return + + coins = data.get("coins", []) + if not coins: + console.print("[yellow]No crypto data available[/yellow]") + return + + table = Table(title=f"Top {limit} Cryptocurrencies", box=box.SIMPLE_HEAVY) + table.add_column("#", justify="right") + table.add_column("Symbol", style="bold") + table.add_column("Price", justify="right") + table.add_column("24h %", justify="right") + table.add_column("Market Cap", justify="right") + + for i, c in enumerate(coins[:limit], 1): + chg = c.get("price_change_percentage_24h", 0) or 0 + style = "green" if chg >= 0 else "red" + mcap = c.get("market_cap", 0) or 0 + table.add_row( + str(i), + c.get("symbol", "?").upper(), + f"${c.get('current_price', 0):,.2f}", + f"[{style}]{chg:+.2f}%[/{style}]", + f"${mcap:,.0f}", + ) + console.print(table) + + +@main.command() +@click.pass_context +def macro(ctx: click.Context) -> None: + """7-signal macro dashboard.""" + f = _get_fetcher() + data = _run(markets.fetch_macro_signals(f)) + + if ctx.obj.get("json"): + _print_json(data) + return + + signals = data.get("signals", {}) + table = Table(title="Macro Signals", box=box.SIMPLE_HEAVY) + table.add_column("Signal", style="bold") + table.add_column("Value", justify="right") + table.add_column("Detail") + + for name, info in signals.items(): + if info is None: + table.add_row(name, "[dim]unavailable[/dim]", "") + elif isinstance(info, dict): + val = info.get("value", info.get("price", "?")) + detail = info.get("classification", info.get("label", "")) + table.add_row(name, str(val), str(detail)) + else: + table.add_row(name, str(info), "") + console.print(table) + + +# --------------------------------------------------------------------------- +# Economic +# --------------------------------------------------------------------------- + +@main.command() +@click.pass_context +def energy(ctx: click.Context) -> None: + """Oil and natural gas prices (EIA).""" + f = _get_fetcher() + data = _run(economic.fetch_energy_prices(f)) + + if ctx.obj.get("json") or "error" in data: + _print_json(data) + return + + oil = data.get("oil", {}) + gas = data.get("natural_gas", {}) + table = Table(title="Energy Prices", box=box.SIMPLE_HEAVY) + table.add_column("Commodity", style="bold") + table.add_column("Price", justify="right") + table.add_column("Date") + + for name, info in [("Brent Crude", oil.get("brent")), ("WTI Crude", oil.get("wti")), ("Natural Gas", gas)]: + if info and isinstance(info, dict): + table.add_row(name, f"${info.get('price', '?')}", str(info.get("date", ""))) + console.print(table) + + +@main.command() +@click.argument("series_id") +@click.option("--limit", "-n", default=30, help="Number of observations") +@click.pass_context +def fred(ctx: click.Context, series_id: str, limit: int) -> None: + """FRED economic data series (e.g., UNRATE, GDP, CPIAUCSL).""" + f = _get_fetcher() + data = _run(economic.fetch_fred_series(f, series_id=series_id, limit=limit)) + + if ctx.obj.get("json") or "error" in data: + _print_json(data) + return + + obs = data.get("observations", []) + title = data.get("title", series_id) + table = Table(title=f"FRED: {title}", box=box.SIMPLE_HEAVY) + table.add_column("Date", style="bold") + table.add_column("Value", justify="right") + + for o in obs[:20]: + table.add_row(o.get("date", ""), str(o.get("value", ""))) + console.print(table) + + +# --------------------------------------------------------------------------- +# Natural +# --------------------------------------------------------------------------- + +@main.command() +@click.option("--min-mag", "-m", default=4.5, help="Minimum magnitude") +@click.option("--hours", "-h", default=24, help="Lookback hours") +@click.pass_context +def earthquakes(ctx: click.Context, min_mag: float, hours: int) -> None: + """Recent earthquakes (USGS).""" + f = _get_fetcher() + data = _run(seismology.fetch_earthquakes(f, min_magnitude=min_mag, hours=hours)) + + if ctx.obj.get("json"): + _print_json(data) + return + + quakes = data.get("earthquakes", []) + console.print(f"[bold]{data.get('count', 0)} earthquakes[/bold] (M{min_mag}+ in last {hours}h)\n") + + table = Table(box=box.SIMPLE_HEAVY) + table.add_column("Mag", justify="right", style="bold") + table.add_column("Location") + table.add_column("Depth (km)", justify="right") + table.add_column("Time") + table.add_column("Alert") + + for q in quakes[:25]: + mag = q.get("magnitude", 0) + style = "red bold" if mag >= 6.0 else "yellow" if mag >= 5.0 else "" + alert = q.get("alert_level") or "" + table.add_row( + f"[{style}]{mag:.1f}[/{style}]" if style else f"{mag:.1f}", + q.get("place", "Unknown"), + f"{q.get('depth_km', 0):.1f}", + q.get("time", "")[:19], + alert, + ) + console.print(table) + + +@main.command() +@click.option("--region", "-r", default=None, help="Region name (e.g., north_america)") +@click.pass_context +def fires(ctx: click.Context, region: str | None) -> None: + """Active wildfires (NASA FIRMS).""" + f = _get_fetcher() + data = _run(wildfire.fetch_wildfires(f, region=region)) + + if ctx.obj.get("json") or "error" in data: + _print_json(data) + return + + console.print(f"[bold]{data.get('total_fires', 0)} high-confidence fires detected[/bold]\n") + + for reg_name, reg_data in data.get("fires_by_region", {}).items(): + count = reg_data.get("count", 0) + if count == 0: + continue + console.print(f" [bold]{reg_name}[/bold]: {count} fires") + for cluster in reg_data.get("top_clusters", [])[:5]: + console.print( + f" ({cluster.get('lat', 0):.1f}, {cluster.get('lon', 0):.1f}) " + f"— {cluster.get('fire_count', 0)} fires, FRP max {cluster.get('max_frp', 0):.0f}" + ) + + +# --------------------------------------------------------------------------- +# Conflict +# --------------------------------------------------------------------------- + +@main.command() +@click.option("--country", "-c", default=None, help="Country name") +@click.option("--days", "-d", default=7, help="Lookback days") +@click.pass_context +def conflicts(ctx: click.Context, country: str | None, days: int) -> None: + """Armed conflict events (ACLED).""" + f = _get_fetcher() + data = _run(conflict.fetch_acled_events(f, country=country, days=days)) + + if ctx.obj.get("json") or "error" in data: + _print_json(data) + return + + events = data.get("events", []) + console.print(f"[bold]{data.get('count', 0)} conflict events[/bold] (last {days}d)\n") + + table = Table(box=box.SIMPLE_HEAVY) + table.add_column("Date", style="bold") + table.add_column("Type") + table.add_column("Country") + table.add_column("Location") + table.add_column("Fatalities", justify="right") + + for e in events[:25]: + fat = e.get("fatalities", 0) or 0 + style = "red bold" if fat >= 10 else "yellow" if fat > 0 else "" + fat_str = f"[{style}]{fat}[/{style}]" if style else str(fat) + table.add_row( + str(e.get("event_date", ""))[:10], + e.get("event_type", ""), + e.get("country", ""), + e.get("location", ""), + fat_str, + ) + console.print(table) + + +# --------------------------------------------------------------------------- +# Military +# --------------------------------------------------------------------------- + +@main.command() +@click.option("--bbox", "-b", default=None, help="Bounding box: lamin,lomin,lamax,lomax") +@click.pass_context +def flights(ctx: click.Context, bbox: str | None) -> None: + """Military aircraft tracking (OpenSky).""" + f = _get_fetcher() + data = _run(military.fetch_military_flights(f, bbox=bbox)) + + if ctx.obj.get("json"): + _print_json(data) + return + + aircraft = data.get("aircraft", []) + console.print(f"[bold]{data.get('count', 0)} military aircraft detected[/bold]\n") + + table = Table(box=box.SIMPLE_HEAVY) + table.add_column("Callsign", style="bold") + table.add_column("ICAO24") + table.add_column("Country") + table.add_column("Alt (m)", justify="right") + table.add_column("Speed (m/s)", justify="right") + + for a in aircraft[:30]: + table.add_row( + a.get("callsign", "?"), + a.get("icao24", ""), + a.get("origin_country", ""), + f"{a.get('altitude_m') or 0:,.0f}", + f"{a.get('velocity_ms') or 0:.0f}", + ) + console.print(table) + + +@main.command() +@click.pass_context +def posture(ctx: click.Context) -> None: + """Military theater posture (5 theaters).""" + f = _get_fetcher() + data = _run(military.fetch_theater_posture(f)) + + if ctx.obj.get("json"): + _print_json(data) + return + + console.print(f"[bold]{data.get('total_military_aircraft', 0)} total military aircraft[/bold]\n") + theaters = data.get("theaters", {}) + + table = Table(title="Theater Posture", box=box.SIMPLE_HEAVY) + table.add_column("Theater", style="bold") + table.add_column("Aircraft", justify="right") + table.add_column("Countries") + table.add_column("Sample Callsigns") + + for name, info in theaters.items(): + count = info.get("count", 0) + style = "red bold" if count >= 20 else "yellow" if count >= 5 else "" + count_str = f"[{style}]{count}[/{style}]" if style else str(count) + table.add_row( + name.replace("_", " ").title(), + count_str, + ", ".join(info.get("countries", [])[:5]), + ", ".join(info.get("sample_callsigns", [])[:3]), + ) + console.print(table) + + +# --------------------------------------------------------------------------- +# Infrastructure +# --------------------------------------------------------------------------- + +@main.command() +@click.pass_context +def outages(ctx: click.Context) -> None: + """Internet outages (Cloudflare Radar).""" + f = _get_fetcher() + data = _run(infrastructure.fetch_internet_outages(f)) + + if ctx.obj.get("json"): + _print_json(data) + return + + console.print(f"[bold]{data.get('ongoing_count', 0)} ongoing outages[/bold], " + f"{data.get('total_7d', 0)} in last 7 days\n") + + for o in data.get("outages", [])[:15]: + ongoing = "[red]ONGOING[/red]" if o.get("is_ongoing") else "" + countries = ", ".join(o.get("countries", [])[:5]) if o.get("countries") else "" + console.print(f" {o.get('start', '')[:16]} {countries} {o.get('description', '')[:80]} {ongoing}") + + +@main.command() +@click.pass_context +def cables(ctx: click.Context) -> None: + """Undersea cable corridor health (NGA).""" + f = _get_fetcher() + data = _run(infrastructure.fetch_cable_health(f)) + + if ctx.obj.get("json"): + _print_json(data) + return + + status_labels = {0: "[green]Clear[/green]", 1: "[yellow]Advisory[/yellow]", + 2: "[red]At Risk[/red]", 3: "[red bold]Disrupted[/red bold]"} + + table = Table(title="Undersea Cable Health", box=box.SIMPLE_HEAVY) + table.add_column("Corridor", style="bold") + table.add_column("Status") + table.add_column("Cables") + table.add_column("Warnings", justify="right") + + for name, info in data.get("corridors", {}).items(): + score = info.get("status_score", 0) + table.add_row( + name.replace("_", " ").title(), + status_labels.get(score, str(score)), + ", ".join(info.get("cables", [])[:3]), + str(len(info.get("relevant_warnings", []))), + ) + console.print(table) + + +# --------------------------------------------------------------------------- +# Maritime +# --------------------------------------------------------------------------- + +@main.command() +@click.option("--navarea", "-n", default=None, help="NAVAREA number (e.g., IV)") +@click.pass_context +def warnings(ctx: click.Context, navarea: str | None) -> None: + """Navigational warnings (NGA Maritime Safety).""" + f = _get_fetcher() + data = _run(maritime.fetch_nav_warnings(f, navarea=navarea)) + + if ctx.obj.get("json"): + _print_json(data) + return + + console.print(f"[bold]{data.get('count', 0)} active warnings[/bold]\n") + + by_area = data.get("by_navarea", {}) + if by_area: + console.print(" By NAVAREA: " + ", ".join(f"{k}:{v}" for k, v in sorted(by_area.items()))) + console.print() + + for w in data.get("warnings", [])[:20]: + console.print(f" [{w.get('navarea', '?')}] {w.get('id', '')} {w.get('text', '')[:100]}") + + +# --------------------------------------------------------------------------- +# Climate +# --------------------------------------------------------------------------- + +@main.command(name="climate") +@click.pass_context +def climate_cmd(ctx: click.Context) -> None: + """Climate anomalies (15 global zones vs. prior year).""" + f = _get_fetcher() + data = _run(climate.fetch_climate_anomalies(f)) + + if ctx.obj.get("json"): + _print_json(data) + return + + zones = data.get("zones", {}) + sig = data.get("significant_anomalies", []) + + table = Table(title="Climate Anomalies", box=box.SIMPLE_HEAVY) + table.add_column("Zone", style="bold") + table.add_column("Temp Anomaly", justify="right") + table.add_column("Precip Anomaly", justify="right") + table.add_column("Flag") + + for key, z in zones.items(): + temp_a = z.get("temp_anomaly_c", 0) + prec_a = z.get("precip_anomaly_pct", 0) + t_style = "red" if temp_a > 3 else "blue" if temp_a < -3 else "" + flag = "[red bold]SIG[/red bold]" if key in sig else "" + t_str = f"[{t_style}]{temp_a:+.1f}C[/{t_style}]" if t_style else f"{temp_a:+.1f}C" + table.add_row(z.get("name", key), t_str, f"{prec_a:+.0f}%", flag) + console.print(table) + + +# --------------------------------------------------------------------------- +# News +# --------------------------------------------------------------------------- + +@main.command(name="news") +@click.option("--category", "-c", default=None, + type=click.Choice(["geopolitics", "security", "technology", "finance", "military", "science"]), + help="Category filter") +@click.option("--limit", "-n", default=30, help="Max items") +@click.pass_context +def news_cmd(ctx: click.Context, category: str | None, limit: int) -> None: + """Intelligence news from 20+ RSS feeds.""" + f = _get_fetcher() + data = _run(news.fetch_news_feed(f, category=category, limit=limit)) + + if ctx.obj.get("json"): + _print_json(data) + return + + items = data.get("items", []) + if not items: + console.print("[yellow]No news items available[/yellow]") + return + + console.print(f"[bold]{data.get('count', 0)} items[/bold] from {', '.join(data.get('categories_fetched', []))}\n") + + for item in items: + cat = item.get("category", "") + title = item.get("title", "") + feed = item.get("feed_name", "") + pub = (item.get("published") or "")[:16] + console.print(f" [{cat}] [bold]{title}[/bold]") + console.print(f" {feed} — {pub}") + + +@main.command() +@click.option("--min-count", "-m", default=3, help="Minimum keyword occurrences") +@click.pass_context +def trending(ctx: click.Context, min_count: int) -> None: + """Trending keywords from recent news.""" + f = _get_fetcher() + data = _run(news.fetch_trending_keywords(f, min_count=min_count)) + + if ctx.obj.get("json"): + _print_json(data) + return + + keywords = data.get("keywords", []) + console.print(f"[bold]Trending keywords[/bold] (from {data.get('total_items_analyzed', 0)} items)\n") + + table = Table(box=box.SIMPLE_HEAVY) + table.add_column("#", justify="right") + table.add_column("Keyword", style="bold") + table.add_column("Count", justify="right") + + for i, kw in enumerate(keywords[:30], 1): + table.add_row(str(i), kw["word"], str(kw["count"])) + console.print(table) + + +@main.command() +@click.argument("query", default="conflict") +@click.option("--mode", "-m", default="artlist", type=click.Choice(["artlist", "timelinevol"])) +@click.option("--limit", "-n", default=20, help="Max records") +@click.pass_context +def gdelt(ctx: click.Context, query: str, mode: str, limit: int) -> None: + """Search GDELT 2.0 global news database.""" + f = _get_fetcher() + data = _run(news.fetch_gdelt_search(f, query=query, mode=mode, limit=limit)) + + if ctx.obj.get("json"): + _print_json(data) + return + + if mode == "artlist": + articles = data.get("articles", []) + console.print(f"[bold]{len(articles)} articles[/bold] for '{query}'\n") + for a in articles[:20]: + title = a.get("title", "")[:80] + domain = a.get("domain", "") + console.print(f" [bold]{title}[/bold] ({domain})") + else: + console.print(f"[bold]Timeline volume for '{query}'[/bold]") + _print_json(data) + + +# --------------------------------------------------------------------------- +# Prediction +# --------------------------------------------------------------------------- + +@main.command() +@click.option("--limit", "-n", default=20, help="Number of markets") +@click.pass_context +def predictions(ctx: click.Context, limit: int) -> None: + """Prediction market movers (Polymarket).""" + f = _get_fetcher() + data = _run(prediction.fetch_prediction_markets(f, limit=limit)) + + if ctx.obj.get("json"): + _print_json(data) + return + + mkts = data.get("markets", []) + if not mkts: + console.print("[yellow]No prediction market data available[/yellow]") + return + + table = Table(title="Prediction Markets", box=box.SIMPLE_HEAVY) + table.add_column("Question", max_width=50) + table.add_column("YES %", justify="right") + table.add_column("Sentiment") + table.add_column("24h Vol", justify="right") + + for m in mkts: + yes_pct = (m.get("yes_probability", 0) or 0) * 100 + sentiment = m.get("sentiment", "") + vol = m.get("volume_24h", 0) or 0 + s_style = "green" if "yes" in sentiment else "red" if "no" in sentiment else "yellow" + table.add_row( + (m.get("question", "")[:50]), + f"{yes_pct:.0f}%", + f"[{s_style}]{sentiment}[/{s_style}]", + f"${vol:,.0f}", + ) + console.print(table) + + +# --------------------------------------------------------------------------- +# Displacement +# --------------------------------------------------------------------------- + +@main.command(name="displacement") +@click.option("--year", "-y", default=None, type=int, help="Reporting year") +@click.pass_context +def displacement_cmd(ctx: click.Context, year: int | None) -> None: + """UNHCR displacement statistics.""" + f = _get_fetcher() + data = _run(displacement.fetch_displacement_summary(f, year=year)) + + if ctx.obj.get("json"): + _print_json(data) + return + + totals = data.get("global_totals", {}) + console.print(f"[bold]Global Displacement ({data.get('year', '?')})[/bold]") + console.print(f" Grand total: {totals.get('grand_total', 0):,}\n") + + by_origin = data.get("by_origin", []) + table = Table(title="Top Countries of Origin", box=box.SIMPLE_HEAVY) + table.add_column("Country", style="bold") + table.add_column("Total Displaced", justify="right") + table.add_column("Refugees", justify="right") + table.add_column("IDPs", justify="right") + + for c in by_origin[:15]: + table.add_row( + c.get("country", ""), + f"{c.get('total_displaced', 0):,}", + f"{c.get('refugees', 0):,}", + f"{c.get('internally_displaced', 0):,}", + ) + console.print(table) + + +# --------------------------------------------------------------------------- +# Aviation +# --------------------------------------------------------------------------- + +@main.command() +@click.pass_context +def delays(ctx: click.Context) -> None: + """US airport delays (FAA).""" + f = _get_fetcher() + data = _run(aviation.fetch_airport_delays(f)) + + if ctx.obj.get("json"): + _print_json(data) + return + + delayed = data.get("delayed", []) + console.print(f"[bold]{data.get('delayed_count', 0)} airports with delays[/bold] " + f"(checked {data.get('total_checked', 0)})\n") + + if not delayed: + console.print("[green]No major airport delays![/green]") + return + + table = Table(box=box.SIMPLE_HEAVY) + table.add_column("Airport", style="bold") + table.add_column("Name") + table.add_column("Delay Info") + + for d in delayed: + statuses = d.get("status", []) + info = "; ".join( + f"{s.get('type', '')} - {s.get('reason', '')} ({s.get('avg_delay', '')})" + for s in statuses + ) if statuses else "Details unavailable" + table.add_row(d.get("code", ""), d.get("name", ""), info[:80]) + console.print(table) + + +# --------------------------------------------------------------------------- +# Cyber +# --------------------------------------------------------------------------- + +@main.command() +@click.option("--limit", "-n", default=30, help="Max threats") +@click.pass_context +def threats(ctx: click.Context, limit: int) -> None: + """Cyber threat intelligence (4 feeds).""" + f = _get_fetcher() + data = _run(cyber.fetch_cyber_threats(f, limit=limit)) + + if ctx.obj.get("json"): + _print_json(data) + return + + by_sev = data.get("by_severity", {}) + console.print(f"[bold]{data.get('count', 0)} threats[/bold] " + f"({data.get('feeds_successful', 0)}/{data.get('feeds_attempted', 0)} feeds)") + console.print(f" [red]Critical: {by_sev.get('critical', 0)}[/red] " + f"[yellow]High: {by_sev.get('high', 0)}[/yellow] " + f"Medium: {by_sev.get('medium', 0)} " + f"[dim]Low: {by_sev.get('low', 0)}[/dim]\n") + + table = Table(box=box.SIMPLE_HEAVY) + table.add_column("Severity") + table.add_column("Type") + table.add_column("Indicator", max_width=40) + table.add_column("Threat") + table.add_column("Feed") + + for t in data.get("threats", [])[:limit]: + sev = t.get("severity", "") + sev_style = {"critical": "red bold", "high": "yellow", "medium": "", "low": "dim"}.get(sev, "") + sev_str = f"[{sev_style}]{sev}[/{sev_style}]" if sev_style else sev + table.add_row( + sev_str, + t.get("type", ""), + (t.get("indicator", ""))[:40], + (t.get("threat", ""))[:30], + t.get("source_feed", ""), + ) + console.print(table) + + +# --------------------------------------------------------------------------- +# Intelligence +# --------------------------------------------------------------------------- + +@main.command() +@click.argument("country_code", default="US") +@click.pass_context +def brief(ctx: click.Context, country_code: str) -> None: + """Country intelligence brief (LLM + data).""" + f = _get_fetcher() + data = _run(intelligence.fetch_country_brief(f, country_code=country_code)) + + if ctx.obj.get("json"): + _print_json(data) + return + + llm_tag = "[green]LLM[/green]" if data.get("llm_available") else "[yellow]data-only[/yellow]" + console.print(f"[bold]Intelligence Brief: {country_code}[/bold] ({llm_tag})\n") + console.print(data.get("brief", "No brief available.")) + + d = data.get("data", {}) + if d.get("gdp") or d.get("recent_events"): + console.print(f"\n[dim]GDP data points: {len(d.get('gdp', []))} | " + f"Recent conflict events: {d.get('recent_events', 0)}[/dim]") + + +@main.command() +@click.option("--limit", "-n", default=20, help="Top N countries") +@click.pass_context +def risk(ctx: click.Context, limit: int) -> None: + """Country risk scores (ACLED-based).""" + f = _get_fetcher() + data = _run(intelligence.fetch_risk_scores(f, limit=limit)) + + if ctx.obj.get("json") or "error" in data: + _print_json(data) + return + + table = Table(title="Country Risk Scores", box=box.SIMPLE_HEAVY) + table.add_column("#", justify="right") + table.add_column("Country", style="bold") + table.add_column("Events (30d)", justify="right") + table.add_column("Risk Score", justify="right") + table.add_column("Level") + + for i, c in enumerate(data.get("countries", []), 1): + level = c.get("risk_level", "") + l_style = {"critical": "red bold", "elevated": "yellow", "moderate": "", "low": "dim"}.get(level, "") + l_str = f"[{l_style}]{level}[/{l_style}]" if l_style else level + table.add_row( + str(i), + c.get("country", ""), + str(c.get("events_30d", 0)), + f"{c.get('risk_score', 0):.0f}", + l_str, + ) + console.print(table) + + +@main.command() +@click.argument("country_code", required=False, default=None) +@click.pass_context +def instability(ctx: click.Context, country_code: str | None) -> None: + """Country Instability Index (0-100).""" + f = _get_fetcher() + data = _run(intelligence.fetch_instability_index(f, country_code=country_code)) + + if ctx.obj.get("json") or "error" in data: + _print_json(data) + return + + if country_code: + idx = data.get("instability_index", 0) + level = data.get("risk_level", "") + console.print(f"[bold]{country_code} Instability Index: {idx}/100 ({level})[/bold]\n") + components = data.get("components", {}) + for name, score in components.items(): + bar = "█" * int(score) + "░" * (20 - int(score)) + console.print(f" {name:30s} {bar} {score:.1f}/20") + else: + table = Table(title="Instability Index (Focus Countries)", box=box.SIMPLE_HEAVY) + table.add_column("Country", style="bold") + table.add_column("Index", justify="right") + table.add_column("Events (30d)", justify="right") + table.add_column("Risk Level") + + for c in data.get("countries", []): + level = c.get("risk_level", "") + l_style = {"critical": "red bold", "high": "yellow", "medium": "", "low": "dim"}.get(level, "") + l_str = f"[{l_style}]{level}[/{l_style}]" if l_style else level + table.add_row( + f"{c.get('country_name', '')} ({c.get('country_code', '')})", + f"{c.get('instability_index', 0):.0f}", + str(c.get("events_30d", 0)), + l_str, + ) + console.print(table) + + +# --------------------------------------------------------------------------- +# Reports +# --------------------------------------------------------------------------- + +@main.group() +def report() -> None: + """Generate intelligence reports (HTML).""" + + +main.add_command(report) + + +@report.command(name="daily") +@click.option("--output-dir", "-o", default=None, help="Output directory") +def report_daily(output_dir: str | None) -> None: + """Generate daily intelligence brief (HTML).""" + console.print("[bold]Generating daily brief...[/bold]") + result = _run(report_gen.generate_daily_brief(output_dir=output_dir)) + console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") + summary = result.get("summary", {}) + console.print(f" Quotes: {summary.get('market_quotes', 0)} | " + f"Conflicts: {summary.get('conflict_events', 0)} | " + f"Threats: {summary.get('cyber_threats', 0)} | " + f"Quakes: {summary.get('earthquakes', 0)}") + + +@report.command(name="threat") +@click.option("--output-dir", "-o", default=None, help="Output directory") +def report_threat(output_dir: str | None) -> None: + """Generate threat landscape report (HTML).""" + console.print("[bold]Generating threat landscape...[/bold]") + result = _run(report_gen.generate_threat_landscape(output_dir=output_dir)) + console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") + + +@report.command(name="market") +@click.option("--output-dir", "-o", default=None, help="Output directory") +def report_market(output_dir: str | None) -> None: + """Generate market overview report (HTML).""" + console.print("[bold]Generating market overview...[/bold]") + result = _run(report_gen.generate_market_overview(output_dir=output_dir)) + console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") + + +@report.command(name="dossier") +@click.argument("country_code") +@click.option("--output-dir", "-o", default=None, help="Output directory") +def report_dossier(country_code: str, output_dir: str | None) -> None: + """Generate country dossier report (HTML).""" + console.print(f"[bold]Generating dossier for {country_code}...[/bold]") + result = _run(report_gen.generate_country_dossier( + country_code=country_code, output_dir=output_dir, + )) + console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") + + +# --------------------------------------------------------------------------- +# System +# --------------------------------------------------------------------------- + +@main.command() +@click.pass_context +def status(ctx: click.Context) -> None: + """Data source health and cache stats.""" + f = _get_fetcher() + breaker_status = f.breaker.status() + cache_stats = f.cache.stats() + + if ctx.obj.get("json"): + _print_json({"circuit_breakers": breaker_status, "cache": cache_stats}) + return + + console.print(Panel("[bold]World Intelligence Status[/bold]")) + + # Cache + console.print(f"\n[bold]Cache:[/bold] {cache_stats.get('active_entries', 0)} active, " + f"{cache_stats.get('expired_entries', 0)} expired") + + # Circuit breakers + if breaker_status: + table = Table(title="Circuit Breakers", box=box.SIMPLE_HEAVY) + table.add_column("Source", style="bold") + table.add_column("Status") + table.add_column("Failures", justify="right") + table.add_column("Cooldown", justify="right") + + for source, info in sorted(breaker_status.items()): + s = info.get("status", "closed") + style = "green" if s == "closed" else "yellow" if s == "half-open" else "red" + table.add_row( + source, + f"[{style}]{s}[/{style}]", + str(info.get("failures", 0)), + f"{info.get('cooldown_remaining_s', 0):.0f}s" if info.get("cooldown_remaining_s") else "", + ) + console.print(table) + else: + console.print("\n[dim]No circuit breaker data yet (no requests made)[/dim]") + + +@main.command(name="sync") +@click.argument("source", required=False) +def sync_cmd(source: str | None) -> None: + """Force refresh a data source cache.""" + f = _get_fetcher() + if source: + # Delete all cache entries matching this source prefix + # Simple approach: evict expired, then note we can't selectively clear yet + console.print(f"[yellow]Force sync not yet implemented for specific source '{source}'[/yellow]") + console.print("[dim]Workaround: cache entries expire naturally based on TTL[/dim]") + else: + removed = f.cache.evict_expired() + console.print(f"Evicted {removed} expired cache entries") + + +if __name__ == "__main__": + main() diff --git a/src/world_intel_mcp/fetcher.py b/src/world_intel_mcp/fetcher.py new file mode 100644 index 0000000..f651e0a --- /dev/null +++ b/src/world_intel_mcp/fetcher.py @@ -0,0 +1,191 @@ +"""Async HTTP fetcher with timeout, retry, rate limiting, and circuit breaker integration. + +All external HTTP calls in world-intel-mcp go through this module. +""" + +import asyncio +import logging +import time +from typing import Any + +import httpx + +from .cache import Cache +from .circuit_breaker import CircuitBreaker + +logger = logging.getLogger("world-intel-mcp.fetcher") + +# Yahoo Finance requires serialized access (600ms gap) +_yahoo_lock = asyncio.Lock() +_yahoo_last_call: float = 0.0 +_YAHOO_MIN_INTERVAL = 0.6 # seconds + + +class Fetcher: + """Centralized HTTP fetcher with caching, retries, and circuit breaking.""" + + def __init__( + self, + cache: Cache, + breaker: CircuitBreaker, + default_timeout: float = 15.0, + max_retries: int = 2, + client: httpx.AsyncClient | None = None, + ): + self.cache = cache + self.breaker = breaker + self.default_timeout = default_timeout + self.max_retries = max_retries + self._client: httpx.AsyncClient | None = client + + async def _get_client(self) -> httpx.AsyncClient: + if self._client is None or self._client.is_closed: + self._client = httpx.AsyncClient( + timeout=httpx.Timeout(self.default_timeout), + follow_redirects=True, + limits=httpx.Limits(max_connections=50, max_keepalive_connections=20), + headers={"User-Agent": "PhoenixAGI-WorldIntel/0.1"}, + proxy=None, # never inherit system SOCKS proxy + ) + return self._client + + async def close(self) -> None: + if self._client and not self._client.is_closed: + await self._client.aclose() + self._client = None + + async def get_json( + self, + url: str, + source: str, + cache_key: str | None = None, + cache_ttl: int = 300, + headers: dict[str, str] | None = None, + params: dict[str, Any] | None = None, + timeout: float | None = None, + yahoo_rate_limit: bool = False, + ) -> dict | list | None: + """Fetch JSON with caching, circuit breaking, and retries. + + Args: + url: Target URL. + source: Source name for circuit breaker tracking. + cache_key: Cache key. If None, uses url+params hash. + cache_ttl: Cache TTL in seconds. + headers: Extra HTTP headers. + params: Query parameters. + timeout: Per-request timeout override. + yahoo_rate_limit: If True, enforce Yahoo Finance 600ms serialization. + + Returns: + Parsed JSON or None on failure. + """ + # Check circuit breaker + if not self.breaker.is_available(source): + logger.debug("Circuit open for %s, skipping", source) + return None + + # Check cache + effective_key = cache_key or f"{source}:{url}:{params}" + cached = self.cache.get(effective_key) + if cached is not None: + return cached + + # Yahoo rate limiting + if yahoo_rate_limit: + await self._yahoo_throttle() + + # Fetch with retries + client = await self._get_client() + last_error: Exception | None = None + + for attempt in range(self.max_retries + 1): + try: + resp = await client.get( + url, + headers=headers, + params=params, + timeout=timeout or self.default_timeout, + ) + resp.raise_for_status() + data = resp.json() + self.breaker.record_success(source) + self.cache.set(effective_key, data, cache_ttl) + return data + except (httpx.HTTPStatusError, httpx.RequestError, Exception) as exc: + last_error = exc + if attempt < self.max_retries: + wait = 1.0 * (attempt + 1) + logger.debug("Retry %d/%d for %s (%s), waiting %.1fs", + attempt + 1, self.max_retries, source, exc, wait) + await asyncio.sleep(wait) + + # All retries failed + self.breaker.record_failure(source) + logger.warning("Fetch failed for %s: %s (url=%s)", source, last_error, url) + return None + + async def get_text( + self, + url: str, + source: str, + cache_key: str | None = None, + cache_ttl: int = 300, + headers: dict[str, str] | None = None, + params: dict[str, Any] | None = None, + timeout: float | None = None, + ) -> str | None: + """Fetch raw text with caching and circuit breaking.""" + if not self.breaker.is_available(source): + return None + + effective_key = cache_key or f"{source}:text:{url}:{params}" + cached = self.cache.get(effective_key) + if cached is not None: + return cached + + client = await self._get_client() + last_error: Exception | None = None + + for attempt in range(self.max_retries + 1): + try: + resp = await client.get( + url, + headers=headers, + params=params, + timeout=timeout or self.default_timeout, + ) + resp.raise_for_status() + text = resp.text + self.breaker.record_success(source) + self.cache.set(effective_key, text, cache_ttl) + return text + except (httpx.HTTPStatusError, httpx.RequestError, Exception) as exc: + last_error = exc + if attempt < self.max_retries: + await asyncio.sleep(1.0 * (attempt + 1)) + + self.breaker.record_failure(source) + logger.warning("Text fetch failed for %s: %s", source, last_error) + return None + + async def get_xml( + self, + url: str, + source: str, + cache_key: str | None = None, + cache_ttl: int = 300, + timeout: float | None = None, + ) -> str | None: + """Fetch XML content (returns raw text for feedparser/ET parsing).""" + return await self.get_text(url, source, cache_key, cache_ttl, timeout=timeout) + + async def _yahoo_throttle(self) -> None: + """Enforce Yahoo Finance rate limit (600ms between calls).""" + global _yahoo_last_call + async with _yahoo_lock: + now = time.time() + elapsed = now - _yahoo_last_call + if elapsed < _YAHOO_MIN_INTERVAL: + await asyncio.sleep(_YAHOO_MIN_INTERVAL - elapsed) + _yahoo_last_call = time.time() diff --git a/src/world_intel_mcp/reports/__init__.py b/src/world_intel_mcp/reports/__init__.py new file mode 100644 index 0000000..5353a30 --- /dev/null +++ b/src/world_intel_mcp/reports/__init__.py @@ -0,0 +1 @@ +"""Report generation for world-intel-mcp.""" diff --git a/src/world_intel_mcp/reports/generator.py b/src/world_intel_mcp/reports/generator.py new file mode 100644 index 0000000..e9807fe --- /dev/null +++ b/src/world_intel_mcp/reports/generator.py @@ -0,0 +1,299 @@ +"""Report orchestrator for world-intel-mcp. + +Gathers data from multiple intelligence sources in parallel and renders +HTML or Markdown reports via Jinja2 templates. +""" + +import asyncio +import logging +import os +from datetime import datetime, timezone +from pathlib import Path + +from ..cache import Cache +from ..circuit_breaker import CircuitBreaker +from ..fetcher import Fetcher +from ..sources import ( + markets, economic, seismology, wildfire, conflict, military, + infrastructure, maritime, climate, news, intelligence, + prediction, displacement, aviation, cyber, +) + +logger = logging.getLogger("world-intel-mcp.reports.generator") + +# Default output directory +_DEFAULT_OUTPUT_DIR = os.environ.get( + "INTEL_REPORT_DIR", + os.path.join(os.environ.get("STORAGE_BASE", "/tmp"), "reports", "intel"), +) + + +def _ensure_output_dir(path: str | None = None) -> Path: + """Create output directory if needed and return Path.""" + output_dir = Path(path or _DEFAULT_OUTPUT_DIR) + output_dir.mkdir(parents=True, exist_ok=True) + return output_dir + + +async def generate_daily_brief(output_dir: str | None = None) -> dict: + """Generate a daily intelligence brief HTML report. + + Gathers: market quotes, macro signals, conflict events, cyber threats, + earthquakes, wildfires, prediction markets, trending keywords. + Renders to HTML and returns the file path + summary. + """ + cache = Cache() + breaker = CircuitBreaker() + fetcher = Fetcher(cache=cache, breaker=breaker) + + try: + # Gather all data in parallel + ( + market_data, + macro_data, + conflict_data, + cyber_data, + quake_data, + fire_data, + predict_data, + keyword_data, + ) = await asyncio.gather( + markets.fetch_market_quotes(fetcher), + markets.fetch_macro_signals(fetcher), + conflict.fetch_acled_events(fetcher, days=1, limit=50), + cyber.fetch_cyber_threats(fetcher, limit=30), + seismology.fetch_earthquakes(fetcher, min_magnitude=4.5, hours=24), + wildfire.fetch_wildfires(fetcher), + prediction.fetch_prediction_markets(fetcher, limit=10), + news.fetch_trending_keywords(fetcher, min_count=3), + ) + + now = datetime.now(timezone.utc) + context = { + "title": "Daily Intelligence Brief", + "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "market_summary": { + "quotes": market_data.get("quotes", []), + "macro_signals": macro_data.get("signals", {}), + }, + "conflict_summary": { + "events": conflict_data.get("events", []), + "count": conflict_data.get("count", 0), + }, + "cyber_summary": { + "threats": cyber_data.get("threats", []), + "by_severity": cyber_data.get("by_severity", {}), + }, + "natural_summary": { + "earthquakes": quake_data.get("earthquakes", []), + "fire_count": fire_data.get("total_fires", 0), + }, + "prediction_highlights": predict_data.get("markets", []), + "trending_keywords": keyword_data.get("keywords", []), + } + + # Render HTML + from .html_report import render_template + html = render_template("daily_brief.html", context) + + # Write to file + out_dir = _ensure_output_dir(output_dir) + filename = f"daily_brief_{now.strftime('%Y%m%d_%H%M%S')}.html" + filepath = out_dir / filename + filepath.write_text(html, encoding="utf-8") + + logger.info("Daily brief generated: %s", filepath) + + return { + "report_type": "daily_brief", + "file_path": str(filepath), + "generated_at": context["generated_at"], + "summary": { + "market_quotes": len(context["market_summary"]["quotes"]), + "conflict_events": context["conflict_summary"]["count"], + "cyber_threats": len(context["cyber_summary"]["threats"]), + "earthquakes": len(context["natural_summary"]["earthquakes"]), + "predictions": len(context["prediction_highlights"]), + "keywords": len(context["trending_keywords"]), + }, + } + finally: + await fetcher.close() + + +async def generate_country_dossier( + country_code: str, + output_dir: str | None = None, +) -> dict: + """Generate a country dossier HTML report.""" + cache = Cache() + breaker = CircuitBreaker() + fetcher = Fetcher(cache=cache, breaker=breaker) + + try: + ( + brief_data, + instability_data, + conflict_data, + displacement_data, + ) = await asyncio.gather( + intelligence.fetch_country_brief(fetcher, country_code=country_code), + intelligence.fetch_instability_index(fetcher, country_code=country_code), + conflict.fetch_acled_events(fetcher, country=country_code, days=30, limit=50), + displacement.fetch_displacement_summary(fetcher), + ) + + now = datetime.now(timezone.utc) + context = { + "title": f"Country Dossier: {country_code}", + "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "country_code": country_code, + "brief": brief_data.get("brief", ""), + "instability": { + "instability_index": instability_data.get("instability_index", 0), + "components": instability_data.get("components", {}), + "risk_level": instability_data.get("risk_level", "unknown"), + }, + "conflict_events": conflict_data.get("events", []), + "displacement": { + "by_origin": displacement_data.get("by_origin", []), + "global_totals": displacement_data.get("global_totals", {}), + }, + "economic": { + "gdp": brief_data.get("data", {}).get("gdp", []), + "inflation": brief_data.get("data", {}).get("inflation", []), + }, + } + + from .html_report import render_template + html = render_template("country_dossier.html", context) + + out_dir = _ensure_output_dir(output_dir) + filename = f"dossier_{country_code}_{now.strftime('%Y%m%d_%H%M%S')}.html" + filepath = out_dir / filename + filepath.write_text(html, encoding="utf-8") + + logger.info("Country dossier generated: %s", filepath) + + return { + "report_type": "country_dossier", + "country_code": country_code, + "file_path": str(filepath), + "generated_at": context["generated_at"], + } + finally: + await fetcher.close() + + +async def generate_threat_landscape(output_dir: str | None = None) -> dict: + """Generate a threat landscape HTML report.""" + cache = Cache() + breaker = CircuitBreaker() + fetcher = Fetcher(cache=cache, breaker=breaker) + + try: + ( + cyber_data, + conflict_data, + military_data, + cable_data, + outage_data, + ) = await asyncio.gather( + cyber.fetch_cyber_threats(fetcher, limit=50), + conflict.fetch_acled_events(fetcher, days=7, limit=100), + military.fetch_theater_posture(fetcher), + infrastructure.fetch_cable_health(fetcher), + infrastructure.fetch_internet_outages(fetcher), + ) + + now = datetime.now(timezone.utc) + context = { + "title": "Threat Landscape Report", + "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "cyber_threats": { + "threats": cyber_data.get("threats", []), + "by_severity": cyber_data.get("by_severity", {}), + "by_type": cyber_data.get("by_type", {}), + }, + "conflict_events": conflict_data.get("events", []), + "military_activity": { + "theaters": military_data.get("theaters", {}), + "total_military_aircraft": military_data.get("total_military_aircraft", 0), + }, + "cable_health": { + "corridors": cable_data.get("corridors", {}), + }, + "outages": { + "outages": outage_data.get("outages", []), + "ongoing_count": outage_data.get("ongoing_count", 0), + }, + } + + from .html_report import render_template + html = render_template("threat_landscape.html", context) + + out_dir = _ensure_output_dir(output_dir) + filename = f"threat_landscape_{now.strftime('%Y%m%d_%H%M%S')}.html" + filepath = out_dir / filename + filepath.write_text(html, encoding="utf-8") + + logger.info("Threat landscape generated: %s", filepath) + + return { + "report_type": "threat_landscape", + "file_path": str(filepath), + "generated_at": context["generated_at"], + } + finally: + await fetcher.close() + + +async def generate_market_overview(output_dir: str | None = None) -> dict: + """Generate a market overview HTML report.""" + cache = Cache() + breaker = CircuitBreaker() + fetcher = Fetcher(cache=cache, breaker=breaker) + + try: + ( + quote_data, + crypto_data, + macro_data, + sector_data, + etf_data, + ) = await asyncio.gather( + markets.fetch_market_quotes(fetcher), + markets.fetch_crypto_quotes(fetcher, limit=20), + markets.fetch_macro_signals(fetcher), + markets.fetch_sector_heatmap(fetcher), + markets.fetch_etf_flows(fetcher), + ) + + now = datetime.now(timezone.utc) + context = { + "title": "Market Overview", + "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "quotes": quote_data.get("quotes", []), + "crypto": crypto_data.get("coins", []), + "macro_signals": macro_data.get("signals", {}), + "sector_heatmap": sector_data.get("sectors", []), + "etf_flows": etf_data, + } + + from .html_report import render_template + html = render_template("market_overview.html", context) + + out_dir = _ensure_output_dir(output_dir) + filename = f"market_overview_{now.strftime('%Y%m%d_%H%M%S')}.html" + filepath = out_dir / filename + filepath.write_text(html, encoding="utf-8") + + logger.info("Market overview generated: %s", filepath) + + return { + "report_type": "market_overview", + "file_path": str(filepath), + "generated_at": context["generated_at"], + } + finally: + await fetcher.close() diff --git a/src/world_intel_mcp/reports/html_report.py b/src/world_intel_mcp/reports/html_report.py new file mode 100644 index 0000000..f673289 --- /dev/null +++ b/src/world_intel_mcp/reports/html_report.py @@ -0,0 +1,35 @@ +"""HTML report renderer using Jinja2 templates. + +Loads templates from the ``templates/`` subdirectory and renders them +with the provided context data. +""" + +import logging +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +logger = logging.getLogger("world-intel-mcp.reports.html_report") + +_TEMPLATE_DIR = Path(__file__).parent / "templates" + +_env = Environment( + loader=FileSystemLoader(str(_TEMPLATE_DIR)), + autoescape=select_autoescape(["html"]), + trim_blocks=True, + lstrip_blocks=True, +) + + +def render_template(template_name: str, context: dict) -> str: + """Render a Jinja2 HTML template with the given context. + + Args: + template_name: Name of the template file in ``templates/``. + context: Dict of variables to pass to the template. + + Returns: + Rendered HTML string. + """ + template = _env.get_template(template_name) + return template.render(**context) diff --git a/src/world_intel_mcp/reports/markdown_report.py b/src/world_intel_mcp/reports/markdown_report.py new file mode 100644 index 0000000..a8b0453 --- /dev/null +++ b/src/world_intel_mcp/reports/markdown_report.py @@ -0,0 +1,161 @@ +"""Markdown report generator for world-intel-mcp. + +Generates Markdown reports with optional Mermaid diagrams. +""" + +import logging +from datetime import datetime, timezone + +logger = logging.getLogger("world-intel-mcp.reports.markdown_report") + + +def generate_daily_brief_md( + market_summary: dict, + conflict_summary: dict, + cyber_summary: dict, + natural_summary: dict, + prediction_highlights: list, + trending_keywords: list, +) -> str: + """Generate a daily intelligence brief in Markdown format.""" + now = datetime.now(timezone.utc) + lines = [ + f"# Daily Intelligence Brief", + f"*Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}*", + "", + "## Markets", + ] + + quotes = market_summary.get("quotes", []) + if quotes: + lines.append("| Symbol | Price | Change |") + lines.append("|--------|------:|-------:|") + for q in quotes[:8]: + chg = q.get("change_pct") or 0 + lines.append(f"| {q.get('symbol', '?')} | {q.get('price', 0):,.2f} | {chg:+.2f}% |") + lines.append("") + + # Conflict + events = conflict_summary.get("events", []) + lines.append(f"## Conflict ({conflict_summary.get('count', 0)} events)") + if events: + lines.append("| Date | Type | Country | Fatalities |") + lines.append("|------|------|---------|----------:|") + for e in events[:10]: + lines.append( + f"| {(e.get('event_date') or '')[:10]} " + f"| {e.get('event_type', '')} " + f"| {e.get('country', '')} " + f"| {e.get('fatalities', 0)} |" + ) + lines.append("") + + # Cyber + by_sev = cyber_summary.get("by_severity", {}) + lines.append(f"## Cyber Threats") + lines.append(f"Critical: {by_sev.get('critical', 0)} | " + f"High: {by_sev.get('high', 0)} | " + f"Medium: {by_sev.get('medium', 0)}") + lines.append("") + + # Natural + quakes = natural_summary.get("earthquakes", []) + lines.append(f"## Natural Events") + lines.append(f"Earthquakes: {len(quakes)} | Fires: {natural_summary.get('fire_count', 0)}") + if quakes: + lines.append("") + lines.append("| Mag | Location | Depth |") + lines.append("|----:|----------|------:|") + for q in quakes[:5]: + lines.append(f"| {q.get('magnitude', 0):.1f} | {(q.get('place') or '')[:40]} | {q.get('depth_km', 0):.0f}km |") + lines.append("") + + # Predictions + if prediction_highlights: + lines.append("## Prediction Markets") + for p in prediction_highlights[:5]: + yes = (p.get("yes_probability", 0) or 0) * 100 + lines.append(f"- **{(p.get('question') or '')[:60]}** — YES: {yes:.0f}% ({p.get('sentiment', '')})") + lines.append("") + + # Trending + if trending_keywords: + lines.append("## Trending Keywords") + kw_str = ", ".join(f"**{k['word']}** ({k['count']})" for k in trending_keywords[:15]) + lines.append(kw_str) + lines.append("") + + lines.append(f"---\n*Phoenix AGI System — World Intelligence*") + return "\n".join(lines) + + +def generate_threat_landscape_md( + cyber_threats: dict, + conflict_events: list, + military_activity: dict, + cable_health: dict, + outages: dict, +) -> str: + """Generate a threat landscape report in Markdown with Mermaid diagram.""" + now = datetime.now(timezone.utc) + lines = [ + "# Threat Landscape Report", + f"*Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}*", + "", + ] + + # Mermaid threat overview + by_sev = cyber_threats.get("by_severity", {}) + lines.append("## Threat Overview") + lines.append("```mermaid") + lines.append("pie title Threat Severity Distribution") + for level in ["critical", "high", "medium", "low"]: + count = by_sev.get(level, 0) + if count > 0: + lines.append(f' "{level.title()}" : {count}') + lines.append("```") + lines.append("") + + # Cyber section + lines.append(f"## Cyber Threats ({len(cyber_threats.get('threats', []))})") + for t in cyber_threats.get("threats", [])[:10]: + lines.append(f"- [{t.get('severity', '').upper()}] **{(t.get('indicator') or '')[:40]}** — {t.get('threat', '')} (via {t.get('source_feed', '')})") + lines.append("") + + # Military + theaters = military_activity.get("theaters", {}) + total = military_activity.get("total_military_aircraft", 0) + lines.append(f"## Military Activity ({total} aircraft)") + for name, info in theaters.items(): + count = info.get("count", 0) + lines.append(f"- **{name.replace('_', ' ').title()}**: {count} aircraft") + lines.append("") + + # Conflict events count + conflict_count = len(conflict_events) + if conflict_count: + lines.append(f"## Active Conflicts ({conflict_count} events, 7 days)") + for e in conflict_events[:10]: + lines.append( + f"- [{(e.get('event_date') or '')[:10]}] **{e.get('country', '')}** — " + f"{e.get('event_type', '')} ({e.get('fatalities', 0)} fatalities)" + ) + lines.append("") + + # Infrastructure + corridors = cable_health.get("corridors", {}) + if corridors: + lines.append("## Cable Health") + status_map = {0: "Clear", 1: "Advisory", 2: "At Risk", 3: "Disrupted"} + for name, info in corridors.items(): + score = info.get("status_score", 0) + lines.append(f"- **{name.replace('_', ' ').title()}**: {status_map.get(score, '?')}") + lines.append("") + + # Outages + ongoing = outages.get("ongoing_count", 0) + lines.append(f"## Internet Outages ({ongoing} ongoing)") + lines.append("") + + lines.append(f"---\n*Phoenix AGI System — Threat Intelligence*") + return "\n".join(lines) diff --git a/src/world_intel_mcp/reports/templates/country_dossier.html b/src/world_intel_mcp/reports/templates/country_dossier.html new file mode 100644 index 0000000..6304997 --- /dev/null +++ b/src/world_intel_mcp/reports/templates/country_dossier.html @@ -0,0 +1,270 @@ + + + + + + {{ title }} — {{ generated_at[:10] }} + + + + +
+

{{ title }}

+

Country Code: {{ country_code }} | Generated: {{ generated_at }} | Phoenix AGI System

+ + +

Instability Assessment

+ {% if instability %} +
+
+

Instability Index

+
{{ "%.1f"|format(instability.instability_index|default(0)) }}
+
+ {{ instability.risk_level|default('Unknown') }} +
+
+
+

Instability Components

+
+ +
+
+
+ {% else %} +

No instability data available for this country.

+ {% endif %} + + +

Intelligence Brief

+ {% if brief %} +
{{ brief }}
+ {% else %} +

No brief generated.

+ {% endif %} + + +

Conflict Events

+ {% if conflict_events %} +
+ + + {% for e in conflict_events[:15] %} + + + + + + + + {% endfor %} +
DateTypeSub-TypeLocationFatalities
{{ (e.event_date or '')[:10] }}{{ e.event_type|default('') }}{{ e.sub_event_type|default('') }}{{ e.location|default('') }}{{ e.fatalities|default(0) }}
+
+ {% else %} +

No conflict events reported for this country.

+ {% endif %} + + +

Displacement & Refugees

+ {% if displacement %} +
+
+

Displacement by Origin

+ {% if displacement.by_origin %} + + + {% for d in displacement.by_origin[:10] %} + + + + + + + {% endfor %} +
YearRefugeesIDPsAsylum Seekers
{{ d.year|default('') }}{{ "{:,}".format(d.refugees|default(0)|int) }}{{ "{:,}".format(d.idps|default(0)|int) }}{{ "{:,}".format(d.asylum_seekers|default(0)|int) }}
+ {% else %} +

No displacement data by origin.

+ {% endif %} +
+ {% if displacement.global_totals %} +
+

Global Context

+ + + {% for key, val in displacement.global_totals.items() %} + + + + + {% endfor %} +
MetricTotal
{{ key }}{{ "{:,}".format(val|int) if val is not none else 'N/A' }}
+
+ {% endif %} +
+ {% else %} +

No displacement data available.

+ {% endif %} + + +

Economic Indicators

+ {% if economic %} +
+
+

GDP Trend

+ {% if economic.gdp %} +
+ +
+ {% else %} +

No GDP data available.

+ {% endif %} +
+
+

Inflation Trend

+ {% if economic.inflation %} + + + {% for entry in economic.inflation[:10] %} + + + + + {% endfor %} +
YearInflation (%)
{{ entry.year|default(entry.date|default('')) }}{{ "%.1f"|format(entry.value or 0) }}%
+ {% else %} +

No inflation data available.

+ {% endif %} +
+
+ {% else %} +

No economic data available.

+ {% endif %} + +
Phoenix AGI System — Country Dossier: {{ country_code }} — {{ generated_at[:10] }}
+
+ + + + diff --git a/src/world_intel_mcp/reports/templates/daily_brief.html b/src/world_intel_mcp/reports/templates/daily_brief.html new file mode 100644 index 0000000..828441b --- /dev/null +++ b/src/world_intel_mcp/reports/templates/daily_brief.html @@ -0,0 +1,186 @@ + + + + + + {{ title }} — {{ generated_at[:10] }} + + + + +
+

{{ title }}

+

Generated: {{ generated_at }} | Phoenix AGI System

+ + +

Markets

+ {% if market_summary %} +
+
+

Index Quotes

+ {% if market_summary.quotes %} + + + {% for q in market_summary.quotes[:8] %} + + + + + + {% endfor %} +
SymbolPriceChange
{{ q.symbol|default('—') }}{{ "%.2f"|format(q.price or 0) }}{{ "%+.2f"|format(q.change_pct or 0) }}%
+ {% else %} +

No quote data available.

+ {% endif %} +
+
+

Macro Signals

+ {% if market_summary.macro_signals %} + + + {% for name, info in market_summary.macro_signals.items() %} + + + + + {% endfor %} +
SignalValue
{{ name }}{% if info is mapping %}{{ info.value|default(info.price|default('?')) }}{% elif info is not none %}{{ info }}{% else %}N/A{% endif %}
+ {% else %} +

No macro signals available.

+ {% endif %} +
+
+ {% else %} +

Market data unavailable.

+ {% endif %} + + +

Conflict & Security

+
+
+

Recent Events ({{ conflict_summary.count|default(0) }})

+ {% if conflict_summary and conflict_summary.events %} + + + {% for e in conflict_summary.events[:10] %} + + + + + + + {% endfor %} +
DateTypeCountryFatalities
{{ (e.event_date or '')[:10] }}{{ e.event_type|default('') }}{{ e.country|default('') }}{{ e.fatalities|default(0) }}
+ {% else %} +

No conflict events reported.

+ {% endif %} +
+
+

Cyber Threats

+ {% if cyber_summary and cyber_summary.threats %} +

+ Critical: {{ cyber_summary.by_severity.critical|default(0) }} | + High: {{ cyber_summary.by_severity.high|default(0) }} | + Medium: {{ cyber_summary.by_severity.medium|default(0) }} +

+ + + {% for t in cyber_summary.threats[:8] %} + + + + + + {% endfor %} +
SeverityIndicatorThreat
{{ t.severity|default('unknown') }}{{ (t.indicator or '')[:30] }}{{ (t.threat or '')[:30] }}
+ {% else %} +

No cyber threat data available.

+ {% endif %} +
+
+ + +

Natural Events

+
+
+

Earthquakes

+ {% if natural_summary and natural_summary.earthquakes %} + + + {% for q in natural_summary.earthquakes[:8] %} + + + + + + {% endfor %} +
MagLocationDepth
{{ "%.1f"|format(q.magnitude or 0) }}{{ (q.place or '')[:40] }}{{ "%.0f"|format(q.depth_km or 0) }}km
+ {% else %} +

No earthquake data available.

+ {% endif %} +
+
+

Wildfires

+

{{ natural_summary.fire_count|default(0) }} high-confidence fires detected

+
+
+ + +

Signals

+
+
+

Prediction Markets

+ {% if prediction_highlights %} + + + {% for p in prediction_highlights[:8] %} + + + + + + {% endfor %} +
QuestionYESSentiment
{{ (p.question or '')[:40] }}{{ "%.0f"|format((p.yes_probability or 0) * 100) }}%{{ p.sentiment|default('') }}
+ {% else %} +

No prediction data available.

+ {% endif %} +
+
+

Trending Keywords

+ {% if trending_keywords %} + {% for kw in trending_keywords[:20] %} + {{ kw.word }} ({{ kw.count }}) + {% endfor %} + {% else %} +

No trending keyword data.

+ {% endif %} +
+
+ +
Phoenix AGI System — World Intelligence Report — {{ generated_at[:10] }}
+
+ + diff --git a/src/world_intel_mcp/reports/templates/market_overview.html b/src/world_intel_mcp/reports/templates/market_overview.html new file mode 100644 index 0000000..ee6814d --- /dev/null +++ b/src/world_intel_mcp/reports/templates/market_overview.html @@ -0,0 +1,277 @@ + + + + + + {{ title }} — {{ generated_at[:10] }} + + + + +
+

{{ title }}

+

Generated: {{ generated_at }} | Phoenix AGI System

+ + +

Equity & Index Quotes

+ {% if quotes %} +
+ + + {% for q in quotes %} + + + + + + + + + {% endfor %} +
SymbolNamePriceChange% ChangeVolume
{{ q.symbol|default('—') }}{{ q.name|default(q.shortName|default('')) }}{{ "%.2f"|format(q.price or q.regularMarketPrice or 0) }} + {{ "%+.2f"|format(q.change or q.regularMarketChange or 0) }} + + {{ "%+.2f"|format(q.change_pct or q.regularMarketChangePercent or 0) }}% + {{ "{:,}".format((q.volume or q.regularMarketVolume or 0)|int) }}
+
+ {% else %} +

No equity quote data available.

+ {% endif %} + + +

Sector Performance

+ {% if sector_heatmap %} +
+
+

Sector Heatmap

+
+ {% for s in sector_heatmap %} + {% set pct = s.change_pct|default(0) %} + {% if pct >= 2 %} + {% set bg = '#166534' %} + {% elif pct >= 0.5 %} + {% set bg = '#14532d' %} + {% elif pct >= 0 %} + {% set bg = '#1a2e1a' %} + {% elif pct >= -0.5 %} + {% set bg = '#2e1a1a' %} + {% elif pct >= -2 %} + {% set bg = '#7f1d1d' %} + {% else %} + {% set bg = '#991b1b' %} + {% endif %} +
+ {{ s.symbol|default(s.name|default('?')) }}
{{ "%+.1f"|format(pct) }}% +
+ {% endfor %} +
+ +

Sector Bar Chart

+
+ +
+
+
+ {% else %} +

No sector data available.

+ {% endif %} + + +

Cryptocurrency

+ {% if crypto %} +
+ + + {% for c in crypto[:15] %} + + + + + + + + {% endfor %} +
CoinPrice (USD)24h ChangeMarket Cap24h Volume
{{ c.symbol|default(c.id|default(''))|upper }}{% if c.name %} {{ c.name }}{% endif %}{{ "${:,.2f}".format(c.current_price|default(c.price|default(0))) }} + {{ "%+.2f"|format(c.price_change_percentage_24h|default(c.change_24h|default(0))) }}% + + {% set mcap = c.market_cap|default(0) %} + {% if mcap >= 1e12 %}${{ "%.1f"|format(mcap / 1e12) }}T + {% elif mcap >= 1e9 %}${{ "%.1f"|format(mcap / 1e9) }}B + {% elif mcap >= 1e6 %}${{ "%.1f"|format(mcap / 1e6) }}M + {% elif mcap > 0 %}${{ "{:,.0f}".format(mcap) }} + {% else %}—{% endif %} + + {% set vol = c.total_volume|default(c.volume_24h|default(0)) %} + {% if vol >= 1e9 %}${{ "%.1f"|format(vol / 1e9) }}B + {% elif vol >= 1e6 %}${{ "%.1f"|format(vol / 1e6) }}M + {% elif vol > 0 %}${{ "{:,.0f}".format(vol) }} + {% else %}—{% endif %} +
+
+ {% else %} +

No cryptocurrency data available.

+ {% endif %} + + +

Macro Signals

+ {% if macro_signals %} +
+ {% for name, info in macro_signals.items() %} +
+

{{ name }}

+ {% if info is mapping %} + + {% for key, val in info.items() %} + + + + + {% endfor %} +
{{ key.replace('_', ' ')|title }} + {% if val is number %} + {% if val >= 1e9 %}{{ "%.2f"|format(val / 1e9) }}B + {% elif val >= 1e6 %}{{ "%.2f"|format(val / 1e6) }}M + {% else %}{{ "%.4f"|format(val) if val < 1 and val > -1 else "%.2f"|format(val) }}{% endif %} + {% elif val is not none %}{{ val }} + {% else %}N/A{% endif %} +
+ {% elif info is not none %} +

{{ info }}

+ {% else %} +

N/A

+ {% endif %} +
+ {% endfor %} +
+ {% else %} +

No macro signal data available.

+ {% endif %} + + +

ETF Flows

+ {% if etf_flows %} +
+ {% if etf_flows is mapping %} + + + {% for name, data in etf_flows.items() %} + + + {% if data is mapping %} + + + {% else %} + + + {% endif %} + + {% endfor %} +
ETF / CategoryFlowDetails
{{ name }} + {% set flow = data.flow|default(data.net_flow|default(0)) %} + {% if flow >= 1e9 %}${{ "%+.1f"|format(flow / 1e9) }}B + {% elif flow >= 1e6 %}${{ "%+.1f"|format(flow / 1e6) }}M + {% else %}{{ "%+.2f"|format(flow) }}{% endif %} + {{ data.description|default(data.period|default('')) }}{{ data }}
+ {% elif etf_flows is iterable and etf_flows is not string %} + + + {% for item in etf_flows %} + + + + + {% endfor %} +
ETFFlow
{{ item.name|default(item.symbol|default('')) }}{{ item.flow|default(item.value|default('')) }}
+ {% else %} +

{{ etf_flows }}

+ {% endif %} +
+ {% else %} +

No ETF flow data available.

+ {% endif %} + +
Phoenix AGI System — Market Overview — {{ generated_at[:10] }}
+
+ + + + diff --git a/src/world_intel_mcp/reports/templates/threat_landscape.html b/src/world_intel_mcp/reports/templates/threat_landscape.html new file mode 100644 index 0000000..f84de26 --- /dev/null +++ b/src/world_intel_mcp/reports/templates/threat_landscape.html @@ -0,0 +1,265 @@ + + + + + + {{ title }} — {{ generated_at[:10] }} + + + + +
+

{{ title }}

+

Generated: {{ generated_at }} | Phoenix AGI System

+ + +

Threat Overview

+
+
+
{{ cyber_threats.by_severity.critical|default(0) if cyber_threats else 0 }}
+
Critical
+
+
+
{{ cyber_threats.by_severity.high|default(0) if cyber_threats else 0 }}
+
High
+
+
+
{{ cyber_threats.by_severity.medium|default(0) if cyber_threats else 0 }}
+
Medium
+
+
+
{{ (cyber_threats.threats|length) if cyber_threats and cyber_threats.threats else 0 }}
+
Total Indicators
+
+
+
{{ outages.ongoing_count|default(0) if outages else 0 }}
+
Active Outages
+
+
+ + +

Cyber Threats

+ {% if cyber_threats and cyber_threats.threats %} +
+
+

Severity Distribution

+
+ +
+
+
+

Threat Types

+ {% if cyber_threats.by_type %} + + + {% for type_name, count in cyber_threats.by_type.items() %} + + + + + {% endfor %} +
TypeCount
{{ type_name }}{{ count }}
+ {% else %} +

No threat type breakdown available.

+ {% endif %} +
+
+ +
+

Threat Indicators

+ + + {% for t in cyber_threats.threats[:20] %} + + + + + + + + {% endfor %} +
SeverityIndicatorTypeThreatFirst Seen
{{ t.severity|default('unknown') }}{{ (t.indicator or '')[:40] }}{{ t.type|default(t.indicator_type|default('')) }}{{ (t.threat or '')[:30] }}{{ (t.first_seen or '')[:10] }}
+
+ {% else %} +

No cyber threat data available.

+ {% endif %} + + +

Conflict Events

+ {% if conflict_events %} +
+ + + {% for e in conflict_events[:15] %} + + + + + + + + {% endfor %} +
DateTypeCountryLocationFatalities
{{ (e.event_date or '')[:10] }}{{ e.event_type|default('') }}{{ e.country|default('') }}{{ e.location|default('') }}{{ e.fatalities|default(0) }}
+
+ {% else %} +

No conflict events reported.

+ {% endif %} + + +

Military Activity

+ {% if military_activity %} +
+ {% if military_activity.theaters %} + {% for theater_name, theater_data in military_activity.theaters.items() %} +
+

{{ theater_name }}

+ {% if theater_data is mapping %} + + + {% for key, val in theater_data.items() %} + + + + + {% endfor %} +
MetricValue
{{ key.replace('_', ' ')|title }}{{ val }}
+ {% else %} +

{{ theater_data }}

+ {% endif %} +
+ {% endfor %} + {% endif %} +
+

Summary

+
{{ military_activity.total_military_aircraft|default(0) }}
+
Military Aircraft Tracked
+
+
+ {% else %} +

No military activity data available.

+ {% endif %} + + +

Submarine Cable Health

+ {% if cable_health and cable_health.corridors %} +
+ {% for corridor_name, corridor_data in cable_health.corridors.items() %} +
+

{{ corridor_name }}

+ {% if corridor_data is mapping %} + + {% for key, val in corridor_data.items() %} + + + + + {% endfor %} +
{{ key.replace('_', ' ')|title }}{{ val }}
+ {% else %} +

{{ corridor_data }}

+ {% endif %} +
+ {% endfor %} +
+ {% else %} +

No cable health data available.

+ {% endif %} + + +

Infrastructure Outages

+ {% if outages and outages.outages %} +
+ + + {% for o in outages.outages[:15] %} + + + + + + + + {% endfor %} +
ServiceStatusRegionSinceImpact
{{ o.service|default(o.name|default('')) }}{{ o.status|default('unknown') }}{{ o.region|default('') }}{{ (o.since or o.started or '')[:16] }}{{ o.impact|default('') }}
+
+ {% else %} +

No active outages reported.

+ {% endif %} + +
Phoenix AGI System — Threat Landscape Report — {{ generated_at[:10] }}
+
+ + + + diff --git a/src/world_intel_mcp/server.py b/src/world_intel_mcp/server.py new file mode 100644 index 0000000..30fa72e --- /dev/null +++ b/src/world_intel_mcp/server.py @@ -0,0 +1,628 @@ +#!/usr/bin/env python3 +""" +World Intelligence MCP Server +============================== + +Real-time global intelligence across 17 domains: +financial markets, economic indicators, earthquakes, wildfires, +conflict, military flights, infrastructure, and more. + +Phase 1: Markets, Economic, Seismology, Wildfire (12 tools). +Phase 2: Conflict, Military, Infrastructure, Maritime, Climate (+10 = 22 tools). +Phase 3: News, Intelligence, Prediction, Displacement, Aviation, Cyber (+14 = 36 tools). +Phase 4: Reports — daily brief, country dossier, threat landscape (+3 = 39 tools). +""" + +import asyncio +import json +import logging +import os +from typing import Any + +from mcp.server import Server +from mcp.server.stdio import stdio_server +from mcp.types import Tool, TextContent + +from .cache import Cache +from .circuit_breaker import CircuitBreaker +from .fetcher import Fetcher +from .sources import markets, economic, seismology, wildfire, conflict, military, infrastructure, maritime, climate, news, intelligence, prediction, displacement, aviation, cyber +from .reports import generator as report_gen + +logging.basicConfig( + level=os.environ.get("WORLD_INTEL_LOG_LEVEL", "INFO"), + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger("world-intel-mcp") + +server = Server("world-intel-mcp") +cache = Cache() +breaker = CircuitBreaker(failure_threshold=3, cooldown_seconds=300) +fetcher = Fetcher(cache=cache, breaker=breaker) + +# --------------------------------------------------------------------------- +# Tool definitions +# --------------------------------------------------------------------------- + +TOOLS: list[Tool] = [ + # --- Markets (6 tools) --- + Tool( + name="intel_market_quotes", + description="Get real-time stock market index quotes (S&P 500, Dow, Nasdaq, FTSE, Nikkei, etc.). Optional: symbols (list of ticker symbols).", + inputSchema={ + "type": "object", + "properties": { + "symbols": { + "type": "array", + "items": {"type": "string"}, + "description": "Ticker symbols (default: major indices)", + }, + }, + }, + ), + Tool( + name="intel_crypto_quotes", + description="Get top cryptocurrency prices, market caps, and 7-day sparklines from CoinGecko. Optional: limit (int, default 20).", + inputSchema={ + "type": "object", + "properties": { + "limit": {"type": "integer", "description": "Number of coins (default 20)", "default": 20}, + }, + }, + ), + Tool( + name="intel_stablecoin_status", + description="Check stablecoin peg health (USDT, USDC, DAI, FDUSD). Flags depegs >0.5%.", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="intel_etf_flows", + description="Get Bitcoin spot ETF prices and volumes (IBIT, FBTC, GBTC, ARKB, BITB).", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="intel_sector_heatmap", + description="Get US equity sector performance heatmap (11 SPDR sector ETFs).", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="intel_macro_signals", + description="Get 7 key macro signals: Fear & Greed, mempool fees, DXY, VIX, gold, 10Y Treasury, BTC dominance.", + inputSchema={"type": "object", "properties": {}}, + ), + # --- Economic (3 tools) --- + Tool( + name="intel_energy_prices", + description="Get crude oil (Brent, WTI) and natural gas prices from EIA. Requires EIA_API_KEY.", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="intel_fred_series", + description="Get Federal Reserve economic data series (GDP, UNRATE, CPIAUCSL, DFF, T10YIE, etc.). Requires FRED_API_KEY.", + inputSchema={ + "type": "object", + "properties": { + "series_id": {"type": "string", "description": "FRED series ID (e.g., 'UNRATE')"}, + "limit": {"type": "integer", "description": "Number of observations", "default": 30}, + }, + "required": ["series_id"], + }, + ), + Tool( + name="intel_world_bank_indicators", + description="Get World Bank development indicators (GDP, inflation, unemployment) for a country.", + inputSchema={ + "type": "object", + "properties": { + "country": {"type": "string", "description": "ISO country code (default: USA)", "default": "USA"}, + "indicators": { + "type": "array", + "items": {"type": "string"}, + "description": "World Bank indicator codes", + }, + }, + }, + ), + # --- Natural (2 tools) --- + Tool( + name="intel_earthquakes", + description="Get recent earthquakes from USGS. No API key needed.", + inputSchema={ + "type": "object", + "properties": { + "min_magnitude": {"type": "number", "description": "Minimum magnitude (default 4.5)", "default": 4.5}, + "hours": {"type": "integer", "description": "Lookback hours (default 24)", "default": 24}, + "limit": {"type": "integer", "description": "Max results (default 50)", "default": 50}, + }, + }, + ), + Tool( + name="intel_wildfires", + description="Get active wildfires from NASA FIRMS (9 global regions). Requires NASA_FIRMS_API_KEY.", + inputSchema={ + "type": "object", + "properties": { + "region": { + "type": "string", + "description": "Specific region (north_america, europe, etc.) or omit for all 9", + "enum": [ + "north_america", "south_america", "europe", "africa", + "middle_east", "south_asia", "east_asia", "southeast_asia", "oceania", + ], + }, + }, + }, + ), + # --- Conflict (3 tools) --- + Tool( + name="intel_acled_events", + description="Get armed conflict events from ACLED. Optional: country (name), days (default 7), limit (default 100). Requires ACLED_ACCESS_TOKEN.", + inputSchema={ + "type": "object", + "properties": { + "country": {"type": "string", "description": "Country name filter"}, + "days": {"type": "integer", "description": "Lookback days (default 7)", "default": 7}, + "limit": {"type": "integer", "description": "Max results (default 100)", "default": 100}, + }, + }, + ), + Tool( + name="intel_ucdp_events", + description="Get state-based violence events from UCDP GED. No API key needed. Optional: days (default 30).", + inputSchema={ + "type": "object", + "properties": { + "days": {"type": "integer", "description": "Lookback days (default 30)", "default": 30}, + "limit": {"type": "integer", "description": "Max results (default 100)", "default": 100}, + }, + }, + ), + Tool( + name="intel_humanitarian_summary", + description="Get humanitarian crisis datasets from HDX. No API key needed. Optional: country code.", + inputSchema={ + "type": "object", + "properties": { + "country": {"type": "string", "description": "ISO country code filter"}, + }, + }, + ), + # --- Military (3 tools) --- + Tool( + name="intel_military_flights", + description="Track military aircraft via OpenSky Network (ICAO hex + callsign filtering). Optional: bbox (lamin,lomin,lamax,lomax).", + inputSchema={ + "type": "object", + "properties": { + "bbox": {"type": "string", "description": "Bounding box: lamin,lomin,lamax,lomax"}, + }, + }, + ), + Tool( + name="intel_theater_posture", + description="Get military aircraft presence across 5 theaters: European, Indo-Pacific, Middle East, Arctic, Korean Peninsula.", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="intel_aircraft_details", + description="Get aircraft details from Wingbits by ICAO24 hex code. Requires WINGBITS_API_KEY.", + inputSchema={ + "type": "object", + "properties": { + "icao24": {"type": "string", "description": "ICAO24 hex code"}, + }, + "required": ["icao24"], + }, + ), + # --- Infrastructure (2 tools) --- + Tool( + name="intel_internet_outages", + description="Get internet outages from Cloudflare Radar (last 7 days). Optional: CLOUDFLARE_API_TOKEN for higher limits.", + inputSchema={"type": "object", "properties": {}}, + ), + Tool( + name="intel_cable_health", + description="Assess undersea cable corridor health from NGA navigational warnings. 6 corridors scored 0-3.", + inputSchema={"type": "object", "properties": {}}, + ), + # --- Maritime (1 tool) --- + Tool( + name="intel_nav_warnings", + description="Get active navigational warnings from NGA Maritime Safety. Optional: navarea (I-XVI).", + inputSchema={ + "type": "object", + "properties": { + "navarea": {"type": "string", "description": "NAVAREA number (e.g., IV, XII)"}, + }, + }, + ), + # --- Climate (1 tool) --- + Tool( + name="intel_climate_anomalies", + description="Detect temperature and precipitation anomalies across 15 global climate zones (vs. prior year baseline).", + inputSchema={ + "type": "object", + "properties": { + "zones": { + "type": "array", + "items": {"type": "string"}, + "description": "Zone keys to check (default: all 15)", + }, + }, + }, + ), + # --- Prediction (1 tool) --- + Tool( + name="intel_prediction_markets", + description="Get active prediction markets from Polymarket (questions, YES probabilities, volumes, sentiment).", + inputSchema={ + "type": "object", + "properties": { + "limit": {"type": "integer", "description": "Number of markets (default 20)", "default": 20}, + }, + }, + ), + # --- Displacement (1 tool) --- + Tool( + name="intel_displacement_summary", + description="Get UNHCR refugee/displacement statistics by country of origin. No API key needed.", + inputSchema={ + "type": "object", + "properties": { + "year": {"type": "integer", "description": "Reporting year (default: last year)"}, + }, + }, + ), + # --- Aviation (1 tool) --- + Tool( + name="intel_airport_delays", + description="Get current US airport delays from FAA (20 major airports). No API key needed.", + inputSchema={"type": "object", "properties": {}}, + ), + # --- Cyber (1 tool) --- + Tool( + name="intel_cyber_threats", + description="Get aggregated cyber threat intelligence from 4 feeds (Feodo, CISA KEV, SANS, URLhaus). No API key needed.", + inputSchema={ + "type": "object", + "properties": { + "limit": {"type": "integer", "description": "Max threats (default 50)", "default": 50}, + }, + }, + ), + # --- News (3 tools) --- + Tool( + name="intel_news_feed", + description="Get aggregated intelligence news from 20+ RSS feeds across 6 categories (geopolitics, security, tech, finance, military, science).", + inputSchema={ + "type": "object", + "properties": { + "category": { + "type": "string", + "description": "Category filter", + "enum": ["geopolitics", "security", "technology", "finance", "military", "science"], + }, + "limit": {"type": "integer", "description": "Max items (default 50)", "default": 50}, + }, + }, + ), + Tool( + name="intel_trending_keywords", + description="Detect trending keywords from recent news headlines. Keyword spike detection across 20+ feeds.", + inputSchema={ + "type": "object", + "properties": { + "min_count": {"type": "integer", "description": "Minimum occurrences (default 3)", "default": 3}, + }, + }, + ), + Tool( + name="intel_gdelt_search", + description="Search GDELT 2.0 for global news articles or volume timelines. No API key needed.", + inputSchema={ + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query (default: 'conflict')", "default": "conflict"}, + "mode": { + "type": "string", + "description": "artlist (articles) or timelinevol (volume timeline)", + "enum": ["artlist", "timelinevol"], + "default": "artlist", + }, + "limit": {"type": "integer", "description": "Max records (default 50)", "default": 50}, + }, + }, + ), + # --- Intelligence (4 tools) --- + Tool( + name="intel_country_brief", + description="Generate a country intelligence brief using Ollama LLM + World Bank + ACLED data. Falls back to data-only if LLM unavailable.", + inputSchema={ + "type": "object", + "properties": { + "country_code": {"type": "string", "description": "ISO country code (default: US)", "default": "US"}, + }, + }, + ), + Tool( + name="intel_risk_scores", + description="Get country risk scores computed from ACLED conflict data vs historical baselines. Requires ACLED_ACCESS_TOKEN.", + inputSchema={ + "type": "object", + "properties": { + "limit": {"type": "integer", "description": "Top N countries (default 20)", "default": 20}, + }, + }, + ), + Tool( + name="intel_instability_index", + description="Compute Country Instability Index (0-100) from conflict, economic, humanitarian, infrastructure, and military signals.", + inputSchema={ + "type": "object", + "properties": { + "country_code": {"type": "string", "description": "ISO alpha-3 code (e.g., UKR). Omit for top-10 focus countries."}, + }, + }, + ), + Tool( + name="intel_signal_convergence", + description="Detect geographic convergence of multi-domain signals (earthquakes, conflict, military) in hotspot regions.", + inputSchema={ + "type": "object", + "properties": { + "lat": {"type": "number", "description": "Center latitude (omit for 5 global hotspots)"}, + "lon": {"type": "number", "description": "Center longitude"}, + "radius_deg": {"type": "number", "description": "Radius in degrees (default 5.0)", "default": 5.0}, + }, + }, + ), + # --- Reports (3 tools) --- + Tool( + name="intel_daily_brief", + description="Generate a daily intelligence brief HTML report (markets, conflict, cyber, natural, predictions, trending). Returns file path.", + inputSchema={ + "type": "object", + "properties": { + "output_dir": {"type": "string", "description": "Custom output directory (default: $STORAGE_BASE/reports/intel/)"}, + }, + }, + ), + Tool( + name="intel_country_dossier", + description="Generate a full country dossier HTML report (brief, instability index, conflict, displacement, economic). Returns file path.", + inputSchema={ + "type": "object", + "properties": { + "country_code": {"type": "string", "description": "ISO country code (e.g., UKR, SYR, MMR)"}, + "output_dir": {"type": "string", "description": "Custom output directory"}, + }, + "required": ["country_code"], + }, + ), + Tool( + name="intel_threat_landscape", + description="Generate a threat landscape HTML report (cyber threats, conflict, military, cable health, outages). Returns file path.", + inputSchema={ + "type": "object", + "properties": { + "output_dir": {"type": "string", "description": "Custom output directory"}, + }, + }, + ), + # --- System (1 tool) --- + Tool( + name="intel_status", + description="Get data source health, circuit breaker status, and cache statistics.", + inputSchema={"type": "object", "properties": {}}, + ), +] + + +# --------------------------------------------------------------------------- +# Tool dispatch +# --------------------------------------------------------------------------- + +async def _dispatch(name: str, arguments: dict[str, Any]) -> Any: + """Route tool call to the appropriate source function.""" + match name: + # Markets + case "intel_market_quotes": + return await markets.fetch_market_quotes(fetcher, symbols=arguments.get("symbols")) + case "intel_crypto_quotes": + return await markets.fetch_crypto_quotes(fetcher, limit=arguments.get("limit", 20)) + case "intel_stablecoin_status": + return await markets.fetch_stablecoin_status(fetcher) + case "intel_etf_flows": + return await markets.fetch_etf_flows(fetcher) + case "intel_sector_heatmap": + return await markets.fetch_sector_heatmap(fetcher) + case "intel_macro_signals": + return await markets.fetch_macro_signals(fetcher) + + # Economic + case "intel_energy_prices": + return await economic.fetch_energy_prices(fetcher) + case "intel_fred_series": + return await economic.fetch_fred_series( + fetcher, + series_id=arguments["series_id"], + limit=arguments.get("limit", 30), + ) + case "intel_world_bank_indicators": + return await economic.fetch_world_bank_indicators( + fetcher, + country=arguments.get("country", "USA"), + indicators=arguments.get("indicators"), + ) + + # Natural + case "intel_earthquakes": + return await seismology.fetch_earthquakes( + fetcher, + min_magnitude=arguments.get("min_magnitude", 4.5), + hours=arguments.get("hours", 24), + limit=arguments.get("limit", 50), + ) + case "intel_wildfires": + return await wildfire.fetch_wildfires(fetcher, region=arguments.get("region")) + + # Conflict + case "intel_acled_events": + return await conflict.fetch_acled_events( + fetcher, + country=arguments.get("country"), + days=arguments.get("days", 7), + limit=arguments.get("limit", 100), + ) + case "intel_ucdp_events": + return await conflict.fetch_ucdp_events( + fetcher, + days=arguments.get("days", 30), + limit=arguments.get("limit", 100), + ) + case "intel_humanitarian_summary": + return await conflict.fetch_humanitarian_summary( + fetcher, country=arguments.get("country"), + ) + + # Military + case "intel_military_flights": + return await military.fetch_military_flights(fetcher, bbox=arguments.get("bbox")) + case "intel_theater_posture": + return await military.fetch_theater_posture(fetcher) + case "intel_aircraft_details": + return await military.fetch_aircraft_details(fetcher, icao24=arguments["icao24"]) + + # Infrastructure + case "intel_internet_outages": + return await infrastructure.fetch_internet_outages(fetcher) + case "intel_cable_health": + return await infrastructure.fetch_cable_health(fetcher) + + # Maritime + case "intel_nav_warnings": + return await maritime.fetch_nav_warnings(fetcher, navarea=arguments.get("navarea")) + + # Climate + case "intel_climate_anomalies": + return await climate.fetch_climate_anomalies(fetcher, zones=arguments.get("zones")) + + # Prediction + case "intel_prediction_markets": + return await prediction.fetch_prediction_markets(fetcher, limit=arguments.get("limit", 20)) + + # Displacement + case "intel_displacement_summary": + return await displacement.fetch_displacement_summary(fetcher, year=arguments.get("year")) + + # Aviation + case "intel_airport_delays": + return await aviation.fetch_airport_delays(fetcher) + + # Cyber + case "intel_cyber_threats": + return await cyber.fetch_cyber_threats(fetcher, limit=arguments.get("limit", 50)) + + # News + case "intel_news_feed": + return await news.fetch_news_feed( + fetcher, + category=arguments.get("category"), + limit=arguments.get("limit", 50), + ) + case "intel_trending_keywords": + return await news.fetch_trending_keywords(fetcher, min_count=arguments.get("min_count", 3)) + case "intel_gdelt_search": + return await news.fetch_gdelt_search( + fetcher, + query=arguments.get("query", "conflict"), + mode=arguments.get("mode", "artlist"), + limit=arguments.get("limit", 50), + ) + + # Intelligence + case "intel_country_brief": + return await intelligence.fetch_country_brief(fetcher, country_code=arguments.get("country_code", "US")) + case "intel_risk_scores": + return await intelligence.fetch_risk_scores(fetcher, limit=arguments.get("limit", 20)) + case "intel_instability_index": + return await intelligence.fetch_instability_index(fetcher, country_code=arguments.get("country_code")) + case "intel_signal_convergence": + return await intelligence.fetch_signal_convergence( + fetcher, + lat=arguments.get("lat"), + lon=arguments.get("lon"), + radius_deg=arguments.get("radius_deg", 5.0), + ) + + # Reports + case "intel_daily_brief": + return await report_gen.generate_daily_brief(output_dir=arguments.get("output_dir")) + case "intel_country_dossier": + return await report_gen.generate_country_dossier( + country_code=arguments["country_code"], + output_dir=arguments.get("output_dir"), + ) + case "intel_threat_landscape": + return await report_gen.generate_threat_landscape(output_dir=arguments.get("output_dir")) + + # System + case "intel_status": + return { + "circuit_breakers": breaker.status(), + "cache": cache.stats(), + "sources": { + "markets": ["yahoo-finance", "coingecko", "alternative-me", "mempool"], + "economic": ["eia", "fred", "world-bank"], + "natural": ["usgs", "nasa-firms"], + "conflict": ["acled", "ucdp", "hdx"], + "military": ["opensky", "wingbits"], + "infrastructure": ["cloudflare-radar", "nga-msi"], + "maritime": ["nga-msi"], + "climate": ["open-meteo"], + "news": ["rss-aggregator", "gdelt"], + "intelligence": ["ollama", "acled", "world-bank", "hdx", "usgs"], + "prediction": ["polymarket"], + "displacement": ["unhcr"], + "aviation": ["faa"], + "cyber": ["feodo-tracker", "cisa-kev", "sans-dshield", "urlhaus"], + }, + } + + case _: + return {"error": f"Unknown tool: {name}"} + + +# --------------------------------------------------------------------------- +# MCP handlers +# --------------------------------------------------------------------------- + +@server.list_tools() +async def list_tools() -> list[Tool]: + return TOOLS + + +@server.call_tool() +async def call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextContent]: + args = arguments or {} + logger.info("Tool call: %s(%s)", name, json.dumps(args, default=str)[:200]) + result = await _dispatch(name, args) + return [TextContent(type="text", text=json.dumps(result, indent=2, default=str))] + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +async def _run() -> None: + logger.info("World Intelligence MCP Server starting (%d tools)", len(TOOLS)) + async with stdio_server() as (read_stream, write_stream): + await server.run(read_stream, write_stream, server.create_initialization_options()) + + +def run() -> None: + asyncio.run(_run()) + + +if __name__ == "__main__": + run() diff --git a/src/world_intel_mcp/sources/__init__.py b/src/world_intel_mcp/sources/__init__.py new file mode 100644 index 0000000..b2eae35 --- /dev/null +++ b/src/world_intel_mcp/sources/__init__.py @@ -0,0 +1 @@ +"""Data source modules for world-intel-mcp.""" diff --git a/src/world_intel_mcp/sources/aviation.py b/src/world_intel_mcp/sources/aviation.py new file mode 100644 index 0000000..2af095e --- /dev/null +++ b/src/world_intel_mcp/sources/aviation.py @@ -0,0 +1,133 @@ +"""FAA airport delay data source for world-intel-mcp. + +Provides real-time US airport delay information from the FAA Airport +Status Web Service (ASWS) API. No API key required. +""" + +import asyncio +import logging +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.aviation") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_FAA_STATUS_URL = "https://soa.smext.faa.gov/asws/api/airport/status" + +_MAJOR_AIRPORTS = [ + "ATL", "LAX", "ORD", "DFW", "DEN", "JFK", "SFO", "SEA", "LAS", "MCO", + "EWR", "CLT", "PHX", "IAH", "MIA", "BOS", "MSP", "FLL", "DTW", "PHL", +] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _parse_airport_status(code: str, data: dict) -> dict: + """Extract structured fields from a single FAA airport status response.""" + name = data.get("Name", code) + delay = data.get("Delay", False) + + # Normalize delay to boolean (API may return string "true"/"false") + if isinstance(delay, str): + delay = delay.lower() == "true" + + status_items = data.get("Status", []) + if not isinstance(status_items, list): + status_items = [status_items] if isinstance(status_items, dict) else [] + + parsed_statuses = [] + for item in status_items: + if not isinstance(item, dict): + continue + parsed_statuses.append({ + "type": item.get("Type", ""), + "reason": item.get("Reason", ""), + "avg_delay": item.get("AvgDelay", ""), + "closure_begin": item.get("ClosureBegin", ""), + "closure_end": item.get("ClosureEnd", ""), + }) + + return { + "code": code, + "name": name, + "delay": delay, + "status": parsed_statuses, + } + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_airport_delays(fetcher: Fetcher) -> dict: + """Fetch current US airport delays from the FAA Airport Status API. + + Queries the FAA ASWS API for each major US airport in parallel and + returns a summary of which airports currently have active delays. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + + Returns: + Dict with delayed airports list, counts, source, and timestamp. + """ + + async def _fetch_one(code: str) -> tuple[str, dict | None]: + """Fetch status for a single airport, returning (code, data|None).""" + data = await fetcher.get_json( + url=f"{_FAA_STATUS_URL}/{code}", + source="faa", + cache_key=f"aviation:faa:{code}", + cache_ttl=300, + ) + return code, data + + # Fetch all airports in parallel + results = await asyncio.gather( + *[_fetch_one(code) for code in _MAJOR_AIRPORTS], + return_exceptions=True, + ) + + now_iso = _utc_now_iso() + + delayed: list[dict] = [] + all_airports: list[dict] = [] + errors = 0 + + for result in results: + if isinstance(result, Exception): + logger.warning("Exception fetching airport status: %s", result) + errors += 1 + continue + + code, data = result + + if data is None: + logger.debug("No data returned for airport %s", code) + errors += 1 + continue + + parsed = _parse_airport_status(code, data) + all_airports.append(parsed) + + if parsed["delay"]: + delayed.append(parsed) + + return { + "delayed": delayed, + "delayed_count": len(delayed), + "total_checked": len(_MAJOR_AIRPORTS), + "errors": errors, + "source": "faa", + "timestamp": now_iso, + } diff --git a/src/world_intel_mcp/sources/climate.py b/src/world_intel_mcp/sources/climate.py new file mode 100644 index 0000000..5136adf --- /dev/null +++ b/src/world_intel_mcp/sources/climate.py @@ -0,0 +1,266 @@ +"""Climate anomaly data source for world-intel-mcp. + +Provides temperature and precipitation anomaly monitoring across 15 global +climate zones using the Open-Meteo Archive API. No API key required. + +Anomalies are computed by comparing the most recent 7-day period against +the same 7-day window from the previous year (baseline). +""" + +import asyncio +import logging +from datetime import datetime, timezone, timedelta + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.climate") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive" + +CLIMATE_ZONES = { + "arctic": {"lat": 80.0, "lon": 0.0, "name": "Arctic (Svalbard)"}, + "subarctic_siberia": {"lat": 65.0, "lon": 100.0, "name": "Subarctic Siberia"}, + "northern_europe": {"lat": 55.0, "lon": 15.0, "name": "Northern Europe"}, + "mediterranean": {"lat": 38.0, "lon": 20.0, "name": "Mediterranean Basin"}, + "sahel": {"lat": 14.0, "lon": 0.0, "name": "Sahel Region"}, + "middle_east": {"lat": 30.0, "lon": 45.0, "name": "Middle East"}, + "south_asia": {"lat": 25.0, "lon": 78.0, "name": "South Asia (India)"}, + "east_asia": {"lat": 35.0, "lon": 116.0, "name": "East Asia (Beijing)"}, + "southeast_asia": {"lat": 2.0, "lon": 105.0, "name": "Southeast Asia"}, + "australia": {"lat": -25.0, "lon": 135.0, "name": "Central Australia"}, + "amazon": {"lat": -3.0, "lon": -60.0, "name": "Amazon Basin"}, + "us_midwest": {"lat": 40.0, "lon": -90.0, "name": "US Midwest"}, + "us_southwest": {"lat": 33.0, "lon": -112.0, "name": "US Southwest"}, + "antarctica": {"lat": -75.0, "lon": 0.0, "name": "Antarctica"}, + "pacific_enso": {"lat": 0.0, "lon": -150.0, "name": "Pacific ENSO Region"}, +} + +_CACHE_TTL = 1800 # 30 minutes + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _safe_avg(values: list[float | None]) -> float: + """Return mean of non-None numeric values, or 0.0 if empty.""" + clean = [v for v in values if v is not None] + if not clean: + return 0.0 + return sum(clean) / len(clean) + + +def _safe_sum(values: list[float | None]) -> float: + """Return sum of non-None numeric values, or 0.0 if empty.""" + return sum(v for v in values if v is not None) + + +def _compute_anomalies( + current_data: dict, + baseline_data: dict, +) -> dict: + """Compute temperature and precipitation anomalies between two periods. + + Returns dict with current/baseline averages and anomaly values. + """ + cur_daily = current_data.get("daily", {}) + base_daily = baseline_data.get("daily", {}) + + cur_max = cur_daily.get("temperature_2m_max", []) + cur_min = cur_daily.get("temperature_2m_min", []) + cur_precip = cur_daily.get("precipitation_sum", []) + + base_max = base_daily.get("temperature_2m_max", []) + base_min = base_daily.get("temperature_2m_min", []) + base_precip = base_daily.get("precipitation_sum", []) + + # Average temperature: mean of (daily_max + daily_min) / 2 across the period + cur_temps = [ + (mx + mn) / 2 + for mx, mn in zip(cur_max, cur_min) + if mx is not None and mn is not None + ] + base_temps = [ + (mx + mn) / 2 + for mx, mn in zip(base_max, base_min) + if mx is not None and mn is not None + ] + + current_avg_temp = _safe_avg(cur_temps) + baseline_avg_temp = _safe_avg(base_temps) + temp_anomaly = round(current_avg_temp - baseline_avg_temp, 2) + + # Precipitation totals + current_precip_mm = round(_safe_sum(cur_precip), 2) + baseline_precip_mm = round(_safe_sum(base_precip), 2) + + # Percentage anomaly (guard against near-zero baseline) + precip_anomaly_pct = round( + ((current_precip_mm - baseline_precip_mm) + / max(baseline_precip_mm, 0.1)) + * 100, + 1, + ) + + # Significant anomaly flags + is_significant = abs(temp_anomaly) > 3.0 or abs(precip_anomaly_pct) > 100 + + return { + "current_avg_temp_c": round(current_avg_temp, 2), + "baseline_avg_temp_c": round(baseline_avg_temp, 2), + "temp_anomaly_c": temp_anomaly, + "current_precip_mm": current_precip_mm, + "baseline_precip_mm": baseline_precip_mm, + "precip_anomaly_pct": precip_anomaly_pct, + "is_significant": is_significant, + } + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_climate_anomalies( + fetcher: Fetcher, + zones: list[str] | None = None, +) -> dict: + """Fetch temperature and precipitation anomalies for global climate zones. + + Compares the most recent 7-day window against the same dates from the + previous year using the Open-Meteo Archive API. No API key required. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + zones: Optional list of zone keys to fetch. If None, all 15 zones + are fetched in parallel. + + Returns: + Dict with per-zone anomaly data, list of significant anomalies, + source identifier, and ISO 8601 UTC timestamp. + """ + now = datetime.now(timezone.utc) + current_end = (now - timedelta(days=1)).strftime("%Y-%m-%d") + current_start = (now - timedelta(days=7)).strftime("%Y-%m-%d") + + baseline_end_dt = now - timedelta(days=1) - timedelta(days=365) + baseline_start_dt = now - timedelta(days=7) - timedelta(days=365) + baseline_end = baseline_end_dt.strftime("%Y-%m-%d") + baseline_start = baseline_start_dt.strftime("%Y-%m-%d") + + # Determine which zones to fetch + if zones is not None: + target_zones = { + k: v for k, v in CLIMATE_ZONES.items() if k in zones + } + if not target_zones: + logger.warning("No valid zone keys in %s", zones) + return { + "zones": {}, + "significant_anomalies": [], + "source": "open-meteo", + "timestamp": _utc_now_iso(), + } + else: + target_zones = dict(CLIMATE_ZONES) + + async def _fetch_zone(zone_key: str, zone_info: dict) -> tuple[str, dict | None]: + """Fetch current and baseline data for a single zone, compute anomalies.""" + lat = zone_info["lat"] + lon = zone_info["lon"] + + common_params = { + "latitude": lat, + "longitude": lon, + "daily": "temperature_2m_max,temperature_2m_min,precipitation_sum", + "timezone": "UTC", + } + + current_params = { + **common_params, + "start_date": current_start, + "end_date": current_end, + } + baseline_params = { + **common_params, + "start_date": baseline_start, + "end_date": baseline_end, + } + + # Fetch current and baseline periods in parallel + current_data, baseline_data = await asyncio.gather( + fetcher.get_json( + url=_ARCHIVE_URL, + source="open-meteo", + cache_key=f"climate:anomalies:{zone_key}:current", + cache_ttl=_CACHE_TTL, + params=current_params, + ), + fetcher.get_json( + url=_ARCHIVE_URL, + source="open-meteo", + cache_key=f"climate:anomalies:{zone_key}:baseline", + cache_ttl=_CACHE_TTL, + params=baseline_params, + ), + ) + + if current_data is None or baseline_data is None: + logger.warning( + "Open-Meteo returned no data for zone %s (current=%s, baseline=%s)", + zone_key, + current_data is not None, + baseline_data is not None, + ) + return (zone_key, None) + + anomalies = _compute_anomalies(current_data, baseline_data) + + return (zone_key, { + "name": zone_info["name"], + "lat": lat, + "lon": lon, + **anomalies, + }) + + # Fetch all target zones in parallel + tasks = [ + _fetch_zone(zone_key, zone_info) + for zone_key, zone_info in target_zones.items() + ] + results = await asyncio.gather(*tasks) + + # Assemble response + zone_results: dict[str, dict] = {} + significant_anomalies: list[str] = [] + + for zone_key, zone_data in results: + if zone_data is None: + continue + zone_results[zone_key] = zone_data + if zone_data["is_significant"]: + significant_anomalies.append(zone_key) + + response = { + "zones": zone_results, + "significant_anomalies": significant_anomalies, + "source": "open-meteo", + "timestamp": _utc_now_iso(), + } + + # Cache composite result + cache_label = "selected" if zones is not None else "all" + fetcher.cache.set( + f"climate:anomalies:{cache_label}", + response, + _CACHE_TTL, + ) + + return response diff --git a/src/world_intel_mcp/sources/conflict.py b/src/world_intel_mcp/sources/conflict.py new file mode 100644 index 0000000..6069314 --- /dev/null +++ b/src/world_intel_mcp/sources/conflict.py @@ -0,0 +1,356 @@ +"""Conflict and crisis data sources for world-intel-mcp. + +Fetches armed conflict events (ACLED), state-based violence data (UCDP), +and humanitarian dataset metadata (HDX) for the world-intel-mcp server. +""" + +import asyncio +import logging +import os +from datetime import datetime, timezone, timedelta + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.conflict") + + +# --------------------------------------------------------------------------- +# ACLED: Armed Conflict Location & Event Data +# --------------------------------------------------------------------------- + +_ACLED_URL = "https://api.acleddata.com/acled/read" + + +async def fetch_acled_events( + fetcher: Fetcher, + country: str | None = None, + days: int = 7, + limit: int = 100, +) -> dict: + """Fetch recent armed conflict events from the ACLED API v3. + + Requires ``ACLED_ACCESS_TOKEN`` in the environment. Free academic + access can be obtained at https://acleddata.com. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + country: Optional country name to filter events. + days: How far back to search (in days from now). + limit: Maximum number of results. + + Returns: + Dict with events list, count, query params, source, and timestamp. + """ + now = datetime.now(timezone.utc) + + access_token = os.environ.get("ACLED_ACCESS_TOKEN") + if not access_token: + return { + "error": "ACLED_ACCESS_TOKEN not configured", + "note": "Free academic access at acleddata.com", + } + + start_date = (now - timedelta(days=days)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + params: dict = { + "key": access_token, + "email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"), + "limit": limit, + "event_date": f"{start_date}|{end_date}", + "event_date_where": "BETWEEN", + } + if country: + params["country"] = country + + cache_country = country or "global" + data = await fetcher.get_json( + _ACLED_URL, + source="acled", + cache_key=f"conflict:acled:{cache_country}:{days}", + cache_ttl=900, + params=params, + ) + + if data is None: + logger.warning("ACLED API returned no data") + return { + "events": [], + "count": 0, + "query": {"country": country, "days": days}, + "source": "acled", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + events = [] + for event in data.get("data", []): + fatalities_raw = event.get("fatalities") + fatalities = int(fatalities_raw) if fatalities_raw is not None else 0 + events.append({ + "event_id_cnty": event.get("event_id_cnty"), + "event_date": event.get("event_date"), + "event_type": event.get("event_type"), + "sub_event_type": event.get("sub_event_type"), + "actor1": event.get("actor1"), + "actor2": event.get("actor2"), + "country": event.get("country"), + "admin1": event.get("admin1"), + "admin2": event.get("admin2"), + "location": event.get("location"), + "latitude": event.get("latitude"), + "longitude": event.get("longitude"), + "fatalities": fatalities, + "notes": event.get("notes"), + "source": event.get("source"), + }) + + # Sort by fatalities descending, then event_date descending + # Negate fatalities for desc; negate date lexicographically is not possible, + # so use a two-pass stable sort: secondary key first, primary key second. + events.sort(key=lambda e: e.get("event_date") or "", reverse=True) + events.sort(key=lambda e: e.get("fatalities") or 0, reverse=True) + + return { + "events": events, + "count": len(events), + "query": {"country": country, "days": days}, + "source": "acled", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +# --------------------------------------------------------------------------- +# UCDP: Uppsala Conflict Data Program (GED API) +# --------------------------------------------------------------------------- + +_UCDP_GED_URL = "https://ucdpapi.pcr.uu.se/api/gedevents/24.1" + +_VIOLENCE_TYPES = { + 1: "state-based", + 2: "non-state", + 3: "one-sided", +} + + +async def fetch_ucdp_events( + fetcher: Fetcher, + days: int = 30, + limit: int = 100, +) -> dict: + """Fetch recent conflict events from the UCDP GED API. + + No API key required. Fetches multiple pages in parallel when the + dataset spans more than one page. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + days: Only include events whose ``date_start`` is within this + many days from now. + limit: Page size (max 1000 per page). + + Returns: + Dict with events list, count, total fatalities, query params, + source, and timestamp. + """ + now = datetime.now(timezone.utc) + cutoff = now - timedelta(days=days) + + # Fetch page 1 to discover total pages + page1_data = await fetcher.get_json( + _UCDP_GED_URL, + source="ucdp", + cache_key=f"conflict:ucdp:{days}:page1", + cache_ttl=21600, + params={"pagesize": min(limit, 1000), "page": 0}, + ) + + if page1_data is None: + logger.warning("UCDP GED API returned no data") + return { + "events": [], + "count": 0, + "total_fatalities_best": 0, + "query": {"days": days}, + "source": "ucdp", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + total_pages = page1_data.get("TotalPages", 1) + all_results = list(page1_data.get("Result", [])) + + # Fetch remaining pages in parallel + if total_pages > 1: + page_tasks = [ + fetcher.get_json( + _UCDP_GED_URL, + source="ucdp", + cache_key=f"conflict:ucdp:{days}:page{p}", + cache_ttl=21600, + params={"pagesize": min(limit, 1000), "page": p}, + ) + for p in range(1, total_pages) + ] + page_results = await asyncio.gather(*page_tasks) + for page_data in page_results: + if page_data is not None: + all_results.extend(page_data.get("Result", [])) + + # Filter to recent events and extract fields + events = [] + total_fatalities_best = 0 + + for record in all_results: + date_start = record.get("date_start") + if date_start is None: + continue + + # Parse date_start and filter by cutoff + try: + event_date = datetime.strptime(date_start[:10], "%Y-%m-%d").replace( + tzinfo=timezone.utc, + ) + except (ValueError, TypeError): + continue + + if event_date < cutoff: + continue + + violence_type_raw = record.get("type_of_violence") + violence_type_int = None + if violence_type_raw is not None: + try: + violence_type_int = int(violence_type_raw) + except (ValueError, TypeError): + pass + + best = 0 + if record.get("best") is not None: + try: + best = int(record["best"]) + except (ValueError, TypeError): + pass + + high = 0 + if record.get("high") is not None: + try: + high = int(record["high"]) + except (ValueError, TypeError): + pass + + low = 0 + if record.get("low") is not None: + try: + low = int(record["low"]) + except (ValueError, TypeError): + pass + + total_fatalities_best += best + + events.append({ + "id": record.get("id"), + "relid": record.get("relid"), + "year": record.get("year"), + "date_start": date_start, + "date_end": record.get("date_end"), + "country": record.get("country"), + "region": record.get("region"), + "type_of_violence": violence_type_int, + "type_of_violence_label": _VIOLENCE_TYPES.get(violence_type_int, "unknown") if violence_type_int is not None else "unknown", + "side_a": record.get("side_a"), + "side_b": record.get("side_b"), + "best": best, + "high": high, + "low": low, + "latitude": record.get("latitude"), + "longitude": record.get("longitude"), + "source_article": record.get("source_article"), + "source_headline": record.get("source_headline"), + }) + + return { + "events": events, + "count": len(events), + "total_fatalities_best": total_fatalities_best, + "query": {"days": days}, + "source": "ucdp", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +# --------------------------------------------------------------------------- +# HDX: Humanitarian Data Exchange (CKAN API) +# --------------------------------------------------------------------------- + +_HDX_SEARCH_URL = "https://data.humdata.org/api/3/action/package_search" + + +async def fetch_humanitarian_summary( + fetcher: Fetcher, + country: str | None = None, +) -> dict: + """Fetch recent humanitarian crisis datasets from HDX (Humanitarian Data Exchange). + + No API key required. Uses the CKAN package_search API. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + country: Optional ISO 3166-1 alpha-3 country code (lowercase) to + filter datasets by geographic group. + + Returns: + Dict with datasets list, count, source, and timestamp. + """ + now = datetime.now(timezone.utc) + + params: dict = { + "q": "crisis", + "rows": 20, + "sort": "metadata_modified desc", + } + if country: + params["fq"] = f"groups:{country.lower()}" + + cache_country = country or "global" + data = await fetcher.get_json( + _HDX_SEARCH_URL, + source="hdx", + cache_key=f"conflict:humanitarian:{cache_country}", + cache_ttl=21600, + params=params, + ) + + if data is None: + logger.warning("HDX API returned no data") + return { + "datasets": [], + "count": 0, + "source": "hdx", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + datasets = [] + results = data.get("result", {}).get("results", []) + + for dataset in results: + notes = dataset.get("notes") or "" + if len(notes) > 200: + notes = notes[:200] + "..." + + org = dataset.get("organization") or {} + org_title = org.get("title") if isinstance(org, dict) else None + + datasets.append({ + "name": dataset.get("name"), + "title": dataset.get("title"), + "organization": org_title, + "metadata_modified": dataset.get("metadata_modified"), + "num_resources": dataset.get("num_resources"), + "notes": notes, + }) + + return { + "datasets": datasets, + "count": len(datasets), + "source": "hdx", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } diff --git a/src/world_intel_mcp/sources/cyber.py b/src/world_intel_mcp/sources/cyber.py new file mode 100644 index 0000000..da4272f --- /dev/null +++ b/src/world_intel_mcp/sources/cyber.py @@ -0,0 +1,278 @@ +"""Cyber threat intelligence feed aggregation for world-intel-mcp. + +Aggregates threat data from multiple free public feeds: +- Feodo Tracker (abuse.ch) -- C2 botnet IPs +- CISA Known Exploited Vulnerabilities -- actively exploited CVEs +- SANS ISC DShield -- top attacking IPs +- URLhaus (abuse.ch) -- malware distribution URLs + +No API keys required. +""" + +import asyncio +import logging +from datetime import datetime, timezone, timedelta + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.cyber") + +# --------------------------------------------------------------------------- +# Feed URLs +# --------------------------------------------------------------------------- + +_FEODO_URL = "https://feodotracker.abuse.ch/downloads/ipblocklist_recommended.json" +_CISA_KEV_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" +_SANS_ISC_URL = "https://isc.sans.edu/api/topips/records/20?json" +_URLHAUS_RECENT_URL = "https://urlhaus-api.abuse.ch/v1/urls/recent/limit/25/" + +# --------------------------------------------------------------------------- +# Severity ranking for sorting +# --------------------------------------------------------------------------- + +_SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _normalize_feodo(data: list | None) -> list[dict]: + """Normalize Feodo Tracker C2 IP entries.""" + if not data or not isinstance(data, list): + return [] + + items: list[dict] = [] + for entry in data: + status = entry.get("status", "").lower() + severity = "critical" if status == "online" else "medium" + items.append({ + "type": "c2_ip", + "indicator": entry.get("ip_address", ""), + "threat": entry.get("malware", "unknown"), + "severity": severity, + "source_feed": "feodo-tracker", + "first_seen": entry.get("first_seen", ""), + "details": { + "port": entry.get("port"), + "status": entry.get("status"), + "hostname": entry.get("hostname"), + "as_number": entry.get("as_number"), + "as_name": entry.get("as_name"), + "country": entry.get("country"), + "last_online": entry.get("last_online"), + }, + }) + return items + + +def _normalize_cisa_kev(data: dict | None) -> list[dict]: + """Normalize CISA Known Exploited Vulnerabilities, last 30 days only.""" + if not data or not isinstance(data, dict): + return [] + + cutoff = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%d") + vulnerabilities = data.get("vulnerabilities", []) + + items: list[dict] = [] + for vuln in vulnerabilities: + date_added = vuln.get("dateAdded", "") + if date_added < cutoff: + continue + + ransomware = vuln.get("knownRansomwareCampaignUse", "").lower() + severity = "critical" if ransomware == "known" else "high" + + cve_id = vuln.get("cveID", "") + vendor = vuln.get("vendorProject", "") + product = vuln.get("product", "") + vuln_name = vuln.get("vulnerabilityName", "") + + items.append({ + "type": "vulnerability", + "indicator": cve_id, + "threat": f"{vendor} {product}: {vuln_name}", + "severity": severity, + "source_feed": "cisa-kev", + "first_seen": date_added, + "details": { + "vendor": vendor, + "product": product, + "vulnerability_name": vuln_name, + "due_date": vuln.get("dueDate"), + "ransomware_use": vuln.get("knownRansomwareCampaignUse"), + "required_action": vuln.get("requiredAction"), + "notes": vuln.get("notes"), + }, + }) + return items + + +def _normalize_sans_isc(data: list | None) -> list[dict]: + """Normalize SANS ISC DShield top attacking IPs.""" + if not data or not isinstance(data, list): + return [] + + items: list[dict] = [] + for entry in data: + ip = entry.get("ip", "") + if not ip: + continue + items.append({ + "type": "attack_ip", + "indicator": ip, + "threat": f"DShield top attacker ({entry.get('attacks', 0)} attacks)", + "severity": "high", + "source_feed": "sans-dshield", + "first_seen": entry.get("firstseen", ""), + "details": { + "count": entry.get("count"), + "attacks": entry.get("attacks"), + "first_seen": entry.get("firstseen"), + "last_seen": entry.get("lastseen"), + "as_name": entry.get("asname"), + "as_country": entry.get("ascountry"), + }, + }) + return items + + +def _normalize_urlhaus(data: dict | None) -> list[dict]: + """Normalize URLhaus recent malware URLs.""" + if not data or not isinstance(data, dict): + return [] + + urls = data.get("urls", []) + if not isinstance(urls, list): + return [] + + items: list[dict] = [] + for entry in urls: + status = (entry.get("url_status") or "").lower() + severity = "high" if status == "online" else "low" + items.append({ + "type": "malware_url", + "indicator": entry.get("url", ""), + "threat": entry.get("threat", "unknown"), + "severity": severity, + "source_feed": "urlhaus", + "first_seen": entry.get("dateadded", ""), + "details": { + "url_status": entry.get("url_status"), + "tags": entry.get("tags"), + "reporter": entry.get("reporter"), + }, + }) + return items + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_cyber_threats( + fetcher: Fetcher, + limit: int = 50, +) -> dict: + """Aggregate cyber threat intelligence from free public feeds. + + Fetches data from Feodo Tracker, CISA KEV, SANS ISC DShield, and + URLhaus in parallel, normalizes into a common format, and returns + the top threats sorted by severity and recency. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + limit: Maximum number of threat items to return. + + Returns: + Dict with threats list, counts, breakdowns by type and severity, + source, and timestamp. + """ + feodo_data, cisa_data, sans_data, urlhaus_data = await asyncio.gather( + fetcher.get_json( + _FEODO_URL, + source="feodo-tracker", + cache_key="cyber:feodo", + cache_ttl=1800, + ), + fetcher.get_json( + _CISA_KEV_URL, + source="cisa-kev", + cache_key="cyber:cisa_kev", + cache_ttl=3600, + ), + fetcher.get_json( + _SANS_ISC_URL, + source="sans-dshield", + cache_key="cyber:sans_isc", + cache_ttl=1800, + ), + fetcher.get_json( + _URLHAUS_RECENT_URL, + source="urlhaus", + cache_key="cyber:urlhaus", + cache_ttl=900, + ), + ) + + # Track which feeds returned data + feeds_successful = 0 + + feodo_items = _normalize_feodo(feodo_data) + if feodo_data is not None: + feeds_successful += 1 + else: + logger.warning("Feodo Tracker feed returned no data") + + cisa_items = _normalize_cisa_kev(cisa_data) + if cisa_data is not None: + feeds_successful += 1 + else: + logger.warning("CISA KEV feed returned no data") + + sans_items = _normalize_sans_isc(sans_data) + if sans_data is not None: + feeds_successful += 1 + else: + logger.warning("SANS ISC DShield feed returned no data") + + urlhaus_items = _normalize_urlhaus(urlhaus_data) + if urlhaus_data is not None: + feeds_successful += 1 + else: + logger.warning("URLhaus feed returned no data") + + # Merge all threats + all_threats = feodo_items + cisa_items + sans_items + urlhaus_items + + # Sort by severity (critical first), then by recency (newest first_seen first). + # Use stable sort: first by date descending, then by severity ascending. + all_threats.sort(key=lambda t: t.get("first_seen", ""), reverse=True) + all_threats.sort(key=lambda t: _SEVERITY_ORDER.get(t.get("severity", "low"), 3)) + + # Apply limit + threats = all_threats[:limit] + + # Compute breakdowns + by_type: dict[str, int] = {} + by_severity: dict[str, int] = {} + for t in threats: + threat_type = t.get("type", "unknown") + by_type[threat_type] = by_type.get(threat_type, 0) + 1 + sev = t.get("severity", "low") + by_severity[sev] = by_severity.get(sev, 0) + 1 + + return { + "threats": threats, + "count": len(threats), + "feeds_successful": feeds_successful, + "feeds_attempted": 4, + "by_type": by_type, + "by_severity": by_severity, + "source": "cyber-feeds", + "timestamp": _utc_now_iso(), + } diff --git a/src/world_intel_mcp/sources/displacement.py b/src/world_intel_mcp/sources/displacement.py new file mode 100644 index 0000000..e41fd50 --- /dev/null +++ b/src/world_intel_mcp/sources/displacement.py @@ -0,0 +1,169 @@ +"""UNHCR displacement data source for world-intel-mcp. + +Provides refugee and displacement population statistics from the UNHCR +Population Statistics API. No API key required. +""" + +import logging +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.displacement") + +_UNHCR_POPULATION_URL = "https://api.unhcr.org/population/v1/population/" + + +async def fetch_displacement_summary( + fetcher: Fetcher, + year: int | None = None, +) -> dict: + """Fetch refugee/displacement population statistics from the UNHCR API. + + Aggregates displacement data by country of origin and computes global + totals across refugees, asylum seekers, IDPs, stateless persons, and + others of concern. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + year: Reporting year to query. Defaults to the previous calendar + year since UNHCR data typically lags by one year. + + Returns: + Dict with top-30 countries by displacement (sorted descending), + global totals, year, source, and timestamp. + """ + now = datetime.now(timezone.utc) + + # Default to previous year since UNHCR data lags + if year is None: + year = now.year - 1 + else: + # Ensure year is an int even if passed as string + year = int(year) + + params = { + "limit": 100, + "yearFrom": year, + "yearTo": year, + "cf_type": "ISO", + } + + data = await fetcher.get_json( + url=_UNHCR_POPULATION_URL, + source="unhcr", + cache_key=f"displacement:unhcr:{year}", + cache_ttl=43200, + params=params, + ) + + if data is None: + logger.warning("UNHCR API returned no data") + return { + "by_origin": [], + "global_totals": { + "total_refugees": 0, + "total_asylum_seekers": 0, + "total_idps": 0, + "total_stateless": 0, + "total_ooc": 0, + "grand_total": 0, + }, + "year": year, + "count": 0, + "source": "unhcr", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + items = data.get("items", []) + + # Aggregate by country of origin (coo_name) + by_country: dict[str, dict[str, int]] = {} + + for item in items: + coo_name = item.get("coo_name") or "Unknown" + + refugees = _safe_int(item.get("refugees")) + asylum_seekers = _safe_int(item.get("asylum_seekers")) + idps = _safe_int(item.get("idps")) + stateless = _safe_int(item.get("stateless")) + ooc = _safe_int(item.get("ooc")) + + if coo_name not in by_country: + by_country[coo_name] = { + "refugees": 0, + "asylum_seekers": 0, + "idps": 0, + "stateless": 0, + "ooc": 0, + } + + entry = by_country[coo_name] + entry["refugees"] += refugees + entry["asylum_seekers"] += asylum_seekers + entry["idps"] += idps + entry["stateless"] += stateless + entry["ooc"] += ooc + + # Build sorted list with total_displaced + by_origin = [] + for country, totals in by_country.items(): + total_displaced = ( + totals["refugees"] + + totals["asylum_seekers"] + + totals["idps"] + + totals["stateless"] + + totals["ooc"] + ) + by_origin.append({ + "country": country, + "refugees": totals["refugees"], + "asylum_seekers": totals["asylum_seekers"], + "internally_displaced": totals["idps"], + "stateless": totals["stateless"], + "others_of_concern": totals["ooc"], + "total_displaced": total_displaced, + }) + + # Sort by total_displaced descending, take top 30 + by_origin.sort(key=lambda e: e["total_displaced"], reverse=True) + by_origin = by_origin[:30] + + # Compute global totals across all countries (not just top 30) + global_refugees = sum(t["refugees"] for t in by_country.values()) + global_asylum_seekers = sum(t["asylum_seekers"] for t in by_country.values()) + global_idps = sum(t["idps"] for t in by_country.values()) + global_stateless = sum(t["stateless"] for t in by_country.values()) + global_ooc = sum(t["ooc"] for t in by_country.values()) + + return { + "by_origin": by_origin, + "global_totals": { + "total_refugees": global_refugees, + "total_asylum_seekers": global_asylum_seekers, + "total_idps": global_idps, + "total_stateless": global_stateless, + "total_ooc": global_ooc, + "grand_total": ( + global_refugees + + global_asylum_seekers + + global_idps + + global_stateless + + global_ooc + ), + }, + "year": year, + "count": len(by_origin), + "source": "unhcr", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +def _safe_int(value: int | float | str | None) -> int: + """Convert a value to int, returning 0 for None or unparseable values.""" + if value is None: + return 0 + try: + return int(value) + except (ValueError, TypeError): + return 0 diff --git a/src/world_intel_mcp/sources/economic.py b/src/world_intel_mcp/sources/economic.py new file mode 100644 index 0000000..f22a9ad --- /dev/null +++ b/src/world_intel_mcp/sources/economic.py @@ -0,0 +1,308 @@ +"""Economic indicator data sources. + +Fetches energy prices (EIA), macroeconomic series (FRED), and +development indicators (World Bank) for the world-intel-mcp server. +""" + +import asyncio +import logging +import os +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.economic") + + +# --------------------------------------------------------------------------- +# EIA: Energy prices (oil & natural gas) +# --------------------------------------------------------------------------- + +_EIA_OIL_URL = "https://api.eia.gov/v2/petroleum/pri/spt/data/" +_EIA_GAS_URL = "https://api.eia.gov/v2/natural-gas/pri/fut/data/" + + +async def fetch_energy_prices( + fetcher: Fetcher, + api_key: str | None = None, +) -> dict: + """Fetch latest crude oil (Brent & WTI) and natural gas spot prices. + + Uses the EIA API v2. Requires an API key via *api_key* or the + ``EIA_API_KEY`` environment variable. + + Returns a dict with ``oil`` and ``natural_gas`` sub-keys, plus + metadata (``fetched_at``, ``source``). + """ + key = api_key or os.environ.get("EIA_API_KEY") + if not key: + return {"error": "EIA_API_KEY not configured"} + + oil_params = { + "api_key": key, + "frequency": "daily", + "data[0]": "value", + "sort[0][column]": "period", + "sort[0][direction]": "desc", + "length": 5, + "facets[product][]": ["EPCBRENT", "EPCWTI"], + } + + gas_params = { + "api_key": key, + "frequency": "daily", + "data[0]": "value", + "sort[0][column]": "period", + "sort[0][direction]": "desc", + "length": 5, + "facets[process][]": "PRC", + } + + oil_data, gas_data = await asyncio.gather( + fetcher.get_json( + _EIA_OIL_URL, + source="eia", + cache_key="economic:energy:oil", + cache_ttl=3600, + params=oil_params, + ), + fetcher.get_json( + _EIA_GAS_URL, + source="eia", + cache_key="economic:energy:gas", + cache_ttl=3600, + params=gas_params, + ), + ) + + result: dict = { + "oil": {"brent": None, "wti": None}, + "natural_gas": None, + "fetched_at": datetime.now(timezone.utc).isoformat(), + "source": "eia", + } + + # --- Parse oil prices --------------------------------------------------- + if oil_data: + try: + records = ( + oil_data.get("response", {}).get("data", []) + ) + for rec in records: + product = rec.get("product") + value = rec.get("value") + period = rec.get("period") + if value is None or period is None: + continue + entry = {"price": float(value), "date": period} + if product == "EPCBRENT" and result["oil"]["brent"] is None: + result["oil"]["brent"] = entry + elif product == "EPCWTI" and result["oil"]["wti"] is None: + result["oil"]["wti"] = entry + except (KeyError, TypeError, ValueError) as exc: + logger.warning("Failed to parse EIA oil data: %s", exc) + + # --- Parse natural gas price -------------------------------------------- + if gas_data: + try: + records = ( + gas_data.get("response", {}).get("data", []) + ) + if records: + rec = records[0] + value = rec.get("value") + period = rec.get("period") + if value is not None and period is not None: + result["natural_gas"] = { + "price": float(value), + "date": period, + } + except (KeyError, TypeError, ValueError) as exc: + logger.warning("Failed to parse EIA natural gas data: %s", exc) + + return result + + +# --------------------------------------------------------------------------- +# FRED: Federal Reserve Economic Data +# --------------------------------------------------------------------------- + +_FRED_URL = "https://api.stlouisfed.org/fred/series/observations" + + +async def fetch_fred_series( + fetcher: Fetcher, + series_id: str, + api_key: str | None = None, + limit: int = 30, +) -> dict: + """Fetch observations for a FRED series. + + Common series IDs: + * ``GDP`` -- Gross Domestic Product + * ``UNRATE`` -- Unemployment Rate + * ``CPIAUCSL`` -- Consumer Price Index + * ``DFF`` -- Federal Funds Effective Rate + * ``T10YIE`` -- 10-Year Breakeven Inflation Rate + + Returns a dict with ``series_id``, ``title``, ``observations``, plus + metadata. + """ + key = api_key or os.environ.get("FRED_API_KEY") + if not key: + return {"error": "FRED_API_KEY not configured"} + + params = { + "series_id": series_id, + "api_key": key, + "file_type": "json", + "sort_order": "desc", + "limit": limit, + } + + data = await fetcher.get_json( + _FRED_URL, + source="fred", + cache_key=f"economic:fred:{series_id}:{limit}", + cache_ttl=3600, + params=params, + ) + + result: dict = { + "series_id": series_id, + "title": series_id, + "observations": [], + "fetched_at": datetime.now(timezone.utc).isoformat(), + "source": "fred", + } + + if data is None: + return result + + try: + raw_obs = data.get("observations", []) + for obs in raw_obs: + date = obs.get("date") + value = obs.get("value") + if date is None: + continue + # FRED uses "." for missing values + parsed_value: float | None = None + if value not in (None, ".", ""): + try: + parsed_value = float(value) + except (ValueError, TypeError): + pass + result["observations"].append({"date": date, "value": parsed_value}) + + # FRED embeds series metadata when available + if "realtime_start" in data: + result["realtime_start"] = data["realtime_start"] + if "realtime_end" in data: + result["realtime_end"] = data["realtime_end"] + except (KeyError, TypeError) as exc: + logger.warning("Failed to parse FRED series %s: %s", series_id, exc) + + return result + + +# --------------------------------------------------------------------------- +# World Bank: Development indicators +# --------------------------------------------------------------------------- + +_WB_BASE = "https://api.worldbank.org/v2/country" + +_DEFAULT_INDICATORS = [ + "NY.GDP.MKTP.CD", # GDP (current US$) + "FP.CPI.TOTL.ZG", # Inflation, consumer prices (annual %) + "SL.UEM.TOTL.ZS", # Unemployment, total (% of labor force) +] + + +async def fetch_world_bank_indicators( + fetcher: Fetcher, + country: str = "USA", + indicators: list[str] | None = None, +) -> dict: + """Fetch World Bank development indicators for a country. + + Defaults to GDP, inflation, and unemployment for the USA. + Indicators are fetched in parallel. + + Returns a dict with ``country``, ``indicators`` (list of dicts with + ``id``, ``name``, and ``values``), plus metadata. + """ + indicator_ids = indicators or _DEFAULT_INDICATORS + + async def _fetch_one(indicator: str) -> dict | None: + url = f"{_WB_BASE}/{country}/indicator/{indicator}" + params = { + "format": "json", + "per_page": 5, + "date": "2020:2025", + } + return await fetcher.get_json( + url, + source="world-bank", + cache_key=f"economic:wb:{country}:{indicator}", + cache_ttl=86400, + params=params, + ) + + responses = await asyncio.gather( + *[_fetch_one(ind) for ind in indicator_ids] + ) + + parsed_indicators: list[dict] = [] + + for indicator_id, raw in zip(indicator_ids, responses): + entry: dict = { + "id": indicator_id, + "name": indicator_id, + "values": [], + } + + if raw is None: + parsed_indicators.append(entry) + continue + + try: + # World Bank v2 JSON returns a 2-element list: + # [metadata_dict, data_records_list] + if isinstance(raw, list) and len(raw) >= 2: + records = raw[1] + if records and isinstance(records, list): + # Extract human-readable indicator name from first record + first = records[0] + ind_info = first.get("indicator", {}) + if isinstance(ind_info, dict): + entry["name"] = ind_info.get("value", indicator_id) + + for rec in records: + year = rec.get("date") + value = rec.get("value") + if year is not None: + parsed_value: float | None = None + if value is not None: + try: + parsed_value = float(value) + except (ValueError, TypeError): + pass + entry["values"].append({ + "year": year, + "value": parsed_value, + }) + except (KeyError, TypeError, IndexError) as exc: + logger.warning( + "Failed to parse World Bank indicator %s for %s: %s", + indicator_id, country, exc, + ) + + parsed_indicators.append(entry) + + return { + "country": country, + "indicators": parsed_indicators, + "fetched_at": datetime.now(timezone.utc).isoformat(), + "source": "world-bank", + } diff --git a/src/world_intel_mcp/sources/infrastructure.py b/src/world_intel_mcp/sources/infrastructure.py new file mode 100644 index 0000000..e9814f4 --- /dev/null +++ b/src/world_intel_mcp/sources/infrastructure.py @@ -0,0 +1,406 @@ +"""Internet infrastructure monitoring for world-intel-mcp. + +Provides real-time data on internet outages (Cloudflare Radar) and +undersea cable corridor health (NGA Maritime Safety Information). +""" + +import asyncio +import logging +import os +import re +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.infrastructure") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_CF_RADAR_OUTAGES_URL = ( + "https://radar.cloudflare.com/api/v1/annotations/outages" +) +_CF_RADAR_TAMPERING_URL = ( + "https://api.cloudflare.com/client/v4/radar/connection_tampering/summary" +) + +_NGA_BROADCAST_WARN_URL = ( + "https://msi.nga.mil/api/publications/broadcast-warn" +) + +# Known undersea cable corridors as rough lat/lon bounding boxes. +CABLE_CORRIDORS: dict[str, dict] = { + "transatlantic_north": { + "lat_range": (35, 55), + "lon_range": (-70, -5), + "cables": ["TAT-14", "AC-1", "MAREA", "Dunant"], + }, + "transatlantic_south": { + "lat_range": (-5, 35), + "lon_range": (-50, 0), + "cables": ["SAex1", "SACS", "EllaLink"], + }, + "transpacific": { + "lat_range": (20, 50), + "lon_range": (120, -120), + "cables": ["FASTER", "Unity", "SJC"], + }, + "asia_europe": { + "lat_range": (-5, 35), + "lon_range": (30, 120), + "cables": ["SEA-ME-WE 6", "AAE-1", "FLAG"], + }, + "red_sea": { + "lat_range": (10, 30), + "lon_range": (30, 50), + "cables": ["FALCON", "EIG", "AAE-1"], + }, + "mediterranean": { + "lat_range": (30, 45), + "lon_range": (-5, 35), + "cables": ["SEA-ME-WE 3/4", "MedNautilus"], + }, +} + +# Keywords that indicate a navigational warning is cable-related. +_CABLE_KEYWORDS = re.compile( + r"\b(cable|submarine|fiber|anchor|dredg)\b", re.IGNORECASE +) + +# Pattern for DMS coordinates like "32-15.5N/044-30.2E" or "32-15N 044-30E". +_DMS_PATTERN = re.compile( + r"(\d{1,3})-(\d{1,2}(?:\.\d+)?)\s*([NS])\s*[/ ]\s*" + r"(\d{1,3})-(\d{1,2}(?:\.\d+)?)\s*([EW])" +) + +# Pattern for decimal lat/lon like "32.258N 44.503E" or "32.258/44.503". +_DECIMAL_PATTERN = re.compile( + r"(\d{1,3}(?:\.\d+)?)\s*([NS])\s*[/ ]\s*" + r"(\d{1,3}(?:\.\d+)?)\s*([EW])" +) + +_STATUS_LABELS = {0: "clear", 1: "advisory", 2: "at_risk", 3: "disrupted"} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _extract_coordinates(text: str) -> list[tuple[float, float]]: + """Extract (lat, lon) pairs from navigational warning text. + + Handles both DMS notation (DD-MM.M[N/S]/DDD-MM.M[E/W]) and simple + decimal notation (DD.DDD[N/S] DDD.DDD[E/W]). + + Returns a list of (latitude, longitude) tuples in decimal degrees. + """ + coords: list[tuple[float, float]] = [] + + # Try DMS pattern first + for match in _DMS_PATTERN.finditer(text): + deg_lat, min_lat, ns, deg_lon, min_lon, ew = match.groups() + lat = float(deg_lat) + float(min_lat) / 60.0 + lon = float(deg_lon) + float(min_lon) / 60.0 + if ns == "S": + lat = -lat + if ew == "W": + lon = -lon + coords.append((lat, lon)) + + # Try decimal pattern + for match in _DECIMAL_PATTERN.finditer(text): + lat_val, ns, lon_val, ew = match.groups() + lat = float(lat_val) + lon = float(lon_val) + if ns == "S": + lat = -lat + if ew == "W": + lon = -lon + coords.append((lat, lon)) + + return coords + + +def _point_in_corridor( + lat: float, + lon: float, + corridor: dict, +) -> bool: + """Check whether a (lat, lon) point falls within a cable corridor bbox. + + The transpacific corridor wraps across the antimeridian (lon_range + has min > max), so it requires special handling. + """ + lat_lo, lat_hi = corridor["lat_range"] + lon_lo, lon_hi = corridor["lon_range"] + + if not (lat_lo <= lat <= lat_hi): + return False + + # Handle antimeridian wrap (e.g. 120 to -120 means 120..180 or -180..-120) + if lon_lo > lon_hi: + return lon >= lon_lo or lon <= lon_hi + return lon_lo <= lon <= lon_hi + + +# --------------------------------------------------------------------------- +# Internet Outages (Cloudflare Radar) +# --------------------------------------------------------------------------- + +async def fetch_internet_outages(fetcher: Fetcher) -> dict: + """Fetch recent internet outage annotations from Cloudflare Radar. + + Uses the public Radar annotations API for the last 7 days. If the + ``CLOUDFLARE_API_TOKEN`` environment variable is set, authenticated + requests are made which may yield higher rate limits. + + Returns: + Dict with outages list, ongoing/total counts, source, and timestamp. + """ + token = os.environ.get("CLOUDFLARE_API_TOKEN") + + headers: dict[str, str] | None = None + if token: + headers = {"Authorization": f"Bearer {token}"} + + params = { + "limit": 20, + "dateRange": "7d", + } + + data = await fetcher.get_json( + url=_CF_RADAR_OUTAGES_URL, + source="cloudflare-radar", + cache_key="infra:outages", + cache_ttl=300, + headers=headers, + params=params, + ) + + now_iso = _utc_now_iso() + + if data is None: + logger.warning("Cloudflare Radar outages API returned no data") + return { + "outages": [], + "ongoing_count": 0, + "total_7d": 0, + "source": "cloudflare-radar", + "timestamp": now_iso, + } + + outages: list[dict] = [] + + # The public annotations endpoint returns: + # { "annotations": [ { "id", "dateStart", "dateEnd", "description", + # "scope", "asns", "locations", ... } ] } + # Adapt to both envelope shapes: top-level list or nested under a key. + raw_annotations: list = [] + if isinstance(data, list): + raw_annotations = data + elif isinstance(data, dict): + # Try common envelope keys + for key in ("annotations", "result", "data"): + candidate = data.get(key) + if isinstance(candidate, list): + raw_annotations = candidate + break + + ongoing_count = 0 + + for ann in raw_annotations: + if not isinstance(ann, dict): + continue + + date_end = ann.get("dateEnd") or ann.get("endDate") + is_ongoing = date_end is None or date_end == "" + + if is_ongoing: + ongoing_count += 1 + + # Normalize ASN list + raw_asns = ann.get("asns", []) + if isinstance(raw_asns, str): + raw_asns = [a.strip() for a in raw_asns.split(",") if a.strip()] + elif not isinstance(raw_asns, list): + raw_asns = [] + + # Normalize locations / country codes + raw_locations = ann.get("locations", ann.get("countries", [])) + if isinstance(raw_locations, str): + raw_locations = [ + loc.strip() for loc in raw_locations.split(",") if loc.strip() + ] + elif not isinstance(raw_locations, list): + raw_locations = [] + + outages.append({ + "id": ann.get("id"), + "start": ann.get("dateStart") or ann.get("startDate"), + "end": date_end if not is_ongoing else None, + "description": ann.get("description", ""), + "scope": ann.get("scope", "unknown"), + "countries": raw_locations, + "asns": raw_asns, + "is_ongoing": is_ongoing, + }) + + return { + "outages": outages, + "ongoing_count": ongoing_count, + "total_7d": len(outages), + "source": "cloudflare-radar", + "timestamp": now_iso, + } + + +# --------------------------------------------------------------------------- +# Undersea Cable Corridor Health (NGA Maritime Safety Information) +# --------------------------------------------------------------------------- + +async def fetch_cable_health(fetcher: Fetcher) -> dict: + """Assess undersea cable corridor health using NGA broadcast warnings. + + Fetches navigational warnings from the NGA Maritime Safety Information + (MSI) API, extracts coordinates from the warning text, and scores each + known cable corridor: + + 0 = clear (no warnings) + 1 = advisory (warning nearby the corridor) + 2 = at_risk (warning inside the corridor) + 3 = disrupted (multiple warnings or explicit cable keyword) + + Returns: + Dict with corridor statuses, warning counts, source, and timestamp. + """ + data = await fetcher.get_json( + url=_NGA_BROADCAST_WARN_URL, + source="nga-msi", + cache_key="infra:cables", + cache_ttl=180, + params={"output": "json"}, + ) + + now_iso = _utc_now_iso() + + if data is None: + logger.warning("NGA MSI broadcast warnings API returned no data") + # Return all corridors as clear when data is unavailable + corridors_result = {} + for name, info in CABLE_CORRIDORS.items(): + corridors_result[name] = { + "status_score": 0, + "status_label": "clear", + "cables": info["cables"], + "relevant_warnings": [], + } + return { + "corridors": corridors_result, + "warnings_total": 0, + "cable_related_warnings": 0, + "source": "nga-msi", + "timestamp": now_iso, + } + + # Parse warnings from the response. The NGA API may return: + # - A top-level list of warnings + # - A dict with a "broadcast-warn" or "broadcastWarn" key + raw_warnings: list = [] + if isinstance(data, list): + raw_warnings = data + elif isinstance(data, dict): + for key in ( + "broadcast-warn", "broadcastWarn", "warnings", + "result", "data", + ): + candidate = data.get(key) + if isinstance(candidate, list): + raw_warnings = candidate + break + + # Initialize corridor tracking + corridor_warnings: dict[str, list[dict]] = { + name: [] for name in CABLE_CORRIDORS + } + cable_related_count = 0 + + for warning in raw_warnings: + if not isinstance(warning, dict): + continue + + text = warning.get("text", "") or "" + msg_year = warning.get("msgYear") + msg_number = warning.get("msgNumber") + nav_area = warning.get("navArea", "") + subregion = warning.get("subregion", "") + status = warning.get("status", "") + issue_date = warning.get("issueDate", "") + + has_cable_keyword = bool(_CABLE_KEYWORDS.search(text)) + if has_cable_keyword: + cable_related_count += 1 + + # Extract coordinates from the warning text + coords = _extract_coordinates(text) + + warning_summary = { + "msgYear": msg_year, + "msgNumber": msg_number, + "navArea": nav_area, + "subregion": subregion, + "status": status, + "issueDate": issue_date, + "has_cable_keyword": has_cable_keyword, + "text_snippet": text[:200] if text else "", + } + + # Check each coordinate against cable corridors + for lat, lon in coords: + for corridor_name, corridor_info in CABLE_CORRIDORS.items(): + if _point_in_corridor(lat, lon, corridor_info): + corridor_warnings[corridor_name].append(warning_summary) + # A warning can match multiple corridors but should + # only appear once per corridor. + break + + # Score each corridor + corridors_result: dict[str, dict] = {} + for name, info in CABLE_CORRIDORS.items(): + warnings_in_corridor = corridor_warnings[name] + cable_mentions = sum( + 1 for w in warnings_in_corridor if w["has_cable_keyword"] + ) + + # Scoring logic: + # 0 = clear: no warnings in corridor + # 1 = advisory: 1 warning, no cable keywords + # 2 = at_risk: 1 warning with cable keyword, or 2+ warnings + # 3 = disrupted: multiple warnings with cable keywords or 3+ total + if not warnings_in_corridor: + score = 0 + elif cable_mentions >= 2 or len(warnings_in_corridor) >= 3: + score = 3 + elif cable_mentions >= 1 or len(warnings_in_corridor) >= 2: + score = 2 + else: + score = 1 + + corridors_result[name] = { + "status_score": score, + "status_label": _STATUS_LABELS[score], + "cables": info["cables"], + "relevant_warnings": warnings_in_corridor, + } + + return { + "corridors": corridors_result, + "warnings_total": len(raw_warnings), + "cable_related_warnings": cable_related_count, + "source": "nga-msi", + "timestamp": now_iso, + } diff --git a/src/world_intel_mcp/sources/intelligence.py b/src/world_intel_mcp/sources/intelligence.py new file mode 100644 index 0000000..fc5a305 --- /dev/null +++ b/src/world_intel_mcp/sources/intelligence.py @@ -0,0 +1,723 @@ +"""Country intelligence, risk scoring, and signal convergence sources. + +Provides higher-level analytical functions that combine data from multiple +APIs (ACLED, World Bank, USGS, Ollama) into country briefs, risk scores, +instability indices, and geographic signal convergence assessments. +""" + +import asyncio +import logging +import os +from datetime import datetime, timezone, timedelta + +import httpx + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.intelligence") + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_ACLED_URL = "https://api.acleddata.com/acled/read" +_WB_BASE = "https://api.worldbank.org/v2/country" +_HDX_SEARCH_URL = "https://data.humdata.org/api/3/action/package_search" +_USGS_ENDPOINT = "https://earthquake.usgs.gov/fdsnws/event/1/query" +_OPENSKY_STATES_URL = "https://opensky-network.org/api/states/all" + +_BASELINES = { + "Syria": 5000, "Yemen": 3000, "Ukraine": 8000, "Myanmar": 4000, + "Somalia": 2500, "Nigeria": 3500, "DR Congo": 3000, "Afghanistan": 2000, + "Iraq": 1500, "Mali": 2000, "Burkina Faso": 2500, "Ethiopia": 2000, + "Sudan": 3000, "South Sudan": 1500, "Cameroon": 1000, "Mozambique": 800, + "Pakistan": 1200, "India": 1000, "Colombia": 1500, "Mexico": 4000, +} + +_FOCUS_COUNTRIES = [ + "SYR", "UKR", "YEM", "MMR", "SDN", "ETH", "NGA", "COD", "AFG", "IRQ", +] + +_HOTSPOTS = { + "middle_east": (33.0, 44.0), + "east_africa": (5.0, 38.0), + "south_asia": (30.0, 70.0), + "eastern_europe": (48.0, 35.0), + "sahel": (15.0, 2.0), +} + +# ISO-3166 alpha-3 to country name for ACLED queries and display. +_ISO3_TO_NAME = { + "SYR": "Syria", "UKR": "Ukraine", "YEM": "Yemen", "MMR": "Myanmar", + "SDN": "Sudan", "ETH": "Ethiopia", "NGA": "Nigeria", "COD": "DR Congo", + "AFG": "Afghanistan", "IRQ": "Iraq", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _risk_level(score: float) -> str: + if score > 150: + return "critical" + if score > 100: + return "elevated" + if score > 50: + return "moderate" + return "low" + + +def _instability_level(index: float) -> str: + if index >= 75: + return "critical" + if index >= 50: + return "high" + if index >= 25: + return "medium" + return "low" + + +# --------------------------------------------------------------------------- +# Function 1: Country Intelligence Brief +# --------------------------------------------------------------------------- + +async def fetch_country_brief( + fetcher: Fetcher, + country_code: str = "US", +) -> dict: + """Generate a country intelligence brief using local LLM and public data. + + Gathers economic indicators from World Bank and conflict data from ACLED + in parallel, then optionally enriches with an Ollama-generated analytical + brief. Falls back to a data-only summary when Ollama is unavailable. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + country_code: ISO 3166-1 alpha-2 country code (e.g. ``US``, ``UA``). + + Returns: + Dict with brief text, supporting data, LLM availability flag, + source, and timestamp. + """ + now = datetime.now(timezone.utc) + + # --- Gather background data in parallel -------------------------------- + async def _fetch_gdp() -> list: + url = f"{_WB_BASE}/{country_code}/indicator/NY.GDP.MKTP.CD" + params = { + "format": "json", + "per_page": 5, + "date": "2020:2025", + } + data = await fetcher.get_json( + url, + source="world-bank", + cache_key=f"intel:wb:gdp:{country_code}", + cache_ttl=86400, + params=params, + ) + if data is None: + return [] + + values = [] + try: + if isinstance(data, list) and len(data) >= 2 and isinstance(data[1], list): + for rec in data[1]: + year = rec.get("date") + value = rec.get("value") + if year is not None and value is not None: + try: + values.append({"year": year, "value": float(value)}) + except (ValueError, TypeError): + pass + except (KeyError, TypeError, IndexError) as exc: + logger.warning("Failed to parse World Bank GDP for %s: %s", country_code, exc) + return values + + async def _fetch_inflation() -> list: + url = f"{_WB_BASE}/{country_code}/indicator/FP.CPI.TOTL.ZG" + params = { + "format": "json", + "per_page": 5, + "date": "2020:2025", + } + data = await fetcher.get_json( + url, + source="world-bank", + cache_key=f"intel:wb:inflation:{country_code}", + cache_ttl=86400, + params=params, + ) + if data is None: + return [] + + values = [] + try: + if isinstance(data, list) and len(data) >= 2 and isinstance(data[1], list): + for rec in data[1]: + year = rec.get("date") + value = rec.get("value") + if year is not None and value is not None: + try: + values.append({"year": year, "value": float(value)}) + except (ValueError, TypeError): + pass + except (KeyError, TypeError, IndexError) as exc: + logger.warning("Failed to parse World Bank inflation for %s: %s", country_code, exc) + return values + + async def _fetch_acled_count() -> int: + access_token = os.environ.get("ACLED_ACCESS_TOKEN") + if not access_token: + return 0 + + start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + params: dict = { + "key": access_token, + "email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"), + "limit": 0, + "event_date": f"{start_date}|{end_date}", + "event_date_where": "BETWEEN", + "country": country_code, + } + data = await fetcher.get_json( + _ACLED_URL, + source="acled", + cache_key=f"intel:acled:count:{country_code}", + cache_ttl=900, + params=params, + ) + if data is None: + return 0 + + # ACLED returns a count field when limit=0 + try: + return int(data.get("count", len(data.get("data", [])))) + except (ValueError, TypeError): + return len(data.get("data", [])) + + gdp_values, inflation_values, event_count = await asyncio.gather( + _fetch_gdp(), + _fetch_inflation(), + _fetch_acled_count(), + ) + + # --- Attempt Ollama-generated brief ------------------------------------ + llm_available = False + brief_text = "LLM brief unavailable. Data summary below." + + prompt = ( + f"Provide a concise 3-paragraph intelligence brief for {country_code}. " + "Cover: (1) current political stability and governance, " + "(2) economic outlook and risks, " + "(3) security concerns and regional dynamics. " + "Be factual and analytical." + ) + + ollama_url = os.environ.get("OLLAMA_API_URL", "http://localhost:11434") + model = os.environ.get("OLLAMA_MODEL", "llama3.2:latest") + + try: + async with httpx.AsyncClient(timeout=30.0, proxy=None) as client: + resp = await client.post( + f"{ollama_url}/api/generate", + json={ + "model": model, + "prompt": prompt, + "stream": False, + }, + ) + resp.raise_for_status() + resp_data = resp.json() + generated = resp_data.get("response", "") + if generated and generated.strip(): + brief_text = generated.strip() + llm_available = True + except (httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError) as exc: + logger.info("Ollama unavailable for country brief (%s): %s", country_code, exc) + except Exception as exc: + logger.warning("Unexpected error calling Ollama: %s", exc) + + return { + "country_code": country_code, + "brief": brief_text, + "data": { + "gdp": gdp_values, + "inflation": inflation_values, + "recent_events": event_count, + }, + "llm_available": llm_available, + "source": "country-intelligence", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +# --------------------------------------------------------------------------- +# Function 2: Country Risk Scores +# --------------------------------------------------------------------------- + +async def fetch_risk_scores( + fetcher: Fetcher, + limit: int = 20, +) -> dict: + """Compute country risk scores from ACLED conflict data and baselines. + + Fetches recent global conflict events, counts per country, and computes + a risk score as ``(events_30d / monthly_baseline) * 100``. Higher + scores indicate conflict above historical norms. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + limit: Maximum number of countries to return (sorted by risk). + + Returns: + Dict with ranked country list, count, source, and timestamp. + """ + now = datetime.now(timezone.utc) + + access_token = os.environ.get("ACLED_ACCESS_TOKEN") + if not access_token: + return { + "error": "ACLED_ACCESS_TOKEN not configured", + "note": "Free academic access at acleddata.com", + "source": "risk-analysis", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + params: dict = { + "key": access_token, + "email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"), + "limit": 500, + "event_date": f"{start_date}|{end_date}", + "event_date_where": "BETWEEN", + } + + data = await fetcher.get_json( + _ACLED_URL, + source="acled", + cache_key="intel:risk:global:30d", + cache_ttl=1800, + params=params, + ) + + if data is None: + logger.warning("ACLED API returned no data for risk scoring") + return { + "countries": [], + "count": 0, + "source": "risk-analysis", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + # Count events per country + country_counts: dict[str, int] = {} + for event in data.get("data", []): + country_name = event.get("country") + if country_name: + country_counts[country_name] = country_counts.get(country_name, 0) + 1 + + # Compute risk scores + countries: list[dict] = [] + for country_name, events_30d in country_counts.items(): + baseline_annual = _BASELINES.get(country_name, 500) + monthly_baseline = baseline_annual / 12.0 + risk_score = (events_30d / monthly_baseline) * 100 if monthly_baseline > 0 else 0.0 + + countries.append({ + "country": country_name, + "events_30d": events_30d, + "monthly_baseline": round(monthly_baseline, 1), + "risk_score": round(risk_score, 1), + "risk_level": _risk_level(risk_score), + }) + + # Sort by risk_score descending, take top N + countries.sort(key=lambda c: c["risk_score"], reverse=True) + countries = countries[:limit] + + return { + "countries": countries, + "count": len(countries), + "source": "risk-analysis", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +# --------------------------------------------------------------------------- +# Function 3: Country Instability Index +# --------------------------------------------------------------------------- + +async def fetch_instability_index( + fetcher: Fetcher, + country_code: str | None = None, +) -> dict: + """Compute a Country Instability Index (CII) from multiple signals. + + Combines conflict intensity, economic stress, humanitarian crisis data, + internet disruption indicators, and military activity into a 0-100 + composite score. Higher values indicate greater instability. + + When *country_code* is ``None``, returns a simplified index for 10 + focus countries using ACLED data as the primary signal. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + country_code: Optional ISO 3166-1 alpha-3 code (e.g. ``UKR``). + + Returns: + Dict with instability index, component scores, risk level, source, + and timestamp. + """ + now = datetime.now(timezone.utc) + + if country_code is not None: + return await _instability_single(fetcher, country_code, now) + + return await _instability_multi(fetcher, now) + + +async def _instability_single( + fetcher: Fetcher, + country_code: str, + now: datetime, +) -> dict: + """Compute full 5-component instability index for a single country.""" + + # --- Parallel data gathering ------------------------------------------- + + async def _conflict_score() -> float: + """Score 0-20 based on ACLED event count in the last 30 days.""" + access_token = os.environ.get("ACLED_ACCESS_TOKEN") + if not access_token: + return 0.0 + + country_name = _ISO3_TO_NAME.get(country_code, country_code) + start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + data = await fetcher.get_json( + _ACLED_URL, + source="acled", + cache_key=f"intel:cii:conflict:{country_code}", + cache_ttl=1800, + params={ + "key": access_token, + "email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"), + "limit": 500, + "event_date": f"{start_date}|{end_date}", + "event_date_where": "BETWEEN", + "country": country_name, + }, + ) + if data is None: + return 0.0 + + count = len(data.get("data", [])) + # Thresholds: 0 events = 0, 500+ = 20 + return min(20.0, (count / 500.0) * 20.0) + + async def _economic_score() -> float: + """Score 0-20 based on World Bank inflation rate.""" + url = f"{_WB_BASE}/{country_code}/indicator/FP.CPI.TOTL.ZG" + data = await fetcher.get_json( + url, + source="world-bank", + cache_key=f"intel:cii:inflation:{country_code}", + cache_ttl=86400, + params={"format": "json", "per_page": 1, "date": "2023:2025"}, + ) + if data is None: + return 0.0 + + try: + if isinstance(data, list) and len(data) >= 2 and isinstance(data[1], list): + for rec in data[1]: + value = rec.get("value") + if value is not None: + inflation = float(value) + # Thresholds: 0% = 0, 50%+ = 20 + return min(20.0, max(0.0, (abs(inflation) / 50.0) * 20.0)) + except (ValueError, TypeError, KeyError, IndexError): + pass + return 0.0 + + async def _humanitarian_score() -> float: + """Score 0-20 based on HDX crisis dataset count.""" + params: dict = { + "q": "crisis", + "rows": 50, + "sort": "metadata_modified desc", + "fq": f"groups:{country_code.lower()}", + } + data = await fetcher.get_json( + _HDX_SEARCH_URL, + source="hdx", + cache_key=f"intel:cii:humanitarian:{country_code}", + cache_ttl=21600, + params=params, + ) + if data is None: + return 0.0 + + try: + count = data.get("result", {}).get("count", 0) + # Thresholds: 0 datasets = 0, 200+ = 20 + return min(20.0, (int(count) / 200.0) * 20.0) + except (ValueError, TypeError): + return 0.0 + + async def _internet_score() -> float: + """Score 0-20 based on Cloudflare Radar connectivity data. + + This is a best-effort check; Cloudflare Radar's public API may + not be available or may require auth. Returns 0 on failure. + """ + # Cloudflare Radar does not have an easy free API for this. + # Placeholder: return 0 (no disruption data). + return 0.0 + + async def _military_score() -> float: + """Score 0-20 based on OpenSky military flight density near country.""" + # Use a rough bounding box for the country. For simplicity, + # we only score countries in _ISO3_TO_NAME with known hotspot + # regions. + _COUNTRY_BBOX = { + "SYR": "32,35,37,42", "UKR": "44,22,52,40", + "YEM": "12,42,19,55", "MMR": "10,92,28,101", + "SDN": "8,21,23,39", "ETH": "3,33,15,48", + "NGA": "4,3,14,15", "COD": "-13,12,5,31", + "AFG": "29,60,38,75", "IRQ": "29,39,37,49", + } + bbox = _COUNTRY_BBOX.get(country_code) + if bbox is None: + return 0.0 + + parts = bbox.split(",") + params: dict[str, str] = {} + if len(parts) == 4: + params["lamin"] = parts[0] + params["lomin"] = parts[1] + params["lamax"] = parts[2] + params["lomax"] = parts[3] + + data = await fetcher.get_json( + _OPENSKY_STATES_URL, + source="opensky", + cache_key=f"intel:cii:military:{country_code}", + cache_ttl=300, + params=params if params else None, + ) + if data is None: + return 0.0 + + states = data.get("states") or [] + # Count all aircraft (military filtering adds complexity; using + # total density as a proxy for activity). + count = len(states) + # Thresholds: 0 = 0, 200+ = 20 + return min(20.0, (count / 200.0) * 20.0) + + conflict, economic, humanitarian, internet, military = await asyncio.gather( + _conflict_score(), + _economic_score(), + _humanitarian_score(), + _internet_score(), + _military_score(), + ) + + instability_index = round(conflict + economic + humanitarian + internet + military, 1) + + return { + "country_code": country_code, + "instability_index": instability_index, + "components": { + "conflict_intensity": round(conflict, 1), + "economic_stress": round(economic, 1), + "humanitarian_crisis": round(humanitarian, 1), + "internet_disruptions": round(internet, 1), + "military_activity": round(military, 1), + }, + "risk_level": _instability_level(instability_index), + "source": "instability-index", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +async def _instability_multi(fetcher: Fetcher, now: datetime) -> dict: + """Compute simplified instability index for focus countries using ACLED.""" + access_token = os.environ.get("ACLED_ACCESS_TOKEN") + if not access_token: + return { + "error": "ACLED_ACCESS_TOKEN not configured", + "source": "instability-index", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + # Fetch global events and bucket by country + data = await fetcher.get_json( + _ACLED_URL, + source="acled", + cache_key="intel:cii:multi:global:30d", + cache_ttl=1800, + params={ + "key": access_token, + "email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"), + "limit": 500, + "event_date": f"{start_date}|{end_date}", + "event_date_where": "BETWEEN", + }, + ) + + country_counts: dict[str, int] = {} + if data is not None: + for event in data.get("data", []): + country_name = event.get("country") + if country_name: + country_counts[country_name] = country_counts.get(country_name, 0) + 1 + + # Map focus country codes to names and compute simplified index + results: list[dict] = [] + for code in _FOCUS_COUNTRIES: + name = _ISO3_TO_NAME.get(code, code) + events = country_counts.get(name, 0) + baseline_annual = _BASELINES.get(name, 500) + monthly_baseline = baseline_annual / 12.0 + + # Simplified CII: conflict component scaled to 0-100 + if monthly_baseline > 0: + ratio = events / monthly_baseline + else: + ratio = 0.0 + instability = min(100.0, round(ratio * 50.0, 1)) + + results.append({ + "country_code": code, + "country_name": name, + "instability_index": instability, + "events_30d": events, + "risk_level": _instability_level(instability), + }) + + results.sort(key=lambda r: r["instability_index"], reverse=True) + + return { + "countries": results, + "count": len(results), + "note": "Simplified index based on ACLED conflict data only. " + "Use country_code parameter for full 5-component analysis.", + "source": "instability-index", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +# --------------------------------------------------------------------------- +# Function 4: Signal Convergence +# --------------------------------------------------------------------------- + +async def fetch_signal_convergence( + fetcher: Fetcher, + lat: float | None = None, + lon: float | None = None, + radius_deg: float = 5.0, +) -> dict: + """Detect geographic convergence of signals in hotspot regions. + + Checks for overlapping seismic activity and other observable signals + within a radius of known or specified hotspot coordinates. Higher + convergence scores indicate multiple signal types in close proximity, + which may warrant deeper investigation. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + lat: Latitude of center point. If ``None``, scans 5 global + hotspot regions. + lon: Longitude of center point. + radius_deg: Radius in degrees for bounding box queries. + + Returns: + Dict with hotspot list, convergence scores, source, and timestamp. + """ + now = datetime.now(timezone.utc) + + if lat is not None and lon is not None: + regions = {"custom": (lat, lon)} + else: + regions = dict(_HOTSPOTS) + + async def _assess_hotspot(name: str, center: tuple[float, float]) -> dict: + center_lat, center_lon = center + + # Earthquake count within bounding box + min_lat = center_lat - radius_deg + max_lat = center_lat + radius_deg + min_lon = center_lon - radius_deg + max_lon = center_lon + radius_deg + + starttime = (now - timedelta(days=30)).strftime("%Y-%m-%dT%H:%M:%S") + + quake_data = await fetcher.get_json( + _USGS_ENDPOINT, + source="usgs", + cache_key=f"intel:convergence:usgs:{name}:{radius_deg}", + cache_ttl=600, + params={ + "format": "geojson", + "minmagnitude": 2.5, + "starttime": starttime, + "minlatitude": min_lat, + "maxlatitude": max_lat, + "minlongitude": min_lon, + "maxlongitude": max_lon, + "limit": 100, + }, + ) + + earthquake_count = 0 + if quake_data is not None: + earthquake_count = len(quake_data.get("features", [])) + + # Convergence score heuristic (0-10) + # Each signal type present adds to the score. + score = 0.0 + + # Earthquakes: 0-5 points based on count + if earthquake_count > 0: + score += min(5.0, (earthquake_count / 20.0) * 5.0) + + # Hotspot presence bonus (known conflict zones get a baseline) + if name in _HOTSPOTS: + score += 2.0 + + score = min(10.0, round(score, 1)) + + return { + "name": name, + "lat": center_lat, + "lon": center_lon, + "signals": { + "earthquakes": earthquake_count, + }, + "convergence_score": score, + } + + tasks = [_assess_hotspot(name, center) for name, center in regions.items()] + results = await asyncio.gather(*tasks) + + # Sort by convergence score descending + hotspots = sorted(results, key=lambda h: h["convergence_score"], reverse=True) + + return { + "hotspots": hotspots, + "source": "signal-convergence", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } diff --git a/src/world_intel_mcp/sources/maritime.py b/src/world_intel_mcp/sources/maritime.py new file mode 100644 index 0000000..9f6095f --- /dev/null +++ b/src/world_intel_mcp/sources/maritime.py @@ -0,0 +1,168 @@ +"""NGA Maritime Safety Information source for world-intel-mcp. + +Provides navigational warnings from the National Geospatial-Intelligence Agency +(NGA) Maritime Safety Information (MSI) broadcast warnings API. +No API key required. +""" + +import logging +import os +import re +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.maritime") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_NGA_WARNINGS_URL = "https://msi.nga.mil/api/publications/broadcast-warn?output=json" + +NAVAREAS = { + "I": "UK (NE Atlantic, North Sea)", + "II": "France (Bay of Biscay, W Africa)", + "III": "Spain (Mediterranean W)", + "IV": "United States (W Atlantic, Caribbean, Gulf of Mexico)", + "V": "Brazil (SW Atlantic)", + "VI": "Argentina (SE Atlantic)", + "VII": "South Africa (Indian Ocean W)", + "VIII": "India (Indian Ocean N)", + "IX": "Pakistan (Arabian Sea, Persian Gulf)", + "X": "Australia (Indian Ocean S)", + "XI": "Japan (NW Pacific)", + "XII": "United States (NE Pacific)", + "XIII": "Russia (Arctic)", + "XIV": "New Zealand (SW Pacific)", + "XV": "Chile (SE Pacific)", + "XVI": "Peru (E Pacific)", +} + +_MAX_TEXT_LENGTH = 500 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _is_active(warning: dict) -> bool: + """Return True if the warning has no cancelDate (i.e. still active).""" + cancel = warning.get("cancelDate") + return cancel is None or cancel == "" + + +def _parse_warning(warning: dict) -> dict: + """Extract structured fields from a single NGA broadcast warning.""" + msg_year = warning.get("msgYear", "") + msg_number = warning.get("msgNumber", "") + warning_id = f"{msg_year}-{msg_number}" if msg_year and msg_number else None + + cancel_date = warning.get("cancelDate") + status = "cancelled" if cancel_date else "active" + + text = warning.get("text", "") or "" + if len(text) > _MAX_TEXT_LENGTH: + text = text[:_MAX_TEXT_LENGTH] + "..." + + return { + "id": warning_id, + "navarea": warning.get("navArea", ""), + "subregion": warning.get("subregion", ""), + "status": status, + "issue_date": warning.get("issueDate"), + "cancel_date": cancel_date if cancel_date else None, + "text": text, + "authority": warning.get("authority", "NGA"), + } + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_nav_warnings( + fetcher: Fetcher, + navarea: str | None = None, +) -> dict: + """Fetch active navigational warnings from NGA Maritime Safety Information. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + navarea: Optional NAVAREA identifier (e.g. "IV", "XII") to filter by. + + Returns: + Dict with warnings list, count, per-navarea breakdown, NAVAREA + definitions, source, and timestamp. + """ + cache_label = navarea.upper() if navarea else "all" + cache_key = f"maritime:warnings:{cache_label}" + + data = await fetcher.get_json( + url=_NGA_WARNINGS_URL, + source="nga-msi", + cache_key=cache_key, + cache_ttl=3600, + ) + + now_iso = _utc_now_iso() + + if data is None: + logger.warning("NGA MSI API returned no data") + return { + "warnings": [], + "count": 0, + "by_navarea": {}, + "navareas": NAVAREAS, + "source": "nga-msi", + "timestamp": now_iso, + } + + # The API returns either a list directly or a dict with a data key + if isinstance(data, list): + raw_warnings = data + elif isinstance(data, dict): + raw_warnings = data.get("broadcast-warn", data.get("data", [])) + if isinstance(raw_warnings, dict): + raw_warnings = [raw_warnings] + else: + raw_warnings = [] + + # Filter to active warnings only + active_raw = [w for w in raw_warnings if _is_active(w)] + + # Filter by NAVAREA if specified + if navarea is not None: + navarea_upper = navarea.upper() + active_raw = [ + w for w in active_raw + if str(w.get("navArea", "")).upper() == navarea_upper + ] + + # Parse each warning into structured format + warnings = [_parse_warning(w) for w in active_raw] + + # Sort by issue date descending (most recent first) + def _issue_sort_key(w: dict) -> str: + return w.get("issue_date") or "" + + warnings.sort(key=_issue_sort_key, reverse=True) + + # Summary stats: count per navarea + by_navarea: dict[str, int] = {} + for w in warnings: + na = w.get("navarea", "UNKNOWN") + by_navarea[na] = by_navarea.get(na, 0) + 1 + + return { + "warnings": warnings, + "count": len(warnings), + "by_navarea": by_navarea, + "navareas": NAVAREAS, + "source": "nga-msi", + "timestamp": now_iso, + } diff --git a/src/world_intel_mcp/sources/markets.py b/src/world_intel_mcp/sources/markets.py new file mode 100644 index 0000000..914f16f --- /dev/null +++ b/src/world_intel_mcp/sources/markets.py @@ -0,0 +1,384 @@ +"""Financial market data sources for world-intel-mcp. + +Provides async functions for equities, crypto, stablecoins, ETF flows, +sector heatmaps, and macro signals. Every function takes a Fetcher +instance as its first argument and returns a dict (or None-safe partial +results when individual upstream calls fail). +""" + +import asyncio +import logging +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.markets") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_DEFAULT_INDEX_SYMBOLS = ["^GSPC", "^DJI", "^IXIC", "^FTSE", "^N225", "^HSI", "^GDAXI"] + +_YAHOO_CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}" + +_COINGECKO_MARKETS_URL = "https://api.coingecko.com/api/v3/coins/markets" +_COINGECKO_GLOBAL_URL = "https://api.coingecko.com/api/v3/global" + +_STABLECOIN_IDS = "tether,usd-coin,dai,first-digital-usd" +_STABLECOIN_DEPEG_THRESHOLD = 0.005 # 0.5% + +_BTC_ETF_SYMBOLS = ["IBIT", "FBTC", "GBTC", "ARKB", "BITB"] + +_SECTOR_ETFS: dict[str, str] = { + "XLK": "Technology", + "XLF": "Financials", + "XLE": "Energy", + "XLV": "Healthcare", + "XLI": "Industrials", + "XLC": "Communication", + "XLY": "Consumer Disc", + "XLP": "Consumer Staples", + "XLRE": "Real Estate", + "XLU": "Utilities", + "XLB": "Materials", +} + +_FEAR_GREED_URL = "https://api.alternative.me/fng/?limit=1" +_MEMPOOL_FEES_URL = "https://mempool.space/api/v1/fees/recommended" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +async def _fetch_yahoo_quote(fetcher: Fetcher, symbol: str, cache_key: str, cache_ttl: int) -> dict | None: + """Fetch a single Yahoo Finance v8 chart quote and extract meta fields.""" + url = _YAHOO_CHART_URL.format(symbol=symbol) + data = await fetcher.get_json( + url, + source="yahoo-finance", + cache_key=cache_key, + cache_ttl=cache_ttl, + params={"range": "1d", "interval": "5m"}, + yahoo_rate_limit=True, + ) + if data is None: + return None + + try: + meta = data["chart"]["result"][0]["meta"] + return { + "symbol": symbol, + "price": meta.get("regularMarketPrice"), + "change_pct": meta.get("regularMarketChangePercent"), + "currency": meta.get("currency"), + } + except (KeyError, IndexError, TypeError): + logger.warning("Unexpected Yahoo chart structure for %s", symbol) + return None + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_market_quotes(fetcher: Fetcher, symbols: list[str] | None = None) -> dict: + """Fetch major index quotes from Yahoo Finance. + + Returns:: + + {"quotes": [...], "source": "yahoo-finance", "timestamp": ""} + """ + symbols = symbols or _DEFAULT_INDEX_SYMBOLS + + tasks = [ + _fetch_yahoo_quote(fetcher, sym, f"markets:quotes:{sym}", 120) + for sym in symbols + ] + results = await asyncio.gather(*tasks) + + quotes = [q for q in results if q is not None] + return { + "quotes": quotes, + "source": "yahoo-finance", + "timestamp": _utc_now_iso(), + } + + +async def fetch_crypto_quotes(fetcher: Fetcher, limit: int = 20) -> dict: + """Fetch top crypto coins by market cap from CoinGecko. + + Returns:: + + {"coins": [...], "source": "coingecko", "timestamp": ""} + """ + data = await fetcher.get_json( + _COINGECKO_MARKETS_URL, + source="coingecko", + cache_key=f"markets:crypto:{limit}", + cache_ttl=180, + params={ + "vs_currency": "usd", + "order": "market_cap_desc", + "per_page": str(limit), + "sparkline": "true", + "price_change_percentage": "1h,24h,7d", + }, + ) + + coins: list[dict] = [] + if data is not None and isinstance(data, list): + for coin in data: + coins.append({ + "id": coin.get("id"), + "symbol": coin.get("symbol"), + "name": coin.get("name"), + "current_price": coin.get("current_price"), + "market_cap": coin.get("market_cap"), + "price_change_percentage_24h": coin.get("price_change_percentage_24h"), + "sparkline_in_7d": coin.get("sparkline_in_7d"), + }) + + return { + "coins": coins, + "source": "coingecko", + "timestamp": _utc_now_iso(), + } + + +async def fetch_stablecoin_status(fetcher: Fetcher) -> dict: + """Detect stablecoin de-peg events via CoinGecko. + + Returns:: + + {"stablecoins": [{id, price, peg_deviation_pct, is_depegged}], ...} + """ + data = await fetcher.get_json( + _COINGECKO_MARKETS_URL, + source="coingecko", + cache_key="markets:stablecoins", + cache_ttl=180, + params={ + "vs_currency": "usd", + "ids": _STABLECOIN_IDS, + "sparkline": "false", + }, + ) + + stablecoins: list[dict] = [] + if data is not None and isinstance(data, list): + for coin in data: + price = coin.get("current_price") + if price is not None: + deviation = abs(price - 1.0) / 1.0 + stablecoins.append({ + "id": coin.get("id"), + "price": price, + "peg_deviation_pct": round(deviation * 100, 4), + "is_depegged": deviation > _STABLECOIN_DEPEG_THRESHOLD, + }) + + return { + "stablecoins": stablecoins, + "source": "coingecko", + "timestamp": _utc_now_iso(), + } + + +async def fetch_etf_flows(fetcher: Fetcher) -> dict: + """Fetch BTC spot ETF volume and price changes from Yahoo Finance. + + Returns:: + + {"etfs": [{symbol, price, change_pct, volume}], ...} + """ + tasks = [ + _fetch_yahoo_quote(fetcher, sym, f"markets:etf_flows:{sym}", 600) + for sym in _BTC_ETF_SYMBOLS + ] + results = await asyncio.gather(*tasks) + + # Re-fetch with volume info (chart meta includes volume in some responses, + # but we need the raw data). We already have price/change from the helper; + # pull volume from the same cached chart payload. + etfs: list[dict] = [] + for sym, quote in zip(_BTC_ETF_SYMBOLS, results): + if quote is None: + continue + # Volume: re-read from cache (already populated by _fetch_yahoo_quote) + cached = fetcher.cache.get(f"markets:etf_flows:{sym}") + volume = None + if cached is not None: + try: + meta = cached["chart"]["result"][0]["meta"] + volume = meta.get("regularMarketVolume") + except (KeyError, IndexError, TypeError): + pass + etfs.append({ + "symbol": sym, + "price": quote["price"], + "change_pct": quote["change_pct"], + "volume": volume, + }) + + return { + "etfs": etfs, + "source": "yahoo-finance", + "timestamp": _utc_now_iso(), + } + + +async def fetch_sector_heatmap(fetcher: Fetcher) -> dict: + """Fetch sector ETF performance for a market heatmap. + + Returns:: + + {"sectors": [{symbol, name, price, change_pct}], ...} + """ + tasks = [ + _fetch_yahoo_quote(fetcher, sym, f"markets:sector_heatmap:{sym}", 300) + for sym in _SECTOR_ETFS + ] + results = await asyncio.gather(*tasks) + + sectors: list[dict] = [] + for sym, quote in zip(_SECTOR_ETFS, results): + if quote is None: + continue + sectors.append({ + "symbol": sym, + "name": _SECTOR_ETFS[sym], + "price": quote["price"], + "change_pct": quote["change_pct"], + }) + + return { + "sectors": sectors, + "source": "yahoo-finance", + "timestamp": _utc_now_iso(), + } + + +# --------------------------------------------------------------------------- +# Macro Signals (aggregated dashboard) +# --------------------------------------------------------------------------- + +async def _fetch_fear_greed(fetcher: Fetcher) -> dict | None: + data = await fetcher.get_json( + _FEAR_GREED_URL, + source="alternative-me", + cache_key="markets:macro:fear_greed", + cache_ttl=300, + ) + if data is None: + return None + try: + entry = data["data"][0] + return { + "value": int(entry["value"]), + "classification": entry.get("value_classification"), + "source": "alternative-me", + } + except (KeyError, IndexError, TypeError, ValueError): + return None + + +async def _fetch_mempool_fees(fetcher: Fetcher) -> dict | None: + data = await fetcher.get_json( + _MEMPOOL_FEES_URL, + source="mempool", + cache_key="markets:macro:mempool_fees", + cache_ttl=300, + ) + if data is None: + return None + return { + "fastest_fee": data.get("fastestFee"), + "half_hour_fee": data.get("halfHourFee"), + "hour_fee": data.get("hourFee"), + "economy_fee": data.get("economyFee"), + "minimum_fee": data.get("minimumFee"), + "source": "mempool", + } + + +async def _fetch_yahoo_macro_symbol(fetcher: Fetcher, symbol: str, label: str) -> dict | None: + quote = await _fetch_yahoo_quote( + fetcher, symbol, f"markets:macro:{label}", 300, + ) + if quote is None: + return None + return { + "symbol": symbol, + "price": quote["price"], + "change_pct": quote["change_pct"], + "source": "yahoo-finance", + } + + +async def _fetch_btc_dominance(fetcher: Fetcher) -> dict | None: + data = await fetcher.get_json( + _COINGECKO_GLOBAL_URL, + source="coingecko", + cache_key="markets:macro:btc_dominance", + cache_ttl=300, + ) + if data is None: + return None + try: + btc_pct = data["data"]["market_cap_percentage"]["btc"] + return { + "btc_dominance_pct": round(btc_pct, 2), + "source": "coingecko", + } + except (KeyError, TypeError): + return None + + +async def fetch_macro_signals(fetcher: Fetcher) -> dict: + """Aggregate 7 macro signals into a single dashboard payload. + + Each signal is fetched independently -- a failure in one does not + affect the others. + + Returns:: + + {"signals": {"fear_greed": {...}, "mempool_fees": {...}, ...}, + "source": "multi", "timestamp": ""} + """ + ( + fear_greed, + mempool_fees, + dxy, + vix, + gold, + treasury_10y, + btc_dominance, + ) = await asyncio.gather( + _fetch_fear_greed(fetcher), + _fetch_mempool_fees(fetcher), + _fetch_yahoo_macro_symbol(fetcher, "DX-Y.NYB", "dxy"), + _fetch_yahoo_macro_symbol(fetcher, "^VIX", "vix"), + _fetch_yahoo_macro_symbol(fetcher, "GC=F", "gold"), + _fetch_yahoo_macro_symbol(fetcher, "^TNX", "treasury_10y"), + _fetch_btc_dominance(fetcher), + ) + + return { + "signals": { + "fear_greed": fear_greed, + "mempool_fees": mempool_fees, + "dxy": dxy, + "vix": vix, + "gold": gold, + "treasury_10y": treasury_10y, + "btc_dominance": btc_dominance, + }, + "source": "multi", + "timestamp": _utc_now_iso(), + } diff --git a/src/world_intel_mcp/sources/military.py b/src/world_intel_mcp/sources/military.py new file mode 100644 index 0000000..6938766 --- /dev/null +++ b/src/world_intel_mcp/sources/military.py @@ -0,0 +1,275 @@ +"""Military aviation tracking source for world-intel-mcp. + +Provides real-time military aircraft tracking via OpenSky Network and +aircraft detail lookups via the Wingbits API. +""" + +import asyncio +import base64 +import logging +import os +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.military") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_OPENSKY_STATES_URL = "https://opensky-network.org/api/states/all" +_WINGBITS_BASE_URL = "https://api.wingbits.com/v1/aircraft" + +# ICAO hex prefix ranges known to be allocated to military operators. +MILITARY_ICAO_PREFIXES = [ + "AE", # US Military + "A8", "A9", "AA", "AB", "AC", "AD", # US Military extended + "43C", "43D", "43E", "43F", # UK Military + "3F", # Germany Military + "3A8", "3A9", "3AA", "3AB", # France Military + "500", "501", "502", # Israel Military + "70", # Pakistan Military + "C0", # Canada Military +] + +# Callsign prefixes commonly used by military flights. +MILITARY_CALLSIGN_PREFIXES = [ + "RCH", "DUKE", "REACH", "EVAC", "JAKE", "TOPCAT", "BOXER", + "IRON", "CASA", "NATO", "LAGR", "TEAL", "SAM", "EXEC", + "SPAR", "VALOR", "BLADE", +] + +# Theater bounding boxes for global military posture assessment. +THEATERS = { + "european": {"bbox": "35,-25,72,45", "desc": "NATO/Russia theater"}, + "indo_pacific": {"bbox": "-10,95,55,155", "desc": "China/Taiwan/SCS"}, + "middle_east": {"bbox": "10,25,45,65", "desc": "Persian Gulf/Red Sea"}, + "arctic": {"bbox": "65,-180,90,180", "desc": "Arctic region"}, + "korean_peninsula": {"bbox": "33,124,43,132", "desc": "Korean DMZ"}, +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _is_military_icao(icao24: str) -> bool: + """Check whether an ICAO24 hex address falls within known military ranges.""" + upper = icao24.upper() + for prefix in MILITARY_ICAO_PREFIXES: + if upper.startswith(prefix): + return True + return False + + +def _is_military_callsign(callsign: str | None) -> bool: + """Check whether a callsign matches known military callsign patterns.""" + if not callsign: + return False + cs = callsign.strip().upper() + for prefix in MILITARY_CALLSIGN_PREFIXES: + if cs.startswith(prefix): + return True + return False + + +def _build_opensky_auth_headers() -> dict[str, str] | None: + """Build HTTP Basic auth header from OpenSky env vars, or None if unset.""" + client_id = os.environ.get("OPENSKY_CLIENT_ID") + client_secret = os.environ.get("OPENSKY_CLIENT_SECRET") + if client_id and client_secret: + credentials = base64.b64encode( + f"{client_id}:{client_secret}".encode() + ).decode() + return {"Authorization": f"Basic {credentials}"} + return None + + +def _extract_aircraft(state: list) -> dict: + """Extract a structured aircraft dict from an OpenSky state vector.""" + return { + "icao24": state[0], + "callsign": state[1].strip() if state[1] else None, + "origin_country": state[2], + "latitude": state[6], + "longitude": state[5], + "altitude_m": state[7], + "velocity_ms": state[9], + "heading": state[10], + "on_ground": state[8], + "squawk": state[14], + } + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_military_flights( + fetcher: Fetcher, + bbox: str | None = None, +) -> dict: + """Fetch current military aircraft positions from the OpenSky Network. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + bbox: Optional bounding box as "lamin,lomin,lamax,lomax". + + Returns: + Dict with aircraft list, count, filter description, source, and timestamp. + """ + params: dict[str, str] = {} + if bbox is not None: + parts = bbox.split(",") + if len(parts) == 4: + params["lamin"] = parts[0] + params["lomin"] = parts[1] + params["lamax"] = parts[2] + params["lomax"] = parts[3] + + headers = _build_opensky_auth_headers() + cache_label = bbox or "global" + + data = await fetcher.get_json( + url=_OPENSKY_STATES_URL, + source="opensky", + cache_key=f"military:flights:{cache_label}", + cache_ttl=120, + headers=headers, + params=params if params else None, + ) + + if data is None: + logger.warning("OpenSky API returned no data (bbox=%s)", cache_label) + return { + "aircraft": [], + "count": 0, + "military_filter": "icao_prefix+callsign", + "source": "opensky", + "timestamp": _utc_now_iso(), + } + + states = data.get("states") or [] + military_aircraft: list[dict] = [] + + for state in states: + if not state or len(state) < 17: + continue + + icao24 = state[0] or "" + callsign = state[1] or "" + + if _is_military_icao(icao24) or _is_military_callsign(callsign): + military_aircraft.append(_extract_aircraft(state)) + + return { + "aircraft": military_aircraft, + "count": len(military_aircraft), + "military_filter": "icao_prefix+callsign", + "source": "opensky", + "timestamp": _utc_now_iso(), + } + + +async def fetch_theater_posture(fetcher: Fetcher) -> dict: + """Assess global military air posture across 5 theater regions. + + Calls fetch_military_flights for each theater in parallel and aggregates + the results into a per-theater summary. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + + Returns: + Dict with per-theater summaries, total count, source, and timestamp. + """ + # Check cache first to avoid redundant parallel fetches. + cached = fetcher.cache.get("military:posture") + if cached is not None: + return cached + + async def _fetch_theater(name: str, info: dict) -> tuple[str, dict]: + result = await fetch_military_flights(fetcher, bbox=info["bbox"]) + aircraft_list = result.get("aircraft", []) + + countries: set[str] = set() + callsigns: list[str] = [] + for ac in aircraft_list: + if ac.get("origin_country"): + countries.add(ac["origin_country"]) + if ac.get("callsign"): + callsigns.append(ac["callsign"]) + + return (name, { + "count": len(aircraft_list), + "countries": sorted(countries), + "sample_callsigns": callsigns[:5], + "bbox": info["bbox"], + "description": info["desc"], + }) + + tasks = [ + _fetch_theater(name, info) for name, info in THEATERS.items() + ] + results = await asyncio.gather(*tasks) + + theaters: dict[str, dict] = {} + total = 0 + for name, summary in results: + theaters[name] = summary + total += summary["count"] + + response = { + "theaters": theaters, + "total_military_aircraft": total, + "source": "opensky", + "timestamp": _utc_now_iso(), + } + + fetcher.cache.set("military:posture", response, 300) + return response + + +async def fetch_aircraft_details(fetcher: Fetcher, icao24: str) -> dict: + """Look up detailed aircraft information from the Wingbits API. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + icao24: The ICAO 24-bit hex address of the aircraft. + + Returns: + Dict with aircraft detail payload, source, and timestamp. + """ + api_key = os.environ.get("WINGBITS_API_KEY") + if not api_key: + return {"error": "WINGBITS_API_KEY not configured"} + + url = f"{_WINGBITS_BASE_URL}/{icao24}" + headers = {"X-API-Key": api_key} + + data = await fetcher.get_json( + url=url, + source="wingbits", + cache_key=f"military:aircraft:{icao24}", + cache_ttl=3600, + headers=headers, + ) + + if data is None: + logger.warning("Wingbits returned no data for icao24=%s", icao24) + return { + "aircraft": {}, + "source": "wingbits", + "timestamp": _utc_now_iso(), + } + + return { + "aircraft": data, + "source": "wingbits", + "timestamp": _utc_now_iso(), + } diff --git a/src/world_intel_mcp/sources/news.py b/src/world_intel_mcp/sources/news.py new file mode 100644 index 0000000..14e274d --- /dev/null +++ b/src/world_intel_mcp/sources/news.py @@ -0,0 +1,396 @@ +"""RSS news aggregation, keyword trending, and GDELT search for world-intel-mcp. + +Provides multi-category RSS feed aggregation from 20+ high-quality intelligence +and news sources, keyword spike detection from recent headlines, and full-text +search via the GDELT 2.0 Doc API. No API keys required. +""" + +import asyncio +import logging +import re +import string +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +try: + import feedparser +except ImportError: + feedparser = None # type: ignore[assignment] + +logger = logging.getLogger("world-intel-mcp.sources.news") + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_RSS_FEEDS: dict[str, list[tuple[str, str]]] = { + "geopolitics": [ + ("Reuters World", "https://feeds.reuters.com/Reuters/worldNews"), + ("AP Top News", "https://rsshub.app/apnews/topics/apf-topnews"), + ("BBC World", "https://feeds.bbci.co.uk/news/world/rss.xml"), + ("Al Jazeera", "https://www.aljazeera.com/xml/rss/all.xml"), + ], + "security": [ + ("BleepingComputer", "https://www.bleepingcomputer.com/feed/"), + ("Krebs on Security", "https://krebsonsecurity.com/feed/"), + ("The Hacker News", "https://feeds.feedburner.com/TheHackersNews"), + ("Schneier on Security", "https://www.schneier.com/feed/atom/"), + ], + "technology": [ + ("Ars Technica", "https://feeds.arstechnica.com/arstechnica/index"), + ("TechCrunch", "https://techcrunch.com/feed/"), + ("The Verge", "https://www.theverge.com/rss/index.xml"), + ], + "finance": [ + ("CNBC", "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114"), + ("MarketWatch", "https://feeds.content.dowjones.io/public/rss/mw_topstories"), + ("FT World", "https://www.ft.com/rss/home/uk"), + ], + "military": [ + ("Defense One", "https://www.defenseone.com/rss/"), + ("War on the Rocks", "https://warontherocks.com/feed/"), + ("The War Zone", "https://www.thedrive.com/the-war-zone/feed"), + ], + "science": [ + ("Nature", "https://www.nature.com/nature.rss"), + ("Science", "https://www.science.org/action/showFeed?type=etoc&feed=rss&jc=science"), + ("Phys.org", "https://phys.org/rss-feed/"), + ], +} + +_STOPWORDS: set[str] = { + "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", + "have", "has", "had", "do", "does", "did", "will", "would", "shall", + "should", "may", "might", "must", "can", "could", "in", "on", "at", + "to", "for", "of", "and", "or", "but", "nor", "not", "no", "so", + "yet", "both", "either", "neither", "with", "from", "by", "as", + "into", "through", "during", "before", "after", "above", "below", + "between", "under", "over", "about", "against", "out", "off", "up", + "down", "then", "once", "here", "there", "when", "where", "why", + "how", "all", "each", "every", "any", "few", "more", "most", "some", + "such", "only", "own", "same", "than", "too", "very", "just", "also", + "now", "it", "its", "he", "she", "they", "them", "their", "his", + "her", "we", "you", "your", "our", "my", "me", "him", "us", + "that", "this", "these", "those", "which", "who", "whom", "what", + "if", "while", "because", "until", "although", "since", "whether", + "new", "says", "said", "one", "two", "first", "last", "many", + "much", "get", "got", "back", "even", "still", "well", "way", + "s", "t", "re", "ve", "d", "ll", "m", +} + +_GDELT_DOC_URL = "https://api.gdeltproject.org/api/v2/doc/doc" + +# Regex to strip punctuation from words +_PUNCT_RE = re.compile(f"[{re.escape(string.punctuation)}]") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _parse_published(entry: dict) -> str | None: + """Parse an RSS entry's published date to ISO 8601 UTC string. + + feedparser stores the parsed time struct in ``published_parsed``. + Falls back to the raw ``published`` string if parsing fails. + """ + import time as _time + + parsed_tuple = entry.get("published_parsed") + if parsed_tuple is not None: + try: + epoch = _time.mktime(parsed_tuple[:9]) + dt = datetime.fromtimestamp(epoch, tz=timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + except (ValueError, TypeError, OverflowError): + pass + + # Fallback: try updated_parsed + updated_tuple = entry.get("updated_parsed") + if updated_tuple is not None: + try: + epoch = _time.mktime(updated_tuple[:9]) + dt = datetime.fromtimestamp(epoch, tz=timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + except (ValueError, TypeError, OverflowError): + pass + + # Last resort: return raw string or None + return entry.get("published") or entry.get("updated") + + +def _truncate(text: str | None, max_len: int = 200) -> str: + """Truncate text to max_len characters, appending '...' if trimmed.""" + if not text: + return "" + if len(text) <= max_len: + return text + return text[:max_len] + "..." + + +# --------------------------------------------------------------------------- +# Function 1: RSS News Feed Aggregation +# --------------------------------------------------------------------------- + +async def fetch_news_feed( + fetcher: Fetcher, + category: str | None = None, + limit: int = 50, +) -> dict: + """Aggregate news from 20+ RSS feeds across intelligence/news categories. + + Uses ``feedparser`` to parse RSS/Atom feeds fetched via the shared HTTP + fetcher. Feeds within each category are fetched in parallel. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + category: Optional category key (geopolitics, security, technology, + finance, military, science). If None, fetches all categories. + limit: Maximum number of items to return. + + Returns: + Dict with items list, count, categories fetched, source, and timestamp. + """ + if feedparser is None: + return { + "error": "feedparser not installed — run: pip install feedparser", + "items": [], + "count": 0, + } + + now = datetime.now(timezone.utc) + + # Determine which categories to fetch + if category is not None: + if category not in _RSS_FEEDS: + return { + "items": [], + "count": 0, + "categories_fetched": [], + "error": f"Unknown category '{category}'. Valid: {list(_RSS_FEEDS.keys())}", + "source": "rss-aggregator", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + categories_to_fetch = {category: _RSS_FEEDS[category]} + else: + categories_to_fetch = dict(_RSS_FEEDS) + + all_items: list[dict] = [] + + async def _fetch_single_feed( + feed_name: str, + url: str, + cat: str, + ) -> list[dict]: + """Fetch and parse a single RSS feed, returning extracted items.""" + safe_name = feed_name.lower().replace(" ", "_") + xml_text = await fetcher.get_xml( + url, + source="rss", + cache_key=f"news:rss:{safe_name}", + cache_ttl=300, + ) + + if xml_text is None: + logger.debug("No data from RSS feed %s (%s)", feed_name, url) + return [] + + parsed = feedparser.parse(xml_text) + items: list[dict] = [] + + for entry in parsed.get("entries", []): + published = _parse_published(entry) + summary_raw = entry.get("summary") or entry.get("description") or "" + items.append({ + "title": entry.get("title", ""), + "link": entry.get("link", ""), + "published": published, + "summary": _truncate(summary_raw, 200), + "feed_name": feed_name, + "category": cat, + }) + + return items + + # Fetch all feeds within each category in parallel + for cat, feeds in categories_to_fetch.items(): + tasks = [ + _fetch_single_feed(feed_name, url, cat) + for feed_name, url in feeds + ] + results = await asyncio.gather(*tasks) + for items in results: + all_items.extend(items) + + # Sort by published date descending (entries without dates go last) + all_items.sort( + key=lambda item: item.get("published") or "", + reverse=True, + ) + + # Apply limit + all_items = all_items[:limit] + + return { + "items": all_items, + "count": len(all_items), + "categories_fetched": list(categories_to_fetch.keys()), + "source": "rss-aggregator", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +# --------------------------------------------------------------------------- +# Function 2: Keyword Trending / Spike Detection +# --------------------------------------------------------------------------- + +async def fetch_trending_keywords( + fetcher: Fetcher, + hours: int = 6, + min_count: int = 3, +) -> dict: + """Detect trending keywords from recent news headlines. + + Fetches up to 200 recent news items via ``fetch_news_feed``, extracts + words from titles, removes stopwords and short tokens, and returns the + most frequently occurring keywords sorted by count. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + hours: Not used for time-windowing (RSS feeds are inherently recent), + but kept for API symmetry with other source functions. + min_count: Minimum occurrences for a keyword to be included. + + Returns: + Dict with keywords list (word + count), total items analyzed, + source, and timestamp. + """ + now = datetime.now(timezone.utc) + + # Fetch a broad set of recent items + feed_data = await fetch_news_feed(fetcher, limit=200) + items = feed_data.get("items", []) + + # Count word frequencies across all titles + word_counts: dict[str, int] = {} + for item in items: + title = item.get("title") or "" + # Lowercase, strip punctuation, split into words + cleaned = _PUNCT_RE.sub(" ", title.lower()) + words = cleaned.split() + for word in words: + word = word.strip() + if len(word) < 3: + continue + if word in _STOPWORDS: + continue + word_counts[word] = word_counts.get(word, 0) + 1 + + # Filter by min_count and sort descending + keywords = [ + {"word": word, "count": count} + for word, count in word_counts.items() + if count >= min_count + ] + keywords.sort(key=lambda k: k["count"], reverse=True) + + # Return top 50 + keywords = keywords[:50] + + return { + "keywords": keywords, + "total_items_analyzed": len(items), + "source": "keyword-analysis", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +# --------------------------------------------------------------------------- +# Function 3: GDELT 2.0 Doc API Search +# --------------------------------------------------------------------------- + +async def fetch_gdelt_search( + fetcher: Fetcher, + query: str = "conflict", + mode: str = "artlist", + limit: int = 50, +) -> dict: + """Search the GDELT 2.0 Doc API for articles or volume timelines. + + No API key required. Supports article list and timeline volume modes. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + query: Search query string. + mode: Either ``artlist`` (article list) or ``timelinevol`` + (volume timeline). + limit: Maximum number of records (artlist mode). + + Returns: + Dict with articles/timeline data, count, query info, source, + and timestamp. + """ + now = datetime.now(timezone.utc) + + params: dict = { + "query": query, + "mode": mode, + "maxrecords": limit, + "format": "json", + } + + safe_query = re.sub(r"[^a-zA-Z0-9_-]", "_", query)[:64] + data = await fetcher.get_json( + _GDELT_DOC_URL, + source="gdelt", + cache_key=f"news:gdelt:{safe_query}:{mode}", + cache_ttl=600, + params=params, + ) + + if data is None: + logger.warning("GDELT API returned no data for query=%s mode=%s", query, mode) + return { + "articles": [] if mode == "artlist" else None, + "timeline": None if mode == "artlist" else [], + "count": 0, + "query": query, + "mode": mode, + "source": "gdelt", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + if mode == "artlist": + articles = [] + for article in data.get("articles", []): + articles.append({ + "title": article.get("title"), + "url": article.get("url"), + "seendate": article.get("seendate"), + "socialimage": article.get("socialimage"), + "domain": article.get("domain"), + "language": article.get("language"), + "sourcecountry": article.get("sourcecountry"), + }) + + return { + "articles": articles, + "count": len(articles), + "query": query, + "mode": mode, + "source": "gdelt", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + # timelinevol mode — return timeline data as-is + timeline = data.get("timeline", []) + return { + "timeline": timeline, + "count": len(timeline) if isinstance(timeline, list) else 0, + "query": query, + "mode": mode, + "source": "gdelt", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } diff --git a/src/world_intel_mcp/sources/prediction.py b/src/world_intel_mcp/sources/prediction.py new file mode 100644 index 0000000..5eb872c --- /dev/null +++ b/src/world_intel_mcp/sources/prediction.py @@ -0,0 +1,155 @@ +"""Prediction market data sources for world-intel-mcp. + +Fetches active prediction markets from Polymarket via their public +Gamma API. Every function takes a Fetcher instance as its first +argument and returns a dict (or empty results when upstream calls fail). +""" + +import json +import logging +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.prediction") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_GAMMA_MARKETS_URL = "https://gamma-api.polymarket.com/markets" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _parse_outcome_prices(raw: str | None) -> float | None: + """Parse the outcomePrices JSON string and return YES probability. + + ``outcomePrices`` is a JSON-encoded list like ``"[0.85, 0.15]"`` + where the first element represents the YES probability. + Returns None if the value is missing or unparseable. + """ + if raw is None: + return None + try: + prices = json.loads(raw) + if isinstance(prices, list) and len(prices) > 0: + return float(prices[0]) + except (json.JSONDecodeError, ValueError, TypeError, IndexError): + logger.debug("Failed to parse outcomePrices: %r", raw) + return None + + +def _classify_sentiment(yes_probability: float) -> str: + """Classify a YES probability into a human-readable sentiment label.""" + if yes_probability > 0.85: + return "strong_yes" + if yes_probability > 0.65: + return "leaning_yes" + if yes_probability < 0.15: + return "strong_no" + if yes_probability < 0.35: + return "leaning_no" + return "uncertain" + + +def _safe_float(value) -> float: + """Convert a value to float, returning 0.0 on failure.""" + if value is None: + return 0.0 + try: + return float(value) + except (ValueError, TypeError): + return 0.0 + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_prediction_markets(fetcher: Fetcher, limit: int = 20) -> dict: + """Fetch active prediction markets from Polymarket sorted by 24h volume. + + Uses the Polymarket Gamma API to retrieve currently active (non-closed) + markets ordered by trading volume. + + Returns:: + + {"markets": [...], "count": N, "source": "polymarket", + "timestamp": ""} + """ + params = { + "limit": str(limit), + "active": "true", + "closed": "false", + "order": "volume24hr", + "ascending": "false", + } + + data = await fetcher.get_json( + _GAMMA_MARKETS_URL, + source="polymarket", + cache_key=f"prediction:polymarket:{limit}", + cache_ttl=300, + params=params, + ) + + if data is None: + return { + "markets": [], + "count": 0, + "source": "polymarket", + "timestamp": _utc_now_iso(), + } + + if not isinstance(data, list): + logger.warning("Unexpected Polymarket response type: %s", type(data).__name__) + return { + "markets": [], + "count": 0, + "source": "polymarket", + "timestamp": _utc_now_iso(), + } + + markets: list[dict] = [] + for item in data: + if not isinstance(item, dict): + continue + + yes_probability = _parse_outcome_prices(item.get("outcomePrices")) + if yes_probability is None: + # Skip markets with unparseable outcome data + continue + + volume_24h = _safe_float(item.get("volume24hr")) + total_volume = _safe_float(item.get("volume")) + liquidity = _safe_float(item.get("liquidity")) + slug = item.get("slug", "") + + markets.append({ + "question": item.get("question", ""), + "yes_probability": round(yes_probability, 4), + "sentiment": _classify_sentiment(yes_probability), + "volume_24h": volume_24h, + "total_volume": total_volume, + "liquidity": liquidity, + "category": item.get("category", ""), + "url": f"https://polymarket.com/event/{slug}" if slug else "", + }) + + # Ensure descending sort by 24h volume (API should already return + # sorted, but enforce it defensively) + markets.sort(key=lambda m: m["volume_24h"], reverse=True) + + return { + "markets": markets, + "count": len(markets), + "source": "polymarket", + "timestamp": _utc_now_iso(), + } diff --git a/src/world_intel_mcp/sources/seismology.py b/src/world_intel_mcp/sources/seismology.py new file mode 100644 index 0000000..d0aa545 --- /dev/null +++ b/src/world_intel_mcp/sources/seismology.py @@ -0,0 +1,98 @@ +"""USGS earthquake data source for world-intel-mcp. + +Provides real-time earthquake data from the USGS GeoJSON API. +No API key required. +""" + +import logging +from datetime import datetime, timezone, timedelta + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.seismology") + +_USGS_ENDPOINT = "https://earthquake.usgs.gov/fdsnws/event/1/query" + + +async def fetch_earthquakes( + fetcher: Fetcher, + min_magnitude: float = 4.5, + hours: int = 24, + limit: int = 50, +) -> dict: + """Fetch recent earthquakes from the USGS GeoJSON API. + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + min_magnitude: Minimum earthquake magnitude to include. + hours: How far back to search (in hours from now). + limit: Maximum number of results. + + Returns: + Dict with earthquakes list, count, query params, source, and timestamp. + """ + now = datetime.now(timezone.utc) + starttime = (now - timedelta(hours=hours)).strftime("%Y-%m-%dT%H:%M:%S") + + params = { + "format": "geojson", + "minmagnitude": min_magnitude, + "orderby": "time", + "limit": limit, + "starttime": starttime, + } + + data = await fetcher.get_json( + url=_USGS_ENDPOINT, + source="usgs", + cache_key=f"seismology:quakes:{min_magnitude}:{hours}", + cache_ttl=300, + params=params, + ) + + if data is None: + logger.warning("USGS API returned no data") + return { + "earthquakes": [], + "count": 0, + "query": {"min_magnitude": min_magnitude, "hours": hours}, + "source": "usgs", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + earthquakes = [] + for feature in data.get("features", []): + props = feature.get("properties", {}) + geom = feature.get("geometry", {}) + coords = geom.get("coordinates", [0, 0, 0]) + + # Convert epoch milliseconds to ISO 8601 UTC + epoch_ms = props.get("time") + if epoch_ms is not None: + eq_time = datetime.fromtimestamp( + epoch_ms / 1000, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + else: + eq_time = None + + earthquakes.append({ + "id": feature.get("id"), + "magnitude": props.get("mag"), + "place": props.get("place"), + "time": eq_time, + "depth_km": coords[2] if len(coords) > 2 else None, + "latitude": coords[1] if len(coords) > 1 else None, + "longitude": coords[0] if len(coords) > 0 else None, + "tsunami_alert": props.get("tsunami"), + "felt_reports": props.get("felt"), + "alert_level": props.get("alert"), + "url": props.get("url"), + }) + + return { + "earthquakes": earthquakes, + "count": len(earthquakes), + "query": {"min_magnitude": min_magnitude, "hours": hours}, + "source": "usgs", + "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } diff --git a/src/world_intel_mcp/sources/wildfire.py b/src/world_intel_mcp/sources/wildfire.py new file mode 100644 index 0000000..8537ced --- /dev/null +++ b/src/world_intel_mcp/sources/wildfire.py @@ -0,0 +1,218 @@ +"""NASA FIRMS wildfire data source for world-intel-mcp. + +Provides real-time active fire detections from the VIIRS/SNPP sensor +via the NASA Fire Information for Resource Management System (FIRMS) API. +Requires a NASA FIRMS API key (env: NASA_FIRMS_API_KEY). +""" + +import asyncio +import logging +import os +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +logger = logging.getLogger("world-intel-mcp.sources.wildfire") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_FIRMS_BASE_URL = "https://firms.modaps.eosdis.nasa.gov/api/area/csv" + +# Bounding boxes: west,south,east,north +REGIONS = { + "north_america": "-170,15,-50,75", + "south_america": "-85,-60,-30,15", + "europe": "-25,35,45,72", + "africa": "-20,-37,55,38", + "middle_east": "25,10,65,45", + "south_asia": "60,5,100,40", + "east_asia": "95,15,150,55", + "southeast_asia": "90,-15,155,25", + "oceania": "105,-50,180,-5", +} + +# CSV columns from FIRMS VIIRS SNPP NRT (1-day) +_CSV_COLUMNS = [ + "latitude", "longitude", "bright_ti4", "scan", "track", + "acq_date", "acq_time", "satellite", "confidence", "version", + "bright_ti5", "frp", "daynight", +] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _parse_fires_csv(csv_text: str) -> list[dict]: + """Parse FIRMS CSV text into a list of fire dicts. + + Only returns high-confidence detections. + """ + lines = csv_text.strip().split("\n") + if len(lines) < 2: + return [] + + # Skip the header line + fires: list[dict] = [] + for line in lines[1:]: + fields = line.split(",") + if len(fields) < len(_CSV_COLUMNS): + continue + + confidence = fields[8].strip() + if confidence.lower() not in ("high", "h"): + continue + + try: + lat = float(fields[0]) + lon = float(fields[1]) + brightness = float(fields[2]) + frp_val = float(fields[11]) if fields[11].strip() else 0.0 + except (ValueError, IndexError): + continue + + fires.append({ + "latitude": lat, + "longitude": lon, + "brightness": brightness, + "frp": frp_val, + "confidence": confidence, + "acq_date": fields[5].strip(), + "acq_time": fields[6].strip(), + "daynight": fields[12].strip(), + "satellite": fields[7].strip(), + }) + + return fires + + +def _cluster_fires(fires: list[dict], top_n: int = 20) -> list[dict]: + """Group fires by a 0.5-degree grid and return the top N clusters by count.""" + grid: dict[tuple[float, float], list[dict]] = {} + + for fire in fires: + # Round to nearest 0.5 degrees + grid_lat = round(fire["latitude"] * 2) / 2 + grid_lon = round(fire["longitude"] * 2) / 2 + key = (grid_lat, grid_lon) + + if key not in grid: + grid[key] = [] + grid[key].append(fire) + + # Sort clusters by fire count descending + sorted_clusters = sorted(grid.items(), key=lambda item: len(item[1]), reverse=True) + + clusters: list[dict] = [] + for (lat, lon), cell_fires in sorted_clusters[:top_n]: + max_frp = max(f["frp"] for f in cell_fires) + clusters.append({ + "lat": lat, + "lon": lon, + "fire_count": len(cell_fires), + "max_frp": max_frp, + }) + + return clusters + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def fetch_wildfires( + fetcher: Fetcher, + region: str | None = None, + api_key: str | None = None, +) -> dict: + """Fetch active wildfire data from NASA FIRMS (VIIRS SNPP NRT, last 24h). + + Args: + fetcher: Shared HTTP fetcher with caching and circuit breaking. + region: Region name (key from REGIONS dict). None fetches all regions. + api_key: NASA FIRMS API key. Falls back to NASA_FIRMS_API_KEY env var. + + Returns: + Dict with fires_by_region, total_fires, source, and timestamp. + """ + key = api_key or os.environ.get("NASA_FIRMS_API_KEY") + if not key: + return {"error": "NASA_FIRMS_API_KEY not configured"} + + if region is not None: + bbox = REGIONS.get(region) + if bbox is None: + return { + "error": f"Unknown region '{region}'. Valid: {', '.join(REGIONS.keys())}", + } + regions_to_fetch: dict[str, str] = {region: bbox} + else: + regions_to_fetch = dict(REGIONS) + + async def _fetch_region(region_name: str, bbox: str) -> tuple[str, list[dict] | None]: + """Fetch fires for a single region, return (name, fires_list_or_None).""" + url = f"{_FIRMS_BASE_URL}/{key}/VIIRS_SNPP_NRT/{bbox}/1" + + csv_text = await fetcher.get_text( + url=url, + source="nasa-firms", + cache_key=f"wildfire:fires:{region_name}", + cache_ttl=1800, + ) + + if csv_text is None: + logger.warning("FIRMS returned no data for region %s", region_name) + return (region_name, None) + + fires = _parse_fires_csv(csv_text) + return (region_name, fires) + + # Fetch all requested regions in parallel + tasks = [ + _fetch_region(name, bbox) + for name, bbox in regions_to_fetch.items() + ] + results = await asyncio.gather(*tasks) + + # Assemble response + fires_by_region: dict[str, dict] = {} + total_fires = 0 + + for region_name, fires in results: + if fires is None: + fires_by_region[region_name] = {"count": 0, "top_clusters": []} + continue + + count = len(fires) + total_fires += count + top_clusters = _cluster_fires(fires) + fires_by_region[region_name] = { + "count": count, + "top_clusters": top_clusters, + } + + cache_label = region if region is not None else "global" + # Store assembled result in cache under the composite key + fetcher.cache.set( + f"wildfire:fires:{cache_label}", + { + "fires_by_region": fires_by_region, + "total_fires": total_fires, + "source": "nasa-firms", + "timestamp": _utc_now_iso(), + }, + 1800, + ) + + return { + "fires_by_region": fires_by_region, + "total_fires": total_fires, + "source": "nasa-firms", + "timestamp": _utc_now_iso(), + } diff --git a/src/world_intel_mcp/tests/__init__.py b/src/world_intel_mcp/tests/__init__.py new file mode 100644 index 0000000..ae516af --- /dev/null +++ b/src/world_intel_mcp/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for world-intel-mcp.""" diff --git a/src/world_intel_mcp/tests/conftest.py b/src/world_intel_mcp/tests/conftest.py new file mode 100644 index 0000000..97cb28a --- /dev/null +++ b/src/world_intel_mcp/tests/conftest.py @@ -0,0 +1,18 @@ +"""Test configuration — strips proxy env vars so httpx doesn't try SOCKS.""" + +import os + +import pytest + +_PROXY_VARS = [ + "ALL_PROXY", "all_proxy", "HTTP_PROXY", "http_proxy", + "HTTPS_PROXY", "https_proxy", "FTP_PROXY", "ftp_proxy", + "GRPC_PROXY", "grpc_proxy", +] + + +@pytest.fixture(autouse=True) +def _strip_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Remove system proxy env vars so httpx creates clean connections.""" + for var in _PROXY_VARS: + monkeypatch.delenv(var, raising=False) diff --git a/src/world_intel_mcp/tests/test_analysis.py b/src/world_intel_mcp/tests/test_analysis.py new file mode 100644 index 0000000..963f228 --- /dev/null +++ b/src/world_intel_mcp/tests/test_analysis.py @@ -0,0 +1,59 @@ +"""Tests for circuit breaker and infrastructure.""" + +import time + +import pytest + +from world_intel_mcp.circuit_breaker import CircuitBreaker + + +def test_circuit_starts_closed() -> None: + cb = CircuitBreaker(failure_threshold=3, cooldown_seconds=1.0) + assert cb.is_available("test-source") + + +def test_circuit_trips_after_threshold() -> None: + cb = CircuitBreaker(failure_threshold=2, cooldown_seconds=10.0) + cb.record_failure("src") + assert cb.is_available("src") # 1 failure, threshold is 2 + cb.record_failure("src") + assert not cb.is_available("src") # tripped + + +def test_circuit_recovers_after_cooldown() -> None: + cb = CircuitBreaker(failure_threshold=1, cooldown_seconds=0.5) + cb.record_failure("src") + assert not cb.is_available("src") + time.sleep(0.6) + assert cb.is_available("src") # half-open, allows probe + + +def test_success_resets_failures() -> None: + cb = CircuitBreaker(failure_threshold=3, cooldown_seconds=10.0) + cb.record_failure("src") + cb.record_failure("src") + cb.record_success("src") + assert cb.is_available("src") + # Even after 2 more failures, need 3 consecutive + cb.record_failure("src") + cb.record_failure("src") + assert cb.is_available("src") # only 2 since reset + + +def test_status_output() -> None: + cb = CircuitBreaker(failure_threshold=2, cooldown_seconds=60.0) + cb.record_success("healthy") + cb.record_failure("unhealthy") + cb.record_failure("unhealthy") + + status = cb.status() + assert status["healthy"]["status"] == "closed" + assert status["unhealthy"]["status"] == "open" + assert status["unhealthy"]["total_trips"] == 1 + + +def test_independent_sources() -> None: + cb = CircuitBreaker(failure_threshold=1, cooldown_seconds=60.0) + cb.record_failure("source_a") + assert not cb.is_available("source_a") + assert cb.is_available("source_b") # independent diff --git a/src/world_intel_mcp/tests/test_cache.py b/src/world_intel_mcp/tests/test_cache.py new file mode 100644 index 0000000..5c97652 --- /dev/null +++ b/src/world_intel_mcp/tests/test_cache.py @@ -0,0 +1,66 @@ +"""Tests for SQLite TTL cache.""" + +import time +from pathlib import Path + +import pytest + +from world_intel_mcp.cache import Cache + + +@pytest.fixture +def cache(tmp_path: Path) -> Cache: + return Cache(db_path=tmp_path / "test_cache.db") + + +def test_set_and_get(cache: Cache) -> None: + cache.set("key1", {"value": 42}, ttl_seconds=60) + result = cache.get("key1") + assert result == {"value": 42} + + +def test_get_missing_key(cache: Cache) -> None: + assert cache.get("nonexistent") is None + + +def test_ttl_expiration(cache: Cache) -> None: + cache.set("ephemeral", "data", ttl_seconds=1) + assert cache.get("ephemeral") == "data" + time.sleep(1.1) + assert cache.get("ephemeral") is None + + +def test_overwrite(cache: Cache) -> None: + cache.set("key", "v1", ttl_seconds=60) + cache.set("key", "v2", ttl_seconds=60) + assert cache.get("key") == "v2" + + +def test_delete(cache: Cache) -> None: + cache.set("key", "val", ttl_seconds=60) + cache.delete("key") + assert cache.get("key") is None + + +def test_evict_expired(cache: Cache) -> None: + cache.set("fresh", "yes", ttl_seconds=300) + cache.set("stale", "no", ttl_seconds=1) + time.sleep(1.1) + removed = cache.evict_expired() + assert removed == 1 + assert cache.get("fresh") == "yes" + assert cache.get("stale") is None + + +def test_stats(cache: Cache) -> None: + cache.set("a", 1, ttl_seconds=300) + cache.set("b", 2, ttl_seconds=300) + stats = cache.stats() + assert stats["total_entries"] == 2 + assert stats["active_entries"] == 2 + + +def test_complex_values(cache: Cache) -> None: + data = {"nested": {"list": [1, 2, 3], "bool": True, "null": None}} + cache.set("complex", data, ttl_seconds=60) + assert cache.get("complex") == data diff --git a/src/world_intel_mcp/tests/test_sources.py b/src/world_intel_mcp/tests/test_sources.py new file mode 100644 index 0000000..1e988af --- /dev/null +++ b/src/world_intel_mcp/tests/test_sources.py @@ -0,0 +1,218 @@ +"""Tests for source modules — uses respx to mock HTTP calls.""" + +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +import respx + +from world_intel_mcp.cache import Cache +from world_intel_mcp.circuit_breaker import CircuitBreaker +from world_intel_mcp.fetcher import Fetcher + + +@pytest.fixture +def cache(tmp_path: Path) -> Cache: + return Cache(db_path=tmp_path / "test_cache.db") + + +@pytest.fixture +def fetcher(cache: Cache) -> Fetcher: + breaker = CircuitBreaker() + return Fetcher(cache=cache, breaker=breaker, default_timeout=5.0) + + +# --------------------------------------------------------------------------- +# Markets +# --------------------------------------------------------------------------- + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_market_quotes(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.markets import fetch_market_quotes + + # Mock Yahoo Finance v8 chart response for ^GSPC + chart_response = { + "chart": { + "result": [{ + "meta": { + "symbol": "^GSPC", + "regularMarketPrice": 5123.45, + "regularMarketChangePercent": 0.42, + "currency": "USD", + } + }] + } + } + + respx.get("https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC").mock( + return_value=httpx.Response(200, json=chart_response) + ) + + result = await fetch_market_quotes(fetcher, symbols=["^GSPC"]) + assert "quotes" in result + assert len(result["quotes"]) == 1 + assert result["quotes"][0]["symbol"] == "^GSPC" + assert result["quotes"][0]["price"] == 5123.45 + assert result["source"] == "yahoo-finance" + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_crypto_quotes(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.markets import fetch_crypto_quotes + + coins = [ + { + "id": "bitcoin", + "symbol": "btc", + "name": "Bitcoin", + "current_price": 98000, + "market_cap": 1900000000000, + "price_change_percentage_24h": 2.5, + "sparkline_in_7d": {"price": [95000, 96000, 97000, 98000]}, + } + ] + + respx.get("https://api.coingecko.com/api/v3/coins/markets").mock( + return_value=httpx.Response(200, json=coins) + ) + + result = await fetch_crypto_quotes(fetcher, limit=5) + assert "coins" in result + assert len(result["coins"]) == 1 + assert result["coins"][0]["symbol"] == "btc" + assert result["source"] == "coingecko" + + +# --------------------------------------------------------------------------- +# Seismology +# --------------------------------------------------------------------------- + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_earthquakes(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.seismology import fetch_earthquakes + + geojson = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "us7000abc1", + "properties": { + "mag": 5.2, + "place": "100km SSW of Somewhere", + "time": 1708700000000, + "tsunami": 0, + "felt": 15, + "alert": "green", + "url": "https://earthquake.usgs.gov/earthquakes/eventpage/us7000abc1", + }, + "geometry": { + "type": "Point", + "coordinates": [-120.5, 35.2, 10.0], + }, + } + ], + } + + respx.get("https://earthquake.usgs.gov/fdsnws/event/1/query").mock( + return_value=httpx.Response(200, json=geojson) + ) + + result = await fetch_earthquakes(fetcher, min_magnitude=4.0, hours=24) + assert result["count"] == 1 + eq = result["earthquakes"][0] + assert eq["magnitude"] == 5.2 + assert eq["id"] == "us7000abc1" + assert eq["depth_km"] == 10.0 + assert eq["latitude"] == 35.2 + assert eq["longitude"] == -120.5 + assert result["source"] == "usgs" + + +# --------------------------------------------------------------------------- +# Wildfire +# --------------------------------------------------------------------------- + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_wildfires_no_api_key(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.wildfire import fetch_wildfires + + with patch.dict("os.environ", {}, clear=False): + # Remove the key if present + import os + os.environ.pop("NASA_FIRMS_API_KEY", None) + result = await fetch_wildfires(fetcher, api_key=None) + + assert "error" in result + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_wildfires_single_region(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.wildfire import fetch_wildfires + + csv_data = ( + "latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,confidence,version,bright_ti5,frp,daynight\n" + "34.5,-118.2,350.0,0.5,0.5,2024-02-23,1200,N,high,2.0,290.0,45.0,D\n" + "34.6,-118.3,360.0,0.5,0.5,2024-02-23,1200,N,high,2.0,295.0,55.0,D\n" + "34.5,-118.2,340.0,0.5,0.5,2024-02-23,1200,N,low,2.0,285.0,30.0,D\n" + ) + + respx.get(url__regex=r".*firms\.modaps\.eosdis\.nasa\.gov.*").mock( + return_value=httpx.Response(200, text=csv_data) + ) + + result = await fetch_wildfires(fetcher, region="north_america", api_key="testkey") + assert "fires_by_region" in result + na = result["fires_by_region"]["north_america"] + assert na["count"] == 2 # only 2 high-confidence + assert result["total_fires"] == 2 + + +# --------------------------------------------------------------------------- +# Economic +# --------------------------------------------------------------------------- + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_fred_series_no_key(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.economic import fetch_fred_series + + import os + os.environ.pop("FRED_API_KEY", None) + result = await fetch_fred_series(fetcher, series_id="UNRATE", api_key=None) + assert "error" in result + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_world_bank_indicators(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.economic import fetch_world_bank_indicators + + wb_response = [ + {"page": 1, "pages": 1, "per_page": 5, "total": 2}, + [ + {"indicator": {"id": "NY.GDP.MKTP.CD", "value": "GDP"}, "date": "2023", "value": 25000000000000}, + {"indicator": {"id": "NY.GDP.MKTP.CD", "value": "GDP"}, "date": "2022", "value": 24000000000000}, + ], + ] + + respx.get(url__regex=r".*api\.worldbank\.org.*").mock( + return_value=httpx.Response(200, json=wb_response) + ) + + result = await fetch_world_bank_indicators(fetcher, country="USA", indicators=["NY.GDP.MKTP.CD"]) + assert "indicators" in result + assert len(result["indicators"]) == 1 + assert result["indicators"][0]["id"] == "NY.GDP.MKTP.CD" + assert result["source"] == "world-bank"