feat: add world-intel-mcp server — 36 tools across 14 intelligence domains

New MCP server providing real-time global intelligence: financial markets
(Yahoo Finance, CoinGecko), economic indicators (FRED, EIA, World Bank),
conflict tracking (ACLED, UCDP, HDX), military flights (OpenSky),
infrastructure monitoring (Cloudflare Radar, NGA cable health), maritime
warnings (NGA), climate anomalies (Open-Meteo), news aggregation (RSS,
GDELT), prediction markets (Polymarket), displacement data (UNHCR),
aviation delays (FAA), cyber threats (Feodo, CISA KEV, SANS, URLhaus),
country intelligence briefs (Ollama LLM), and HTML report generation
(Jinja2 + Chart.js).

Includes: SQLite TTL cache, per-source circuit breakers, async HTTP
fetcher with retry/rate-limiting, Click CLI with 27 commands, 4 analysis
modules (instability scoring, geo-convergence, signal aggregation, news
clustering), 4 HTML report templates, and 28 unit tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Marc Shade
2026-02-23 10:33:57 -05:00
commit 01cb4255ef
43 changed files with 8643 additions and 0 deletions
+12
View File
@@ -0,0 +1,12 @@
# world-intel-mcp
Real-time global intelligence MCP server for Phoenix AGI. Covers financial markets, economic indicators, earthquakes, wildfires, conflict, military flights, infrastructure health, and more.
## Quick Start
```bash
pip install -e .
world-intel-mcp # Run as MCP server
intel markets # CLI: stock indices
intel earthquakes # CLI: recent quakes
```
+46
View File
@@ -0,0 +1,46 @@
[project]
name = "world-intel-mcp"
version = "0.1.0"
description = "World Intelligence MCP Server - real-time global intelligence across 17 domains"
readme = "README.md"
requires-python = ">=3.11"
license = {text = "MIT"}
authors = [
{name = "Phoenix AGI", email = "phoenix@2acrestudios.com"}
]
keywords = ["mcp", "intelligence", "osint", "markets", "geopolitical", "phoenix"]
dependencies = [
"mcp>=1.0.0",
"httpx>=0.27.0",
"pydantic>=2.0.0",
"feedparser>=6.0.0",
"click>=8.1.0",
"jinja2>=3.1.0",
"rich>=13.0.0",
]
[project.optional-dependencies]
pdf = ["weasyprint>=62.0"]
dev = [
"pytest>=8.0.0",
"pytest-asyncio>=0.23.0",
"pytest-cov>=4.0.0",
"respx>=0.21.0",
]
[project.scripts]
world-intel-mcp = "world_intel_mcp.server:run"
intel = "world_intel_mcp.cli:main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/world_intel_mcp"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["src/world_intel_mcp/tests"]
addopts = "-v --tb=short"
+7
View File
@@ -0,0 +1,7 @@
mcp==1.8.1
httpx[socks]==0.28.1
pydantic==2.10.6
feedparser==6.0.11
click==8.1.8
jinja2==3.1.5
rich==13.9.4
+3
View File
@@ -0,0 +1,3 @@
"""World Intelligence MCP Server — real-time global intelligence across 17 domains."""
__version__ = "0.1.0"
+1
View File
@@ -0,0 +1 @@
"""Analysis modules for world-intel-mcp."""
@@ -0,0 +1,98 @@
"""News article clustering using Jaccard similarity.
Groups news articles by content similarity to reduce noise
and identify story clusters.
"""
import logging
import re
logger = logging.getLogger("world-intel-mcp.analysis.clustering")
_STOPWORDS = frozenset({
"the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
"have", "has", "had", "do", "does", "did", "will", "would", "could",
"should", "may", "might", "can", "shall", "in", "on", "at", "to",
"for", "of", "and", "or", "but", "not", "no", "nor", "with", "from",
"by", "it", "its", "that", "this", "these", "those", "he", "she",
"they", "we", "you", "i", "me", "my", "his", "her", "our", "your",
"their", "what", "which", "who", "whom", "how", "when", "where",
"why", "if", "then", "than", "so", "as", "up", "out", "about",
"into", "over", "after", "before", "between", "under", "again",
"also", "just", "more", "most", "other", "some", "such", "very",
"new", "said", "says", "one", "two", "first",
})
def _tokenize(text: str) -> set[str]:
"""Extract meaningful word tokens from text."""
words = re.findall(r'[a-z]{3,}', text.lower())
return {w for w in words if w not in _STOPWORDS}
def jaccard_similarity(set_a: set, set_b: set) -> float:
"""Compute Jaccard similarity between two sets."""
if not set_a or not set_b:
return 0.0
intersection = len(set_a & set_b)
union = len(set_a | set_b)
return intersection / union if union > 0 else 0.0
def cluster_articles(
articles: list[dict],
similarity_threshold: float = 0.3,
title_field: str = "title",
) -> list[dict]:
"""Cluster articles by title similarity.
Args:
articles: List of article dicts.
similarity_threshold: Minimum Jaccard similarity to merge.
title_field: Key containing the article title.
Returns:
List of cluster dicts with representative article and member count.
"""
if not articles:
return []
# Tokenize all titles
tokenized = []
for article in articles:
title = article.get(title_field, "") or ""
tokens = _tokenize(title)
tokenized.append(tokens)
# Greedy single-linkage clustering
assigned = [False] * len(articles)
clusters: list[dict] = []
for i in range(len(articles)):
if assigned[i]:
continue
cluster_members = [i]
assigned[i] = True
for j in range(i + 1, len(articles)):
if assigned[j]:
continue
# Check similarity against cluster representative (first member)
sim = jaccard_similarity(tokenized[i], tokenized[j])
if sim >= similarity_threshold:
cluster_members.append(j)
assigned[j] = True
# Build cluster output
representative = articles[cluster_members[0]]
clusters.append({
"representative": representative,
"member_count": len(cluster_members),
"member_indices": cluster_members,
})
# Sort by member count descending
clusters.sort(key=lambda c: c["member_count"], reverse=True)
return clusters
@@ -0,0 +1,84 @@
"""Geo-convergence detection for multi-domain signal overlap.
Detects when signals from different domains (conflict, natural disasters,
military activity) converge geographically, suggesting elevated risk.
"""
import logging
from collections import defaultdict
from math import floor
logger = logging.getLogger("world-intel-mcp.analysis.convergence")
def _grid_key(lat: float, lon: float, resolution: float = 1.0) -> tuple[int, int]:
"""Convert lat/lon to grid cell key."""
return (int(floor(lat / resolution)), int(floor(lon / resolution)))
def detect_convergence(
events: list[dict],
resolution: float = 1.0,
min_types: int = 2,
min_total: int = 3,
) -> list[dict]:
"""Detect geographic convergence of multi-domain signals.
Args:
events: List of dicts with 'lat', 'lon', 'type' (domain), and optional 'weight'.
resolution: Grid cell size in degrees.
min_types: Minimum number of different signal types for convergence.
min_total: Minimum total events in cell for convergence.
Returns:
List of convergence hotspots sorted by score descending.
"""
grid: dict[tuple[int, int], list[dict]] = defaultdict(list)
for event in events:
lat = event.get("lat")
lon = event.get("lon")
if lat is None or lon is None:
continue
try:
lat_f = float(lat)
lon_f = float(lon)
except (ValueError, TypeError):
continue
key = _grid_key(lat_f, lon_f, resolution)
grid[key].append(event)
hotspots = []
for (grid_lat, grid_lon), cell_events in grid.items():
if len(cell_events) < min_total:
continue
types = set()
total_weight = 0.0
for e in cell_events:
types.add(e.get("type", "unknown"))
total_weight += e.get("weight", 1.0)
if len(types) < min_types:
continue
# Center of grid cell
center_lat = (grid_lat + 0.5) * resolution
center_lon = (grid_lon + 0.5) * resolution
# Score: number of types * log of total events * weight
score = len(types) * (1 + len(cell_events) ** 0.5) * (total_weight / len(cell_events))
hotspots.append({
"lat": round(center_lat, 2),
"lon": round(center_lon, 2),
"event_count": len(cell_events),
"signal_types": sorted(types),
"type_count": len(types),
"total_weight": round(total_weight, 1),
"convergence_score": round(score, 2),
})
hotspots.sort(key=lambda h: h["convergence_score"], reverse=True)
return hotspots
+110
View File
@@ -0,0 +1,110 @@
"""Country Instability Index (CII) computation.
Pure scoring functions that transform raw data into instability scores.
"""
import logging
logger = logging.getLogger("world-intel-mcp.analysis.instability")
def score_conflict_intensity(event_count: int, days: int = 30) -> float:
"""Score conflict intensity 0-20 based on event count per period."""
daily_rate = event_count / max(days, 1)
# 0 events = 0, 10+/day = 20
return min(20.0, daily_rate * 2.0)
def score_economic_stress(inflation_rate: float | None, gdp_growth: float | None) -> float:
"""Score economic stress 0-20 from inflation and GDP growth."""
score = 0.0
if inflation_rate is not None:
# High inflation (>20%) = 10, moderate (>10%) = 5
if inflation_rate > 20:
score += 10.0
elif inflation_rate > 10:
score += 5.0
elif inflation_rate > 5:
score += 2.0
if gdp_growth is not None:
# Negative GDP = 10, stagnant = 5
if gdp_growth < -5:
score += 10.0
elif gdp_growth < 0:
score += 7.0
elif gdp_growth < 1:
score += 3.0
return min(20.0, score)
def score_humanitarian_crisis(dataset_count: int, displacement_total: int = 0) -> float:
"""Score humanitarian crisis 0-20 based on HDX datasets and displacement."""
score = 0.0
# Many crisis datasets = high concern
score += min(10.0, dataset_count * 0.5)
# Displacement
if displacement_total > 1_000_000:
score += 10.0
elif displacement_total > 100_000:
score += 5.0
return min(20.0, score)
def score_infrastructure_disruption(outage_count: int, cable_warnings: int = 0) -> float:
"""Score infrastructure disruption 0-20."""
score = 0.0
score += min(10.0, outage_count * 2.0)
score += min(10.0, cable_warnings * 2.5)
return min(20.0, score)
def score_military_activity(aircraft_count: int, flight_density: float = 0.0) -> float:
"""Score military activity 0-20 based on aircraft presence."""
score = 0.0
if aircraft_count > 50:
score = 20.0
elif aircraft_count > 20:
score = 15.0
elif aircraft_count > 10:
score = 10.0
elif aircraft_count > 5:
score = 5.0
elif aircraft_count > 0:
score = 2.0
return score
def compute_cii(
conflict: float = 0.0,
economic: float = 0.0,
humanitarian: float = 0.0,
infrastructure: float = 0.0,
military: float = 0.0,
) -> dict:
"""Compute Country Instability Index (0-100) from component scores.
Returns dict with index, components, and risk_level.
"""
total = conflict + economic + humanitarian + infrastructure + military
total = min(100.0, max(0.0, total))
if total >= 75:
risk_level = "critical"
elif total >= 50:
risk_level = "high"
elif total >= 25:
risk_level = "medium"
else:
risk_level = "low"
return {
"instability_index": round(total, 1),
"components": {
"conflict_intensity": round(conflict, 1),
"economic_stress": round(economic, 1),
"humanitarian_crisis": round(humanitarian, 1),
"infrastructure_disruption": round(infrastructure, 1),
"military_activity": round(military, 1),
},
"risk_level": risk_level,
}
+98
View File
@@ -0,0 +1,98 @@
"""Signal aggregation per country.
Collects and normalizes signals from multiple domains into
a per-country summary for dashboard and reporting.
"""
import logging
from collections import defaultdict
logger = logging.getLogger("world-intel-mcp.analysis.signals")
def aggregate_country_signals(
conflict_events: list[dict] | None = None,
displacement_data: list[dict] | None = None,
earthquake_data: list[dict] | None = None,
fire_data: dict | None = None,
) -> dict[str, dict]:
"""Aggregate multi-domain signals by country.
Args:
conflict_events: ACLED/UCDP events with 'country' field.
displacement_data: UNHCR data with 'country' field.
earthquake_data: USGS earthquakes (uses reverse geocoding heuristic).
fire_data: Wildfire data by region (maps to countries approximately).
Returns:
Dict mapping country name to signal summary.
"""
countries: dict[str, dict] = defaultdict(lambda: {
"conflict_events": 0,
"fatalities": 0,
"displaced_persons": 0,
"earthquakes": 0,
"max_earthquake_mag": 0.0,
"fires": 0,
"signal_count": 0,
"domains": set(),
})
# Conflict events
if conflict_events:
for event in conflict_events:
country = event.get("country")
if not country:
continue
c = countries[country]
c["conflict_events"] += 1
fat = event.get("fatalities", 0)
if isinstance(fat, (int, float)):
c["fatalities"] += int(fat)
c["domains"].add("conflict")
# Displacement
if displacement_data:
for record in displacement_data:
country = record.get("country")
if not country:
continue
c = countries[country]
total = record.get("total_displaced", 0)
if isinstance(total, (int, float)):
c["displaced_persons"] += int(total)
c["domains"].add("displacement")
# Earthquakes (approximate country from place string)
if earthquake_data:
for quake in earthquake_data:
place = quake.get("place", "") or ""
# USGS format: "123km SSE of City, Country"
parts = place.rsplit(", ", 1)
country = parts[-1] if len(parts) > 1 else "Unknown"
c = countries[country]
c["earthquakes"] += 1
mag = quake.get("magnitude", 0) or 0
if isinstance(mag, (int, float)) and mag > c["max_earthquake_mag"]:
c["max_earthquake_mag"] = float(mag)
c["domains"].add("seismology")
# TODO: fire_data integration (requires region-to-country mapping)
_ = fire_data
# Compute signal counts
result = {}
for country, data in countries.items():
domains = data.pop("domains")
data["signal_count"] = len(domains)
data["active_domains"] = sorted(domains)
result[country] = data
# Sort by signal count, then fatalities
result = dict(sorted(
result.items(),
key=lambda x: (x[1]["signal_count"], x[1]["fatalities"]),
reverse=True,
))
return result
+105
View File
@@ -0,0 +1,105 @@
"""SQLite TTL cache for world-intel-mcp.
Simple key-value cache with per-key TTL, WAL mode, and automatic eviction.
No external deps — just stdlib sqlite3.
"""
import json
import logging
import sqlite3
import time
from pathlib import Path
from typing import Any
logger = logging.getLogger("world-intel-mcp.cache")
_DEFAULT_DB = Path.home() / ".cache" / "world-intel-mcp" / "cache.db"
class Cache:
"""SQLite-backed TTL cache."""
def __init__(self, db_path: Path | None = None):
self.db_path = db_path or _DEFAULT_DB
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._conn: sqlite3.Connection | None = None
self._init_db()
def _init_db(self) -> None:
conn = self._get_conn()
conn.execute("""
CREATE TABLE IF NOT EXISTS cache (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
expires_at REAL NOT NULL,
created_at REAL NOT NULL
)
""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_cache_expires ON cache(expires_at)")
conn.commit()
def _get_conn(self) -> sqlite3.Connection:
if self._conn is None:
self._conn = sqlite3.connect(str(self.db_path), timeout=10)
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.execute("PRAGMA busy_timeout=5000")
self._conn.execute("PRAGMA synchronous=NORMAL")
return self._conn
def get(self, key: str) -> Any | None:
"""Get a cached value. Returns None if missing or expired."""
conn = self._get_conn()
row = conn.execute(
"SELECT value, expires_at FROM cache WHERE key = ?", (key,)
).fetchone()
if row is None:
return None
value, expires_at = row
if time.time() > expires_at:
conn.execute("DELETE FROM cache WHERE key = ?", (key,))
conn.commit()
return None
return json.loads(value)
def set(self, key: str, value: Any, ttl_seconds: int) -> None:
"""Store a value with TTL in seconds."""
conn = self._get_conn()
now = time.time()
conn.execute(
"INSERT OR REPLACE INTO cache (key, value, expires_at, created_at) VALUES (?, ?, ?, ?)",
(key, json.dumps(value, default=str), now + ttl_seconds, now),
)
conn.commit()
def delete(self, key: str) -> None:
"""Delete a specific key."""
conn = self._get_conn()
conn.execute("DELETE FROM cache WHERE key = ?", (key,))
conn.commit()
def evict_expired(self) -> int:
"""Remove all expired entries. Returns count removed."""
conn = self._get_conn()
cursor = conn.execute("DELETE FROM cache WHERE expires_at < ?", (time.time(),))
conn.commit()
return cursor.rowcount
def stats(self) -> dict[str, Any]:
"""Cache statistics."""
conn = self._get_conn()
now = time.time()
total = conn.execute("SELECT COUNT(*) FROM cache").fetchone()[0]
expired = conn.execute(
"SELECT COUNT(*) FROM cache WHERE expires_at < ?", (now,)
).fetchone()[0]
return {
"total_entries": total,
"active_entries": total - expired,
"expired_entries": expired,
"db_path": str(self.db_path),
}
def close(self) -> None:
if self._conn:
self._conn.close()
self._conn = None
+92
View File
@@ -0,0 +1,92 @@
"""Per-source circuit breaker for world-intel-mcp.
Tracks failures per data source. Trips after N consecutive failures,
blocks calls for a cooldown period, then allows a single probe.
"""
import logging
import time
from dataclasses import dataclass, field
logger = logging.getLogger("world-intel-mcp.circuit_breaker")
@dataclass
class _State:
failures: int = 0
last_failure: float = 0.0
tripped_at: float = 0.0
is_open: bool = False
total_trips: int = 0
total_successes: int = 0
total_failures: int = 0
class CircuitBreaker:
"""Circuit breaker with configurable thresholds per source."""
def __init__(
self,
failure_threshold: int = 3,
cooldown_seconds: float = 300.0,
):
self.failure_threshold = failure_threshold
self.cooldown_seconds = cooldown_seconds
self._states: dict[str, _State] = {}
def _get(self, source: str) -> _State:
if source not in self._states:
self._states[source] = _State()
return self._states[source]
def is_available(self, source: str) -> bool:
"""Check if source is available (circuit closed or cooldown elapsed)."""
state = self._get(source)
if not state.is_open:
return True
if time.time() - state.tripped_at >= self.cooldown_seconds:
return True # allow probe
return False
def record_success(self, source: str) -> None:
"""Record successful call — resets failure count, closes circuit."""
state = self._get(source)
state.failures = 0
state.is_open = False
state.total_successes += 1
def record_failure(self, source: str) -> None:
"""Record failed call — increments counter, may trip breaker."""
state = self._get(source)
state.failures += 1
state.last_failure = time.time()
state.total_failures += 1
if state.failures >= self.failure_threshold and not state.is_open:
state.is_open = True
state.tripped_at = time.time()
state.total_trips += 1
logger.warning(
"Circuit breaker TRIPPED for %s (failures=%d, cooldown=%.0fs)",
source, state.failures, self.cooldown_seconds,
)
def status(self) -> dict[str, dict]:
"""Return status of all tracked sources."""
now = time.time()
result = {}
for source, state in self._states.items():
if state.is_open:
remaining = max(0, self.cooldown_seconds - (now - state.tripped_at))
status = "open" if remaining > 0 else "half-open"
else:
remaining = 0
status = "closed"
result[source] = {
"status": status,
"failures": state.failures,
"cooldown_remaining_s": round(remaining, 1),
"total_trips": state.total_trips,
"total_successes": state.total_successes,
"total_failures": state.total_failures,
}
return result
+978
View File
@@ -0,0 +1,978 @@
#!/usr/bin/env python3
"""
intel — CLI for World Intelligence MCP.
Calls source functions directly (no MCP protocol overhead).
"""
import asyncio
import json
from typing import Any
import click
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from rich import box
from .cache import Cache
from .circuit_breaker import CircuitBreaker
from .fetcher import Fetcher
from .sources import markets, economic, seismology, wildfire, conflict, military, infrastructure, maritime, climate, news, intelligence, prediction, displacement, aviation, cyber
from .reports import generator as report_gen
console = Console()
# Shared infrastructure — lazily initialized
_fetcher: Fetcher | None = None
def _get_fetcher() -> Fetcher:
global _fetcher
if _fetcher is None:
cache = Cache()
breaker = CircuitBreaker()
_fetcher = Fetcher(cache=cache, breaker=breaker)
return _fetcher
def _run(coro: Any) -> Any:
"""Run an async coroutine from sync CLI context."""
return asyncio.run(coro)
def _print_json(data: dict) -> None:
"""Print raw JSON if --json flag, otherwise formatted."""
console.print_json(json.dumps(data, default=str))
# ---------------------------------------------------------------------------
# Root group
# ---------------------------------------------------------------------------
@click.group()
@click.option("--json-output", is_flag=True, help="Output raw JSON")
@click.pass_context
def main(ctx: click.Context, json_output: bool) -> None:
"""World Intelligence CLI — real-time global intelligence."""
ctx.ensure_object(dict)
ctx.obj["json"] = json_output
# ---------------------------------------------------------------------------
# Markets
# ---------------------------------------------------------------------------
@main.command(name="markets")
@click.option("--symbols", "-s", multiple=True, help="Ticker symbols")
@click.pass_context
def markets_cmd(ctx: click.Context, symbols: tuple[str, ...]) -> None:
"""Stock market index quotes."""
f = _get_fetcher()
sym_list = list(symbols) if symbols else None
data = _run(markets.fetch_market_quotes(f, symbols=sym_list))
if ctx.obj.get("json"):
_print_json(data)
return
quotes = data.get("quotes", [])
if not quotes:
console.print("[yellow]No market data available[/yellow]")
return
table = Table(title="Market Indices", box=box.SIMPLE_HEAVY)
table.add_column("Symbol", style="bold")
table.add_column("Price", justify="right")
table.add_column("Change %", justify="right")
table.add_column("Currency")
for q in quotes:
chg = q.get("change_pct") or 0
price = q.get("price") or 0
style = "green" if chg >= 0 else "red"
table.add_row(
q.get("symbol", "?"),
f"{price:,.2f}",
f"[{style}]{chg:+.2f}%[/{style}]",
q.get("currency", ""),
)
console.print(table)
@main.command()
@click.option("--limit", "-n", default=20, help="Number of coins")
@click.pass_context
def crypto(ctx: click.Context, limit: int) -> None:
"""Top cryptocurrency prices."""
f = _get_fetcher()
data = _run(markets.fetch_crypto_quotes(f, limit=limit))
if ctx.obj.get("json"):
_print_json(data)
return
coins = data.get("coins", [])
if not coins:
console.print("[yellow]No crypto data available[/yellow]")
return
table = Table(title=f"Top {limit} Cryptocurrencies", box=box.SIMPLE_HEAVY)
table.add_column("#", justify="right")
table.add_column("Symbol", style="bold")
table.add_column("Price", justify="right")
table.add_column("24h %", justify="right")
table.add_column("Market Cap", justify="right")
for i, c in enumerate(coins[:limit], 1):
chg = c.get("price_change_percentage_24h", 0) or 0
style = "green" if chg >= 0 else "red"
mcap = c.get("market_cap", 0) or 0
table.add_row(
str(i),
c.get("symbol", "?").upper(),
f"${c.get('current_price', 0):,.2f}",
f"[{style}]{chg:+.2f}%[/{style}]",
f"${mcap:,.0f}",
)
console.print(table)
@main.command()
@click.pass_context
def macro(ctx: click.Context) -> None:
"""7-signal macro dashboard."""
f = _get_fetcher()
data = _run(markets.fetch_macro_signals(f))
if ctx.obj.get("json"):
_print_json(data)
return
signals = data.get("signals", {})
table = Table(title="Macro Signals", box=box.SIMPLE_HEAVY)
table.add_column("Signal", style="bold")
table.add_column("Value", justify="right")
table.add_column("Detail")
for name, info in signals.items():
if info is None:
table.add_row(name, "[dim]unavailable[/dim]", "")
elif isinstance(info, dict):
val = info.get("value", info.get("price", "?"))
detail = info.get("classification", info.get("label", ""))
table.add_row(name, str(val), str(detail))
else:
table.add_row(name, str(info), "")
console.print(table)
# ---------------------------------------------------------------------------
# Economic
# ---------------------------------------------------------------------------
@main.command()
@click.pass_context
def energy(ctx: click.Context) -> None:
"""Oil and natural gas prices (EIA)."""
f = _get_fetcher()
data = _run(economic.fetch_energy_prices(f))
if ctx.obj.get("json") or "error" in data:
_print_json(data)
return
oil = data.get("oil", {})
gas = data.get("natural_gas", {})
table = Table(title="Energy Prices", box=box.SIMPLE_HEAVY)
table.add_column("Commodity", style="bold")
table.add_column("Price", justify="right")
table.add_column("Date")
for name, info in [("Brent Crude", oil.get("brent")), ("WTI Crude", oil.get("wti")), ("Natural Gas", gas)]:
if info and isinstance(info, dict):
table.add_row(name, f"${info.get('price', '?')}", str(info.get("date", "")))
console.print(table)
@main.command()
@click.argument("series_id")
@click.option("--limit", "-n", default=30, help="Number of observations")
@click.pass_context
def fred(ctx: click.Context, series_id: str, limit: int) -> None:
"""FRED economic data series (e.g., UNRATE, GDP, CPIAUCSL)."""
f = _get_fetcher()
data = _run(economic.fetch_fred_series(f, series_id=series_id, limit=limit))
if ctx.obj.get("json") or "error" in data:
_print_json(data)
return
obs = data.get("observations", [])
title = data.get("title", series_id)
table = Table(title=f"FRED: {title}", box=box.SIMPLE_HEAVY)
table.add_column("Date", style="bold")
table.add_column("Value", justify="right")
for o in obs[:20]:
table.add_row(o.get("date", ""), str(o.get("value", "")))
console.print(table)
# ---------------------------------------------------------------------------
# Natural
# ---------------------------------------------------------------------------
@main.command()
@click.option("--min-mag", "-m", default=4.5, help="Minimum magnitude")
@click.option("--hours", "-h", default=24, help="Lookback hours")
@click.pass_context
def earthquakes(ctx: click.Context, min_mag: float, hours: int) -> None:
"""Recent earthquakes (USGS)."""
f = _get_fetcher()
data = _run(seismology.fetch_earthquakes(f, min_magnitude=min_mag, hours=hours))
if ctx.obj.get("json"):
_print_json(data)
return
quakes = data.get("earthquakes", [])
console.print(f"[bold]{data.get('count', 0)} earthquakes[/bold] (M{min_mag}+ in last {hours}h)\n")
table = Table(box=box.SIMPLE_HEAVY)
table.add_column("Mag", justify="right", style="bold")
table.add_column("Location")
table.add_column("Depth (km)", justify="right")
table.add_column("Time")
table.add_column("Alert")
for q in quakes[:25]:
mag = q.get("magnitude", 0)
style = "red bold" if mag >= 6.0 else "yellow" if mag >= 5.0 else ""
alert = q.get("alert_level") or ""
table.add_row(
f"[{style}]{mag:.1f}[/{style}]" if style else f"{mag:.1f}",
q.get("place", "Unknown"),
f"{q.get('depth_km', 0):.1f}",
q.get("time", "")[:19],
alert,
)
console.print(table)
@main.command()
@click.option("--region", "-r", default=None, help="Region name (e.g., north_america)")
@click.pass_context
def fires(ctx: click.Context, region: str | None) -> None:
"""Active wildfires (NASA FIRMS)."""
f = _get_fetcher()
data = _run(wildfire.fetch_wildfires(f, region=region))
if ctx.obj.get("json") or "error" in data:
_print_json(data)
return
console.print(f"[bold]{data.get('total_fires', 0)} high-confidence fires detected[/bold]\n")
for reg_name, reg_data in data.get("fires_by_region", {}).items():
count = reg_data.get("count", 0)
if count == 0:
continue
console.print(f" [bold]{reg_name}[/bold]: {count} fires")
for cluster in reg_data.get("top_clusters", [])[:5]:
console.print(
f" ({cluster.get('lat', 0):.1f}, {cluster.get('lon', 0):.1f}) "
f"{cluster.get('fire_count', 0)} fires, FRP max {cluster.get('max_frp', 0):.0f}"
)
# ---------------------------------------------------------------------------
# Conflict
# ---------------------------------------------------------------------------
@main.command()
@click.option("--country", "-c", default=None, help="Country name")
@click.option("--days", "-d", default=7, help="Lookback days")
@click.pass_context
def conflicts(ctx: click.Context, country: str | None, days: int) -> None:
"""Armed conflict events (ACLED)."""
f = _get_fetcher()
data = _run(conflict.fetch_acled_events(f, country=country, days=days))
if ctx.obj.get("json") or "error" in data:
_print_json(data)
return
events = data.get("events", [])
console.print(f"[bold]{data.get('count', 0)} conflict events[/bold] (last {days}d)\n")
table = Table(box=box.SIMPLE_HEAVY)
table.add_column("Date", style="bold")
table.add_column("Type")
table.add_column("Country")
table.add_column("Location")
table.add_column("Fatalities", justify="right")
for e in events[:25]:
fat = e.get("fatalities", 0) or 0
style = "red bold" if fat >= 10 else "yellow" if fat > 0 else ""
fat_str = f"[{style}]{fat}[/{style}]" if style else str(fat)
table.add_row(
str(e.get("event_date", ""))[:10],
e.get("event_type", ""),
e.get("country", ""),
e.get("location", ""),
fat_str,
)
console.print(table)
# ---------------------------------------------------------------------------
# Military
# ---------------------------------------------------------------------------
@main.command()
@click.option("--bbox", "-b", default=None, help="Bounding box: lamin,lomin,lamax,lomax")
@click.pass_context
def flights(ctx: click.Context, bbox: str | None) -> None:
"""Military aircraft tracking (OpenSky)."""
f = _get_fetcher()
data = _run(military.fetch_military_flights(f, bbox=bbox))
if ctx.obj.get("json"):
_print_json(data)
return
aircraft = data.get("aircraft", [])
console.print(f"[bold]{data.get('count', 0)} military aircraft detected[/bold]\n")
table = Table(box=box.SIMPLE_HEAVY)
table.add_column("Callsign", style="bold")
table.add_column("ICAO24")
table.add_column("Country")
table.add_column("Alt (m)", justify="right")
table.add_column("Speed (m/s)", justify="right")
for a in aircraft[:30]:
table.add_row(
a.get("callsign", "?"),
a.get("icao24", ""),
a.get("origin_country", ""),
f"{a.get('altitude_m') or 0:,.0f}",
f"{a.get('velocity_ms') or 0:.0f}",
)
console.print(table)
@main.command()
@click.pass_context
def posture(ctx: click.Context) -> None:
"""Military theater posture (5 theaters)."""
f = _get_fetcher()
data = _run(military.fetch_theater_posture(f))
if ctx.obj.get("json"):
_print_json(data)
return
console.print(f"[bold]{data.get('total_military_aircraft', 0)} total military aircraft[/bold]\n")
theaters = data.get("theaters", {})
table = Table(title="Theater Posture", box=box.SIMPLE_HEAVY)
table.add_column("Theater", style="bold")
table.add_column("Aircraft", justify="right")
table.add_column("Countries")
table.add_column("Sample Callsigns")
for name, info in theaters.items():
count = info.get("count", 0)
style = "red bold" if count >= 20 else "yellow" if count >= 5 else ""
count_str = f"[{style}]{count}[/{style}]" if style else str(count)
table.add_row(
name.replace("_", " ").title(),
count_str,
", ".join(info.get("countries", [])[:5]),
", ".join(info.get("sample_callsigns", [])[:3]),
)
console.print(table)
# ---------------------------------------------------------------------------
# Infrastructure
# ---------------------------------------------------------------------------
@main.command()
@click.pass_context
def outages(ctx: click.Context) -> None:
"""Internet outages (Cloudflare Radar)."""
f = _get_fetcher()
data = _run(infrastructure.fetch_internet_outages(f))
if ctx.obj.get("json"):
_print_json(data)
return
console.print(f"[bold]{data.get('ongoing_count', 0)} ongoing outages[/bold], "
f"{data.get('total_7d', 0)} in last 7 days\n")
for o in data.get("outages", [])[:15]:
ongoing = "[red]ONGOING[/red]" if o.get("is_ongoing") else ""
countries = ", ".join(o.get("countries", [])[:5]) if o.get("countries") else ""
console.print(f" {o.get('start', '')[:16]} {countries} {o.get('description', '')[:80]} {ongoing}")
@main.command()
@click.pass_context
def cables(ctx: click.Context) -> None:
"""Undersea cable corridor health (NGA)."""
f = _get_fetcher()
data = _run(infrastructure.fetch_cable_health(f))
if ctx.obj.get("json"):
_print_json(data)
return
status_labels = {0: "[green]Clear[/green]", 1: "[yellow]Advisory[/yellow]",
2: "[red]At Risk[/red]", 3: "[red bold]Disrupted[/red bold]"}
table = Table(title="Undersea Cable Health", box=box.SIMPLE_HEAVY)
table.add_column("Corridor", style="bold")
table.add_column("Status")
table.add_column("Cables")
table.add_column("Warnings", justify="right")
for name, info in data.get("corridors", {}).items():
score = info.get("status_score", 0)
table.add_row(
name.replace("_", " ").title(),
status_labels.get(score, str(score)),
", ".join(info.get("cables", [])[:3]),
str(len(info.get("relevant_warnings", []))),
)
console.print(table)
# ---------------------------------------------------------------------------
# Maritime
# ---------------------------------------------------------------------------
@main.command()
@click.option("--navarea", "-n", default=None, help="NAVAREA number (e.g., IV)")
@click.pass_context
def warnings(ctx: click.Context, navarea: str | None) -> None:
"""Navigational warnings (NGA Maritime Safety)."""
f = _get_fetcher()
data = _run(maritime.fetch_nav_warnings(f, navarea=navarea))
if ctx.obj.get("json"):
_print_json(data)
return
console.print(f"[bold]{data.get('count', 0)} active warnings[/bold]\n")
by_area = data.get("by_navarea", {})
if by_area:
console.print(" By NAVAREA: " + ", ".join(f"{k}:{v}" for k, v in sorted(by_area.items())))
console.print()
for w in data.get("warnings", [])[:20]:
console.print(f" [{w.get('navarea', '?')}] {w.get('id', '')} {w.get('text', '')[:100]}")
# ---------------------------------------------------------------------------
# Climate
# ---------------------------------------------------------------------------
@main.command(name="climate")
@click.pass_context
def climate_cmd(ctx: click.Context) -> None:
"""Climate anomalies (15 global zones vs. prior year)."""
f = _get_fetcher()
data = _run(climate.fetch_climate_anomalies(f))
if ctx.obj.get("json"):
_print_json(data)
return
zones = data.get("zones", {})
sig = data.get("significant_anomalies", [])
table = Table(title="Climate Anomalies", box=box.SIMPLE_HEAVY)
table.add_column("Zone", style="bold")
table.add_column("Temp Anomaly", justify="right")
table.add_column("Precip Anomaly", justify="right")
table.add_column("Flag")
for key, z in zones.items():
temp_a = z.get("temp_anomaly_c", 0)
prec_a = z.get("precip_anomaly_pct", 0)
t_style = "red" if temp_a > 3 else "blue" if temp_a < -3 else ""
flag = "[red bold]SIG[/red bold]" if key in sig else ""
t_str = f"[{t_style}]{temp_a:+.1f}C[/{t_style}]" if t_style else f"{temp_a:+.1f}C"
table.add_row(z.get("name", key), t_str, f"{prec_a:+.0f}%", flag)
console.print(table)
# ---------------------------------------------------------------------------
# News
# ---------------------------------------------------------------------------
@main.command(name="news")
@click.option("--category", "-c", default=None,
type=click.Choice(["geopolitics", "security", "technology", "finance", "military", "science"]),
help="Category filter")
@click.option("--limit", "-n", default=30, help="Max items")
@click.pass_context
def news_cmd(ctx: click.Context, category: str | None, limit: int) -> None:
"""Intelligence news from 20+ RSS feeds."""
f = _get_fetcher()
data = _run(news.fetch_news_feed(f, category=category, limit=limit))
if ctx.obj.get("json"):
_print_json(data)
return
items = data.get("items", [])
if not items:
console.print("[yellow]No news items available[/yellow]")
return
console.print(f"[bold]{data.get('count', 0)} items[/bold] from {', '.join(data.get('categories_fetched', []))}\n")
for item in items:
cat = item.get("category", "")
title = item.get("title", "")
feed = item.get("feed_name", "")
pub = (item.get("published") or "")[:16]
console.print(f" [{cat}] [bold]{title}[/bold]")
console.print(f" {feed}{pub}")
@main.command()
@click.option("--min-count", "-m", default=3, help="Minimum keyword occurrences")
@click.pass_context
def trending(ctx: click.Context, min_count: int) -> None:
"""Trending keywords from recent news."""
f = _get_fetcher()
data = _run(news.fetch_trending_keywords(f, min_count=min_count))
if ctx.obj.get("json"):
_print_json(data)
return
keywords = data.get("keywords", [])
console.print(f"[bold]Trending keywords[/bold] (from {data.get('total_items_analyzed', 0)} items)\n")
table = Table(box=box.SIMPLE_HEAVY)
table.add_column("#", justify="right")
table.add_column("Keyword", style="bold")
table.add_column("Count", justify="right")
for i, kw in enumerate(keywords[:30], 1):
table.add_row(str(i), kw["word"], str(kw["count"]))
console.print(table)
@main.command()
@click.argument("query", default="conflict")
@click.option("--mode", "-m", default="artlist", type=click.Choice(["artlist", "timelinevol"]))
@click.option("--limit", "-n", default=20, help="Max records")
@click.pass_context
def gdelt(ctx: click.Context, query: str, mode: str, limit: int) -> None:
"""Search GDELT 2.0 global news database."""
f = _get_fetcher()
data = _run(news.fetch_gdelt_search(f, query=query, mode=mode, limit=limit))
if ctx.obj.get("json"):
_print_json(data)
return
if mode == "artlist":
articles = data.get("articles", [])
console.print(f"[bold]{len(articles)} articles[/bold] for '{query}'\n")
for a in articles[:20]:
title = a.get("title", "")[:80]
domain = a.get("domain", "")
console.print(f" [bold]{title}[/bold] ({domain})")
else:
console.print(f"[bold]Timeline volume for '{query}'[/bold]")
_print_json(data)
# ---------------------------------------------------------------------------
# Prediction
# ---------------------------------------------------------------------------
@main.command()
@click.option("--limit", "-n", default=20, help="Number of markets")
@click.pass_context
def predictions(ctx: click.Context, limit: int) -> None:
"""Prediction market movers (Polymarket)."""
f = _get_fetcher()
data = _run(prediction.fetch_prediction_markets(f, limit=limit))
if ctx.obj.get("json"):
_print_json(data)
return
mkts = data.get("markets", [])
if not mkts:
console.print("[yellow]No prediction market data available[/yellow]")
return
table = Table(title="Prediction Markets", box=box.SIMPLE_HEAVY)
table.add_column("Question", max_width=50)
table.add_column("YES %", justify="right")
table.add_column("Sentiment")
table.add_column("24h Vol", justify="right")
for m in mkts:
yes_pct = (m.get("yes_probability", 0) or 0) * 100
sentiment = m.get("sentiment", "")
vol = m.get("volume_24h", 0) or 0
s_style = "green" if "yes" in sentiment else "red" if "no" in sentiment else "yellow"
table.add_row(
(m.get("question", "")[:50]),
f"{yes_pct:.0f}%",
f"[{s_style}]{sentiment}[/{s_style}]",
f"${vol:,.0f}",
)
console.print(table)
# ---------------------------------------------------------------------------
# Displacement
# ---------------------------------------------------------------------------
@main.command(name="displacement")
@click.option("--year", "-y", default=None, type=int, help="Reporting year")
@click.pass_context
def displacement_cmd(ctx: click.Context, year: int | None) -> None:
"""UNHCR displacement statistics."""
f = _get_fetcher()
data = _run(displacement.fetch_displacement_summary(f, year=year))
if ctx.obj.get("json"):
_print_json(data)
return
totals = data.get("global_totals", {})
console.print(f"[bold]Global Displacement ({data.get('year', '?')})[/bold]")
console.print(f" Grand total: {totals.get('grand_total', 0):,}\n")
by_origin = data.get("by_origin", [])
table = Table(title="Top Countries of Origin", box=box.SIMPLE_HEAVY)
table.add_column("Country", style="bold")
table.add_column("Total Displaced", justify="right")
table.add_column("Refugees", justify="right")
table.add_column("IDPs", justify="right")
for c in by_origin[:15]:
table.add_row(
c.get("country", ""),
f"{c.get('total_displaced', 0):,}",
f"{c.get('refugees', 0):,}",
f"{c.get('internally_displaced', 0):,}",
)
console.print(table)
# ---------------------------------------------------------------------------
# Aviation
# ---------------------------------------------------------------------------
@main.command()
@click.pass_context
def delays(ctx: click.Context) -> None:
"""US airport delays (FAA)."""
f = _get_fetcher()
data = _run(aviation.fetch_airport_delays(f))
if ctx.obj.get("json"):
_print_json(data)
return
delayed = data.get("delayed", [])
console.print(f"[bold]{data.get('delayed_count', 0)} airports with delays[/bold] "
f"(checked {data.get('total_checked', 0)})\n")
if not delayed:
console.print("[green]No major airport delays![/green]")
return
table = Table(box=box.SIMPLE_HEAVY)
table.add_column("Airport", style="bold")
table.add_column("Name")
table.add_column("Delay Info")
for d in delayed:
statuses = d.get("status", [])
info = "; ".join(
f"{s.get('type', '')} - {s.get('reason', '')} ({s.get('avg_delay', '')})"
for s in statuses
) if statuses else "Details unavailable"
table.add_row(d.get("code", ""), d.get("name", ""), info[:80])
console.print(table)
# ---------------------------------------------------------------------------
# Cyber
# ---------------------------------------------------------------------------
@main.command()
@click.option("--limit", "-n", default=30, help="Max threats")
@click.pass_context
def threats(ctx: click.Context, limit: int) -> None:
"""Cyber threat intelligence (4 feeds)."""
f = _get_fetcher()
data = _run(cyber.fetch_cyber_threats(f, limit=limit))
if ctx.obj.get("json"):
_print_json(data)
return
by_sev = data.get("by_severity", {})
console.print(f"[bold]{data.get('count', 0)} threats[/bold] "
f"({data.get('feeds_successful', 0)}/{data.get('feeds_attempted', 0)} feeds)")
console.print(f" [red]Critical: {by_sev.get('critical', 0)}[/red] "
f"[yellow]High: {by_sev.get('high', 0)}[/yellow] "
f"Medium: {by_sev.get('medium', 0)} "
f"[dim]Low: {by_sev.get('low', 0)}[/dim]\n")
table = Table(box=box.SIMPLE_HEAVY)
table.add_column("Severity")
table.add_column("Type")
table.add_column("Indicator", max_width=40)
table.add_column("Threat")
table.add_column("Feed")
for t in data.get("threats", [])[:limit]:
sev = t.get("severity", "")
sev_style = {"critical": "red bold", "high": "yellow", "medium": "", "low": "dim"}.get(sev, "")
sev_str = f"[{sev_style}]{sev}[/{sev_style}]" if sev_style else sev
table.add_row(
sev_str,
t.get("type", ""),
(t.get("indicator", ""))[:40],
(t.get("threat", ""))[:30],
t.get("source_feed", ""),
)
console.print(table)
# ---------------------------------------------------------------------------
# Intelligence
# ---------------------------------------------------------------------------
@main.command()
@click.argument("country_code", default="US")
@click.pass_context
def brief(ctx: click.Context, country_code: str) -> None:
"""Country intelligence brief (LLM + data)."""
f = _get_fetcher()
data = _run(intelligence.fetch_country_brief(f, country_code=country_code))
if ctx.obj.get("json"):
_print_json(data)
return
llm_tag = "[green]LLM[/green]" if data.get("llm_available") else "[yellow]data-only[/yellow]"
console.print(f"[bold]Intelligence Brief: {country_code}[/bold] ({llm_tag})\n")
console.print(data.get("brief", "No brief available."))
d = data.get("data", {})
if d.get("gdp") or d.get("recent_events"):
console.print(f"\n[dim]GDP data points: {len(d.get('gdp', []))} | "
f"Recent conflict events: {d.get('recent_events', 0)}[/dim]")
@main.command()
@click.option("--limit", "-n", default=20, help="Top N countries")
@click.pass_context
def risk(ctx: click.Context, limit: int) -> None:
"""Country risk scores (ACLED-based)."""
f = _get_fetcher()
data = _run(intelligence.fetch_risk_scores(f, limit=limit))
if ctx.obj.get("json") or "error" in data:
_print_json(data)
return
table = Table(title="Country Risk Scores", box=box.SIMPLE_HEAVY)
table.add_column("#", justify="right")
table.add_column("Country", style="bold")
table.add_column("Events (30d)", justify="right")
table.add_column("Risk Score", justify="right")
table.add_column("Level")
for i, c in enumerate(data.get("countries", []), 1):
level = c.get("risk_level", "")
l_style = {"critical": "red bold", "elevated": "yellow", "moderate": "", "low": "dim"}.get(level, "")
l_str = f"[{l_style}]{level}[/{l_style}]" if l_style else level
table.add_row(
str(i),
c.get("country", ""),
str(c.get("events_30d", 0)),
f"{c.get('risk_score', 0):.0f}",
l_str,
)
console.print(table)
@main.command()
@click.argument("country_code", required=False, default=None)
@click.pass_context
def instability(ctx: click.Context, country_code: str | None) -> None:
"""Country Instability Index (0-100)."""
f = _get_fetcher()
data = _run(intelligence.fetch_instability_index(f, country_code=country_code))
if ctx.obj.get("json") or "error" in data:
_print_json(data)
return
if country_code:
idx = data.get("instability_index", 0)
level = data.get("risk_level", "")
console.print(f"[bold]{country_code} Instability Index: {idx}/100 ({level})[/bold]\n")
components = data.get("components", {})
for name, score in components.items():
bar = "" * int(score) + "" * (20 - int(score))
console.print(f" {name:30s} {bar} {score:.1f}/20")
else:
table = Table(title="Instability Index (Focus Countries)", box=box.SIMPLE_HEAVY)
table.add_column("Country", style="bold")
table.add_column("Index", justify="right")
table.add_column("Events (30d)", justify="right")
table.add_column("Risk Level")
for c in data.get("countries", []):
level = c.get("risk_level", "")
l_style = {"critical": "red bold", "high": "yellow", "medium": "", "low": "dim"}.get(level, "")
l_str = f"[{l_style}]{level}[/{l_style}]" if l_style else level
table.add_row(
f"{c.get('country_name', '')} ({c.get('country_code', '')})",
f"{c.get('instability_index', 0):.0f}",
str(c.get("events_30d", 0)),
l_str,
)
console.print(table)
# ---------------------------------------------------------------------------
# Reports
# ---------------------------------------------------------------------------
@main.group()
def report() -> None:
"""Generate intelligence reports (HTML)."""
main.add_command(report)
@report.command(name="daily")
@click.option("--output-dir", "-o", default=None, help="Output directory")
def report_daily(output_dir: str | None) -> None:
"""Generate daily intelligence brief (HTML)."""
console.print("[bold]Generating daily brief...[/bold]")
result = _run(report_gen.generate_daily_brief(output_dir=output_dir))
console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}")
summary = result.get("summary", {})
console.print(f" Quotes: {summary.get('market_quotes', 0)} | "
f"Conflicts: {summary.get('conflict_events', 0)} | "
f"Threats: {summary.get('cyber_threats', 0)} | "
f"Quakes: {summary.get('earthquakes', 0)}")
@report.command(name="threat")
@click.option("--output-dir", "-o", default=None, help="Output directory")
def report_threat(output_dir: str | None) -> None:
"""Generate threat landscape report (HTML)."""
console.print("[bold]Generating threat landscape...[/bold]")
result = _run(report_gen.generate_threat_landscape(output_dir=output_dir))
console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}")
@report.command(name="market")
@click.option("--output-dir", "-o", default=None, help="Output directory")
def report_market(output_dir: str | None) -> None:
"""Generate market overview report (HTML)."""
console.print("[bold]Generating market overview...[/bold]")
result = _run(report_gen.generate_market_overview(output_dir=output_dir))
console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}")
@report.command(name="dossier")
@click.argument("country_code")
@click.option("--output-dir", "-o", default=None, help="Output directory")
def report_dossier(country_code: str, output_dir: str | None) -> None:
"""Generate country dossier report (HTML)."""
console.print(f"[bold]Generating dossier for {country_code}...[/bold]")
result = _run(report_gen.generate_country_dossier(
country_code=country_code, output_dir=output_dir,
))
console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}")
# ---------------------------------------------------------------------------
# System
# ---------------------------------------------------------------------------
@main.command()
@click.pass_context
def status(ctx: click.Context) -> None:
"""Data source health and cache stats."""
f = _get_fetcher()
breaker_status = f.breaker.status()
cache_stats = f.cache.stats()
if ctx.obj.get("json"):
_print_json({"circuit_breakers": breaker_status, "cache": cache_stats})
return
console.print(Panel("[bold]World Intelligence Status[/bold]"))
# Cache
console.print(f"\n[bold]Cache:[/bold] {cache_stats.get('active_entries', 0)} active, "
f"{cache_stats.get('expired_entries', 0)} expired")
# Circuit breakers
if breaker_status:
table = Table(title="Circuit Breakers", box=box.SIMPLE_HEAVY)
table.add_column("Source", style="bold")
table.add_column("Status")
table.add_column("Failures", justify="right")
table.add_column("Cooldown", justify="right")
for source, info in sorted(breaker_status.items()):
s = info.get("status", "closed")
style = "green" if s == "closed" else "yellow" if s == "half-open" else "red"
table.add_row(
source,
f"[{style}]{s}[/{style}]",
str(info.get("failures", 0)),
f"{info.get('cooldown_remaining_s', 0):.0f}s" if info.get("cooldown_remaining_s") else "",
)
console.print(table)
else:
console.print("\n[dim]No circuit breaker data yet (no requests made)[/dim]")
@main.command(name="sync")
@click.argument("source", required=False)
def sync_cmd(source: str | None) -> None:
"""Force refresh a data source cache."""
f = _get_fetcher()
if source:
# Delete all cache entries matching this source prefix
# Simple approach: evict expired, then note we can't selectively clear yet
console.print(f"[yellow]Force sync not yet implemented for specific source '{source}'[/yellow]")
console.print("[dim]Workaround: cache entries expire naturally based on TTL[/dim]")
else:
removed = f.cache.evict_expired()
console.print(f"Evicted {removed} expired cache entries")
if __name__ == "__main__":
main()
+191
View File
@@ -0,0 +1,191 @@
"""Async HTTP fetcher with timeout, retry, rate limiting, and circuit breaker integration.
All external HTTP calls in world-intel-mcp go through this module.
"""
import asyncio
import logging
import time
from typing import Any
import httpx
from .cache import Cache
from .circuit_breaker import CircuitBreaker
logger = logging.getLogger("world-intel-mcp.fetcher")
# Yahoo Finance requires serialized access (600ms gap)
_yahoo_lock = asyncio.Lock()
_yahoo_last_call: float = 0.0
_YAHOO_MIN_INTERVAL = 0.6 # seconds
class Fetcher:
"""Centralized HTTP fetcher with caching, retries, and circuit breaking."""
def __init__(
self,
cache: Cache,
breaker: CircuitBreaker,
default_timeout: float = 15.0,
max_retries: int = 2,
client: httpx.AsyncClient | None = None,
):
self.cache = cache
self.breaker = breaker
self.default_timeout = default_timeout
self.max_retries = max_retries
self._client: httpx.AsyncClient | None = client
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(self.default_timeout),
follow_redirects=True,
limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
headers={"User-Agent": "PhoenixAGI-WorldIntel/0.1"},
proxy=None, # never inherit system SOCKS proxy
)
return self._client
async def close(self) -> None:
if self._client and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def get_json(
self,
url: str,
source: str,
cache_key: str | None = None,
cache_ttl: int = 300,
headers: dict[str, str] | None = None,
params: dict[str, Any] | None = None,
timeout: float | None = None,
yahoo_rate_limit: bool = False,
) -> dict | list | None:
"""Fetch JSON with caching, circuit breaking, and retries.
Args:
url: Target URL.
source: Source name for circuit breaker tracking.
cache_key: Cache key. If None, uses url+params hash.
cache_ttl: Cache TTL in seconds.
headers: Extra HTTP headers.
params: Query parameters.
timeout: Per-request timeout override.
yahoo_rate_limit: If True, enforce Yahoo Finance 600ms serialization.
Returns:
Parsed JSON or None on failure.
"""
# Check circuit breaker
if not self.breaker.is_available(source):
logger.debug("Circuit open for %s, skipping", source)
return None
# Check cache
effective_key = cache_key or f"{source}:{url}:{params}"
cached = self.cache.get(effective_key)
if cached is not None:
return cached
# Yahoo rate limiting
if yahoo_rate_limit:
await self._yahoo_throttle()
# Fetch with retries
client = await self._get_client()
last_error: Exception | None = None
for attempt in range(self.max_retries + 1):
try:
resp = await client.get(
url,
headers=headers,
params=params,
timeout=timeout or self.default_timeout,
)
resp.raise_for_status()
data = resp.json()
self.breaker.record_success(source)
self.cache.set(effective_key, data, cache_ttl)
return data
except (httpx.HTTPStatusError, httpx.RequestError, Exception) as exc:
last_error = exc
if attempt < self.max_retries:
wait = 1.0 * (attempt + 1)
logger.debug("Retry %d/%d for %s (%s), waiting %.1fs",
attempt + 1, self.max_retries, source, exc, wait)
await asyncio.sleep(wait)
# All retries failed
self.breaker.record_failure(source)
logger.warning("Fetch failed for %s: %s (url=%s)", source, last_error, url)
return None
async def get_text(
self,
url: str,
source: str,
cache_key: str | None = None,
cache_ttl: int = 300,
headers: dict[str, str] | None = None,
params: dict[str, Any] | None = None,
timeout: float | None = None,
) -> str | None:
"""Fetch raw text with caching and circuit breaking."""
if not self.breaker.is_available(source):
return None
effective_key = cache_key or f"{source}:text:{url}:{params}"
cached = self.cache.get(effective_key)
if cached is not None:
return cached
client = await self._get_client()
last_error: Exception | None = None
for attempt in range(self.max_retries + 1):
try:
resp = await client.get(
url,
headers=headers,
params=params,
timeout=timeout or self.default_timeout,
)
resp.raise_for_status()
text = resp.text
self.breaker.record_success(source)
self.cache.set(effective_key, text, cache_ttl)
return text
except (httpx.HTTPStatusError, httpx.RequestError, Exception) as exc:
last_error = exc
if attempt < self.max_retries:
await asyncio.sleep(1.0 * (attempt + 1))
self.breaker.record_failure(source)
logger.warning("Text fetch failed for %s: %s", source, last_error)
return None
async def get_xml(
self,
url: str,
source: str,
cache_key: str | None = None,
cache_ttl: int = 300,
timeout: float | None = None,
) -> str | None:
"""Fetch XML content (returns raw text for feedparser/ET parsing)."""
return await self.get_text(url, source, cache_key, cache_ttl, timeout=timeout)
async def _yahoo_throttle(self) -> None:
"""Enforce Yahoo Finance rate limit (600ms between calls)."""
global _yahoo_last_call
async with _yahoo_lock:
now = time.time()
elapsed = now - _yahoo_last_call
if elapsed < _YAHOO_MIN_INTERVAL:
await asyncio.sleep(_YAHOO_MIN_INTERVAL - elapsed)
_yahoo_last_call = time.time()
+1
View File
@@ -0,0 +1 @@
"""Report generation for world-intel-mcp."""
+299
View File
@@ -0,0 +1,299 @@
"""Report orchestrator for world-intel-mcp.
Gathers data from multiple intelligence sources in parallel and renders
HTML or Markdown reports via Jinja2 templates.
"""
import asyncio
import logging
import os
from datetime import datetime, timezone
from pathlib import Path
from ..cache import Cache
from ..circuit_breaker import CircuitBreaker
from ..fetcher import Fetcher
from ..sources import (
markets, economic, seismology, wildfire, conflict, military,
infrastructure, maritime, climate, news, intelligence,
prediction, displacement, aviation, cyber,
)
logger = logging.getLogger("world-intel-mcp.reports.generator")
# Default output directory
_DEFAULT_OUTPUT_DIR = os.environ.get(
"INTEL_REPORT_DIR",
os.path.join(os.environ.get("STORAGE_BASE", "/tmp"), "reports", "intel"),
)
def _ensure_output_dir(path: str | None = None) -> Path:
"""Create output directory if needed and return Path."""
output_dir = Path(path or _DEFAULT_OUTPUT_DIR)
output_dir.mkdir(parents=True, exist_ok=True)
return output_dir
async def generate_daily_brief(output_dir: str | None = None) -> dict:
"""Generate a daily intelligence brief HTML report.
Gathers: market quotes, macro signals, conflict events, cyber threats,
earthquakes, wildfires, prediction markets, trending keywords.
Renders to HTML and returns the file path + summary.
"""
cache = Cache()
breaker = CircuitBreaker()
fetcher = Fetcher(cache=cache, breaker=breaker)
try:
# Gather all data in parallel
(
market_data,
macro_data,
conflict_data,
cyber_data,
quake_data,
fire_data,
predict_data,
keyword_data,
) = await asyncio.gather(
markets.fetch_market_quotes(fetcher),
markets.fetch_macro_signals(fetcher),
conflict.fetch_acled_events(fetcher, days=1, limit=50),
cyber.fetch_cyber_threats(fetcher, limit=30),
seismology.fetch_earthquakes(fetcher, min_magnitude=4.5, hours=24),
wildfire.fetch_wildfires(fetcher),
prediction.fetch_prediction_markets(fetcher, limit=10),
news.fetch_trending_keywords(fetcher, min_count=3),
)
now = datetime.now(timezone.utc)
context = {
"title": "Daily Intelligence Brief",
"generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"market_summary": {
"quotes": market_data.get("quotes", []),
"macro_signals": macro_data.get("signals", {}),
},
"conflict_summary": {
"events": conflict_data.get("events", []),
"count": conflict_data.get("count", 0),
},
"cyber_summary": {
"threats": cyber_data.get("threats", []),
"by_severity": cyber_data.get("by_severity", {}),
},
"natural_summary": {
"earthquakes": quake_data.get("earthquakes", []),
"fire_count": fire_data.get("total_fires", 0),
},
"prediction_highlights": predict_data.get("markets", []),
"trending_keywords": keyword_data.get("keywords", []),
}
# Render HTML
from .html_report import render_template
html = render_template("daily_brief.html", context)
# Write to file
out_dir = _ensure_output_dir(output_dir)
filename = f"daily_brief_{now.strftime('%Y%m%d_%H%M%S')}.html"
filepath = out_dir / filename
filepath.write_text(html, encoding="utf-8")
logger.info("Daily brief generated: %s", filepath)
return {
"report_type": "daily_brief",
"file_path": str(filepath),
"generated_at": context["generated_at"],
"summary": {
"market_quotes": len(context["market_summary"]["quotes"]),
"conflict_events": context["conflict_summary"]["count"],
"cyber_threats": len(context["cyber_summary"]["threats"]),
"earthquakes": len(context["natural_summary"]["earthquakes"]),
"predictions": len(context["prediction_highlights"]),
"keywords": len(context["trending_keywords"]),
},
}
finally:
await fetcher.close()
async def generate_country_dossier(
country_code: str,
output_dir: str | None = None,
) -> dict:
"""Generate a country dossier HTML report."""
cache = Cache()
breaker = CircuitBreaker()
fetcher = Fetcher(cache=cache, breaker=breaker)
try:
(
brief_data,
instability_data,
conflict_data,
displacement_data,
) = await asyncio.gather(
intelligence.fetch_country_brief(fetcher, country_code=country_code),
intelligence.fetch_instability_index(fetcher, country_code=country_code),
conflict.fetch_acled_events(fetcher, country=country_code, days=30, limit=50),
displacement.fetch_displacement_summary(fetcher),
)
now = datetime.now(timezone.utc)
context = {
"title": f"Country Dossier: {country_code}",
"generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"country_code": country_code,
"brief": brief_data.get("brief", ""),
"instability": {
"instability_index": instability_data.get("instability_index", 0),
"components": instability_data.get("components", {}),
"risk_level": instability_data.get("risk_level", "unknown"),
},
"conflict_events": conflict_data.get("events", []),
"displacement": {
"by_origin": displacement_data.get("by_origin", []),
"global_totals": displacement_data.get("global_totals", {}),
},
"economic": {
"gdp": brief_data.get("data", {}).get("gdp", []),
"inflation": brief_data.get("data", {}).get("inflation", []),
},
}
from .html_report import render_template
html = render_template("country_dossier.html", context)
out_dir = _ensure_output_dir(output_dir)
filename = f"dossier_{country_code}_{now.strftime('%Y%m%d_%H%M%S')}.html"
filepath = out_dir / filename
filepath.write_text(html, encoding="utf-8")
logger.info("Country dossier generated: %s", filepath)
return {
"report_type": "country_dossier",
"country_code": country_code,
"file_path": str(filepath),
"generated_at": context["generated_at"],
}
finally:
await fetcher.close()
async def generate_threat_landscape(output_dir: str | None = None) -> dict:
"""Generate a threat landscape HTML report."""
cache = Cache()
breaker = CircuitBreaker()
fetcher = Fetcher(cache=cache, breaker=breaker)
try:
(
cyber_data,
conflict_data,
military_data,
cable_data,
outage_data,
) = await asyncio.gather(
cyber.fetch_cyber_threats(fetcher, limit=50),
conflict.fetch_acled_events(fetcher, days=7, limit=100),
military.fetch_theater_posture(fetcher),
infrastructure.fetch_cable_health(fetcher),
infrastructure.fetch_internet_outages(fetcher),
)
now = datetime.now(timezone.utc)
context = {
"title": "Threat Landscape Report",
"generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"cyber_threats": {
"threats": cyber_data.get("threats", []),
"by_severity": cyber_data.get("by_severity", {}),
"by_type": cyber_data.get("by_type", {}),
},
"conflict_events": conflict_data.get("events", []),
"military_activity": {
"theaters": military_data.get("theaters", {}),
"total_military_aircraft": military_data.get("total_military_aircraft", 0),
},
"cable_health": {
"corridors": cable_data.get("corridors", {}),
},
"outages": {
"outages": outage_data.get("outages", []),
"ongoing_count": outage_data.get("ongoing_count", 0),
},
}
from .html_report import render_template
html = render_template("threat_landscape.html", context)
out_dir = _ensure_output_dir(output_dir)
filename = f"threat_landscape_{now.strftime('%Y%m%d_%H%M%S')}.html"
filepath = out_dir / filename
filepath.write_text(html, encoding="utf-8")
logger.info("Threat landscape generated: %s", filepath)
return {
"report_type": "threat_landscape",
"file_path": str(filepath),
"generated_at": context["generated_at"],
}
finally:
await fetcher.close()
async def generate_market_overview(output_dir: str | None = None) -> dict:
"""Generate a market overview HTML report."""
cache = Cache()
breaker = CircuitBreaker()
fetcher = Fetcher(cache=cache, breaker=breaker)
try:
(
quote_data,
crypto_data,
macro_data,
sector_data,
etf_data,
) = await asyncio.gather(
markets.fetch_market_quotes(fetcher),
markets.fetch_crypto_quotes(fetcher, limit=20),
markets.fetch_macro_signals(fetcher),
markets.fetch_sector_heatmap(fetcher),
markets.fetch_etf_flows(fetcher),
)
now = datetime.now(timezone.utc)
context = {
"title": "Market Overview",
"generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"quotes": quote_data.get("quotes", []),
"crypto": crypto_data.get("coins", []),
"macro_signals": macro_data.get("signals", {}),
"sector_heatmap": sector_data.get("sectors", []),
"etf_flows": etf_data,
}
from .html_report import render_template
html = render_template("market_overview.html", context)
out_dir = _ensure_output_dir(output_dir)
filename = f"market_overview_{now.strftime('%Y%m%d_%H%M%S')}.html"
filepath = out_dir / filename
filepath.write_text(html, encoding="utf-8")
logger.info("Market overview generated: %s", filepath)
return {
"report_type": "market_overview",
"file_path": str(filepath),
"generated_at": context["generated_at"],
}
finally:
await fetcher.close()
@@ -0,0 +1,35 @@
"""HTML report renderer using Jinja2 templates.
Loads templates from the ``templates/`` subdirectory and renders them
with the provided context data.
"""
import logging
from pathlib import Path
from jinja2 import Environment, FileSystemLoader, select_autoescape
logger = logging.getLogger("world-intel-mcp.reports.html_report")
_TEMPLATE_DIR = Path(__file__).parent / "templates"
_env = Environment(
loader=FileSystemLoader(str(_TEMPLATE_DIR)),
autoescape=select_autoescape(["html"]),
trim_blocks=True,
lstrip_blocks=True,
)
def render_template(template_name: str, context: dict) -> str:
"""Render a Jinja2 HTML template with the given context.
Args:
template_name: Name of the template file in ``templates/``.
context: Dict of variables to pass to the template.
Returns:
Rendered HTML string.
"""
template = _env.get_template(template_name)
return template.render(**context)
@@ -0,0 +1,161 @@
"""Markdown report generator for world-intel-mcp.
Generates Markdown reports with optional Mermaid diagrams.
"""
import logging
from datetime import datetime, timezone
logger = logging.getLogger("world-intel-mcp.reports.markdown_report")
def generate_daily_brief_md(
market_summary: dict,
conflict_summary: dict,
cyber_summary: dict,
natural_summary: dict,
prediction_highlights: list,
trending_keywords: list,
) -> str:
"""Generate a daily intelligence brief in Markdown format."""
now = datetime.now(timezone.utc)
lines = [
f"# Daily Intelligence Brief",
f"*Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}*",
"",
"## Markets",
]
quotes = market_summary.get("quotes", [])
if quotes:
lines.append("| Symbol | Price | Change |")
lines.append("|--------|------:|-------:|")
for q in quotes[:8]:
chg = q.get("change_pct") or 0
lines.append(f"| {q.get('symbol', '?')} | {q.get('price', 0):,.2f} | {chg:+.2f}% |")
lines.append("")
# Conflict
events = conflict_summary.get("events", [])
lines.append(f"## Conflict ({conflict_summary.get('count', 0)} events)")
if events:
lines.append("| Date | Type | Country | Fatalities |")
lines.append("|------|------|---------|----------:|")
for e in events[:10]:
lines.append(
f"| {(e.get('event_date') or '')[:10]} "
f"| {e.get('event_type', '')} "
f"| {e.get('country', '')} "
f"| {e.get('fatalities', 0)} |"
)
lines.append("")
# Cyber
by_sev = cyber_summary.get("by_severity", {})
lines.append(f"## Cyber Threats")
lines.append(f"Critical: {by_sev.get('critical', 0)} | "
f"High: {by_sev.get('high', 0)} | "
f"Medium: {by_sev.get('medium', 0)}")
lines.append("")
# Natural
quakes = natural_summary.get("earthquakes", [])
lines.append(f"## Natural Events")
lines.append(f"Earthquakes: {len(quakes)} | Fires: {natural_summary.get('fire_count', 0)}")
if quakes:
lines.append("")
lines.append("| Mag | Location | Depth |")
lines.append("|----:|----------|------:|")
for q in quakes[:5]:
lines.append(f"| {q.get('magnitude', 0):.1f} | {(q.get('place') or '')[:40]} | {q.get('depth_km', 0):.0f}km |")
lines.append("")
# Predictions
if prediction_highlights:
lines.append("## Prediction Markets")
for p in prediction_highlights[:5]:
yes = (p.get("yes_probability", 0) or 0) * 100
lines.append(f"- **{(p.get('question') or '')[:60]}** — YES: {yes:.0f}% ({p.get('sentiment', '')})")
lines.append("")
# Trending
if trending_keywords:
lines.append("## Trending Keywords")
kw_str = ", ".join(f"**{k['word']}** ({k['count']})" for k in trending_keywords[:15])
lines.append(kw_str)
lines.append("")
lines.append(f"---\n*Phoenix AGI System — World Intelligence*")
return "\n".join(lines)
def generate_threat_landscape_md(
cyber_threats: dict,
conflict_events: list,
military_activity: dict,
cable_health: dict,
outages: dict,
) -> str:
"""Generate a threat landscape report in Markdown with Mermaid diagram."""
now = datetime.now(timezone.utc)
lines = [
"# Threat Landscape Report",
f"*Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}*",
"",
]
# Mermaid threat overview
by_sev = cyber_threats.get("by_severity", {})
lines.append("## Threat Overview")
lines.append("```mermaid")
lines.append("pie title Threat Severity Distribution")
for level in ["critical", "high", "medium", "low"]:
count = by_sev.get(level, 0)
if count > 0:
lines.append(f' "{level.title()}" : {count}')
lines.append("```")
lines.append("")
# Cyber section
lines.append(f"## Cyber Threats ({len(cyber_threats.get('threats', []))})")
for t in cyber_threats.get("threats", [])[:10]:
lines.append(f"- [{t.get('severity', '').upper()}] **{(t.get('indicator') or '')[:40]}** — {t.get('threat', '')} (via {t.get('source_feed', '')})")
lines.append("")
# Military
theaters = military_activity.get("theaters", {})
total = military_activity.get("total_military_aircraft", 0)
lines.append(f"## Military Activity ({total} aircraft)")
for name, info in theaters.items():
count = info.get("count", 0)
lines.append(f"- **{name.replace('_', ' ').title()}**: {count} aircraft")
lines.append("")
# Conflict events count
conflict_count = len(conflict_events)
if conflict_count:
lines.append(f"## Active Conflicts ({conflict_count} events, 7 days)")
for e in conflict_events[:10]:
lines.append(
f"- [{(e.get('event_date') or '')[:10]}] **{e.get('country', '')}** — "
f"{e.get('event_type', '')} ({e.get('fatalities', 0)} fatalities)"
)
lines.append("")
# Infrastructure
corridors = cable_health.get("corridors", {})
if corridors:
lines.append("## Cable Health")
status_map = {0: "Clear", 1: "Advisory", 2: "At Risk", 3: "Disrupted"}
for name, info in corridors.items():
score = info.get("status_score", 0)
lines.append(f"- **{name.replace('_', ' ').title()}**: {status_map.get(score, '?')}")
lines.append("")
# Outages
ongoing = outages.get("ongoing_count", 0)
lines.append(f"## Internet Outages ({ongoing} ongoing)")
lines.append("")
lines.append(f"---\n*Phoenix AGI System — Threat Intelligence*")
return "\n".join(lines)
@@ -0,0 +1,270 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{ title }} — {{ generated_at[:10] }}</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #0f172a; color: #e2e8f0; line-height: 1.6; padding: 2rem; }
.container { max-width: 1200px; margin: 0 auto; }
h1 { font-size: 2rem; color: #f8fafc; margin-bottom: 0.5rem; }
h2 { font-size: 1.4rem; color: #94a3b8; margin: 2rem 0 1rem; border-bottom: 1px solid #334155; padding-bottom: 0.5rem; }
.meta { color: #64748b; margin-bottom: 2rem; }
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem; }
.card { background: #1e293b; border-radius: 8px; padding: 1.5rem; border: 1px solid #334155; }
.card h3 { color: #f1f5f9; margin-bottom: 1rem; font-size: 1.1rem; }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 0.5rem; text-align: left; border-bottom: 1px solid #334155; }
th { color: #94a3b8; font-weight: 600; font-size: 0.85rem; text-transform: uppercase; }
td { color: #cbd5e1; }
.severity-critical { color: #ef4444; font-weight: bold; }
.severity-high { color: #f59e0b; }
.severity-medium { color: #6b7280; }
.up { color: #22c55e; }
.down { color: #ef4444; }
.chart-container { position: relative; height: 250px; margin: 1rem 0; }
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.8rem; margin: 2px; background: #334155; }
.empty { color: #64748b; font-style: italic; padding: 1rem 0; }
.brief-text { background: #1e293b; border-radius: 8px; padding: 1.5rem; border: 1px solid #334155; white-space: pre-wrap; margin-bottom: 1.5rem; }
.risk-badge { display: inline-block; padding: 4px 12px; border-radius: 4px; font-weight: bold; font-size: 0.9rem; text-transform: uppercase; }
.risk-critical { background: #7f1d1d; color: #fca5a5; }
.risk-high { background: #78350f; color: #fbbf24; }
.risk-elevated { background: #3f3f46; color: #a1a1aa; }
.risk-moderate { background: #14532d; color: #86efac; }
.risk-low { background: #1e3a5f; color: #93c5fd; }
.stat-number { font-size: 2rem; font-weight: bold; color: #f8fafc; }
.stat-label { font-size: 0.85rem; color: #94a3b8; text-transform: uppercase; }
footer { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid #334155; color: #64748b; font-size: 0.85rem; text-align: center; }
</style>
</head>
<body>
<div class="container">
<h1>{{ title }}</h1>
<p class="meta">Country Code: {{ country_code }} | Generated: {{ generated_at }} | Phoenix AGI System</p>
<!-- Instability Overview -->
<h2>Instability Assessment</h2>
{% if instability %}
<div class="grid">
<div class="card" style="text-align:center;">
<h3>Instability Index</h3>
<div class="stat-number">{{ "%.1f"|format(instability.instability_index|default(0)) }}</div>
<div class="stat-label" style="margin:0.5rem 0;">
<span class="risk-badge risk-{{ instability.risk_level|default('moderate')|lower }}">{{ instability.risk_level|default('Unknown') }}</span>
</div>
</div>
<div class="card">
<h3>Instability Components</h3>
<div class="chart-container">
<canvas id="radarChart"></canvas>
</div>
</div>
</div>
{% else %}
<p class="empty">No instability data available for this country.</p>
{% endif %}
<!-- LLM Brief -->
<h2>Intelligence Brief</h2>
{% if brief %}
<div class="brief-text">{{ brief }}</div>
{% else %}
<p class="empty">No brief generated.</p>
{% endif %}
<!-- Conflict Events -->
<h2>Conflict Events</h2>
{% if conflict_events %}
<div class="card">
<table>
<tr><th>Date</th><th>Type</th><th>Sub-Type</th><th>Location</th><th>Fatalities</th></tr>
{% for e in conflict_events[:15] %}
<tr>
<td>{{ (e.event_date or '')[:10] }}</td>
<td>{{ e.event_type|default('') }}</td>
<td>{{ e.sub_event_type|default('') }}</td>
<td>{{ e.location|default('') }}</td>
<td class="{{ 'severity-critical' if (e.fatalities or 0) >= 10 else 'severity-high' if (e.fatalities or 0) > 0 else '' }}">{{ e.fatalities|default(0) }}</td>
</tr>
{% endfor %}
</table>
</div>
{% else %}
<p class="empty">No conflict events reported for this country.</p>
{% endif %}
<!-- Displacement -->
<h2>Displacement &amp; Refugees</h2>
{% if displacement %}
<div class="grid">
<div class="card">
<h3>Displacement by Origin</h3>
{% if displacement.by_origin %}
<table>
<tr><th>Year</th><th>Refugees</th><th>IDPs</th><th>Asylum Seekers</th></tr>
{% for d in displacement.by_origin[:10] %}
<tr>
<td>{{ d.year|default('') }}</td>
<td>{{ "{:,}".format(d.refugees|default(0)|int) }}</td>
<td>{{ "{:,}".format(d.idps|default(0)|int) }}</td>
<td>{{ "{:,}".format(d.asylum_seekers|default(0)|int) }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No displacement data by origin.</p>
{% endif %}
</div>
{% if displacement.global_totals %}
<div class="card">
<h3>Global Context</h3>
<table>
<tr><th>Metric</th><th>Total</th></tr>
{% for key, val in displacement.global_totals.items() %}
<tr>
<td>{{ key }}</td>
<td>{{ "{:,}".format(val|int) if val is not none else 'N/A' }}</td>
</tr>
{% endfor %}
</table>
</div>
{% endif %}
</div>
{% else %}
<p class="empty">No displacement data available.</p>
{% endif %}
<!-- Economic Indicators -->
<h2>Economic Indicators</h2>
{% if economic %}
<div class="grid">
<div class="card">
<h3>GDP Trend</h3>
{% if economic.gdp %}
<div class="chart-container">
<canvas id="gdpChart"></canvas>
</div>
{% else %}
<p class="empty">No GDP data available.</p>
{% endif %}
</div>
<div class="card">
<h3>Inflation Trend</h3>
{% if economic.inflation %}
<table>
<tr><th>Year</th><th>Inflation (%)</th></tr>
{% for entry in economic.inflation[:10] %}
<tr>
<td>{{ entry.year|default(entry.date|default('')) }}</td>
<td class="{{ 'severity-critical' if (entry.value or 0) > 10 else 'severity-high' if (entry.value or 0) > 5 else '' }}">{{ "%.1f"|format(entry.value or 0) }}%</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No inflation data available.</p>
{% endif %}
</div>
</div>
{% else %}
<p class="empty">No economic data available.</p>
{% endif %}
<footer>Phoenix AGI System &mdash; Country Dossier: {{ country_code }} &mdash; {{ generated_at[:10] }}</footer>
</div>
<script>
// Radar Chart: Instability Components
{% if instability and instability.components %}
(function() {
var components = {{ instability.components|tojson }};
var labels = Object.keys(components);
var values = Object.values(components);
new Chart(document.getElementById('radarChart'), {
type: 'radar',
data: {
labels: labels.map(function(l) {
return l.replace(/_/g, ' ').replace(/\b\w/g, function(c) { return c.toUpperCase(); });
}),
datasets: [{
label: 'Instability',
data: values,
backgroundColor: 'rgba(239, 68, 68, 0.2)',
borderColor: '#ef4444',
borderWidth: 2,
pointBackgroundColor: '#ef4444',
pointBorderColor: '#ef4444',
pointRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
r: {
beginAtZero: true,
max: 10,
ticks: { color: '#94a3b8', backdropColor: 'transparent', stepSize: 2 },
grid: { color: '#334155' },
angleLines: { color: '#334155' },
pointLabels: { color: '#cbd5e1', font: { size: 11 } }
}
},
plugins: {
legend: { display: false }
}
}
});
})();
{% endif %}
// Bar Chart: GDP Trend
{% if economic and economic.gdp %}
(function() {
var gdpData = {{ economic.gdp|tojson }};
var labels = gdpData.map(function(d) { return d.year || d.date || ''; });
var values = gdpData.map(function(d) { return d.value || 0; });
new Chart(document.getElementById('gdpChart'), {
type: 'bar',
data: {
labels: labels,
datasets: [{
label: 'GDP (current USD)',
data: values,
backgroundColor: '#3b82f6',
borderColor: '#2563eb',
borderWidth: 1,
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
x: { ticks: { color: '#94a3b8' }, grid: { color: '#1e293b' } },
y: {
ticks: {
color: '#94a3b8',
callback: function(v) {
if (v >= 1e12) return (v / 1e12).toFixed(1) + 'T';
if (v >= 1e9) return (v / 1e9).toFixed(1) + 'B';
if (v >= 1e6) return (v / 1e6).toFixed(1) + 'M';
return v;
}
},
grid: { color: '#334155' }
}
},
plugins: {
legend: { labels: { color: '#cbd5e1' } }
}
}
});
})();
{% endif %}
</script>
</body>
</html>
@@ -0,0 +1,186 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{ title }} — {{ generated_at[:10] }}</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #0f172a; color: #e2e8f0; line-height: 1.6; padding: 2rem; }
.container { max-width: 1200px; margin: 0 auto; }
h1 { font-size: 2rem; color: #f8fafc; margin-bottom: 0.5rem; }
h2 { font-size: 1.4rem; color: #94a3b8; margin: 2rem 0 1rem; border-bottom: 1px solid #334155; padding-bottom: 0.5rem; }
.meta { color: #64748b; margin-bottom: 2rem; }
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem; }
.card { background: #1e293b; border-radius: 8px; padding: 1.5rem; border: 1px solid #334155; }
.card h3 { color: #f1f5f9; margin-bottom: 1rem; font-size: 1.1rem; }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 0.5rem; text-align: left; border-bottom: 1px solid #334155; }
th { color: #94a3b8; font-weight: 600; font-size: 0.85rem; text-transform: uppercase; }
td { color: #cbd5e1; }
.severity-critical { color: #ef4444; font-weight: bold; }
.severity-high { color: #f59e0b; }
.severity-medium { color: #6b7280; }
.up { color: #22c55e; }
.down { color: #ef4444; }
.chart-container { position: relative; height: 250px; margin: 1rem 0; }
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.8rem; margin: 2px; background: #334155; }
.empty { color: #64748b; font-style: italic; padding: 1rem 0; }
footer { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid #334155; color: #64748b; font-size: 0.85rem; text-align: center; }
</style>
</head>
<body>
<div class="container">
<h1>{{ title }}</h1>
<p class="meta">Generated: {{ generated_at }} | Phoenix AGI System</p>
<!-- Market Summary -->
<h2>Markets</h2>
{% if market_summary %}
<div class="grid">
<div class="card">
<h3>Index Quotes</h3>
{% if market_summary.quotes %}
<table>
<tr><th>Symbol</th><th>Price</th><th>Change</th></tr>
{% for q in market_summary.quotes[:8] %}
<tr>
<td>{{ q.symbol|default('—') }}</td>
<td>{{ "%.2f"|format(q.price or 0) }}</td>
<td class="{{ 'up' if (q.change_pct or 0) >= 0 else 'down' }}">{{ "%+.2f"|format(q.change_pct or 0) }}%</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No quote data available.</p>
{% endif %}
</div>
<div class="card">
<h3>Macro Signals</h3>
{% if market_summary.macro_signals %}
<table>
<tr><th>Signal</th><th>Value</th></tr>
{% for name, info in market_summary.macro_signals.items() %}
<tr>
<td>{{ name }}</td>
<td>{% if info is mapping %}{{ info.value|default(info.price|default('?')) }}{% elif info is not none %}{{ info }}{% else %}<span style="color:#64748b">N/A</span>{% endif %}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No macro signals available.</p>
{% endif %}
</div>
</div>
{% else %}
<p class="empty">Market data unavailable.</p>
{% endif %}
<!-- Conflict & Security -->
<h2>Conflict &amp; Security</h2>
<div class="grid">
<div class="card">
<h3>Recent Events ({{ conflict_summary.count|default(0) }})</h3>
{% if conflict_summary and conflict_summary.events %}
<table>
<tr><th>Date</th><th>Type</th><th>Country</th><th>Fatalities</th></tr>
{% for e in conflict_summary.events[:10] %}
<tr>
<td>{{ (e.event_date or '')[:10] }}</td>
<td>{{ e.event_type|default('') }}</td>
<td>{{ e.country|default('') }}</td>
<td class="{{ 'severity-critical' if (e.fatalities or 0) >= 10 else 'severity-high' if (e.fatalities or 0) > 0 else '' }}">{{ e.fatalities|default(0) }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No conflict events reported.</p>
{% endif %}
</div>
<div class="card">
<h3>Cyber Threats</h3>
{% if cyber_summary and cyber_summary.threats %}
<p style="margin-bottom:1rem;">
<span class="severity-critical">Critical: {{ cyber_summary.by_severity.critical|default(0) }}</span> |
<span class="severity-high">High: {{ cyber_summary.by_severity.high|default(0) }}</span> |
Medium: {{ cyber_summary.by_severity.medium|default(0) }}
</p>
<table>
<tr><th>Severity</th><th>Indicator</th><th>Threat</th></tr>
{% for t in cyber_summary.threats[:8] %}
<tr>
<td class="severity-{{ t.severity|default('medium') }}">{{ t.severity|default('unknown') }}</td>
<td>{{ (t.indicator or '')[:30] }}</td>
<td>{{ (t.threat or '')[:30] }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No cyber threat data available.</p>
{% endif %}
</div>
</div>
<!-- Natural Events -->
<h2>Natural Events</h2>
<div class="grid">
<div class="card">
<h3>Earthquakes</h3>
{% if natural_summary and natural_summary.earthquakes %}
<table>
<tr><th>Mag</th><th>Location</th><th>Depth</th></tr>
{% for q in natural_summary.earthquakes[:8] %}
<tr>
<td class="{{ 'severity-critical' if (q.magnitude or 0) >= 6 else 'severity-high' if (q.magnitude or 0) >= 5 else '' }}">{{ "%.1f"|format(q.magnitude or 0) }}</td>
<td>{{ (q.place or '')[:40] }}</td>
<td>{{ "%.0f"|format(q.depth_km or 0) }}km</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No earthquake data available.</p>
{% endif %}
</div>
<div class="card">
<h3>Wildfires</h3>
<p>{{ natural_summary.fire_count|default(0) }} high-confidence fires detected</p>
</div>
</div>
<!-- Predictions & Trending -->
<h2>Signals</h2>
<div class="grid">
<div class="card">
<h3>Prediction Markets</h3>
{% if prediction_highlights %}
<table>
<tr><th>Question</th><th>YES</th><th>Sentiment</th></tr>
{% for p in prediction_highlights[:8] %}
<tr>
<td>{{ (p.question or '')[:40] }}</td>
<td>{{ "%.0f"|format((p.yes_probability or 0) * 100) }}%</td>
<td>{{ p.sentiment|default('') }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No prediction data available.</p>
{% endif %}
</div>
<div class="card">
<h3>Trending Keywords</h3>
{% if trending_keywords %}
{% for kw in trending_keywords[:20] %}
<span class="tag">{{ kw.word }} ({{ kw.count }})</span>
{% endfor %}
{% else %}
<p class="empty">No trending keyword data.</p>
{% endif %}
</div>
</div>
<footer>Phoenix AGI System &mdash; World Intelligence Report &mdash; {{ generated_at[:10] }}</footer>
</div>
</body>
</html>
@@ -0,0 +1,277 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{ title }} — {{ generated_at[:10] }}</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #0f172a; color: #e2e8f0; line-height: 1.6; padding: 2rem; }
.container { max-width: 1200px; margin: 0 auto; }
h1 { font-size: 2rem; color: #f8fafc; margin-bottom: 0.5rem; }
h2 { font-size: 1.4rem; color: #94a3b8; margin: 2rem 0 1rem; border-bottom: 1px solid #334155; padding-bottom: 0.5rem; }
.meta { color: #64748b; margin-bottom: 2rem; }
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem; }
.card { background: #1e293b; border-radius: 8px; padding: 1.5rem; border: 1px solid #334155; }
.card h3 { color: #f1f5f9; margin-bottom: 1rem; font-size: 1.1rem; }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 0.5rem; text-align: left; border-bottom: 1px solid #334155; }
th { color: #94a3b8; font-weight: 600; font-size: 0.85rem; text-transform: uppercase; }
td { color: #cbd5e1; }
.severity-critical { color: #ef4444; font-weight: bold; }
.severity-high { color: #f59e0b; }
.severity-medium { color: #6b7280; }
.up { color: #22c55e; }
.down { color: #ef4444; }
.chart-container { position: relative; height: 250px; margin: 1rem 0; }
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.8rem; margin: 2px; background: #334155; }
.empty { color: #64748b; font-style: italic; padding: 1rem 0; }
.stat-row { display: flex; gap: 1.5rem; margin-bottom: 1.5rem; flex-wrap: wrap; }
.stat-box { background: #1e293b; border: 1px solid #334155; border-radius: 8px; padding: 1.2rem; text-align: center; flex: 1; min-width: 140px; }
.stat-number { font-size: 2rem; font-weight: bold; color: #f8fafc; }
.stat-label { font-size: 0.85rem; color: #94a3b8; text-transform: uppercase; }
.heatmap-cell { display: inline-block; padding: 6px 10px; margin: 3px; border-radius: 4px; font-size: 0.85rem; font-weight: 600; min-width: 80px; text-align: center; }
footer { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid #334155; color: #64748b; font-size: 0.85rem; text-align: center; }
</style>
</head>
<body>
<div class="container">
<h1>{{ title }}</h1>
<p class="meta">Generated: {{ generated_at }} | Phoenix AGI System</p>
<!-- Equity Quotes -->
<h2>Equity &amp; Index Quotes</h2>
{% if quotes %}
<div class="card">
<table>
<tr><th>Symbol</th><th>Name</th><th>Price</th><th>Change</th><th>% Change</th><th>Volume</th></tr>
{% for q in quotes %}
<tr>
<td style="font-weight:600;">{{ q.symbol|default('—') }}</td>
<td>{{ q.name|default(q.shortName|default('')) }}</td>
<td>{{ "%.2f"|format(q.price or q.regularMarketPrice or 0) }}</td>
<td class="{{ 'up' if (q.change or q.regularMarketChange or 0) >= 0 else 'down' }}">
{{ "%+.2f"|format(q.change or q.regularMarketChange or 0) }}
</td>
<td class="{{ 'up' if (q.change_pct or q.regularMarketChangePercent or 0) >= 0 else 'down' }}">
{{ "%+.2f"|format(q.change_pct or q.regularMarketChangePercent or 0) }}%
</td>
<td>{{ "{:,}".format((q.volume or q.regularMarketVolume or 0)|int) }}</td>
</tr>
{% endfor %}
</table>
</div>
{% else %}
<p class="empty">No equity quote data available.</p>
{% endif %}
<!-- Sector Heatmap -->
<h2>Sector Performance</h2>
{% if sector_heatmap %}
<div class="grid">
<div class="card" style="grid-column: 1 / -1;">
<h3>Sector Heatmap</h3>
<div style="display:flex;flex-wrap:wrap;gap:4px;margin-bottom:1.5rem;">
{% for s in sector_heatmap %}
{% set pct = s.change_pct|default(0) %}
{% if pct >= 2 %}
{% set bg = '#166534' %}
{% elif pct >= 0.5 %}
{% set bg = '#14532d' %}
{% elif pct >= 0 %}
{% set bg = '#1a2e1a' %}
{% elif pct >= -0.5 %}
{% set bg = '#2e1a1a' %}
{% elif pct >= -2 %}
{% set bg = '#7f1d1d' %}
{% else %}
{% set bg = '#991b1b' %}
{% endif %}
<div class="heatmap-cell" style="background:{{ bg }};color:{{ '#86efac' if pct >= 0 else '#fca5a5' }};">
{{ s.symbol|default(s.name|default('?')) }}<br>{{ "%+.1f"|format(pct) }}%
</div>
{% endfor %}
</div>
<h3>Sector Bar Chart</h3>
<div class="chart-container" style="height:300px;">
<canvas id="sectorChart"></canvas>
</div>
</div>
</div>
{% else %}
<p class="empty">No sector data available.</p>
{% endif %}
<!-- Cryptocurrency -->
<h2>Cryptocurrency</h2>
{% if crypto %}
<div class="card">
<table>
<tr><th>Coin</th><th>Price (USD)</th><th>24h Change</th><th>Market Cap</th><th>24h Volume</th></tr>
{% for c in crypto[:15] %}
<tr>
<td style="font-weight:600;">{{ c.symbol|default(c.id|default(''))|upper }}{% if c.name %} <span style="color:#64748b;font-weight:normal;">{{ c.name }}</span>{% endif %}</td>
<td>{{ "${:,.2f}".format(c.current_price|default(c.price|default(0))) }}</td>
<td class="{{ 'up' if (c.price_change_percentage_24h|default(c.change_24h|default(0))) >= 0 else 'down' }}">
{{ "%+.2f"|format(c.price_change_percentage_24h|default(c.change_24h|default(0))) }}%
</td>
<td>
{% set mcap = c.market_cap|default(0) %}
{% if mcap >= 1e12 %}${{ "%.1f"|format(mcap / 1e12) }}T
{% elif mcap >= 1e9 %}${{ "%.1f"|format(mcap / 1e9) }}B
{% elif mcap >= 1e6 %}${{ "%.1f"|format(mcap / 1e6) }}M
{% elif mcap > 0 %}${{ "{:,.0f}".format(mcap) }}
{% else %}—{% endif %}
</td>
<td>
{% set vol = c.total_volume|default(c.volume_24h|default(0)) %}
{% if vol >= 1e9 %}${{ "%.1f"|format(vol / 1e9) }}B
{% elif vol >= 1e6 %}${{ "%.1f"|format(vol / 1e6) }}M
{% elif vol > 0 %}${{ "{:,.0f}".format(vol) }}
{% else %}—{% endif %}
</td>
</tr>
{% endfor %}
</table>
</div>
{% else %}
<p class="empty">No cryptocurrency data available.</p>
{% endif %}
<!-- Macro Signals -->
<h2>Macro Signals</h2>
{% if macro_signals %}
<div class="grid">
{% for name, info in macro_signals.items() %}
<div class="card">
<h3>{{ name }}</h3>
{% if info is mapping %}
<table>
{% for key, val in info.items() %}
<tr>
<td>{{ key.replace('_', ' ')|title }}</td>
<td>
{% if val is number %}
{% if val >= 1e9 %}{{ "%.2f"|format(val / 1e9) }}B
{% elif val >= 1e6 %}{{ "%.2f"|format(val / 1e6) }}M
{% else %}{{ "%.4f"|format(val) if val < 1 and val > -1 else "%.2f"|format(val) }}{% endif %}
{% elif val is not none %}{{ val }}
{% else %}<span style="color:#64748b">N/A</span>{% endif %}
</td>
</tr>
{% endfor %}
</table>
{% elif info is not none %}
<p style="font-size:1.5rem;font-weight:bold;color:#f8fafc;">{{ info }}</p>
{% else %}
<p class="empty">N/A</p>
{% endif %}
</div>
{% endfor %}
</div>
{% else %}
<p class="empty">No macro signal data available.</p>
{% endif %}
<!-- ETF Flows -->
<h2>ETF Flows</h2>
{% if etf_flows %}
<div class="card">
{% if etf_flows is mapping %}
<table>
<tr><th>ETF / Category</th><th>Flow</th><th>Details</th></tr>
{% for name, data in etf_flows.items() %}
<tr>
<td style="font-weight:600;">{{ name }}</td>
{% if data is mapping %}
<td class="{{ 'up' if (data.flow|default(data.net_flow|default(0))) >= 0 else 'down' }}">
{% set flow = data.flow|default(data.net_flow|default(0)) %}
{% if flow >= 1e9 %}${{ "%+.1f"|format(flow / 1e9) }}B
{% elif flow >= 1e6 %}${{ "%+.1f"|format(flow / 1e6) }}M
{% else %}{{ "%+.2f"|format(flow) }}{% endif %}
</td>
<td>{{ data.description|default(data.period|default('')) }}</td>
{% else %}
<td>{{ data }}</td>
<td></td>
{% endif %}
</tr>
{% endfor %}
</table>
{% elif etf_flows is iterable and etf_flows is not string %}
<table>
<tr><th>ETF</th><th>Flow</th></tr>
{% for item in etf_flows %}
<tr>
<td>{{ item.name|default(item.symbol|default('')) }}</td>
<td>{{ item.flow|default(item.value|default('')) }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p>{{ etf_flows }}</p>
{% endif %}
</div>
{% else %}
<p class="empty">No ETF flow data available.</p>
{% endif %}
<footer>Phoenix AGI System &mdash; Market Overview &mdash; {{ generated_at[:10] }}</footer>
</div>
<script>
// Bar Chart: Sector Performance
{% if sector_heatmap %}
(function() {
var sectors = {{ sector_heatmap|tojson }};
var labels = sectors.map(function(s) { return s.name || s.symbol || '?'; });
var values = sectors.map(function(s) { return s.change_pct || 0; });
var colors = values.map(function(v) { return v >= 0 ? '#22c55e' : '#ef4444'; });
new Chart(document.getElementById('sectorChart'), {
type: 'bar',
data: {
labels: labels,
datasets: [{
label: 'Change %',
data: values,
backgroundColor: colors,
borderColor: colors.map(function(c) { return c === '#22c55e' ? '#16a34a' : '#dc2626'; }),
borderWidth: 1,
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
indexAxis: 'y',
scales: {
x: {
ticks: {
color: '#94a3b8',
callback: function(v) { return v + '%'; }
},
grid: { color: '#334155' }
},
y: {
ticks: { color: '#cbd5e1', font: { size: 11 } },
grid: { display: false }
}
},
plugins: {
legend: { display: false },
tooltip: {
callbacks: {
label: function(ctx) { return ctx.parsed.x.toFixed(2) + '%'; }
}
}
}
}
});
})();
{% endif %}
</script>
</body>
</html>
@@ -0,0 +1,265 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{ title }} — {{ generated_at[:10] }}</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #0f172a; color: #e2e8f0; line-height: 1.6; padding: 2rem; }
.container { max-width: 1200px; margin: 0 auto; }
h1 { font-size: 2rem; color: #f8fafc; margin-bottom: 0.5rem; }
h2 { font-size: 1.4rem; color: #94a3b8; margin: 2rem 0 1rem; border-bottom: 1px solid #334155; padding-bottom: 0.5rem; }
.meta { color: #64748b; margin-bottom: 2rem; }
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem; }
.card { background: #1e293b; border-radius: 8px; padding: 1.5rem; border: 1px solid #334155; }
.card h3 { color: #f1f5f9; margin-bottom: 1rem; font-size: 1.1rem; }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 0.5rem; text-align: left; border-bottom: 1px solid #334155; }
th { color: #94a3b8; font-weight: 600; font-size: 0.85rem; text-transform: uppercase; }
td { color: #cbd5e1; }
.severity-critical { color: #ef4444; font-weight: bold; }
.severity-high { color: #f59e0b; }
.severity-medium { color: #6b7280; }
.up { color: #22c55e; }
.down { color: #ef4444; }
.chart-container { position: relative; height: 250px; margin: 1rem 0; }
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.8rem; margin: 2px; background: #334155; }
.empty { color: #64748b; font-style: italic; padding: 1rem 0; }
.stat-row { display: flex; gap: 1.5rem; margin-bottom: 1.5rem; flex-wrap: wrap; }
.stat-box { background: #1e293b; border: 1px solid #334155; border-radius: 8px; padding: 1.2rem; text-align: center; flex: 1; min-width: 140px; }
.stat-number { font-size: 2rem; font-weight: bold; color: #f8fafc; }
.stat-label { font-size: 0.85rem; color: #94a3b8; text-transform: uppercase; }
footer { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid #334155; color: #64748b; font-size: 0.85rem; text-align: center; }
</style>
</head>
<body>
<div class="container">
<h1>{{ title }}</h1>
<p class="meta">Generated: {{ generated_at }} | Phoenix AGI System</p>
<!-- Threat Summary Stats -->
<h2>Threat Overview</h2>
<div class="stat-row">
<div class="stat-box">
<div class="stat-number severity-critical">{{ cyber_threats.by_severity.critical|default(0) if cyber_threats else 0 }}</div>
<div class="stat-label">Critical</div>
</div>
<div class="stat-box">
<div class="stat-number severity-high">{{ cyber_threats.by_severity.high|default(0) if cyber_threats else 0 }}</div>
<div class="stat-label">High</div>
</div>
<div class="stat-box">
<div class="stat-number">{{ cyber_threats.by_severity.medium|default(0) if cyber_threats else 0 }}</div>
<div class="stat-label">Medium</div>
</div>
<div class="stat-box">
<div class="stat-number">{{ (cyber_threats.threats|length) if cyber_threats and cyber_threats.threats else 0 }}</div>
<div class="stat-label">Total Indicators</div>
</div>
<div class="stat-box">
<div class="stat-number">{{ outages.ongoing_count|default(0) if outages else 0 }}</div>
<div class="stat-label">Active Outages</div>
</div>
</div>
<!-- Cyber Threats -->
<h2>Cyber Threats</h2>
{% if cyber_threats and cyber_threats.threats %}
<div class="grid">
<div class="card">
<h3>Severity Distribution</h3>
<div class="chart-container">
<canvas id="severityChart"></canvas>
</div>
</div>
<div class="card">
<h3>Threat Types</h3>
{% if cyber_threats.by_type %}
<table>
<tr><th>Type</th><th>Count</th></tr>
{% for type_name, count in cyber_threats.by_type.items() %}
<tr>
<td>{{ type_name }}</td>
<td>{{ count }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p class="empty">No threat type breakdown available.</p>
{% endif %}
</div>
</div>
<div class="card" style="margin-top:1.5rem;">
<h3>Threat Indicators</h3>
<table>
<tr><th>Severity</th><th>Indicator</th><th>Type</th><th>Threat</th><th>First Seen</th></tr>
{% for t in cyber_threats.threats[:20] %}
<tr>
<td class="severity-{{ t.severity|default('medium') }}">{{ t.severity|default('unknown') }}</td>
<td style="font-family:monospace;font-size:0.85rem;">{{ (t.indicator or '')[:40] }}</td>
<td>{{ t.type|default(t.indicator_type|default('')) }}</td>
<td>{{ (t.threat or '')[:30] }}</td>
<td>{{ (t.first_seen or '')[:10] }}</td>
</tr>
{% endfor %}
</table>
</div>
{% else %}
<p class="empty">No cyber threat data available.</p>
{% endif %}
<!-- Conflict Events -->
<h2>Conflict Events</h2>
{% if conflict_events %}
<div class="card">
<table>
<tr><th>Date</th><th>Type</th><th>Country</th><th>Location</th><th>Fatalities</th></tr>
{% for e in conflict_events[:15] %}
<tr>
<td>{{ (e.event_date or '')[:10] }}</td>
<td>{{ e.event_type|default('') }}</td>
<td>{{ e.country|default('') }}</td>
<td>{{ e.location|default('') }}</td>
<td class="{{ 'severity-critical' if (e.fatalities or 0) >= 10 else 'severity-high' if (e.fatalities or 0) > 0 else '' }}">{{ e.fatalities|default(0) }}</td>
</tr>
{% endfor %}
</table>
</div>
{% else %}
<p class="empty">No conflict events reported.</p>
{% endif %}
<!-- Military Activity -->
<h2>Military Activity</h2>
{% if military_activity %}
<div class="grid">
{% if military_activity.theaters %}
{% for theater_name, theater_data in military_activity.theaters.items() %}
<div class="card">
<h3>{{ theater_name }}</h3>
{% if theater_data is mapping %}
<table>
<tr><th>Metric</th><th>Value</th></tr>
{% for key, val in theater_data.items() %}
<tr>
<td>{{ key.replace('_', ' ')|title }}</td>
<td>{{ val }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p>{{ theater_data }}</p>
{% endif %}
</div>
{% endfor %}
{% endif %}
<div class="card">
<h3>Summary</h3>
<div class="stat-number" style="margin-bottom:0.5rem;">{{ military_activity.total_military_aircraft|default(0) }}</div>
<div class="stat-label">Military Aircraft Tracked</div>
</div>
</div>
{% else %}
<p class="empty">No military activity data available.</p>
{% endif %}
<!-- Cable Health -->
<h2>Submarine Cable Health</h2>
{% if cable_health and cable_health.corridors %}
<div class="grid">
{% for corridor_name, corridor_data in cable_health.corridors.items() %}
<div class="card">
<h3>{{ corridor_name }}</h3>
{% if corridor_data is mapping %}
<table>
{% for key, val in corridor_data.items() %}
<tr>
<td>{{ key.replace('_', ' ')|title }}</td>
<td>{{ val }}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p>{{ corridor_data }}</p>
{% endif %}
</div>
{% endfor %}
</div>
{% else %}
<p class="empty">No cable health data available.</p>
{% endif %}
<!-- Infrastructure Outages -->
<h2>Infrastructure Outages</h2>
{% if outages and outages.outages %}
<div class="card">
<table>
<tr><th>Service</th><th>Status</th><th>Region</th><th>Since</th><th>Impact</th></tr>
{% for o in outages.outages[:15] %}
<tr>
<td>{{ o.service|default(o.name|default('')) }}</td>
<td class="{{ 'severity-critical' if (o.status or '')|lower in ['major', 'critical'] else 'severity-high' if (o.status or '')|lower in ['partial', 'degraded'] else '' }}">{{ o.status|default('unknown') }}</td>
<td>{{ o.region|default('') }}</td>
<td>{{ (o.since or o.started or '')[:16] }}</td>
<td>{{ o.impact|default('') }}</td>
</tr>
{% endfor %}
</table>
</div>
{% else %}
<p class="empty">No active outages reported.</p>
{% endif %}
<footer>Phoenix AGI System &mdash; Threat Landscape Report &mdash; {{ generated_at[:10] }}</footer>
</div>
<script>
// Doughnut Chart: Threat Severity Distribution
{% if cyber_threats and cyber_threats.by_severity %}
(function() {
var severity = {{ cyber_threats.by_severity|tojson }};
var labels = Object.keys(severity).map(function(l) {
return l.charAt(0).toUpperCase() + l.slice(1);
});
var values = Object.values(severity);
var colorMap = {
'Critical': '#ef4444',
'High': '#f59e0b',
'Medium': '#6b7280',
'Low': '#3b82f6',
'Info': '#64748b'
};
var colors = labels.map(function(l) { return colorMap[l] || '#94a3b8'; });
new Chart(document.getElementById('severityChart'), {
type: 'doughnut',
data: {
labels: labels,
datasets: [{
data: values,
backgroundColor: colors,
borderColor: '#0f172a',
borderWidth: 2
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
cutout: '55%',
plugins: {
legend: {
position: 'bottom',
labels: { color: '#cbd5e1', padding: 12, font: { size: 12 } }
}
}
}
});
})();
{% endif %}
</script>
</body>
</html>
+628
View File
@@ -0,0 +1,628 @@
#!/usr/bin/env python3
"""
World Intelligence MCP Server
==============================
Real-time global intelligence across 17 domains:
financial markets, economic indicators, earthquakes, wildfires,
conflict, military flights, infrastructure, and more.
Phase 1: Markets, Economic, Seismology, Wildfire (12 tools).
Phase 2: Conflict, Military, Infrastructure, Maritime, Climate (+10 = 22 tools).
Phase 3: News, Intelligence, Prediction, Displacement, Aviation, Cyber (+14 = 36 tools).
Phase 4: Reports — daily brief, country dossier, threat landscape (+3 = 39 tools).
"""
import asyncio
import json
import logging
import os
from typing import Any
from mcp.server import Server
from mcp.server.stdio import stdio_server
from mcp.types import Tool, TextContent
from .cache import Cache
from .circuit_breaker import CircuitBreaker
from .fetcher import Fetcher
from .sources import markets, economic, seismology, wildfire, conflict, military, infrastructure, maritime, climate, news, intelligence, prediction, displacement, aviation, cyber
from .reports import generator as report_gen
logging.basicConfig(
level=os.environ.get("WORLD_INTEL_LOG_LEVEL", "INFO"),
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger("world-intel-mcp")
server = Server("world-intel-mcp")
cache = Cache()
breaker = CircuitBreaker(failure_threshold=3, cooldown_seconds=300)
fetcher = Fetcher(cache=cache, breaker=breaker)
# ---------------------------------------------------------------------------
# Tool definitions
# ---------------------------------------------------------------------------
TOOLS: list[Tool] = [
# --- Markets (6 tools) ---
Tool(
name="intel_market_quotes",
description="Get real-time stock market index quotes (S&P 500, Dow, Nasdaq, FTSE, Nikkei, etc.). Optional: symbols (list of ticker symbols).",
inputSchema={
"type": "object",
"properties": {
"symbols": {
"type": "array",
"items": {"type": "string"},
"description": "Ticker symbols (default: major indices)",
},
},
},
),
Tool(
name="intel_crypto_quotes",
description="Get top cryptocurrency prices, market caps, and 7-day sparklines from CoinGecko. Optional: limit (int, default 20).",
inputSchema={
"type": "object",
"properties": {
"limit": {"type": "integer", "description": "Number of coins (default 20)", "default": 20},
},
},
),
Tool(
name="intel_stablecoin_status",
description="Check stablecoin peg health (USDT, USDC, DAI, FDUSD). Flags depegs >0.5%.",
inputSchema={"type": "object", "properties": {}},
),
Tool(
name="intel_etf_flows",
description="Get Bitcoin spot ETF prices and volumes (IBIT, FBTC, GBTC, ARKB, BITB).",
inputSchema={"type": "object", "properties": {}},
),
Tool(
name="intel_sector_heatmap",
description="Get US equity sector performance heatmap (11 SPDR sector ETFs).",
inputSchema={"type": "object", "properties": {}},
),
Tool(
name="intel_macro_signals",
description="Get 7 key macro signals: Fear & Greed, mempool fees, DXY, VIX, gold, 10Y Treasury, BTC dominance.",
inputSchema={"type": "object", "properties": {}},
),
# --- Economic (3 tools) ---
Tool(
name="intel_energy_prices",
description="Get crude oil (Brent, WTI) and natural gas prices from EIA. Requires EIA_API_KEY.",
inputSchema={"type": "object", "properties": {}},
),
Tool(
name="intel_fred_series",
description="Get Federal Reserve economic data series (GDP, UNRATE, CPIAUCSL, DFF, T10YIE, etc.). Requires FRED_API_KEY.",
inputSchema={
"type": "object",
"properties": {
"series_id": {"type": "string", "description": "FRED series ID (e.g., 'UNRATE')"},
"limit": {"type": "integer", "description": "Number of observations", "default": 30},
},
"required": ["series_id"],
},
),
Tool(
name="intel_world_bank_indicators",
description="Get World Bank development indicators (GDP, inflation, unemployment) for a country.",
inputSchema={
"type": "object",
"properties": {
"country": {"type": "string", "description": "ISO country code (default: USA)", "default": "USA"},
"indicators": {
"type": "array",
"items": {"type": "string"},
"description": "World Bank indicator codes",
},
},
},
),
# --- Natural (2 tools) ---
Tool(
name="intel_earthquakes",
description="Get recent earthquakes from USGS. No API key needed.",
inputSchema={
"type": "object",
"properties": {
"min_magnitude": {"type": "number", "description": "Minimum magnitude (default 4.5)", "default": 4.5},
"hours": {"type": "integer", "description": "Lookback hours (default 24)", "default": 24},
"limit": {"type": "integer", "description": "Max results (default 50)", "default": 50},
},
},
),
Tool(
name="intel_wildfires",
description="Get active wildfires from NASA FIRMS (9 global regions). Requires NASA_FIRMS_API_KEY.",
inputSchema={
"type": "object",
"properties": {
"region": {
"type": "string",
"description": "Specific region (north_america, europe, etc.) or omit for all 9",
"enum": [
"north_america", "south_america", "europe", "africa",
"middle_east", "south_asia", "east_asia", "southeast_asia", "oceania",
],
},
},
},
),
# --- Conflict (3 tools) ---
Tool(
name="intel_acled_events",
description="Get armed conflict events from ACLED. Optional: country (name), days (default 7), limit (default 100). Requires ACLED_ACCESS_TOKEN.",
inputSchema={
"type": "object",
"properties": {
"country": {"type": "string", "description": "Country name filter"},
"days": {"type": "integer", "description": "Lookback days (default 7)", "default": 7},
"limit": {"type": "integer", "description": "Max results (default 100)", "default": 100},
},
},
),
Tool(
name="intel_ucdp_events",
description="Get state-based violence events from UCDP GED. No API key needed. Optional: days (default 30).",
inputSchema={
"type": "object",
"properties": {
"days": {"type": "integer", "description": "Lookback days (default 30)", "default": 30},
"limit": {"type": "integer", "description": "Max results (default 100)", "default": 100},
},
},
),
Tool(
name="intel_humanitarian_summary",
description="Get humanitarian crisis datasets from HDX. No API key needed. Optional: country code.",
inputSchema={
"type": "object",
"properties": {
"country": {"type": "string", "description": "ISO country code filter"},
},
},
),
# --- Military (3 tools) ---
Tool(
name="intel_military_flights",
description="Track military aircraft via OpenSky Network (ICAO hex + callsign filtering). Optional: bbox (lamin,lomin,lamax,lomax).",
inputSchema={
"type": "object",
"properties": {
"bbox": {"type": "string", "description": "Bounding box: lamin,lomin,lamax,lomax"},
},
},
),
Tool(
name="intel_theater_posture",
description="Get military aircraft presence across 5 theaters: European, Indo-Pacific, Middle East, Arctic, Korean Peninsula.",
inputSchema={"type": "object", "properties": {}},
),
Tool(
name="intel_aircraft_details",
description="Get aircraft details from Wingbits by ICAO24 hex code. Requires WINGBITS_API_KEY.",
inputSchema={
"type": "object",
"properties": {
"icao24": {"type": "string", "description": "ICAO24 hex code"},
},
"required": ["icao24"],
},
),
# --- Infrastructure (2 tools) ---
Tool(
name="intel_internet_outages",
description="Get internet outages from Cloudflare Radar (last 7 days). Optional: CLOUDFLARE_API_TOKEN for higher limits.",
inputSchema={"type": "object", "properties": {}},
),
Tool(
name="intel_cable_health",
description="Assess undersea cable corridor health from NGA navigational warnings. 6 corridors scored 0-3.",
inputSchema={"type": "object", "properties": {}},
),
# --- Maritime (1 tool) ---
Tool(
name="intel_nav_warnings",
description="Get active navigational warnings from NGA Maritime Safety. Optional: navarea (I-XVI).",
inputSchema={
"type": "object",
"properties": {
"navarea": {"type": "string", "description": "NAVAREA number (e.g., IV, XII)"},
},
},
),
# --- Climate (1 tool) ---
Tool(
name="intel_climate_anomalies",
description="Detect temperature and precipitation anomalies across 15 global climate zones (vs. prior year baseline).",
inputSchema={
"type": "object",
"properties": {
"zones": {
"type": "array",
"items": {"type": "string"},
"description": "Zone keys to check (default: all 15)",
},
},
},
),
# --- Prediction (1 tool) ---
Tool(
name="intel_prediction_markets",
description="Get active prediction markets from Polymarket (questions, YES probabilities, volumes, sentiment).",
inputSchema={
"type": "object",
"properties": {
"limit": {"type": "integer", "description": "Number of markets (default 20)", "default": 20},
},
},
),
# --- Displacement (1 tool) ---
Tool(
name="intel_displacement_summary",
description="Get UNHCR refugee/displacement statistics by country of origin. No API key needed.",
inputSchema={
"type": "object",
"properties": {
"year": {"type": "integer", "description": "Reporting year (default: last year)"},
},
},
),
# --- Aviation (1 tool) ---
Tool(
name="intel_airport_delays",
description="Get current US airport delays from FAA (20 major airports). No API key needed.",
inputSchema={"type": "object", "properties": {}},
),
# --- Cyber (1 tool) ---
Tool(
name="intel_cyber_threats",
description="Get aggregated cyber threat intelligence from 4 feeds (Feodo, CISA KEV, SANS, URLhaus). No API key needed.",
inputSchema={
"type": "object",
"properties": {
"limit": {"type": "integer", "description": "Max threats (default 50)", "default": 50},
},
},
),
# --- News (3 tools) ---
Tool(
name="intel_news_feed",
description="Get aggregated intelligence news from 20+ RSS feeds across 6 categories (geopolitics, security, tech, finance, military, science).",
inputSchema={
"type": "object",
"properties": {
"category": {
"type": "string",
"description": "Category filter",
"enum": ["geopolitics", "security", "technology", "finance", "military", "science"],
},
"limit": {"type": "integer", "description": "Max items (default 50)", "default": 50},
},
},
),
Tool(
name="intel_trending_keywords",
description="Detect trending keywords from recent news headlines. Keyword spike detection across 20+ feeds.",
inputSchema={
"type": "object",
"properties": {
"min_count": {"type": "integer", "description": "Minimum occurrences (default 3)", "default": 3},
},
},
),
Tool(
name="intel_gdelt_search",
description="Search GDELT 2.0 for global news articles or volume timelines. No API key needed.",
inputSchema={
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query (default: 'conflict')", "default": "conflict"},
"mode": {
"type": "string",
"description": "artlist (articles) or timelinevol (volume timeline)",
"enum": ["artlist", "timelinevol"],
"default": "artlist",
},
"limit": {"type": "integer", "description": "Max records (default 50)", "default": 50},
},
},
),
# --- Intelligence (4 tools) ---
Tool(
name="intel_country_brief",
description="Generate a country intelligence brief using Ollama LLM + World Bank + ACLED data. Falls back to data-only if LLM unavailable.",
inputSchema={
"type": "object",
"properties": {
"country_code": {"type": "string", "description": "ISO country code (default: US)", "default": "US"},
},
},
),
Tool(
name="intel_risk_scores",
description="Get country risk scores computed from ACLED conflict data vs historical baselines. Requires ACLED_ACCESS_TOKEN.",
inputSchema={
"type": "object",
"properties": {
"limit": {"type": "integer", "description": "Top N countries (default 20)", "default": 20},
},
},
),
Tool(
name="intel_instability_index",
description="Compute Country Instability Index (0-100) from conflict, economic, humanitarian, infrastructure, and military signals.",
inputSchema={
"type": "object",
"properties": {
"country_code": {"type": "string", "description": "ISO alpha-3 code (e.g., UKR). Omit for top-10 focus countries."},
},
},
),
Tool(
name="intel_signal_convergence",
description="Detect geographic convergence of multi-domain signals (earthquakes, conflict, military) in hotspot regions.",
inputSchema={
"type": "object",
"properties": {
"lat": {"type": "number", "description": "Center latitude (omit for 5 global hotspots)"},
"lon": {"type": "number", "description": "Center longitude"},
"radius_deg": {"type": "number", "description": "Radius in degrees (default 5.0)", "default": 5.0},
},
},
),
# --- Reports (3 tools) ---
Tool(
name="intel_daily_brief",
description="Generate a daily intelligence brief HTML report (markets, conflict, cyber, natural, predictions, trending). Returns file path.",
inputSchema={
"type": "object",
"properties": {
"output_dir": {"type": "string", "description": "Custom output directory (default: $STORAGE_BASE/reports/intel/)"},
},
},
),
Tool(
name="intel_country_dossier",
description="Generate a full country dossier HTML report (brief, instability index, conflict, displacement, economic). Returns file path.",
inputSchema={
"type": "object",
"properties": {
"country_code": {"type": "string", "description": "ISO country code (e.g., UKR, SYR, MMR)"},
"output_dir": {"type": "string", "description": "Custom output directory"},
},
"required": ["country_code"],
},
),
Tool(
name="intel_threat_landscape",
description="Generate a threat landscape HTML report (cyber threats, conflict, military, cable health, outages). Returns file path.",
inputSchema={
"type": "object",
"properties": {
"output_dir": {"type": "string", "description": "Custom output directory"},
},
},
),
# --- System (1 tool) ---
Tool(
name="intel_status",
description="Get data source health, circuit breaker status, and cache statistics.",
inputSchema={"type": "object", "properties": {}},
),
]
# ---------------------------------------------------------------------------
# Tool dispatch
# ---------------------------------------------------------------------------
async def _dispatch(name: str, arguments: dict[str, Any]) -> Any:
"""Route tool call to the appropriate source function."""
match name:
# Markets
case "intel_market_quotes":
return await markets.fetch_market_quotes(fetcher, symbols=arguments.get("symbols"))
case "intel_crypto_quotes":
return await markets.fetch_crypto_quotes(fetcher, limit=arguments.get("limit", 20))
case "intel_stablecoin_status":
return await markets.fetch_stablecoin_status(fetcher)
case "intel_etf_flows":
return await markets.fetch_etf_flows(fetcher)
case "intel_sector_heatmap":
return await markets.fetch_sector_heatmap(fetcher)
case "intel_macro_signals":
return await markets.fetch_macro_signals(fetcher)
# Economic
case "intel_energy_prices":
return await economic.fetch_energy_prices(fetcher)
case "intel_fred_series":
return await economic.fetch_fred_series(
fetcher,
series_id=arguments["series_id"],
limit=arguments.get("limit", 30),
)
case "intel_world_bank_indicators":
return await economic.fetch_world_bank_indicators(
fetcher,
country=arguments.get("country", "USA"),
indicators=arguments.get("indicators"),
)
# Natural
case "intel_earthquakes":
return await seismology.fetch_earthquakes(
fetcher,
min_magnitude=arguments.get("min_magnitude", 4.5),
hours=arguments.get("hours", 24),
limit=arguments.get("limit", 50),
)
case "intel_wildfires":
return await wildfire.fetch_wildfires(fetcher, region=arguments.get("region"))
# Conflict
case "intel_acled_events":
return await conflict.fetch_acled_events(
fetcher,
country=arguments.get("country"),
days=arguments.get("days", 7),
limit=arguments.get("limit", 100),
)
case "intel_ucdp_events":
return await conflict.fetch_ucdp_events(
fetcher,
days=arguments.get("days", 30),
limit=arguments.get("limit", 100),
)
case "intel_humanitarian_summary":
return await conflict.fetch_humanitarian_summary(
fetcher, country=arguments.get("country"),
)
# Military
case "intel_military_flights":
return await military.fetch_military_flights(fetcher, bbox=arguments.get("bbox"))
case "intel_theater_posture":
return await military.fetch_theater_posture(fetcher)
case "intel_aircraft_details":
return await military.fetch_aircraft_details(fetcher, icao24=arguments["icao24"])
# Infrastructure
case "intel_internet_outages":
return await infrastructure.fetch_internet_outages(fetcher)
case "intel_cable_health":
return await infrastructure.fetch_cable_health(fetcher)
# Maritime
case "intel_nav_warnings":
return await maritime.fetch_nav_warnings(fetcher, navarea=arguments.get("navarea"))
# Climate
case "intel_climate_anomalies":
return await climate.fetch_climate_anomalies(fetcher, zones=arguments.get("zones"))
# Prediction
case "intel_prediction_markets":
return await prediction.fetch_prediction_markets(fetcher, limit=arguments.get("limit", 20))
# Displacement
case "intel_displacement_summary":
return await displacement.fetch_displacement_summary(fetcher, year=arguments.get("year"))
# Aviation
case "intel_airport_delays":
return await aviation.fetch_airport_delays(fetcher)
# Cyber
case "intel_cyber_threats":
return await cyber.fetch_cyber_threats(fetcher, limit=arguments.get("limit", 50))
# News
case "intel_news_feed":
return await news.fetch_news_feed(
fetcher,
category=arguments.get("category"),
limit=arguments.get("limit", 50),
)
case "intel_trending_keywords":
return await news.fetch_trending_keywords(fetcher, min_count=arguments.get("min_count", 3))
case "intel_gdelt_search":
return await news.fetch_gdelt_search(
fetcher,
query=arguments.get("query", "conflict"),
mode=arguments.get("mode", "artlist"),
limit=arguments.get("limit", 50),
)
# Intelligence
case "intel_country_brief":
return await intelligence.fetch_country_brief(fetcher, country_code=arguments.get("country_code", "US"))
case "intel_risk_scores":
return await intelligence.fetch_risk_scores(fetcher, limit=arguments.get("limit", 20))
case "intel_instability_index":
return await intelligence.fetch_instability_index(fetcher, country_code=arguments.get("country_code"))
case "intel_signal_convergence":
return await intelligence.fetch_signal_convergence(
fetcher,
lat=arguments.get("lat"),
lon=arguments.get("lon"),
radius_deg=arguments.get("radius_deg", 5.0),
)
# Reports
case "intel_daily_brief":
return await report_gen.generate_daily_brief(output_dir=arguments.get("output_dir"))
case "intel_country_dossier":
return await report_gen.generate_country_dossier(
country_code=arguments["country_code"],
output_dir=arguments.get("output_dir"),
)
case "intel_threat_landscape":
return await report_gen.generate_threat_landscape(output_dir=arguments.get("output_dir"))
# System
case "intel_status":
return {
"circuit_breakers": breaker.status(),
"cache": cache.stats(),
"sources": {
"markets": ["yahoo-finance", "coingecko", "alternative-me", "mempool"],
"economic": ["eia", "fred", "world-bank"],
"natural": ["usgs", "nasa-firms"],
"conflict": ["acled", "ucdp", "hdx"],
"military": ["opensky", "wingbits"],
"infrastructure": ["cloudflare-radar", "nga-msi"],
"maritime": ["nga-msi"],
"climate": ["open-meteo"],
"news": ["rss-aggregator", "gdelt"],
"intelligence": ["ollama", "acled", "world-bank", "hdx", "usgs"],
"prediction": ["polymarket"],
"displacement": ["unhcr"],
"aviation": ["faa"],
"cyber": ["feodo-tracker", "cisa-kev", "sans-dshield", "urlhaus"],
},
}
case _:
return {"error": f"Unknown tool: {name}"}
# ---------------------------------------------------------------------------
# MCP handlers
# ---------------------------------------------------------------------------
@server.list_tools()
async def list_tools() -> list[Tool]:
return TOOLS
@server.call_tool()
async def call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextContent]:
args = arguments or {}
logger.info("Tool call: %s(%s)", name, json.dumps(args, default=str)[:200])
result = await _dispatch(name, args)
return [TextContent(type="text", text=json.dumps(result, indent=2, default=str))]
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
async def _run() -> None:
logger.info("World Intelligence MCP Server starting (%d tools)", len(TOOLS))
async with stdio_server() as (read_stream, write_stream):
await server.run(read_stream, write_stream, server.create_initialization_options())
def run() -> None:
asyncio.run(_run())
if __name__ == "__main__":
run()
+1
View File
@@ -0,0 +1 @@
"""Data source modules for world-intel-mcp."""
+133
View File
@@ -0,0 +1,133 @@
"""FAA airport delay data source for world-intel-mcp.
Provides real-time US airport delay information from the FAA Airport
Status Web Service (ASWS) API. No API key required.
"""
import asyncio
import logging
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.aviation")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_FAA_STATUS_URL = "https://soa.smext.faa.gov/asws/api/airport/status"
_MAJOR_AIRPORTS = [
"ATL", "LAX", "ORD", "DFW", "DEN", "JFK", "SFO", "SEA", "LAS", "MCO",
"EWR", "CLT", "PHX", "IAH", "MIA", "BOS", "MSP", "FLL", "DTW", "PHL",
]
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _parse_airport_status(code: str, data: dict) -> dict:
"""Extract structured fields from a single FAA airport status response."""
name = data.get("Name", code)
delay = data.get("Delay", False)
# Normalize delay to boolean (API may return string "true"/"false")
if isinstance(delay, str):
delay = delay.lower() == "true"
status_items = data.get("Status", [])
if not isinstance(status_items, list):
status_items = [status_items] if isinstance(status_items, dict) else []
parsed_statuses = []
for item in status_items:
if not isinstance(item, dict):
continue
parsed_statuses.append({
"type": item.get("Type", ""),
"reason": item.get("Reason", ""),
"avg_delay": item.get("AvgDelay", ""),
"closure_begin": item.get("ClosureBegin", ""),
"closure_end": item.get("ClosureEnd", ""),
})
return {
"code": code,
"name": name,
"delay": delay,
"status": parsed_statuses,
}
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_airport_delays(fetcher: Fetcher) -> dict:
"""Fetch current US airport delays from the FAA Airport Status API.
Queries the FAA ASWS API for each major US airport in parallel and
returns a summary of which airports currently have active delays.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
Returns:
Dict with delayed airports list, counts, source, and timestamp.
"""
async def _fetch_one(code: str) -> tuple[str, dict | None]:
"""Fetch status for a single airport, returning (code, data|None)."""
data = await fetcher.get_json(
url=f"{_FAA_STATUS_URL}/{code}",
source="faa",
cache_key=f"aviation:faa:{code}",
cache_ttl=300,
)
return code, data
# Fetch all airports in parallel
results = await asyncio.gather(
*[_fetch_one(code) for code in _MAJOR_AIRPORTS],
return_exceptions=True,
)
now_iso = _utc_now_iso()
delayed: list[dict] = []
all_airports: list[dict] = []
errors = 0
for result in results:
if isinstance(result, Exception):
logger.warning("Exception fetching airport status: %s", result)
errors += 1
continue
code, data = result
if data is None:
logger.debug("No data returned for airport %s", code)
errors += 1
continue
parsed = _parse_airport_status(code, data)
all_airports.append(parsed)
if parsed["delay"]:
delayed.append(parsed)
return {
"delayed": delayed,
"delayed_count": len(delayed),
"total_checked": len(_MAJOR_AIRPORTS),
"errors": errors,
"source": "faa",
"timestamp": now_iso,
}
+266
View File
@@ -0,0 +1,266 @@
"""Climate anomaly data source for world-intel-mcp.
Provides temperature and precipitation anomaly monitoring across 15 global
climate zones using the Open-Meteo Archive API. No API key required.
Anomalies are computed by comparing the most recent 7-day period against
the same 7-day window from the previous year (baseline).
"""
import asyncio
import logging
from datetime import datetime, timezone, timedelta
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.climate")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive"
CLIMATE_ZONES = {
"arctic": {"lat": 80.0, "lon": 0.0, "name": "Arctic (Svalbard)"},
"subarctic_siberia": {"lat": 65.0, "lon": 100.0, "name": "Subarctic Siberia"},
"northern_europe": {"lat": 55.0, "lon": 15.0, "name": "Northern Europe"},
"mediterranean": {"lat": 38.0, "lon": 20.0, "name": "Mediterranean Basin"},
"sahel": {"lat": 14.0, "lon": 0.0, "name": "Sahel Region"},
"middle_east": {"lat": 30.0, "lon": 45.0, "name": "Middle East"},
"south_asia": {"lat": 25.0, "lon": 78.0, "name": "South Asia (India)"},
"east_asia": {"lat": 35.0, "lon": 116.0, "name": "East Asia (Beijing)"},
"southeast_asia": {"lat": 2.0, "lon": 105.0, "name": "Southeast Asia"},
"australia": {"lat": -25.0, "lon": 135.0, "name": "Central Australia"},
"amazon": {"lat": -3.0, "lon": -60.0, "name": "Amazon Basin"},
"us_midwest": {"lat": 40.0, "lon": -90.0, "name": "US Midwest"},
"us_southwest": {"lat": 33.0, "lon": -112.0, "name": "US Southwest"},
"antarctica": {"lat": -75.0, "lon": 0.0, "name": "Antarctica"},
"pacific_enso": {"lat": 0.0, "lon": -150.0, "name": "Pacific ENSO Region"},
}
_CACHE_TTL = 1800 # 30 minutes
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _safe_avg(values: list[float | None]) -> float:
"""Return mean of non-None numeric values, or 0.0 if empty."""
clean = [v for v in values if v is not None]
if not clean:
return 0.0
return sum(clean) / len(clean)
def _safe_sum(values: list[float | None]) -> float:
"""Return sum of non-None numeric values, or 0.0 if empty."""
return sum(v for v in values if v is not None)
def _compute_anomalies(
current_data: dict,
baseline_data: dict,
) -> dict:
"""Compute temperature and precipitation anomalies between two periods.
Returns dict with current/baseline averages and anomaly values.
"""
cur_daily = current_data.get("daily", {})
base_daily = baseline_data.get("daily", {})
cur_max = cur_daily.get("temperature_2m_max", [])
cur_min = cur_daily.get("temperature_2m_min", [])
cur_precip = cur_daily.get("precipitation_sum", [])
base_max = base_daily.get("temperature_2m_max", [])
base_min = base_daily.get("temperature_2m_min", [])
base_precip = base_daily.get("precipitation_sum", [])
# Average temperature: mean of (daily_max + daily_min) / 2 across the period
cur_temps = [
(mx + mn) / 2
for mx, mn in zip(cur_max, cur_min)
if mx is not None and mn is not None
]
base_temps = [
(mx + mn) / 2
for mx, mn in zip(base_max, base_min)
if mx is not None and mn is not None
]
current_avg_temp = _safe_avg(cur_temps)
baseline_avg_temp = _safe_avg(base_temps)
temp_anomaly = round(current_avg_temp - baseline_avg_temp, 2)
# Precipitation totals
current_precip_mm = round(_safe_sum(cur_precip), 2)
baseline_precip_mm = round(_safe_sum(base_precip), 2)
# Percentage anomaly (guard against near-zero baseline)
precip_anomaly_pct = round(
((current_precip_mm - baseline_precip_mm)
/ max(baseline_precip_mm, 0.1))
* 100,
1,
)
# Significant anomaly flags
is_significant = abs(temp_anomaly) > 3.0 or abs(precip_anomaly_pct) > 100
return {
"current_avg_temp_c": round(current_avg_temp, 2),
"baseline_avg_temp_c": round(baseline_avg_temp, 2),
"temp_anomaly_c": temp_anomaly,
"current_precip_mm": current_precip_mm,
"baseline_precip_mm": baseline_precip_mm,
"precip_anomaly_pct": precip_anomaly_pct,
"is_significant": is_significant,
}
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_climate_anomalies(
fetcher: Fetcher,
zones: list[str] | None = None,
) -> dict:
"""Fetch temperature and precipitation anomalies for global climate zones.
Compares the most recent 7-day window against the same dates from the
previous year using the Open-Meteo Archive API. No API key required.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
zones: Optional list of zone keys to fetch. If None, all 15 zones
are fetched in parallel.
Returns:
Dict with per-zone anomaly data, list of significant anomalies,
source identifier, and ISO 8601 UTC timestamp.
"""
now = datetime.now(timezone.utc)
current_end = (now - timedelta(days=1)).strftime("%Y-%m-%d")
current_start = (now - timedelta(days=7)).strftime("%Y-%m-%d")
baseline_end_dt = now - timedelta(days=1) - timedelta(days=365)
baseline_start_dt = now - timedelta(days=7) - timedelta(days=365)
baseline_end = baseline_end_dt.strftime("%Y-%m-%d")
baseline_start = baseline_start_dt.strftime("%Y-%m-%d")
# Determine which zones to fetch
if zones is not None:
target_zones = {
k: v for k, v in CLIMATE_ZONES.items() if k in zones
}
if not target_zones:
logger.warning("No valid zone keys in %s", zones)
return {
"zones": {},
"significant_anomalies": [],
"source": "open-meteo",
"timestamp": _utc_now_iso(),
}
else:
target_zones = dict(CLIMATE_ZONES)
async def _fetch_zone(zone_key: str, zone_info: dict) -> tuple[str, dict | None]:
"""Fetch current and baseline data for a single zone, compute anomalies."""
lat = zone_info["lat"]
lon = zone_info["lon"]
common_params = {
"latitude": lat,
"longitude": lon,
"daily": "temperature_2m_max,temperature_2m_min,precipitation_sum",
"timezone": "UTC",
}
current_params = {
**common_params,
"start_date": current_start,
"end_date": current_end,
}
baseline_params = {
**common_params,
"start_date": baseline_start,
"end_date": baseline_end,
}
# Fetch current and baseline periods in parallel
current_data, baseline_data = await asyncio.gather(
fetcher.get_json(
url=_ARCHIVE_URL,
source="open-meteo",
cache_key=f"climate:anomalies:{zone_key}:current",
cache_ttl=_CACHE_TTL,
params=current_params,
),
fetcher.get_json(
url=_ARCHIVE_URL,
source="open-meteo",
cache_key=f"climate:anomalies:{zone_key}:baseline",
cache_ttl=_CACHE_TTL,
params=baseline_params,
),
)
if current_data is None or baseline_data is None:
logger.warning(
"Open-Meteo returned no data for zone %s (current=%s, baseline=%s)",
zone_key,
current_data is not None,
baseline_data is not None,
)
return (zone_key, None)
anomalies = _compute_anomalies(current_data, baseline_data)
return (zone_key, {
"name": zone_info["name"],
"lat": lat,
"lon": lon,
**anomalies,
})
# Fetch all target zones in parallel
tasks = [
_fetch_zone(zone_key, zone_info)
for zone_key, zone_info in target_zones.items()
]
results = await asyncio.gather(*tasks)
# Assemble response
zone_results: dict[str, dict] = {}
significant_anomalies: list[str] = []
for zone_key, zone_data in results:
if zone_data is None:
continue
zone_results[zone_key] = zone_data
if zone_data["is_significant"]:
significant_anomalies.append(zone_key)
response = {
"zones": zone_results,
"significant_anomalies": significant_anomalies,
"source": "open-meteo",
"timestamp": _utc_now_iso(),
}
# Cache composite result
cache_label = "selected" if zones is not None else "all"
fetcher.cache.set(
f"climate:anomalies:{cache_label}",
response,
_CACHE_TTL,
)
return response
+356
View File
@@ -0,0 +1,356 @@
"""Conflict and crisis data sources for world-intel-mcp.
Fetches armed conflict events (ACLED), state-based violence data (UCDP),
and humanitarian dataset metadata (HDX) for the world-intel-mcp server.
"""
import asyncio
import logging
import os
from datetime import datetime, timezone, timedelta
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.conflict")
# ---------------------------------------------------------------------------
# ACLED: Armed Conflict Location & Event Data
# ---------------------------------------------------------------------------
_ACLED_URL = "https://api.acleddata.com/acled/read"
async def fetch_acled_events(
fetcher: Fetcher,
country: str | None = None,
days: int = 7,
limit: int = 100,
) -> dict:
"""Fetch recent armed conflict events from the ACLED API v3.
Requires ``ACLED_ACCESS_TOKEN`` in the environment. Free academic
access can be obtained at https://acleddata.com.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
country: Optional country name to filter events.
days: How far back to search (in days from now).
limit: Maximum number of results.
Returns:
Dict with events list, count, query params, source, and timestamp.
"""
now = datetime.now(timezone.utc)
access_token = os.environ.get("ACLED_ACCESS_TOKEN")
if not access_token:
return {
"error": "ACLED_ACCESS_TOKEN not configured",
"note": "Free academic access at acleddata.com",
}
start_date = (now - timedelta(days=days)).strftime("%Y-%m-%d")
end_date = now.strftime("%Y-%m-%d")
params: dict = {
"key": access_token,
"email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"),
"limit": limit,
"event_date": f"{start_date}|{end_date}",
"event_date_where": "BETWEEN",
}
if country:
params["country"] = country
cache_country = country or "global"
data = await fetcher.get_json(
_ACLED_URL,
source="acled",
cache_key=f"conflict:acled:{cache_country}:{days}",
cache_ttl=900,
params=params,
)
if data is None:
logger.warning("ACLED API returned no data")
return {
"events": [],
"count": 0,
"query": {"country": country, "days": days},
"source": "acled",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
events = []
for event in data.get("data", []):
fatalities_raw = event.get("fatalities")
fatalities = int(fatalities_raw) if fatalities_raw is not None else 0
events.append({
"event_id_cnty": event.get("event_id_cnty"),
"event_date": event.get("event_date"),
"event_type": event.get("event_type"),
"sub_event_type": event.get("sub_event_type"),
"actor1": event.get("actor1"),
"actor2": event.get("actor2"),
"country": event.get("country"),
"admin1": event.get("admin1"),
"admin2": event.get("admin2"),
"location": event.get("location"),
"latitude": event.get("latitude"),
"longitude": event.get("longitude"),
"fatalities": fatalities,
"notes": event.get("notes"),
"source": event.get("source"),
})
# Sort by fatalities descending, then event_date descending
# Negate fatalities for desc; negate date lexicographically is not possible,
# so use a two-pass stable sort: secondary key first, primary key second.
events.sort(key=lambda e: e.get("event_date") or "", reverse=True)
events.sort(key=lambda e: e.get("fatalities") or 0, reverse=True)
return {
"events": events,
"count": len(events),
"query": {"country": country, "days": days},
"source": "acled",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# ---------------------------------------------------------------------------
# UCDP: Uppsala Conflict Data Program (GED API)
# ---------------------------------------------------------------------------
_UCDP_GED_URL = "https://ucdpapi.pcr.uu.se/api/gedevents/24.1"
_VIOLENCE_TYPES = {
1: "state-based",
2: "non-state",
3: "one-sided",
}
async def fetch_ucdp_events(
fetcher: Fetcher,
days: int = 30,
limit: int = 100,
) -> dict:
"""Fetch recent conflict events from the UCDP GED API.
No API key required. Fetches multiple pages in parallel when the
dataset spans more than one page.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
days: Only include events whose ``date_start`` is within this
many days from now.
limit: Page size (max 1000 per page).
Returns:
Dict with events list, count, total fatalities, query params,
source, and timestamp.
"""
now = datetime.now(timezone.utc)
cutoff = now - timedelta(days=days)
# Fetch page 1 to discover total pages
page1_data = await fetcher.get_json(
_UCDP_GED_URL,
source="ucdp",
cache_key=f"conflict:ucdp:{days}:page1",
cache_ttl=21600,
params={"pagesize": min(limit, 1000), "page": 0},
)
if page1_data is None:
logger.warning("UCDP GED API returned no data")
return {
"events": [],
"count": 0,
"total_fatalities_best": 0,
"query": {"days": days},
"source": "ucdp",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
total_pages = page1_data.get("TotalPages", 1)
all_results = list(page1_data.get("Result", []))
# Fetch remaining pages in parallel
if total_pages > 1:
page_tasks = [
fetcher.get_json(
_UCDP_GED_URL,
source="ucdp",
cache_key=f"conflict:ucdp:{days}:page{p}",
cache_ttl=21600,
params={"pagesize": min(limit, 1000), "page": p},
)
for p in range(1, total_pages)
]
page_results = await asyncio.gather(*page_tasks)
for page_data in page_results:
if page_data is not None:
all_results.extend(page_data.get("Result", []))
# Filter to recent events and extract fields
events = []
total_fatalities_best = 0
for record in all_results:
date_start = record.get("date_start")
if date_start is None:
continue
# Parse date_start and filter by cutoff
try:
event_date = datetime.strptime(date_start[:10], "%Y-%m-%d").replace(
tzinfo=timezone.utc,
)
except (ValueError, TypeError):
continue
if event_date < cutoff:
continue
violence_type_raw = record.get("type_of_violence")
violence_type_int = None
if violence_type_raw is not None:
try:
violence_type_int = int(violence_type_raw)
except (ValueError, TypeError):
pass
best = 0
if record.get("best") is not None:
try:
best = int(record["best"])
except (ValueError, TypeError):
pass
high = 0
if record.get("high") is not None:
try:
high = int(record["high"])
except (ValueError, TypeError):
pass
low = 0
if record.get("low") is not None:
try:
low = int(record["low"])
except (ValueError, TypeError):
pass
total_fatalities_best += best
events.append({
"id": record.get("id"),
"relid": record.get("relid"),
"year": record.get("year"),
"date_start": date_start,
"date_end": record.get("date_end"),
"country": record.get("country"),
"region": record.get("region"),
"type_of_violence": violence_type_int,
"type_of_violence_label": _VIOLENCE_TYPES.get(violence_type_int, "unknown") if violence_type_int is not None else "unknown",
"side_a": record.get("side_a"),
"side_b": record.get("side_b"),
"best": best,
"high": high,
"low": low,
"latitude": record.get("latitude"),
"longitude": record.get("longitude"),
"source_article": record.get("source_article"),
"source_headline": record.get("source_headline"),
})
return {
"events": events,
"count": len(events),
"total_fatalities_best": total_fatalities_best,
"query": {"days": days},
"source": "ucdp",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# ---------------------------------------------------------------------------
# HDX: Humanitarian Data Exchange (CKAN API)
# ---------------------------------------------------------------------------
_HDX_SEARCH_URL = "https://data.humdata.org/api/3/action/package_search"
async def fetch_humanitarian_summary(
fetcher: Fetcher,
country: str | None = None,
) -> dict:
"""Fetch recent humanitarian crisis datasets from HDX (Humanitarian Data Exchange).
No API key required. Uses the CKAN package_search API.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
country: Optional ISO 3166-1 alpha-3 country code (lowercase) to
filter datasets by geographic group.
Returns:
Dict with datasets list, count, source, and timestamp.
"""
now = datetime.now(timezone.utc)
params: dict = {
"q": "crisis",
"rows": 20,
"sort": "metadata_modified desc",
}
if country:
params["fq"] = f"groups:{country.lower()}"
cache_country = country or "global"
data = await fetcher.get_json(
_HDX_SEARCH_URL,
source="hdx",
cache_key=f"conflict:humanitarian:{cache_country}",
cache_ttl=21600,
params=params,
)
if data is None:
logger.warning("HDX API returned no data")
return {
"datasets": [],
"count": 0,
"source": "hdx",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
datasets = []
results = data.get("result", {}).get("results", [])
for dataset in results:
notes = dataset.get("notes") or ""
if len(notes) > 200:
notes = notes[:200] + "..."
org = dataset.get("organization") or {}
org_title = org.get("title") if isinstance(org, dict) else None
datasets.append({
"name": dataset.get("name"),
"title": dataset.get("title"),
"organization": org_title,
"metadata_modified": dataset.get("metadata_modified"),
"num_resources": dataset.get("num_resources"),
"notes": notes,
})
return {
"datasets": datasets,
"count": len(datasets),
"source": "hdx",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
+278
View File
@@ -0,0 +1,278 @@
"""Cyber threat intelligence feed aggregation for world-intel-mcp.
Aggregates threat data from multiple free public feeds:
- Feodo Tracker (abuse.ch) -- C2 botnet IPs
- CISA Known Exploited Vulnerabilities -- actively exploited CVEs
- SANS ISC DShield -- top attacking IPs
- URLhaus (abuse.ch) -- malware distribution URLs
No API keys required.
"""
import asyncio
import logging
from datetime import datetime, timezone, timedelta
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.cyber")
# ---------------------------------------------------------------------------
# Feed URLs
# ---------------------------------------------------------------------------
_FEODO_URL = "https://feodotracker.abuse.ch/downloads/ipblocklist_recommended.json"
_CISA_KEV_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
_SANS_ISC_URL = "https://isc.sans.edu/api/topips/records/20?json"
_URLHAUS_RECENT_URL = "https://urlhaus-api.abuse.ch/v1/urls/recent/limit/25/"
# ---------------------------------------------------------------------------
# Severity ranking for sorting
# ---------------------------------------------------------------------------
_SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _normalize_feodo(data: list | None) -> list[dict]:
"""Normalize Feodo Tracker C2 IP entries."""
if not data or not isinstance(data, list):
return []
items: list[dict] = []
for entry in data:
status = entry.get("status", "").lower()
severity = "critical" if status == "online" else "medium"
items.append({
"type": "c2_ip",
"indicator": entry.get("ip_address", ""),
"threat": entry.get("malware", "unknown"),
"severity": severity,
"source_feed": "feodo-tracker",
"first_seen": entry.get("first_seen", ""),
"details": {
"port": entry.get("port"),
"status": entry.get("status"),
"hostname": entry.get("hostname"),
"as_number": entry.get("as_number"),
"as_name": entry.get("as_name"),
"country": entry.get("country"),
"last_online": entry.get("last_online"),
},
})
return items
def _normalize_cisa_kev(data: dict | None) -> list[dict]:
"""Normalize CISA Known Exploited Vulnerabilities, last 30 days only."""
if not data or not isinstance(data, dict):
return []
cutoff = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%d")
vulnerabilities = data.get("vulnerabilities", [])
items: list[dict] = []
for vuln in vulnerabilities:
date_added = vuln.get("dateAdded", "")
if date_added < cutoff:
continue
ransomware = vuln.get("knownRansomwareCampaignUse", "").lower()
severity = "critical" if ransomware == "known" else "high"
cve_id = vuln.get("cveID", "")
vendor = vuln.get("vendorProject", "")
product = vuln.get("product", "")
vuln_name = vuln.get("vulnerabilityName", "")
items.append({
"type": "vulnerability",
"indicator": cve_id,
"threat": f"{vendor} {product}: {vuln_name}",
"severity": severity,
"source_feed": "cisa-kev",
"first_seen": date_added,
"details": {
"vendor": vendor,
"product": product,
"vulnerability_name": vuln_name,
"due_date": vuln.get("dueDate"),
"ransomware_use": vuln.get("knownRansomwareCampaignUse"),
"required_action": vuln.get("requiredAction"),
"notes": vuln.get("notes"),
},
})
return items
def _normalize_sans_isc(data: list | None) -> list[dict]:
"""Normalize SANS ISC DShield top attacking IPs."""
if not data or not isinstance(data, list):
return []
items: list[dict] = []
for entry in data:
ip = entry.get("ip", "")
if not ip:
continue
items.append({
"type": "attack_ip",
"indicator": ip,
"threat": f"DShield top attacker ({entry.get('attacks', 0)} attacks)",
"severity": "high",
"source_feed": "sans-dshield",
"first_seen": entry.get("firstseen", ""),
"details": {
"count": entry.get("count"),
"attacks": entry.get("attacks"),
"first_seen": entry.get("firstseen"),
"last_seen": entry.get("lastseen"),
"as_name": entry.get("asname"),
"as_country": entry.get("ascountry"),
},
})
return items
def _normalize_urlhaus(data: dict | None) -> list[dict]:
"""Normalize URLhaus recent malware URLs."""
if not data or not isinstance(data, dict):
return []
urls = data.get("urls", [])
if not isinstance(urls, list):
return []
items: list[dict] = []
for entry in urls:
status = (entry.get("url_status") or "").lower()
severity = "high" if status == "online" else "low"
items.append({
"type": "malware_url",
"indicator": entry.get("url", ""),
"threat": entry.get("threat", "unknown"),
"severity": severity,
"source_feed": "urlhaus",
"first_seen": entry.get("dateadded", ""),
"details": {
"url_status": entry.get("url_status"),
"tags": entry.get("tags"),
"reporter": entry.get("reporter"),
},
})
return items
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_cyber_threats(
fetcher: Fetcher,
limit: int = 50,
) -> dict:
"""Aggregate cyber threat intelligence from free public feeds.
Fetches data from Feodo Tracker, CISA KEV, SANS ISC DShield, and
URLhaus in parallel, normalizes into a common format, and returns
the top threats sorted by severity and recency.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
limit: Maximum number of threat items to return.
Returns:
Dict with threats list, counts, breakdowns by type and severity,
source, and timestamp.
"""
feodo_data, cisa_data, sans_data, urlhaus_data = await asyncio.gather(
fetcher.get_json(
_FEODO_URL,
source="feodo-tracker",
cache_key="cyber:feodo",
cache_ttl=1800,
),
fetcher.get_json(
_CISA_KEV_URL,
source="cisa-kev",
cache_key="cyber:cisa_kev",
cache_ttl=3600,
),
fetcher.get_json(
_SANS_ISC_URL,
source="sans-dshield",
cache_key="cyber:sans_isc",
cache_ttl=1800,
),
fetcher.get_json(
_URLHAUS_RECENT_URL,
source="urlhaus",
cache_key="cyber:urlhaus",
cache_ttl=900,
),
)
# Track which feeds returned data
feeds_successful = 0
feodo_items = _normalize_feodo(feodo_data)
if feodo_data is not None:
feeds_successful += 1
else:
logger.warning("Feodo Tracker feed returned no data")
cisa_items = _normalize_cisa_kev(cisa_data)
if cisa_data is not None:
feeds_successful += 1
else:
logger.warning("CISA KEV feed returned no data")
sans_items = _normalize_sans_isc(sans_data)
if sans_data is not None:
feeds_successful += 1
else:
logger.warning("SANS ISC DShield feed returned no data")
urlhaus_items = _normalize_urlhaus(urlhaus_data)
if urlhaus_data is not None:
feeds_successful += 1
else:
logger.warning("URLhaus feed returned no data")
# Merge all threats
all_threats = feodo_items + cisa_items + sans_items + urlhaus_items
# Sort by severity (critical first), then by recency (newest first_seen first).
# Use stable sort: first by date descending, then by severity ascending.
all_threats.sort(key=lambda t: t.get("first_seen", ""), reverse=True)
all_threats.sort(key=lambda t: _SEVERITY_ORDER.get(t.get("severity", "low"), 3))
# Apply limit
threats = all_threats[:limit]
# Compute breakdowns
by_type: dict[str, int] = {}
by_severity: dict[str, int] = {}
for t in threats:
threat_type = t.get("type", "unknown")
by_type[threat_type] = by_type.get(threat_type, 0) + 1
sev = t.get("severity", "low")
by_severity[sev] = by_severity.get(sev, 0) + 1
return {
"threats": threats,
"count": len(threats),
"feeds_successful": feeds_successful,
"feeds_attempted": 4,
"by_type": by_type,
"by_severity": by_severity,
"source": "cyber-feeds",
"timestamp": _utc_now_iso(),
}
+169
View File
@@ -0,0 +1,169 @@
"""UNHCR displacement data source for world-intel-mcp.
Provides refugee and displacement population statistics from the UNHCR
Population Statistics API. No API key required.
"""
import logging
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.displacement")
_UNHCR_POPULATION_URL = "https://api.unhcr.org/population/v1/population/"
async def fetch_displacement_summary(
fetcher: Fetcher,
year: int | None = None,
) -> dict:
"""Fetch refugee/displacement population statistics from the UNHCR API.
Aggregates displacement data by country of origin and computes global
totals across refugees, asylum seekers, IDPs, stateless persons, and
others of concern.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
year: Reporting year to query. Defaults to the previous calendar
year since UNHCR data typically lags by one year.
Returns:
Dict with top-30 countries by displacement (sorted descending),
global totals, year, source, and timestamp.
"""
now = datetime.now(timezone.utc)
# Default to previous year since UNHCR data lags
if year is None:
year = now.year - 1
else:
# Ensure year is an int even if passed as string
year = int(year)
params = {
"limit": 100,
"yearFrom": year,
"yearTo": year,
"cf_type": "ISO",
}
data = await fetcher.get_json(
url=_UNHCR_POPULATION_URL,
source="unhcr",
cache_key=f"displacement:unhcr:{year}",
cache_ttl=43200,
params=params,
)
if data is None:
logger.warning("UNHCR API returned no data")
return {
"by_origin": [],
"global_totals": {
"total_refugees": 0,
"total_asylum_seekers": 0,
"total_idps": 0,
"total_stateless": 0,
"total_ooc": 0,
"grand_total": 0,
},
"year": year,
"count": 0,
"source": "unhcr",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
items = data.get("items", [])
# Aggregate by country of origin (coo_name)
by_country: dict[str, dict[str, int]] = {}
for item in items:
coo_name = item.get("coo_name") or "Unknown"
refugees = _safe_int(item.get("refugees"))
asylum_seekers = _safe_int(item.get("asylum_seekers"))
idps = _safe_int(item.get("idps"))
stateless = _safe_int(item.get("stateless"))
ooc = _safe_int(item.get("ooc"))
if coo_name not in by_country:
by_country[coo_name] = {
"refugees": 0,
"asylum_seekers": 0,
"idps": 0,
"stateless": 0,
"ooc": 0,
}
entry = by_country[coo_name]
entry["refugees"] += refugees
entry["asylum_seekers"] += asylum_seekers
entry["idps"] += idps
entry["stateless"] += stateless
entry["ooc"] += ooc
# Build sorted list with total_displaced
by_origin = []
for country, totals in by_country.items():
total_displaced = (
totals["refugees"]
+ totals["asylum_seekers"]
+ totals["idps"]
+ totals["stateless"]
+ totals["ooc"]
)
by_origin.append({
"country": country,
"refugees": totals["refugees"],
"asylum_seekers": totals["asylum_seekers"],
"internally_displaced": totals["idps"],
"stateless": totals["stateless"],
"others_of_concern": totals["ooc"],
"total_displaced": total_displaced,
})
# Sort by total_displaced descending, take top 30
by_origin.sort(key=lambda e: e["total_displaced"], reverse=True)
by_origin = by_origin[:30]
# Compute global totals across all countries (not just top 30)
global_refugees = sum(t["refugees"] for t in by_country.values())
global_asylum_seekers = sum(t["asylum_seekers"] for t in by_country.values())
global_idps = sum(t["idps"] for t in by_country.values())
global_stateless = sum(t["stateless"] for t in by_country.values())
global_ooc = sum(t["ooc"] for t in by_country.values())
return {
"by_origin": by_origin,
"global_totals": {
"total_refugees": global_refugees,
"total_asylum_seekers": global_asylum_seekers,
"total_idps": global_idps,
"total_stateless": global_stateless,
"total_ooc": global_ooc,
"grand_total": (
global_refugees
+ global_asylum_seekers
+ global_idps
+ global_stateless
+ global_ooc
),
},
"year": year,
"count": len(by_origin),
"source": "unhcr",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
def _safe_int(value: int | float | str | None) -> int:
"""Convert a value to int, returning 0 for None or unparseable values."""
if value is None:
return 0
try:
return int(value)
except (ValueError, TypeError):
return 0
+308
View File
@@ -0,0 +1,308 @@
"""Economic indicator data sources.
Fetches energy prices (EIA), macroeconomic series (FRED), and
development indicators (World Bank) for the world-intel-mcp server.
"""
import asyncio
import logging
import os
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.economic")
# ---------------------------------------------------------------------------
# EIA: Energy prices (oil & natural gas)
# ---------------------------------------------------------------------------
_EIA_OIL_URL = "https://api.eia.gov/v2/petroleum/pri/spt/data/"
_EIA_GAS_URL = "https://api.eia.gov/v2/natural-gas/pri/fut/data/"
async def fetch_energy_prices(
fetcher: Fetcher,
api_key: str | None = None,
) -> dict:
"""Fetch latest crude oil (Brent & WTI) and natural gas spot prices.
Uses the EIA API v2. Requires an API key via *api_key* or the
``EIA_API_KEY`` environment variable.
Returns a dict with ``oil`` and ``natural_gas`` sub-keys, plus
metadata (``fetched_at``, ``source``).
"""
key = api_key or os.environ.get("EIA_API_KEY")
if not key:
return {"error": "EIA_API_KEY not configured"}
oil_params = {
"api_key": key,
"frequency": "daily",
"data[0]": "value",
"sort[0][column]": "period",
"sort[0][direction]": "desc",
"length": 5,
"facets[product][]": ["EPCBRENT", "EPCWTI"],
}
gas_params = {
"api_key": key,
"frequency": "daily",
"data[0]": "value",
"sort[0][column]": "period",
"sort[0][direction]": "desc",
"length": 5,
"facets[process][]": "PRC",
}
oil_data, gas_data = await asyncio.gather(
fetcher.get_json(
_EIA_OIL_URL,
source="eia",
cache_key="economic:energy:oil",
cache_ttl=3600,
params=oil_params,
),
fetcher.get_json(
_EIA_GAS_URL,
source="eia",
cache_key="economic:energy:gas",
cache_ttl=3600,
params=gas_params,
),
)
result: dict = {
"oil": {"brent": None, "wti": None},
"natural_gas": None,
"fetched_at": datetime.now(timezone.utc).isoformat(),
"source": "eia",
}
# --- Parse oil prices ---------------------------------------------------
if oil_data:
try:
records = (
oil_data.get("response", {}).get("data", [])
)
for rec in records:
product = rec.get("product")
value = rec.get("value")
period = rec.get("period")
if value is None or period is None:
continue
entry = {"price": float(value), "date": period}
if product == "EPCBRENT" and result["oil"]["brent"] is None:
result["oil"]["brent"] = entry
elif product == "EPCWTI" and result["oil"]["wti"] is None:
result["oil"]["wti"] = entry
except (KeyError, TypeError, ValueError) as exc:
logger.warning("Failed to parse EIA oil data: %s", exc)
# --- Parse natural gas price --------------------------------------------
if gas_data:
try:
records = (
gas_data.get("response", {}).get("data", [])
)
if records:
rec = records[0]
value = rec.get("value")
period = rec.get("period")
if value is not None and period is not None:
result["natural_gas"] = {
"price": float(value),
"date": period,
}
except (KeyError, TypeError, ValueError) as exc:
logger.warning("Failed to parse EIA natural gas data: %s", exc)
return result
# ---------------------------------------------------------------------------
# FRED: Federal Reserve Economic Data
# ---------------------------------------------------------------------------
_FRED_URL = "https://api.stlouisfed.org/fred/series/observations"
async def fetch_fred_series(
fetcher: Fetcher,
series_id: str,
api_key: str | None = None,
limit: int = 30,
) -> dict:
"""Fetch observations for a FRED series.
Common series IDs:
* ``GDP`` -- Gross Domestic Product
* ``UNRATE`` -- Unemployment Rate
* ``CPIAUCSL`` -- Consumer Price Index
* ``DFF`` -- Federal Funds Effective Rate
* ``T10YIE`` -- 10-Year Breakeven Inflation Rate
Returns a dict with ``series_id``, ``title``, ``observations``, plus
metadata.
"""
key = api_key or os.environ.get("FRED_API_KEY")
if not key:
return {"error": "FRED_API_KEY not configured"}
params = {
"series_id": series_id,
"api_key": key,
"file_type": "json",
"sort_order": "desc",
"limit": limit,
}
data = await fetcher.get_json(
_FRED_URL,
source="fred",
cache_key=f"economic:fred:{series_id}:{limit}",
cache_ttl=3600,
params=params,
)
result: dict = {
"series_id": series_id,
"title": series_id,
"observations": [],
"fetched_at": datetime.now(timezone.utc).isoformat(),
"source": "fred",
}
if data is None:
return result
try:
raw_obs = data.get("observations", [])
for obs in raw_obs:
date = obs.get("date")
value = obs.get("value")
if date is None:
continue
# FRED uses "." for missing values
parsed_value: float | None = None
if value not in (None, ".", ""):
try:
parsed_value = float(value)
except (ValueError, TypeError):
pass
result["observations"].append({"date": date, "value": parsed_value})
# FRED embeds series metadata when available
if "realtime_start" in data:
result["realtime_start"] = data["realtime_start"]
if "realtime_end" in data:
result["realtime_end"] = data["realtime_end"]
except (KeyError, TypeError) as exc:
logger.warning("Failed to parse FRED series %s: %s", series_id, exc)
return result
# ---------------------------------------------------------------------------
# World Bank: Development indicators
# ---------------------------------------------------------------------------
_WB_BASE = "https://api.worldbank.org/v2/country"
_DEFAULT_INDICATORS = [
"NY.GDP.MKTP.CD", # GDP (current US$)
"FP.CPI.TOTL.ZG", # Inflation, consumer prices (annual %)
"SL.UEM.TOTL.ZS", # Unemployment, total (% of labor force)
]
async def fetch_world_bank_indicators(
fetcher: Fetcher,
country: str = "USA",
indicators: list[str] | None = None,
) -> dict:
"""Fetch World Bank development indicators for a country.
Defaults to GDP, inflation, and unemployment for the USA.
Indicators are fetched in parallel.
Returns a dict with ``country``, ``indicators`` (list of dicts with
``id``, ``name``, and ``values``), plus metadata.
"""
indicator_ids = indicators or _DEFAULT_INDICATORS
async def _fetch_one(indicator: str) -> dict | None:
url = f"{_WB_BASE}/{country}/indicator/{indicator}"
params = {
"format": "json",
"per_page": 5,
"date": "2020:2025",
}
return await fetcher.get_json(
url,
source="world-bank",
cache_key=f"economic:wb:{country}:{indicator}",
cache_ttl=86400,
params=params,
)
responses = await asyncio.gather(
*[_fetch_one(ind) for ind in indicator_ids]
)
parsed_indicators: list[dict] = []
for indicator_id, raw in zip(indicator_ids, responses):
entry: dict = {
"id": indicator_id,
"name": indicator_id,
"values": [],
}
if raw is None:
parsed_indicators.append(entry)
continue
try:
# World Bank v2 JSON returns a 2-element list:
# [metadata_dict, data_records_list]
if isinstance(raw, list) and len(raw) >= 2:
records = raw[1]
if records and isinstance(records, list):
# Extract human-readable indicator name from first record
first = records[0]
ind_info = first.get("indicator", {})
if isinstance(ind_info, dict):
entry["name"] = ind_info.get("value", indicator_id)
for rec in records:
year = rec.get("date")
value = rec.get("value")
if year is not None:
parsed_value: float | None = None
if value is not None:
try:
parsed_value = float(value)
except (ValueError, TypeError):
pass
entry["values"].append({
"year": year,
"value": parsed_value,
})
except (KeyError, TypeError, IndexError) as exc:
logger.warning(
"Failed to parse World Bank indicator %s for %s: %s",
indicator_id, country, exc,
)
parsed_indicators.append(entry)
return {
"country": country,
"indicators": parsed_indicators,
"fetched_at": datetime.now(timezone.utc).isoformat(),
"source": "world-bank",
}
@@ -0,0 +1,406 @@
"""Internet infrastructure monitoring for world-intel-mcp.
Provides real-time data on internet outages (Cloudflare Radar) and
undersea cable corridor health (NGA Maritime Safety Information).
"""
import asyncio
import logging
import os
import re
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.infrastructure")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_CF_RADAR_OUTAGES_URL = (
"https://radar.cloudflare.com/api/v1/annotations/outages"
)
_CF_RADAR_TAMPERING_URL = (
"https://api.cloudflare.com/client/v4/radar/connection_tampering/summary"
)
_NGA_BROADCAST_WARN_URL = (
"https://msi.nga.mil/api/publications/broadcast-warn"
)
# Known undersea cable corridors as rough lat/lon bounding boxes.
CABLE_CORRIDORS: dict[str, dict] = {
"transatlantic_north": {
"lat_range": (35, 55),
"lon_range": (-70, -5),
"cables": ["TAT-14", "AC-1", "MAREA", "Dunant"],
},
"transatlantic_south": {
"lat_range": (-5, 35),
"lon_range": (-50, 0),
"cables": ["SAex1", "SACS", "EllaLink"],
},
"transpacific": {
"lat_range": (20, 50),
"lon_range": (120, -120),
"cables": ["FASTER", "Unity", "SJC"],
},
"asia_europe": {
"lat_range": (-5, 35),
"lon_range": (30, 120),
"cables": ["SEA-ME-WE 6", "AAE-1", "FLAG"],
},
"red_sea": {
"lat_range": (10, 30),
"lon_range": (30, 50),
"cables": ["FALCON", "EIG", "AAE-1"],
},
"mediterranean": {
"lat_range": (30, 45),
"lon_range": (-5, 35),
"cables": ["SEA-ME-WE 3/4", "MedNautilus"],
},
}
# Keywords that indicate a navigational warning is cable-related.
_CABLE_KEYWORDS = re.compile(
r"\b(cable|submarine|fiber|anchor|dredg)\b", re.IGNORECASE
)
# Pattern for DMS coordinates like "32-15.5N/044-30.2E" or "32-15N 044-30E".
_DMS_PATTERN = re.compile(
r"(\d{1,3})-(\d{1,2}(?:\.\d+)?)\s*([NS])\s*[/ ]\s*"
r"(\d{1,3})-(\d{1,2}(?:\.\d+)?)\s*([EW])"
)
# Pattern for decimal lat/lon like "32.258N 44.503E" or "32.258/44.503".
_DECIMAL_PATTERN = re.compile(
r"(\d{1,3}(?:\.\d+)?)\s*([NS])\s*[/ ]\s*"
r"(\d{1,3}(?:\.\d+)?)\s*([EW])"
)
_STATUS_LABELS = {0: "clear", 1: "advisory", 2: "at_risk", 3: "disrupted"}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _extract_coordinates(text: str) -> list[tuple[float, float]]:
"""Extract (lat, lon) pairs from navigational warning text.
Handles both DMS notation (DD-MM.M[N/S]/DDD-MM.M[E/W]) and simple
decimal notation (DD.DDD[N/S] DDD.DDD[E/W]).
Returns a list of (latitude, longitude) tuples in decimal degrees.
"""
coords: list[tuple[float, float]] = []
# Try DMS pattern first
for match in _DMS_PATTERN.finditer(text):
deg_lat, min_lat, ns, deg_lon, min_lon, ew = match.groups()
lat = float(deg_lat) + float(min_lat) / 60.0
lon = float(deg_lon) + float(min_lon) / 60.0
if ns == "S":
lat = -lat
if ew == "W":
lon = -lon
coords.append((lat, lon))
# Try decimal pattern
for match in _DECIMAL_PATTERN.finditer(text):
lat_val, ns, lon_val, ew = match.groups()
lat = float(lat_val)
lon = float(lon_val)
if ns == "S":
lat = -lat
if ew == "W":
lon = -lon
coords.append((lat, lon))
return coords
def _point_in_corridor(
lat: float,
lon: float,
corridor: dict,
) -> bool:
"""Check whether a (lat, lon) point falls within a cable corridor bbox.
The transpacific corridor wraps across the antimeridian (lon_range
has min > max), so it requires special handling.
"""
lat_lo, lat_hi = corridor["lat_range"]
lon_lo, lon_hi = corridor["lon_range"]
if not (lat_lo <= lat <= lat_hi):
return False
# Handle antimeridian wrap (e.g. 120 to -120 means 120..180 or -180..-120)
if lon_lo > lon_hi:
return lon >= lon_lo or lon <= lon_hi
return lon_lo <= lon <= lon_hi
# ---------------------------------------------------------------------------
# Internet Outages (Cloudflare Radar)
# ---------------------------------------------------------------------------
async def fetch_internet_outages(fetcher: Fetcher) -> dict:
"""Fetch recent internet outage annotations from Cloudflare Radar.
Uses the public Radar annotations API for the last 7 days. If the
``CLOUDFLARE_API_TOKEN`` environment variable is set, authenticated
requests are made which may yield higher rate limits.
Returns:
Dict with outages list, ongoing/total counts, source, and timestamp.
"""
token = os.environ.get("CLOUDFLARE_API_TOKEN")
headers: dict[str, str] | None = None
if token:
headers = {"Authorization": f"Bearer {token}"}
params = {
"limit": 20,
"dateRange": "7d",
}
data = await fetcher.get_json(
url=_CF_RADAR_OUTAGES_URL,
source="cloudflare-radar",
cache_key="infra:outages",
cache_ttl=300,
headers=headers,
params=params,
)
now_iso = _utc_now_iso()
if data is None:
logger.warning("Cloudflare Radar outages API returned no data")
return {
"outages": [],
"ongoing_count": 0,
"total_7d": 0,
"source": "cloudflare-radar",
"timestamp": now_iso,
}
outages: list[dict] = []
# The public annotations endpoint returns:
# { "annotations": [ { "id", "dateStart", "dateEnd", "description",
# "scope", "asns", "locations", ... } ] }
# Adapt to both envelope shapes: top-level list or nested under a key.
raw_annotations: list = []
if isinstance(data, list):
raw_annotations = data
elif isinstance(data, dict):
# Try common envelope keys
for key in ("annotations", "result", "data"):
candidate = data.get(key)
if isinstance(candidate, list):
raw_annotations = candidate
break
ongoing_count = 0
for ann in raw_annotations:
if not isinstance(ann, dict):
continue
date_end = ann.get("dateEnd") or ann.get("endDate")
is_ongoing = date_end is None or date_end == ""
if is_ongoing:
ongoing_count += 1
# Normalize ASN list
raw_asns = ann.get("asns", [])
if isinstance(raw_asns, str):
raw_asns = [a.strip() for a in raw_asns.split(",") if a.strip()]
elif not isinstance(raw_asns, list):
raw_asns = []
# Normalize locations / country codes
raw_locations = ann.get("locations", ann.get("countries", []))
if isinstance(raw_locations, str):
raw_locations = [
loc.strip() for loc in raw_locations.split(",") if loc.strip()
]
elif not isinstance(raw_locations, list):
raw_locations = []
outages.append({
"id": ann.get("id"),
"start": ann.get("dateStart") or ann.get("startDate"),
"end": date_end if not is_ongoing else None,
"description": ann.get("description", ""),
"scope": ann.get("scope", "unknown"),
"countries": raw_locations,
"asns": raw_asns,
"is_ongoing": is_ongoing,
})
return {
"outages": outages,
"ongoing_count": ongoing_count,
"total_7d": len(outages),
"source": "cloudflare-radar",
"timestamp": now_iso,
}
# ---------------------------------------------------------------------------
# Undersea Cable Corridor Health (NGA Maritime Safety Information)
# ---------------------------------------------------------------------------
async def fetch_cable_health(fetcher: Fetcher) -> dict:
"""Assess undersea cable corridor health using NGA broadcast warnings.
Fetches navigational warnings from the NGA Maritime Safety Information
(MSI) API, extracts coordinates from the warning text, and scores each
known cable corridor:
0 = clear (no warnings)
1 = advisory (warning nearby the corridor)
2 = at_risk (warning inside the corridor)
3 = disrupted (multiple warnings or explicit cable keyword)
Returns:
Dict with corridor statuses, warning counts, source, and timestamp.
"""
data = await fetcher.get_json(
url=_NGA_BROADCAST_WARN_URL,
source="nga-msi",
cache_key="infra:cables",
cache_ttl=180,
params={"output": "json"},
)
now_iso = _utc_now_iso()
if data is None:
logger.warning("NGA MSI broadcast warnings API returned no data")
# Return all corridors as clear when data is unavailable
corridors_result = {}
for name, info in CABLE_CORRIDORS.items():
corridors_result[name] = {
"status_score": 0,
"status_label": "clear",
"cables": info["cables"],
"relevant_warnings": [],
}
return {
"corridors": corridors_result,
"warnings_total": 0,
"cable_related_warnings": 0,
"source": "nga-msi",
"timestamp": now_iso,
}
# Parse warnings from the response. The NGA API may return:
# - A top-level list of warnings
# - A dict with a "broadcast-warn" or "broadcastWarn" key
raw_warnings: list = []
if isinstance(data, list):
raw_warnings = data
elif isinstance(data, dict):
for key in (
"broadcast-warn", "broadcastWarn", "warnings",
"result", "data",
):
candidate = data.get(key)
if isinstance(candidate, list):
raw_warnings = candidate
break
# Initialize corridor tracking
corridor_warnings: dict[str, list[dict]] = {
name: [] for name in CABLE_CORRIDORS
}
cable_related_count = 0
for warning in raw_warnings:
if not isinstance(warning, dict):
continue
text = warning.get("text", "") or ""
msg_year = warning.get("msgYear")
msg_number = warning.get("msgNumber")
nav_area = warning.get("navArea", "")
subregion = warning.get("subregion", "")
status = warning.get("status", "")
issue_date = warning.get("issueDate", "")
has_cable_keyword = bool(_CABLE_KEYWORDS.search(text))
if has_cable_keyword:
cable_related_count += 1
# Extract coordinates from the warning text
coords = _extract_coordinates(text)
warning_summary = {
"msgYear": msg_year,
"msgNumber": msg_number,
"navArea": nav_area,
"subregion": subregion,
"status": status,
"issueDate": issue_date,
"has_cable_keyword": has_cable_keyword,
"text_snippet": text[:200] if text else "",
}
# Check each coordinate against cable corridors
for lat, lon in coords:
for corridor_name, corridor_info in CABLE_CORRIDORS.items():
if _point_in_corridor(lat, lon, corridor_info):
corridor_warnings[corridor_name].append(warning_summary)
# A warning can match multiple corridors but should
# only appear once per corridor.
break
# Score each corridor
corridors_result: dict[str, dict] = {}
for name, info in CABLE_CORRIDORS.items():
warnings_in_corridor = corridor_warnings[name]
cable_mentions = sum(
1 for w in warnings_in_corridor if w["has_cable_keyword"]
)
# Scoring logic:
# 0 = clear: no warnings in corridor
# 1 = advisory: 1 warning, no cable keywords
# 2 = at_risk: 1 warning with cable keyword, or 2+ warnings
# 3 = disrupted: multiple warnings with cable keywords or 3+ total
if not warnings_in_corridor:
score = 0
elif cable_mentions >= 2 or len(warnings_in_corridor) >= 3:
score = 3
elif cable_mentions >= 1 or len(warnings_in_corridor) >= 2:
score = 2
else:
score = 1
corridors_result[name] = {
"status_score": score,
"status_label": _STATUS_LABELS[score],
"cables": info["cables"],
"relevant_warnings": warnings_in_corridor,
}
return {
"corridors": corridors_result,
"warnings_total": len(raw_warnings),
"cable_related_warnings": cable_related_count,
"source": "nga-msi",
"timestamp": now_iso,
}
+723
View File
@@ -0,0 +1,723 @@
"""Country intelligence, risk scoring, and signal convergence sources.
Provides higher-level analytical functions that combine data from multiple
APIs (ACLED, World Bank, USGS, Ollama) into country briefs, risk scores,
instability indices, and geographic signal convergence assessments.
"""
import asyncio
import logging
import os
from datetime import datetime, timezone, timedelta
import httpx
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.intelligence")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_ACLED_URL = "https://api.acleddata.com/acled/read"
_WB_BASE = "https://api.worldbank.org/v2/country"
_HDX_SEARCH_URL = "https://data.humdata.org/api/3/action/package_search"
_USGS_ENDPOINT = "https://earthquake.usgs.gov/fdsnws/event/1/query"
_OPENSKY_STATES_URL = "https://opensky-network.org/api/states/all"
_BASELINES = {
"Syria": 5000, "Yemen": 3000, "Ukraine": 8000, "Myanmar": 4000,
"Somalia": 2500, "Nigeria": 3500, "DR Congo": 3000, "Afghanistan": 2000,
"Iraq": 1500, "Mali": 2000, "Burkina Faso": 2500, "Ethiopia": 2000,
"Sudan": 3000, "South Sudan": 1500, "Cameroon": 1000, "Mozambique": 800,
"Pakistan": 1200, "India": 1000, "Colombia": 1500, "Mexico": 4000,
}
_FOCUS_COUNTRIES = [
"SYR", "UKR", "YEM", "MMR", "SDN", "ETH", "NGA", "COD", "AFG", "IRQ",
]
_HOTSPOTS = {
"middle_east": (33.0, 44.0),
"east_africa": (5.0, 38.0),
"south_asia": (30.0, 70.0),
"eastern_europe": (48.0, 35.0),
"sahel": (15.0, 2.0),
}
# ISO-3166 alpha-3 to country name for ACLED queries and display.
_ISO3_TO_NAME = {
"SYR": "Syria", "UKR": "Ukraine", "YEM": "Yemen", "MMR": "Myanmar",
"SDN": "Sudan", "ETH": "Ethiopia", "NGA": "Nigeria", "COD": "DR Congo",
"AFG": "Afghanistan", "IRQ": "Iraq",
}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _risk_level(score: float) -> str:
if score > 150:
return "critical"
if score > 100:
return "elevated"
if score > 50:
return "moderate"
return "low"
def _instability_level(index: float) -> str:
if index >= 75:
return "critical"
if index >= 50:
return "high"
if index >= 25:
return "medium"
return "low"
# ---------------------------------------------------------------------------
# Function 1: Country Intelligence Brief
# ---------------------------------------------------------------------------
async def fetch_country_brief(
fetcher: Fetcher,
country_code: str = "US",
) -> dict:
"""Generate a country intelligence brief using local LLM and public data.
Gathers economic indicators from World Bank and conflict data from ACLED
in parallel, then optionally enriches with an Ollama-generated analytical
brief. Falls back to a data-only summary when Ollama is unavailable.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
country_code: ISO 3166-1 alpha-2 country code (e.g. ``US``, ``UA``).
Returns:
Dict with brief text, supporting data, LLM availability flag,
source, and timestamp.
"""
now = datetime.now(timezone.utc)
# --- Gather background data in parallel --------------------------------
async def _fetch_gdp() -> list:
url = f"{_WB_BASE}/{country_code}/indicator/NY.GDP.MKTP.CD"
params = {
"format": "json",
"per_page": 5,
"date": "2020:2025",
}
data = await fetcher.get_json(
url,
source="world-bank",
cache_key=f"intel:wb:gdp:{country_code}",
cache_ttl=86400,
params=params,
)
if data is None:
return []
values = []
try:
if isinstance(data, list) and len(data) >= 2 and isinstance(data[1], list):
for rec in data[1]:
year = rec.get("date")
value = rec.get("value")
if year is not None and value is not None:
try:
values.append({"year": year, "value": float(value)})
except (ValueError, TypeError):
pass
except (KeyError, TypeError, IndexError) as exc:
logger.warning("Failed to parse World Bank GDP for %s: %s", country_code, exc)
return values
async def _fetch_inflation() -> list:
url = f"{_WB_BASE}/{country_code}/indicator/FP.CPI.TOTL.ZG"
params = {
"format": "json",
"per_page": 5,
"date": "2020:2025",
}
data = await fetcher.get_json(
url,
source="world-bank",
cache_key=f"intel:wb:inflation:{country_code}",
cache_ttl=86400,
params=params,
)
if data is None:
return []
values = []
try:
if isinstance(data, list) and len(data) >= 2 and isinstance(data[1], list):
for rec in data[1]:
year = rec.get("date")
value = rec.get("value")
if year is not None and value is not None:
try:
values.append({"year": year, "value": float(value)})
except (ValueError, TypeError):
pass
except (KeyError, TypeError, IndexError) as exc:
logger.warning("Failed to parse World Bank inflation for %s: %s", country_code, exc)
return values
async def _fetch_acled_count() -> int:
access_token = os.environ.get("ACLED_ACCESS_TOKEN")
if not access_token:
return 0
start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d")
end_date = now.strftime("%Y-%m-%d")
params: dict = {
"key": access_token,
"email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"),
"limit": 0,
"event_date": f"{start_date}|{end_date}",
"event_date_where": "BETWEEN",
"country": country_code,
}
data = await fetcher.get_json(
_ACLED_URL,
source="acled",
cache_key=f"intel:acled:count:{country_code}",
cache_ttl=900,
params=params,
)
if data is None:
return 0
# ACLED returns a count field when limit=0
try:
return int(data.get("count", len(data.get("data", []))))
except (ValueError, TypeError):
return len(data.get("data", []))
gdp_values, inflation_values, event_count = await asyncio.gather(
_fetch_gdp(),
_fetch_inflation(),
_fetch_acled_count(),
)
# --- Attempt Ollama-generated brief ------------------------------------
llm_available = False
brief_text = "LLM brief unavailable. Data summary below."
prompt = (
f"Provide a concise 3-paragraph intelligence brief for {country_code}. "
"Cover: (1) current political stability and governance, "
"(2) economic outlook and risks, "
"(3) security concerns and regional dynamics. "
"Be factual and analytical."
)
ollama_url = os.environ.get("OLLAMA_API_URL", "http://localhost:11434")
model = os.environ.get("OLLAMA_MODEL", "llama3.2:latest")
try:
async with httpx.AsyncClient(timeout=30.0, proxy=None) as client:
resp = await client.post(
f"{ollama_url}/api/generate",
json={
"model": model,
"prompt": prompt,
"stream": False,
},
)
resp.raise_for_status()
resp_data = resp.json()
generated = resp_data.get("response", "")
if generated and generated.strip():
brief_text = generated.strip()
llm_available = True
except (httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError) as exc:
logger.info("Ollama unavailable for country brief (%s): %s", country_code, exc)
except Exception as exc:
logger.warning("Unexpected error calling Ollama: %s", exc)
return {
"country_code": country_code,
"brief": brief_text,
"data": {
"gdp": gdp_values,
"inflation": inflation_values,
"recent_events": event_count,
},
"llm_available": llm_available,
"source": "country-intelligence",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# ---------------------------------------------------------------------------
# Function 2: Country Risk Scores
# ---------------------------------------------------------------------------
async def fetch_risk_scores(
fetcher: Fetcher,
limit: int = 20,
) -> dict:
"""Compute country risk scores from ACLED conflict data and baselines.
Fetches recent global conflict events, counts per country, and computes
a risk score as ``(events_30d / monthly_baseline) * 100``. Higher
scores indicate conflict above historical norms.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
limit: Maximum number of countries to return (sorted by risk).
Returns:
Dict with ranked country list, count, source, and timestamp.
"""
now = datetime.now(timezone.utc)
access_token = os.environ.get("ACLED_ACCESS_TOKEN")
if not access_token:
return {
"error": "ACLED_ACCESS_TOKEN not configured",
"note": "Free academic access at acleddata.com",
"source": "risk-analysis",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d")
end_date = now.strftime("%Y-%m-%d")
params: dict = {
"key": access_token,
"email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"),
"limit": 500,
"event_date": f"{start_date}|{end_date}",
"event_date_where": "BETWEEN",
}
data = await fetcher.get_json(
_ACLED_URL,
source="acled",
cache_key="intel:risk:global:30d",
cache_ttl=1800,
params=params,
)
if data is None:
logger.warning("ACLED API returned no data for risk scoring")
return {
"countries": [],
"count": 0,
"source": "risk-analysis",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# Count events per country
country_counts: dict[str, int] = {}
for event in data.get("data", []):
country_name = event.get("country")
if country_name:
country_counts[country_name] = country_counts.get(country_name, 0) + 1
# Compute risk scores
countries: list[dict] = []
for country_name, events_30d in country_counts.items():
baseline_annual = _BASELINES.get(country_name, 500)
monthly_baseline = baseline_annual / 12.0
risk_score = (events_30d / monthly_baseline) * 100 if monthly_baseline > 0 else 0.0
countries.append({
"country": country_name,
"events_30d": events_30d,
"monthly_baseline": round(monthly_baseline, 1),
"risk_score": round(risk_score, 1),
"risk_level": _risk_level(risk_score),
})
# Sort by risk_score descending, take top N
countries.sort(key=lambda c: c["risk_score"], reverse=True)
countries = countries[:limit]
return {
"countries": countries,
"count": len(countries),
"source": "risk-analysis",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# ---------------------------------------------------------------------------
# Function 3: Country Instability Index
# ---------------------------------------------------------------------------
async def fetch_instability_index(
fetcher: Fetcher,
country_code: str | None = None,
) -> dict:
"""Compute a Country Instability Index (CII) from multiple signals.
Combines conflict intensity, economic stress, humanitarian crisis data,
internet disruption indicators, and military activity into a 0-100
composite score. Higher values indicate greater instability.
When *country_code* is ``None``, returns a simplified index for 10
focus countries using ACLED data as the primary signal.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
country_code: Optional ISO 3166-1 alpha-3 code (e.g. ``UKR``).
Returns:
Dict with instability index, component scores, risk level, source,
and timestamp.
"""
now = datetime.now(timezone.utc)
if country_code is not None:
return await _instability_single(fetcher, country_code, now)
return await _instability_multi(fetcher, now)
async def _instability_single(
fetcher: Fetcher,
country_code: str,
now: datetime,
) -> dict:
"""Compute full 5-component instability index for a single country."""
# --- Parallel data gathering -------------------------------------------
async def _conflict_score() -> float:
"""Score 0-20 based on ACLED event count in the last 30 days."""
access_token = os.environ.get("ACLED_ACCESS_TOKEN")
if not access_token:
return 0.0
country_name = _ISO3_TO_NAME.get(country_code, country_code)
start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d")
end_date = now.strftime("%Y-%m-%d")
data = await fetcher.get_json(
_ACLED_URL,
source="acled",
cache_key=f"intel:cii:conflict:{country_code}",
cache_ttl=1800,
params={
"key": access_token,
"email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"),
"limit": 500,
"event_date": f"{start_date}|{end_date}",
"event_date_where": "BETWEEN",
"country": country_name,
},
)
if data is None:
return 0.0
count = len(data.get("data", []))
# Thresholds: 0 events = 0, 500+ = 20
return min(20.0, (count / 500.0) * 20.0)
async def _economic_score() -> float:
"""Score 0-20 based on World Bank inflation rate."""
url = f"{_WB_BASE}/{country_code}/indicator/FP.CPI.TOTL.ZG"
data = await fetcher.get_json(
url,
source="world-bank",
cache_key=f"intel:cii:inflation:{country_code}",
cache_ttl=86400,
params={"format": "json", "per_page": 1, "date": "2023:2025"},
)
if data is None:
return 0.0
try:
if isinstance(data, list) and len(data) >= 2 and isinstance(data[1], list):
for rec in data[1]:
value = rec.get("value")
if value is not None:
inflation = float(value)
# Thresholds: 0% = 0, 50%+ = 20
return min(20.0, max(0.0, (abs(inflation) / 50.0) * 20.0))
except (ValueError, TypeError, KeyError, IndexError):
pass
return 0.0
async def _humanitarian_score() -> float:
"""Score 0-20 based on HDX crisis dataset count."""
params: dict = {
"q": "crisis",
"rows": 50,
"sort": "metadata_modified desc",
"fq": f"groups:{country_code.lower()}",
}
data = await fetcher.get_json(
_HDX_SEARCH_URL,
source="hdx",
cache_key=f"intel:cii:humanitarian:{country_code}",
cache_ttl=21600,
params=params,
)
if data is None:
return 0.0
try:
count = data.get("result", {}).get("count", 0)
# Thresholds: 0 datasets = 0, 200+ = 20
return min(20.0, (int(count) / 200.0) * 20.0)
except (ValueError, TypeError):
return 0.0
async def _internet_score() -> float:
"""Score 0-20 based on Cloudflare Radar connectivity data.
This is a best-effort check; Cloudflare Radar's public API may
not be available or may require auth. Returns 0 on failure.
"""
# Cloudflare Radar does not have an easy free API for this.
# Placeholder: return 0 (no disruption data).
return 0.0
async def _military_score() -> float:
"""Score 0-20 based on OpenSky military flight density near country."""
# Use a rough bounding box for the country. For simplicity,
# we only score countries in _ISO3_TO_NAME with known hotspot
# regions.
_COUNTRY_BBOX = {
"SYR": "32,35,37,42", "UKR": "44,22,52,40",
"YEM": "12,42,19,55", "MMR": "10,92,28,101",
"SDN": "8,21,23,39", "ETH": "3,33,15,48",
"NGA": "4,3,14,15", "COD": "-13,12,5,31",
"AFG": "29,60,38,75", "IRQ": "29,39,37,49",
}
bbox = _COUNTRY_BBOX.get(country_code)
if bbox is None:
return 0.0
parts = bbox.split(",")
params: dict[str, str] = {}
if len(parts) == 4:
params["lamin"] = parts[0]
params["lomin"] = parts[1]
params["lamax"] = parts[2]
params["lomax"] = parts[3]
data = await fetcher.get_json(
_OPENSKY_STATES_URL,
source="opensky",
cache_key=f"intel:cii:military:{country_code}",
cache_ttl=300,
params=params if params else None,
)
if data is None:
return 0.0
states = data.get("states") or []
# Count all aircraft (military filtering adds complexity; using
# total density as a proxy for activity).
count = len(states)
# Thresholds: 0 = 0, 200+ = 20
return min(20.0, (count / 200.0) * 20.0)
conflict, economic, humanitarian, internet, military = await asyncio.gather(
_conflict_score(),
_economic_score(),
_humanitarian_score(),
_internet_score(),
_military_score(),
)
instability_index = round(conflict + economic + humanitarian + internet + military, 1)
return {
"country_code": country_code,
"instability_index": instability_index,
"components": {
"conflict_intensity": round(conflict, 1),
"economic_stress": round(economic, 1),
"humanitarian_crisis": round(humanitarian, 1),
"internet_disruptions": round(internet, 1),
"military_activity": round(military, 1),
},
"risk_level": _instability_level(instability_index),
"source": "instability-index",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
async def _instability_multi(fetcher: Fetcher, now: datetime) -> dict:
"""Compute simplified instability index for focus countries using ACLED."""
access_token = os.environ.get("ACLED_ACCESS_TOKEN")
if not access_token:
return {
"error": "ACLED_ACCESS_TOKEN not configured",
"source": "instability-index",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
start_date = (now - timedelta(days=30)).strftime("%Y-%m-%d")
end_date = now.strftime("%Y-%m-%d")
# Fetch global events and bucket by country
data = await fetcher.get_json(
_ACLED_URL,
source="acled",
cache_key="intel:cii:multi:global:30d",
cache_ttl=1800,
params={
"key": access_token,
"email": os.environ.get("ACLED_EMAIL", "phoenix@2acrestudios.com"),
"limit": 500,
"event_date": f"{start_date}|{end_date}",
"event_date_where": "BETWEEN",
},
)
country_counts: dict[str, int] = {}
if data is not None:
for event in data.get("data", []):
country_name = event.get("country")
if country_name:
country_counts[country_name] = country_counts.get(country_name, 0) + 1
# Map focus country codes to names and compute simplified index
results: list[dict] = []
for code in _FOCUS_COUNTRIES:
name = _ISO3_TO_NAME.get(code, code)
events = country_counts.get(name, 0)
baseline_annual = _BASELINES.get(name, 500)
monthly_baseline = baseline_annual / 12.0
# Simplified CII: conflict component scaled to 0-100
if monthly_baseline > 0:
ratio = events / monthly_baseline
else:
ratio = 0.0
instability = min(100.0, round(ratio * 50.0, 1))
results.append({
"country_code": code,
"country_name": name,
"instability_index": instability,
"events_30d": events,
"risk_level": _instability_level(instability),
})
results.sort(key=lambda r: r["instability_index"], reverse=True)
return {
"countries": results,
"count": len(results),
"note": "Simplified index based on ACLED conflict data only. "
"Use country_code parameter for full 5-component analysis.",
"source": "instability-index",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# ---------------------------------------------------------------------------
# Function 4: Signal Convergence
# ---------------------------------------------------------------------------
async def fetch_signal_convergence(
fetcher: Fetcher,
lat: float | None = None,
lon: float | None = None,
radius_deg: float = 5.0,
) -> dict:
"""Detect geographic convergence of signals in hotspot regions.
Checks for overlapping seismic activity and other observable signals
within a radius of known or specified hotspot coordinates. Higher
convergence scores indicate multiple signal types in close proximity,
which may warrant deeper investigation.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
lat: Latitude of center point. If ``None``, scans 5 global
hotspot regions.
lon: Longitude of center point.
radius_deg: Radius in degrees for bounding box queries.
Returns:
Dict with hotspot list, convergence scores, source, and timestamp.
"""
now = datetime.now(timezone.utc)
if lat is not None and lon is not None:
regions = {"custom": (lat, lon)}
else:
regions = dict(_HOTSPOTS)
async def _assess_hotspot(name: str, center: tuple[float, float]) -> dict:
center_lat, center_lon = center
# Earthquake count within bounding box
min_lat = center_lat - radius_deg
max_lat = center_lat + radius_deg
min_lon = center_lon - radius_deg
max_lon = center_lon + radius_deg
starttime = (now - timedelta(days=30)).strftime("%Y-%m-%dT%H:%M:%S")
quake_data = await fetcher.get_json(
_USGS_ENDPOINT,
source="usgs",
cache_key=f"intel:convergence:usgs:{name}:{radius_deg}",
cache_ttl=600,
params={
"format": "geojson",
"minmagnitude": 2.5,
"starttime": starttime,
"minlatitude": min_lat,
"maxlatitude": max_lat,
"minlongitude": min_lon,
"maxlongitude": max_lon,
"limit": 100,
},
)
earthquake_count = 0
if quake_data is not None:
earthquake_count = len(quake_data.get("features", []))
# Convergence score heuristic (0-10)
# Each signal type present adds to the score.
score = 0.0
# Earthquakes: 0-5 points based on count
if earthquake_count > 0:
score += min(5.0, (earthquake_count / 20.0) * 5.0)
# Hotspot presence bonus (known conflict zones get a baseline)
if name in _HOTSPOTS:
score += 2.0
score = min(10.0, round(score, 1))
return {
"name": name,
"lat": center_lat,
"lon": center_lon,
"signals": {
"earthquakes": earthquake_count,
},
"convergence_score": score,
}
tasks = [_assess_hotspot(name, center) for name, center in regions.items()]
results = await asyncio.gather(*tasks)
# Sort by convergence score descending
hotspots = sorted(results, key=lambda h: h["convergence_score"], reverse=True)
return {
"hotspots": hotspots,
"source": "signal-convergence",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
+168
View File
@@ -0,0 +1,168 @@
"""NGA Maritime Safety Information source for world-intel-mcp.
Provides navigational warnings from the National Geospatial-Intelligence Agency
(NGA) Maritime Safety Information (MSI) broadcast warnings API.
No API key required.
"""
import logging
import os
import re
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.maritime")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_NGA_WARNINGS_URL = "https://msi.nga.mil/api/publications/broadcast-warn?output=json"
NAVAREAS = {
"I": "UK (NE Atlantic, North Sea)",
"II": "France (Bay of Biscay, W Africa)",
"III": "Spain (Mediterranean W)",
"IV": "United States (W Atlantic, Caribbean, Gulf of Mexico)",
"V": "Brazil (SW Atlantic)",
"VI": "Argentina (SE Atlantic)",
"VII": "South Africa (Indian Ocean W)",
"VIII": "India (Indian Ocean N)",
"IX": "Pakistan (Arabian Sea, Persian Gulf)",
"X": "Australia (Indian Ocean S)",
"XI": "Japan (NW Pacific)",
"XII": "United States (NE Pacific)",
"XIII": "Russia (Arctic)",
"XIV": "New Zealand (SW Pacific)",
"XV": "Chile (SE Pacific)",
"XVI": "Peru (E Pacific)",
}
_MAX_TEXT_LENGTH = 500
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _is_active(warning: dict) -> bool:
"""Return True if the warning has no cancelDate (i.e. still active)."""
cancel = warning.get("cancelDate")
return cancel is None or cancel == ""
def _parse_warning(warning: dict) -> dict:
"""Extract structured fields from a single NGA broadcast warning."""
msg_year = warning.get("msgYear", "")
msg_number = warning.get("msgNumber", "")
warning_id = f"{msg_year}-{msg_number}" if msg_year and msg_number else None
cancel_date = warning.get("cancelDate")
status = "cancelled" if cancel_date else "active"
text = warning.get("text", "") or ""
if len(text) > _MAX_TEXT_LENGTH:
text = text[:_MAX_TEXT_LENGTH] + "..."
return {
"id": warning_id,
"navarea": warning.get("navArea", ""),
"subregion": warning.get("subregion", ""),
"status": status,
"issue_date": warning.get("issueDate"),
"cancel_date": cancel_date if cancel_date else None,
"text": text,
"authority": warning.get("authority", "NGA"),
}
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_nav_warnings(
fetcher: Fetcher,
navarea: str | None = None,
) -> dict:
"""Fetch active navigational warnings from NGA Maritime Safety Information.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
navarea: Optional NAVAREA identifier (e.g. "IV", "XII") to filter by.
Returns:
Dict with warnings list, count, per-navarea breakdown, NAVAREA
definitions, source, and timestamp.
"""
cache_label = navarea.upper() if navarea else "all"
cache_key = f"maritime:warnings:{cache_label}"
data = await fetcher.get_json(
url=_NGA_WARNINGS_URL,
source="nga-msi",
cache_key=cache_key,
cache_ttl=3600,
)
now_iso = _utc_now_iso()
if data is None:
logger.warning("NGA MSI API returned no data")
return {
"warnings": [],
"count": 0,
"by_navarea": {},
"navareas": NAVAREAS,
"source": "nga-msi",
"timestamp": now_iso,
}
# The API returns either a list directly or a dict with a data key
if isinstance(data, list):
raw_warnings = data
elif isinstance(data, dict):
raw_warnings = data.get("broadcast-warn", data.get("data", []))
if isinstance(raw_warnings, dict):
raw_warnings = [raw_warnings]
else:
raw_warnings = []
# Filter to active warnings only
active_raw = [w for w in raw_warnings if _is_active(w)]
# Filter by NAVAREA if specified
if navarea is not None:
navarea_upper = navarea.upper()
active_raw = [
w for w in active_raw
if str(w.get("navArea", "")).upper() == navarea_upper
]
# Parse each warning into structured format
warnings = [_parse_warning(w) for w in active_raw]
# Sort by issue date descending (most recent first)
def _issue_sort_key(w: dict) -> str:
return w.get("issue_date") or ""
warnings.sort(key=_issue_sort_key, reverse=True)
# Summary stats: count per navarea
by_navarea: dict[str, int] = {}
for w in warnings:
na = w.get("navarea", "UNKNOWN")
by_navarea[na] = by_navarea.get(na, 0) + 1
return {
"warnings": warnings,
"count": len(warnings),
"by_navarea": by_navarea,
"navareas": NAVAREAS,
"source": "nga-msi",
"timestamp": now_iso,
}
+384
View File
@@ -0,0 +1,384 @@
"""Financial market data sources for world-intel-mcp.
Provides async functions for equities, crypto, stablecoins, ETF flows,
sector heatmaps, and macro signals. Every function takes a Fetcher
instance as its first argument and returns a dict (or None-safe partial
results when individual upstream calls fail).
"""
import asyncio
import logging
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.markets")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_DEFAULT_INDEX_SYMBOLS = ["^GSPC", "^DJI", "^IXIC", "^FTSE", "^N225", "^HSI", "^GDAXI"]
_YAHOO_CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
_COINGECKO_MARKETS_URL = "https://api.coingecko.com/api/v3/coins/markets"
_COINGECKO_GLOBAL_URL = "https://api.coingecko.com/api/v3/global"
_STABLECOIN_IDS = "tether,usd-coin,dai,first-digital-usd"
_STABLECOIN_DEPEG_THRESHOLD = 0.005 # 0.5%
_BTC_ETF_SYMBOLS = ["IBIT", "FBTC", "GBTC", "ARKB", "BITB"]
_SECTOR_ETFS: dict[str, str] = {
"XLK": "Technology",
"XLF": "Financials",
"XLE": "Energy",
"XLV": "Healthcare",
"XLI": "Industrials",
"XLC": "Communication",
"XLY": "Consumer Disc",
"XLP": "Consumer Staples",
"XLRE": "Real Estate",
"XLU": "Utilities",
"XLB": "Materials",
}
_FEAR_GREED_URL = "https://api.alternative.me/fng/?limit=1"
_MEMPOOL_FEES_URL = "https://mempool.space/api/v1/fees/recommended"
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
async def _fetch_yahoo_quote(fetcher: Fetcher, symbol: str, cache_key: str, cache_ttl: int) -> dict | None:
"""Fetch a single Yahoo Finance v8 chart quote and extract meta fields."""
url = _YAHOO_CHART_URL.format(symbol=symbol)
data = await fetcher.get_json(
url,
source="yahoo-finance",
cache_key=cache_key,
cache_ttl=cache_ttl,
params={"range": "1d", "interval": "5m"},
yahoo_rate_limit=True,
)
if data is None:
return None
try:
meta = data["chart"]["result"][0]["meta"]
return {
"symbol": symbol,
"price": meta.get("regularMarketPrice"),
"change_pct": meta.get("regularMarketChangePercent"),
"currency": meta.get("currency"),
}
except (KeyError, IndexError, TypeError):
logger.warning("Unexpected Yahoo chart structure for %s", symbol)
return None
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_market_quotes(fetcher: Fetcher, symbols: list[str] | None = None) -> dict:
"""Fetch major index quotes from Yahoo Finance.
Returns::
{"quotes": [...], "source": "yahoo-finance", "timestamp": "<iso>"}
"""
symbols = symbols or _DEFAULT_INDEX_SYMBOLS
tasks = [
_fetch_yahoo_quote(fetcher, sym, f"markets:quotes:{sym}", 120)
for sym in symbols
]
results = await asyncio.gather(*tasks)
quotes = [q for q in results if q is not None]
return {
"quotes": quotes,
"source": "yahoo-finance",
"timestamp": _utc_now_iso(),
}
async def fetch_crypto_quotes(fetcher: Fetcher, limit: int = 20) -> dict:
"""Fetch top crypto coins by market cap from CoinGecko.
Returns::
{"coins": [...], "source": "coingecko", "timestamp": "<iso>"}
"""
data = await fetcher.get_json(
_COINGECKO_MARKETS_URL,
source="coingecko",
cache_key=f"markets:crypto:{limit}",
cache_ttl=180,
params={
"vs_currency": "usd",
"order": "market_cap_desc",
"per_page": str(limit),
"sparkline": "true",
"price_change_percentage": "1h,24h,7d",
},
)
coins: list[dict] = []
if data is not None and isinstance(data, list):
for coin in data:
coins.append({
"id": coin.get("id"),
"symbol": coin.get("symbol"),
"name": coin.get("name"),
"current_price": coin.get("current_price"),
"market_cap": coin.get("market_cap"),
"price_change_percentage_24h": coin.get("price_change_percentage_24h"),
"sparkline_in_7d": coin.get("sparkline_in_7d"),
})
return {
"coins": coins,
"source": "coingecko",
"timestamp": _utc_now_iso(),
}
async def fetch_stablecoin_status(fetcher: Fetcher) -> dict:
"""Detect stablecoin de-peg events via CoinGecko.
Returns::
{"stablecoins": [{id, price, peg_deviation_pct, is_depegged}], ...}
"""
data = await fetcher.get_json(
_COINGECKO_MARKETS_URL,
source="coingecko",
cache_key="markets:stablecoins",
cache_ttl=180,
params={
"vs_currency": "usd",
"ids": _STABLECOIN_IDS,
"sparkline": "false",
},
)
stablecoins: list[dict] = []
if data is not None and isinstance(data, list):
for coin in data:
price = coin.get("current_price")
if price is not None:
deviation = abs(price - 1.0) / 1.0
stablecoins.append({
"id": coin.get("id"),
"price": price,
"peg_deviation_pct": round(deviation * 100, 4),
"is_depegged": deviation > _STABLECOIN_DEPEG_THRESHOLD,
})
return {
"stablecoins": stablecoins,
"source": "coingecko",
"timestamp": _utc_now_iso(),
}
async def fetch_etf_flows(fetcher: Fetcher) -> dict:
"""Fetch BTC spot ETF volume and price changes from Yahoo Finance.
Returns::
{"etfs": [{symbol, price, change_pct, volume}], ...}
"""
tasks = [
_fetch_yahoo_quote(fetcher, sym, f"markets:etf_flows:{sym}", 600)
for sym in _BTC_ETF_SYMBOLS
]
results = await asyncio.gather(*tasks)
# Re-fetch with volume info (chart meta includes volume in some responses,
# but we need the raw data). We already have price/change from the helper;
# pull volume from the same cached chart payload.
etfs: list[dict] = []
for sym, quote in zip(_BTC_ETF_SYMBOLS, results):
if quote is None:
continue
# Volume: re-read from cache (already populated by _fetch_yahoo_quote)
cached = fetcher.cache.get(f"markets:etf_flows:{sym}")
volume = None
if cached is not None:
try:
meta = cached["chart"]["result"][0]["meta"]
volume = meta.get("regularMarketVolume")
except (KeyError, IndexError, TypeError):
pass
etfs.append({
"symbol": sym,
"price": quote["price"],
"change_pct": quote["change_pct"],
"volume": volume,
})
return {
"etfs": etfs,
"source": "yahoo-finance",
"timestamp": _utc_now_iso(),
}
async def fetch_sector_heatmap(fetcher: Fetcher) -> dict:
"""Fetch sector ETF performance for a market heatmap.
Returns::
{"sectors": [{symbol, name, price, change_pct}], ...}
"""
tasks = [
_fetch_yahoo_quote(fetcher, sym, f"markets:sector_heatmap:{sym}", 300)
for sym in _SECTOR_ETFS
]
results = await asyncio.gather(*tasks)
sectors: list[dict] = []
for sym, quote in zip(_SECTOR_ETFS, results):
if quote is None:
continue
sectors.append({
"symbol": sym,
"name": _SECTOR_ETFS[sym],
"price": quote["price"],
"change_pct": quote["change_pct"],
})
return {
"sectors": sectors,
"source": "yahoo-finance",
"timestamp": _utc_now_iso(),
}
# ---------------------------------------------------------------------------
# Macro Signals (aggregated dashboard)
# ---------------------------------------------------------------------------
async def _fetch_fear_greed(fetcher: Fetcher) -> dict | None:
data = await fetcher.get_json(
_FEAR_GREED_URL,
source="alternative-me",
cache_key="markets:macro:fear_greed",
cache_ttl=300,
)
if data is None:
return None
try:
entry = data["data"][0]
return {
"value": int(entry["value"]),
"classification": entry.get("value_classification"),
"source": "alternative-me",
}
except (KeyError, IndexError, TypeError, ValueError):
return None
async def _fetch_mempool_fees(fetcher: Fetcher) -> dict | None:
data = await fetcher.get_json(
_MEMPOOL_FEES_URL,
source="mempool",
cache_key="markets:macro:mempool_fees",
cache_ttl=300,
)
if data is None:
return None
return {
"fastest_fee": data.get("fastestFee"),
"half_hour_fee": data.get("halfHourFee"),
"hour_fee": data.get("hourFee"),
"economy_fee": data.get("economyFee"),
"minimum_fee": data.get("minimumFee"),
"source": "mempool",
}
async def _fetch_yahoo_macro_symbol(fetcher: Fetcher, symbol: str, label: str) -> dict | None:
quote = await _fetch_yahoo_quote(
fetcher, symbol, f"markets:macro:{label}", 300,
)
if quote is None:
return None
return {
"symbol": symbol,
"price": quote["price"],
"change_pct": quote["change_pct"],
"source": "yahoo-finance",
}
async def _fetch_btc_dominance(fetcher: Fetcher) -> dict | None:
data = await fetcher.get_json(
_COINGECKO_GLOBAL_URL,
source="coingecko",
cache_key="markets:macro:btc_dominance",
cache_ttl=300,
)
if data is None:
return None
try:
btc_pct = data["data"]["market_cap_percentage"]["btc"]
return {
"btc_dominance_pct": round(btc_pct, 2),
"source": "coingecko",
}
except (KeyError, TypeError):
return None
async def fetch_macro_signals(fetcher: Fetcher) -> dict:
"""Aggregate 7 macro signals into a single dashboard payload.
Each signal is fetched independently -- a failure in one does not
affect the others.
Returns::
{"signals": {"fear_greed": {...}, "mempool_fees": {...}, ...},
"source": "multi", "timestamp": "<iso>"}
"""
(
fear_greed,
mempool_fees,
dxy,
vix,
gold,
treasury_10y,
btc_dominance,
) = await asyncio.gather(
_fetch_fear_greed(fetcher),
_fetch_mempool_fees(fetcher),
_fetch_yahoo_macro_symbol(fetcher, "DX-Y.NYB", "dxy"),
_fetch_yahoo_macro_symbol(fetcher, "^VIX", "vix"),
_fetch_yahoo_macro_symbol(fetcher, "GC=F", "gold"),
_fetch_yahoo_macro_symbol(fetcher, "^TNX", "treasury_10y"),
_fetch_btc_dominance(fetcher),
)
return {
"signals": {
"fear_greed": fear_greed,
"mempool_fees": mempool_fees,
"dxy": dxy,
"vix": vix,
"gold": gold,
"treasury_10y": treasury_10y,
"btc_dominance": btc_dominance,
},
"source": "multi",
"timestamp": _utc_now_iso(),
}
+275
View File
@@ -0,0 +1,275 @@
"""Military aviation tracking source for world-intel-mcp.
Provides real-time military aircraft tracking via OpenSky Network and
aircraft detail lookups via the Wingbits API.
"""
import asyncio
import base64
import logging
import os
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.military")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_OPENSKY_STATES_URL = "https://opensky-network.org/api/states/all"
_WINGBITS_BASE_URL = "https://api.wingbits.com/v1/aircraft"
# ICAO hex prefix ranges known to be allocated to military operators.
MILITARY_ICAO_PREFIXES = [
"AE", # US Military
"A8", "A9", "AA", "AB", "AC", "AD", # US Military extended
"43C", "43D", "43E", "43F", # UK Military
"3F", # Germany Military
"3A8", "3A9", "3AA", "3AB", # France Military
"500", "501", "502", # Israel Military
"70", # Pakistan Military
"C0", # Canada Military
]
# Callsign prefixes commonly used by military flights.
MILITARY_CALLSIGN_PREFIXES = [
"RCH", "DUKE", "REACH", "EVAC", "JAKE", "TOPCAT", "BOXER",
"IRON", "CASA", "NATO", "LAGR", "TEAL", "SAM", "EXEC",
"SPAR", "VALOR", "BLADE",
]
# Theater bounding boxes for global military posture assessment.
THEATERS = {
"european": {"bbox": "35,-25,72,45", "desc": "NATO/Russia theater"},
"indo_pacific": {"bbox": "-10,95,55,155", "desc": "China/Taiwan/SCS"},
"middle_east": {"bbox": "10,25,45,65", "desc": "Persian Gulf/Red Sea"},
"arctic": {"bbox": "65,-180,90,180", "desc": "Arctic region"},
"korean_peninsula": {"bbox": "33,124,43,132", "desc": "Korean DMZ"},
}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _is_military_icao(icao24: str) -> bool:
"""Check whether an ICAO24 hex address falls within known military ranges."""
upper = icao24.upper()
for prefix in MILITARY_ICAO_PREFIXES:
if upper.startswith(prefix):
return True
return False
def _is_military_callsign(callsign: str | None) -> bool:
"""Check whether a callsign matches known military callsign patterns."""
if not callsign:
return False
cs = callsign.strip().upper()
for prefix in MILITARY_CALLSIGN_PREFIXES:
if cs.startswith(prefix):
return True
return False
def _build_opensky_auth_headers() -> dict[str, str] | None:
"""Build HTTP Basic auth header from OpenSky env vars, or None if unset."""
client_id = os.environ.get("OPENSKY_CLIENT_ID")
client_secret = os.environ.get("OPENSKY_CLIENT_SECRET")
if client_id and client_secret:
credentials = base64.b64encode(
f"{client_id}:{client_secret}".encode()
).decode()
return {"Authorization": f"Basic {credentials}"}
return None
def _extract_aircraft(state: list) -> dict:
"""Extract a structured aircraft dict from an OpenSky state vector."""
return {
"icao24": state[0],
"callsign": state[1].strip() if state[1] else None,
"origin_country": state[2],
"latitude": state[6],
"longitude": state[5],
"altitude_m": state[7],
"velocity_ms": state[9],
"heading": state[10],
"on_ground": state[8],
"squawk": state[14],
}
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_military_flights(
fetcher: Fetcher,
bbox: str | None = None,
) -> dict:
"""Fetch current military aircraft positions from the OpenSky Network.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
bbox: Optional bounding box as "lamin,lomin,lamax,lomax".
Returns:
Dict with aircraft list, count, filter description, source, and timestamp.
"""
params: dict[str, str] = {}
if bbox is not None:
parts = bbox.split(",")
if len(parts) == 4:
params["lamin"] = parts[0]
params["lomin"] = parts[1]
params["lamax"] = parts[2]
params["lomax"] = parts[3]
headers = _build_opensky_auth_headers()
cache_label = bbox or "global"
data = await fetcher.get_json(
url=_OPENSKY_STATES_URL,
source="opensky",
cache_key=f"military:flights:{cache_label}",
cache_ttl=120,
headers=headers,
params=params if params else None,
)
if data is None:
logger.warning("OpenSky API returned no data (bbox=%s)", cache_label)
return {
"aircraft": [],
"count": 0,
"military_filter": "icao_prefix+callsign",
"source": "opensky",
"timestamp": _utc_now_iso(),
}
states = data.get("states") or []
military_aircraft: list[dict] = []
for state in states:
if not state or len(state) < 17:
continue
icao24 = state[0] or ""
callsign = state[1] or ""
if _is_military_icao(icao24) or _is_military_callsign(callsign):
military_aircraft.append(_extract_aircraft(state))
return {
"aircraft": military_aircraft,
"count": len(military_aircraft),
"military_filter": "icao_prefix+callsign",
"source": "opensky",
"timestamp": _utc_now_iso(),
}
async def fetch_theater_posture(fetcher: Fetcher) -> dict:
"""Assess global military air posture across 5 theater regions.
Calls fetch_military_flights for each theater in parallel and aggregates
the results into a per-theater summary.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
Returns:
Dict with per-theater summaries, total count, source, and timestamp.
"""
# Check cache first to avoid redundant parallel fetches.
cached = fetcher.cache.get("military:posture")
if cached is not None:
return cached
async def _fetch_theater(name: str, info: dict) -> tuple[str, dict]:
result = await fetch_military_flights(fetcher, bbox=info["bbox"])
aircraft_list = result.get("aircraft", [])
countries: set[str] = set()
callsigns: list[str] = []
for ac in aircraft_list:
if ac.get("origin_country"):
countries.add(ac["origin_country"])
if ac.get("callsign"):
callsigns.append(ac["callsign"])
return (name, {
"count": len(aircraft_list),
"countries": sorted(countries),
"sample_callsigns": callsigns[:5],
"bbox": info["bbox"],
"description": info["desc"],
})
tasks = [
_fetch_theater(name, info) for name, info in THEATERS.items()
]
results = await asyncio.gather(*tasks)
theaters: dict[str, dict] = {}
total = 0
for name, summary in results:
theaters[name] = summary
total += summary["count"]
response = {
"theaters": theaters,
"total_military_aircraft": total,
"source": "opensky",
"timestamp": _utc_now_iso(),
}
fetcher.cache.set("military:posture", response, 300)
return response
async def fetch_aircraft_details(fetcher: Fetcher, icao24: str) -> dict:
"""Look up detailed aircraft information from the Wingbits API.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
icao24: The ICAO 24-bit hex address of the aircraft.
Returns:
Dict with aircraft detail payload, source, and timestamp.
"""
api_key = os.environ.get("WINGBITS_API_KEY")
if not api_key:
return {"error": "WINGBITS_API_KEY not configured"}
url = f"{_WINGBITS_BASE_URL}/{icao24}"
headers = {"X-API-Key": api_key}
data = await fetcher.get_json(
url=url,
source="wingbits",
cache_key=f"military:aircraft:{icao24}",
cache_ttl=3600,
headers=headers,
)
if data is None:
logger.warning("Wingbits returned no data for icao24=%s", icao24)
return {
"aircraft": {},
"source": "wingbits",
"timestamp": _utc_now_iso(),
}
return {
"aircraft": data,
"source": "wingbits",
"timestamp": _utc_now_iso(),
}
+396
View File
@@ -0,0 +1,396 @@
"""RSS news aggregation, keyword trending, and GDELT search for world-intel-mcp.
Provides multi-category RSS feed aggregation from 20+ high-quality intelligence
and news sources, keyword spike detection from recent headlines, and full-text
search via the GDELT 2.0 Doc API. No API keys required.
"""
import asyncio
import logging
import re
import string
from datetime import datetime, timezone
from ..fetcher import Fetcher
try:
import feedparser
except ImportError:
feedparser = None # type: ignore[assignment]
logger = logging.getLogger("world-intel-mcp.sources.news")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_RSS_FEEDS: dict[str, list[tuple[str, str]]] = {
"geopolitics": [
("Reuters World", "https://feeds.reuters.com/Reuters/worldNews"),
("AP Top News", "https://rsshub.app/apnews/topics/apf-topnews"),
("BBC World", "https://feeds.bbci.co.uk/news/world/rss.xml"),
("Al Jazeera", "https://www.aljazeera.com/xml/rss/all.xml"),
],
"security": [
("BleepingComputer", "https://www.bleepingcomputer.com/feed/"),
("Krebs on Security", "https://krebsonsecurity.com/feed/"),
("The Hacker News", "https://feeds.feedburner.com/TheHackersNews"),
("Schneier on Security", "https://www.schneier.com/feed/atom/"),
],
"technology": [
("Ars Technica", "https://feeds.arstechnica.com/arstechnica/index"),
("TechCrunch", "https://techcrunch.com/feed/"),
("The Verge", "https://www.theverge.com/rss/index.xml"),
],
"finance": [
("CNBC", "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114"),
("MarketWatch", "https://feeds.content.dowjones.io/public/rss/mw_topstories"),
("FT World", "https://www.ft.com/rss/home/uk"),
],
"military": [
("Defense One", "https://www.defenseone.com/rss/"),
("War on the Rocks", "https://warontherocks.com/feed/"),
("The War Zone", "https://www.thedrive.com/the-war-zone/feed"),
],
"science": [
("Nature", "https://www.nature.com/nature.rss"),
("Science", "https://www.science.org/action/showFeed?type=etoc&feed=rss&jc=science"),
("Phys.org", "https://phys.org/rss-feed/"),
],
}
_STOPWORDS: set[str] = {
"the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
"have", "has", "had", "do", "does", "did", "will", "would", "shall",
"should", "may", "might", "must", "can", "could", "in", "on", "at",
"to", "for", "of", "and", "or", "but", "nor", "not", "no", "so",
"yet", "both", "either", "neither", "with", "from", "by", "as",
"into", "through", "during", "before", "after", "above", "below",
"between", "under", "over", "about", "against", "out", "off", "up",
"down", "then", "once", "here", "there", "when", "where", "why",
"how", "all", "each", "every", "any", "few", "more", "most", "some",
"such", "only", "own", "same", "than", "too", "very", "just", "also",
"now", "it", "its", "he", "she", "they", "them", "their", "his",
"her", "we", "you", "your", "our", "my", "me", "him", "us",
"that", "this", "these", "those", "which", "who", "whom", "what",
"if", "while", "because", "until", "although", "since", "whether",
"new", "says", "said", "one", "two", "first", "last", "many",
"much", "get", "got", "back", "even", "still", "well", "way",
"s", "t", "re", "ve", "d", "ll", "m",
}
_GDELT_DOC_URL = "https://api.gdeltproject.org/api/v2/doc/doc"
# Regex to strip punctuation from words
_PUNCT_RE = re.compile(f"[{re.escape(string.punctuation)}]")
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _parse_published(entry: dict) -> str | None:
"""Parse an RSS entry's published date to ISO 8601 UTC string.
feedparser stores the parsed time struct in ``published_parsed``.
Falls back to the raw ``published`` string if parsing fails.
"""
import time as _time
parsed_tuple = entry.get("published_parsed")
if parsed_tuple is not None:
try:
epoch = _time.mktime(parsed_tuple[:9])
dt = datetime.fromtimestamp(epoch, tz=timezone.utc)
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
except (ValueError, TypeError, OverflowError):
pass
# Fallback: try updated_parsed
updated_tuple = entry.get("updated_parsed")
if updated_tuple is not None:
try:
epoch = _time.mktime(updated_tuple[:9])
dt = datetime.fromtimestamp(epoch, tz=timezone.utc)
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
except (ValueError, TypeError, OverflowError):
pass
# Last resort: return raw string or None
return entry.get("published") or entry.get("updated")
def _truncate(text: str | None, max_len: int = 200) -> str:
"""Truncate text to max_len characters, appending '...' if trimmed."""
if not text:
return ""
if len(text) <= max_len:
return text
return text[:max_len] + "..."
# ---------------------------------------------------------------------------
# Function 1: RSS News Feed Aggregation
# ---------------------------------------------------------------------------
async def fetch_news_feed(
fetcher: Fetcher,
category: str | None = None,
limit: int = 50,
) -> dict:
"""Aggregate news from 20+ RSS feeds across intelligence/news categories.
Uses ``feedparser`` to parse RSS/Atom feeds fetched via the shared HTTP
fetcher. Feeds within each category are fetched in parallel.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
category: Optional category key (geopolitics, security, technology,
finance, military, science). If None, fetches all categories.
limit: Maximum number of items to return.
Returns:
Dict with items list, count, categories fetched, source, and timestamp.
"""
if feedparser is None:
return {
"error": "feedparser not installed — run: pip install feedparser",
"items": [],
"count": 0,
}
now = datetime.now(timezone.utc)
# Determine which categories to fetch
if category is not None:
if category not in _RSS_FEEDS:
return {
"items": [],
"count": 0,
"categories_fetched": [],
"error": f"Unknown category '{category}'. Valid: {list(_RSS_FEEDS.keys())}",
"source": "rss-aggregator",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
categories_to_fetch = {category: _RSS_FEEDS[category]}
else:
categories_to_fetch = dict(_RSS_FEEDS)
all_items: list[dict] = []
async def _fetch_single_feed(
feed_name: str,
url: str,
cat: str,
) -> list[dict]:
"""Fetch and parse a single RSS feed, returning extracted items."""
safe_name = feed_name.lower().replace(" ", "_")
xml_text = await fetcher.get_xml(
url,
source="rss",
cache_key=f"news:rss:{safe_name}",
cache_ttl=300,
)
if xml_text is None:
logger.debug("No data from RSS feed %s (%s)", feed_name, url)
return []
parsed = feedparser.parse(xml_text)
items: list[dict] = []
for entry in parsed.get("entries", []):
published = _parse_published(entry)
summary_raw = entry.get("summary") or entry.get("description") or ""
items.append({
"title": entry.get("title", ""),
"link": entry.get("link", ""),
"published": published,
"summary": _truncate(summary_raw, 200),
"feed_name": feed_name,
"category": cat,
})
return items
# Fetch all feeds within each category in parallel
for cat, feeds in categories_to_fetch.items():
tasks = [
_fetch_single_feed(feed_name, url, cat)
for feed_name, url in feeds
]
results = await asyncio.gather(*tasks)
for items in results:
all_items.extend(items)
# Sort by published date descending (entries without dates go last)
all_items.sort(
key=lambda item: item.get("published") or "",
reverse=True,
)
# Apply limit
all_items = all_items[:limit]
return {
"items": all_items,
"count": len(all_items),
"categories_fetched": list(categories_to_fetch.keys()),
"source": "rss-aggregator",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# ---------------------------------------------------------------------------
# Function 2: Keyword Trending / Spike Detection
# ---------------------------------------------------------------------------
async def fetch_trending_keywords(
fetcher: Fetcher,
hours: int = 6,
min_count: int = 3,
) -> dict:
"""Detect trending keywords from recent news headlines.
Fetches up to 200 recent news items via ``fetch_news_feed``, extracts
words from titles, removes stopwords and short tokens, and returns the
most frequently occurring keywords sorted by count.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
hours: Not used for time-windowing (RSS feeds are inherently recent),
but kept for API symmetry with other source functions.
min_count: Minimum occurrences for a keyword to be included.
Returns:
Dict with keywords list (word + count), total items analyzed,
source, and timestamp.
"""
now = datetime.now(timezone.utc)
# Fetch a broad set of recent items
feed_data = await fetch_news_feed(fetcher, limit=200)
items = feed_data.get("items", [])
# Count word frequencies across all titles
word_counts: dict[str, int] = {}
for item in items:
title = item.get("title") or ""
# Lowercase, strip punctuation, split into words
cleaned = _PUNCT_RE.sub(" ", title.lower())
words = cleaned.split()
for word in words:
word = word.strip()
if len(word) < 3:
continue
if word in _STOPWORDS:
continue
word_counts[word] = word_counts.get(word, 0) + 1
# Filter by min_count and sort descending
keywords = [
{"word": word, "count": count}
for word, count in word_counts.items()
if count >= min_count
]
keywords.sort(key=lambda k: k["count"], reverse=True)
# Return top 50
keywords = keywords[:50]
return {
"keywords": keywords,
"total_items_analyzed": len(items),
"source": "keyword-analysis",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# ---------------------------------------------------------------------------
# Function 3: GDELT 2.0 Doc API Search
# ---------------------------------------------------------------------------
async def fetch_gdelt_search(
fetcher: Fetcher,
query: str = "conflict",
mode: str = "artlist",
limit: int = 50,
) -> dict:
"""Search the GDELT 2.0 Doc API for articles or volume timelines.
No API key required. Supports article list and timeline volume modes.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
query: Search query string.
mode: Either ``artlist`` (article list) or ``timelinevol``
(volume timeline).
limit: Maximum number of records (artlist mode).
Returns:
Dict with articles/timeline data, count, query info, source,
and timestamp.
"""
now = datetime.now(timezone.utc)
params: dict = {
"query": query,
"mode": mode,
"maxrecords": limit,
"format": "json",
}
safe_query = re.sub(r"[^a-zA-Z0-9_-]", "_", query)[:64]
data = await fetcher.get_json(
_GDELT_DOC_URL,
source="gdelt",
cache_key=f"news:gdelt:{safe_query}:{mode}",
cache_ttl=600,
params=params,
)
if data is None:
logger.warning("GDELT API returned no data for query=%s mode=%s", query, mode)
return {
"articles": [] if mode == "artlist" else None,
"timeline": None if mode == "artlist" else [],
"count": 0,
"query": query,
"mode": mode,
"source": "gdelt",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
if mode == "artlist":
articles = []
for article in data.get("articles", []):
articles.append({
"title": article.get("title"),
"url": article.get("url"),
"seendate": article.get("seendate"),
"socialimage": article.get("socialimage"),
"domain": article.get("domain"),
"language": article.get("language"),
"sourcecountry": article.get("sourcecountry"),
})
return {
"articles": articles,
"count": len(articles),
"query": query,
"mode": mode,
"source": "gdelt",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
# timelinevol mode — return timeline data as-is
timeline = data.get("timeline", [])
return {
"timeline": timeline,
"count": len(timeline) if isinstance(timeline, list) else 0,
"query": query,
"mode": mode,
"source": "gdelt",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
+155
View File
@@ -0,0 +1,155 @@
"""Prediction market data sources for world-intel-mcp.
Fetches active prediction markets from Polymarket via their public
Gamma API. Every function takes a Fetcher instance as its first
argument and returns a dict (or empty results when upstream calls fail).
"""
import json
import logging
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.prediction")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_GAMMA_MARKETS_URL = "https://gamma-api.polymarket.com/markets"
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _parse_outcome_prices(raw: str | None) -> float | None:
"""Parse the outcomePrices JSON string and return YES probability.
``outcomePrices`` is a JSON-encoded list like ``"[0.85, 0.15]"``
where the first element represents the YES probability.
Returns None if the value is missing or unparseable.
"""
if raw is None:
return None
try:
prices = json.loads(raw)
if isinstance(prices, list) and len(prices) > 0:
return float(prices[0])
except (json.JSONDecodeError, ValueError, TypeError, IndexError):
logger.debug("Failed to parse outcomePrices: %r", raw)
return None
def _classify_sentiment(yes_probability: float) -> str:
"""Classify a YES probability into a human-readable sentiment label."""
if yes_probability > 0.85:
return "strong_yes"
if yes_probability > 0.65:
return "leaning_yes"
if yes_probability < 0.15:
return "strong_no"
if yes_probability < 0.35:
return "leaning_no"
return "uncertain"
def _safe_float(value) -> float:
"""Convert a value to float, returning 0.0 on failure."""
if value is None:
return 0.0
try:
return float(value)
except (ValueError, TypeError):
return 0.0
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_prediction_markets(fetcher: Fetcher, limit: int = 20) -> dict:
"""Fetch active prediction markets from Polymarket sorted by 24h volume.
Uses the Polymarket Gamma API to retrieve currently active (non-closed)
markets ordered by trading volume.
Returns::
{"markets": [...], "count": N, "source": "polymarket",
"timestamp": "<iso>"}
"""
params = {
"limit": str(limit),
"active": "true",
"closed": "false",
"order": "volume24hr",
"ascending": "false",
}
data = await fetcher.get_json(
_GAMMA_MARKETS_URL,
source="polymarket",
cache_key=f"prediction:polymarket:{limit}",
cache_ttl=300,
params=params,
)
if data is None:
return {
"markets": [],
"count": 0,
"source": "polymarket",
"timestamp": _utc_now_iso(),
}
if not isinstance(data, list):
logger.warning("Unexpected Polymarket response type: %s", type(data).__name__)
return {
"markets": [],
"count": 0,
"source": "polymarket",
"timestamp": _utc_now_iso(),
}
markets: list[dict] = []
for item in data:
if not isinstance(item, dict):
continue
yes_probability = _parse_outcome_prices(item.get("outcomePrices"))
if yes_probability is None:
# Skip markets with unparseable outcome data
continue
volume_24h = _safe_float(item.get("volume24hr"))
total_volume = _safe_float(item.get("volume"))
liquidity = _safe_float(item.get("liquidity"))
slug = item.get("slug", "")
markets.append({
"question": item.get("question", ""),
"yes_probability": round(yes_probability, 4),
"sentiment": _classify_sentiment(yes_probability),
"volume_24h": volume_24h,
"total_volume": total_volume,
"liquidity": liquidity,
"category": item.get("category", ""),
"url": f"https://polymarket.com/event/{slug}" if slug else "",
})
# Ensure descending sort by 24h volume (API should already return
# sorted, but enforce it defensively)
markets.sort(key=lambda m: m["volume_24h"], reverse=True)
return {
"markets": markets,
"count": len(markets),
"source": "polymarket",
"timestamp": _utc_now_iso(),
}
+98
View File
@@ -0,0 +1,98 @@
"""USGS earthquake data source for world-intel-mcp.
Provides real-time earthquake data from the USGS GeoJSON API.
No API key required.
"""
import logging
from datetime import datetime, timezone, timedelta
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.seismology")
_USGS_ENDPOINT = "https://earthquake.usgs.gov/fdsnws/event/1/query"
async def fetch_earthquakes(
fetcher: Fetcher,
min_magnitude: float = 4.5,
hours: int = 24,
limit: int = 50,
) -> dict:
"""Fetch recent earthquakes from the USGS GeoJSON API.
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
min_magnitude: Minimum earthquake magnitude to include.
hours: How far back to search (in hours from now).
limit: Maximum number of results.
Returns:
Dict with earthquakes list, count, query params, source, and timestamp.
"""
now = datetime.now(timezone.utc)
starttime = (now - timedelta(hours=hours)).strftime("%Y-%m-%dT%H:%M:%S")
params = {
"format": "geojson",
"minmagnitude": min_magnitude,
"orderby": "time",
"limit": limit,
"starttime": starttime,
}
data = await fetcher.get_json(
url=_USGS_ENDPOINT,
source="usgs",
cache_key=f"seismology:quakes:{min_magnitude}:{hours}",
cache_ttl=300,
params=params,
)
if data is None:
logger.warning("USGS API returned no data")
return {
"earthquakes": [],
"count": 0,
"query": {"min_magnitude": min_magnitude, "hours": hours},
"source": "usgs",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
earthquakes = []
for feature in data.get("features", []):
props = feature.get("properties", {})
geom = feature.get("geometry", {})
coords = geom.get("coordinates", [0, 0, 0])
# Convert epoch milliseconds to ISO 8601 UTC
epoch_ms = props.get("time")
if epoch_ms is not None:
eq_time = datetime.fromtimestamp(
epoch_ms / 1000, tz=timezone.utc
).strftime("%Y-%m-%dT%H:%M:%SZ")
else:
eq_time = None
earthquakes.append({
"id": feature.get("id"),
"magnitude": props.get("mag"),
"place": props.get("place"),
"time": eq_time,
"depth_km": coords[2] if len(coords) > 2 else None,
"latitude": coords[1] if len(coords) > 1 else None,
"longitude": coords[0] if len(coords) > 0 else None,
"tsunami_alert": props.get("tsunami"),
"felt_reports": props.get("felt"),
"alert_level": props.get("alert"),
"url": props.get("url"),
})
return {
"earthquakes": earthquakes,
"count": len(earthquakes),
"query": {"min_magnitude": min_magnitude, "hours": hours},
"source": "usgs",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
+218
View File
@@ -0,0 +1,218 @@
"""NASA FIRMS wildfire data source for world-intel-mcp.
Provides real-time active fire detections from the VIIRS/SNPP sensor
via the NASA Fire Information for Resource Management System (FIRMS) API.
Requires a NASA FIRMS API key (env: NASA_FIRMS_API_KEY).
"""
import asyncio
import logging
import os
from datetime import datetime, timezone
from ..fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp.sources.wildfire")
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_FIRMS_BASE_URL = "https://firms.modaps.eosdis.nasa.gov/api/area/csv"
# Bounding boxes: west,south,east,north
REGIONS = {
"north_america": "-170,15,-50,75",
"south_america": "-85,-60,-30,15",
"europe": "-25,35,45,72",
"africa": "-20,-37,55,38",
"middle_east": "25,10,65,45",
"south_asia": "60,5,100,40",
"east_asia": "95,15,150,55",
"southeast_asia": "90,-15,155,25",
"oceania": "105,-50,180,-5",
}
# CSV columns from FIRMS VIIRS SNPP NRT (1-day)
_CSV_COLUMNS = [
"latitude", "longitude", "bright_ti4", "scan", "track",
"acq_date", "acq_time", "satellite", "confidence", "version",
"bright_ti5", "frp", "daynight",
]
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _parse_fires_csv(csv_text: str) -> list[dict]:
"""Parse FIRMS CSV text into a list of fire dicts.
Only returns high-confidence detections.
"""
lines = csv_text.strip().split("\n")
if len(lines) < 2:
return []
# Skip the header line
fires: list[dict] = []
for line in lines[1:]:
fields = line.split(",")
if len(fields) < len(_CSV_COLUMNS):
continue
confidence = fields[8].strip()
if confidence.lower() not in ("high", "h"):
continue
try:
lat = float(fields[0])
lon = float(fields[1])
brightness = float(fields[2])
frp_val = float(fields[11]) if fields[11].strip() else 0.0
except (ValueError, IndexError):
continue
fires.append({
"latitude": lat,
"longitude": lon,
"brightness": brightness,
"frp": frp_val,
"confidence": confidence,
"acq_date": fields[5].strip(),
"acq_time": fields[6].strip(),
"daynight": fields[12].strip(),
"satellite": fields[7].strip(),
})
return fires
def _cluster_fires(fires: list[dict], top_n: int = 20) -> list[dict]:
"""Group fires by a 0.5-degree grid and return the top N clusters by count."""
grid: dict[tuple[float, float], list[dict]] = {}
for fire in fires:
# Round to nearest 0.5 degrees
grid_lat = round(fire["latitude"] * 2) / 2
grid_lon = round(fire["longitude"] * 2) / 2
key = (grid_lat, grid_lon)
if key not in grid:
grid[key] = []
grid[key].append(fire)
# Sort clusters by fire count descending
sorted_clusters = sorted(grid.items(), key=lambda item: len(item[1]), reverse=True)
clusters: list[dict] = []
for (lat, lon), cell_fires in sorted_clusters[:top_n]:
max_frp = max(f["frp"] for f in cell_fires)
clusters.append({
"lat": lat,
"lon": lon,
"fire_count": len(cell_fires),
"max_frp": max_frp,
})
return clusters
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
async def fetch_wildfires(
fetcher: Fetcher,
region: str | None = None,
api_key: str | None = None,
) -> dict:
"""Fetch active wildfire data from NASA FIRMS (VIIRS SNPP NRT, last 24h).
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
region: Region name (key from REGIONS dict). None fetches all regions.
api_key: NASA FIRMS API key. Falls back to NASA_FIRMS_API_KEY env var.
Returns:
Dict with fires_by_region, total_fires, source, and timestamp.
"""
key = api_key or os.environ.get("NASA_FIRMS_API_KEY")
if not key:
return {"error": "NASA_FIRMS_API_KEY not configured"}
if region is not None:
bbox = REGIONS.get(region)
if bbox is None:
return {
"error": f"Unknown region '{region}'. Valid: {', '.join(REGIONS.keys())}",
}
regions_to_fetch: dict[str, str] = {region: bbox}
else:
regions_to_fetch = dict(REGIONS)
async def _fetch_region(region_name: str, bbox: str) -> tuple[str, list[dict] | None]:
"""Fetch fires for a single region, return (name, fires_list_or_None)."""
url = f"{_FIRMS_BASE_URL}/{key}/VIIRS_SNPP_NRT/{bbox}/1"
csv_text = await fetcher.get_text(
url=url,
source="nasa-firms",
cache_key=f"wildfire:fires:{region_name}",
cache_ttl=1800,
)
if csv_text is None:
logger.warning("FIRMS returned no data for region %s", region_name)
return (region_name, None)
fires = _parse_fires_csv(csv_text)
return (region_name, fires)
# Fetch all requested regions in parallel
tasks = [
_fetch_region(name, bbox)
for name, bbox in regions_to_fetch.items()
]
results = await asyncio.gather(*tasks)
# Assemble response
fires_by_region: dict[str, dict] = {}
total_fires = 0
for region_name, fires in results:
if fires is None:
fires_by_region[region_name] = {"count": 0, "top_clusters": []}
continue
count = len(fires)
total_fires += count
top_clusters = _cluster_fires(fires)
fires_by_region[region_name] = {
"count": count,
"top_clusters": top_clusters,
}
cache_label = region if region is not None else "global"
# Store assembled result in cache under the composite key
fetcher.cache.set(
f"wildfire:fires:{cache_label}",
{
"fires_by_region": fires_by_region,
"total_fires": total_fires,
"source": "nasa-firms",
"timestamp": _utc_now_iso(),
},
1800,
)
return {
"fires_by_region": fires_by_region,
"total_fires": total_fires,
"source": "nasa-firms",
"timestamp": _utc_now_iso(),
}
+1
View File
@@ -0,0 +1 @@
"""Tests for world-intel-mcp."""
+18
View File
@@ -0,0 +1,18 @@
"""Test configuration — strips proxy env vars so httpx doesn't try SOCKS."""
import os
import pytest
_PROXY_VARS = [
"ALL_PROXY", "all_proxy", "HTTP_PROXY", "http_proxy",
"HTTPS_PROXY", "https_proxy", "FTP_PROXY", "ftp_proxy",
"GRPC_PROXY", "grpc_proxy",
]
@pytest.fixture(autouse=True)
def _strip_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Remove system proxy env vars so httpx creates clean connections."""
for var in _PROXY_VARS:
monkeypatch.delenv(var, raising=False)
@@ -0,0 +1,59 @@
"""Tests for circuit breaker and infrastructure."""
import time
import pytest
from world_intel_mcp.circuit_breaker import CircuitBreaker
def test_circuit_starts_closed() -> None:
cb = CircuitBreaker(failure_threshold=3, cooldown_seconds=1.0)
assert cb.is_available("test-source")
def test_circuit_trips_after_threshold() -> None:
cb = CircuitBreaker(failure_threshold=2, cooldown_seconds=10.0)
cb.record_failure("src")
assert cb.is_available("src") # 1 failure, threshold is 2
cb.record_failure("src")
assert not cb.is_available("src") # tripped
def test_circuit_recovers_after_cooldown() -> None:
cb = CircuitBreaker(failure_threshold=1, cooldown_seconds=0.5)
cb.record_failure("src")
assert not cb.is_available("src")
time.sleep(0.6)
assert cb.is_available("src") # half-open, allows probe
def test_success_resets_failures() -> None:
cb = CircuitBreaker(failure_threshold=3, cooldown_seconds=10.0)
cb.record_failure("src")
cb.record_failure("src")
cb.record_success("src")
assert cb.is_available("src")
# Even after 2 more failures, need 3 consecutive
cb.record_failure("src")
cb.record_failure("src")
assert cb.is_available("src") # only 2 since reset
def test_status_output() -> None:
cb = CircuitBreaker(failure_threshold=2, cooldown_seconds=60.0)
cb.record_success("healthy")
cb.record_failure("unhealthy")
cb.record_failure("unhealthy")
status = cb.status()
assert status["healthy"]["status"] == "closed"
assert status["unhealthy"]["status"] == "open"
assert status["unhealthy"]["total_trips"] == 1
def test_independent_sources() -> None:
cb = CircuitBreaker(failure_threshold=1, cooldown_seconds=60.0)
cb.record_failure("source_a")
assert not cb.is_available("source_a")
assert cb.is_available("source_b") # independent
+66
View File
@@ -0,0 +1,66 @@
"""Tests for SQLite TTL cache."""
import time
from pathlib import Path
import pytest
from world_intel_mcp.cache import Cache
@pytest.fixture
def cache(tmp_path: Path) -> Cache:
return Cache(db_path=tmp_path / "test_cache.db")
def test_set_and_get(cache: Cache) -> None:
cache.set("key1", {"value": 42}, ttl_seconds=60)
result = cache.get("key1")
assert result == {"value": 42}
def test_get_missing_key(cache: Cache) -> None:
assert cache.get("nonexistent") is None
def test_ttl_expiration(cache: Cache) -> None:
cache.set("ephemeral", "data", ttl_seconds=1)
assert cache.get("ephemeral") == "data"
time.sleep(1.1)
assert cache.get("ephemeral") is None
def test_overwrite(cache: Cache) -> None:
cache.set("key", "v1", ttl_seconds=60)
cache.set("key", "v2", ttl_seconds=60)
assert cache.get("key") == "v2"
def test_delete(cache: Cache) -> None:
cache.set("key", "val", ttl_seconds=60)
cache.delete("key")
assert cache.get("key") is None
def test_evict_expired(cache: Cache) -> None:
cache.set("fresh", "yes", ttl_seconds=300)
cache.set("stale", "no", ttl_seconds=1)
time.sleep(1.1)
removed = cache.evict_expired()
assert removed == 1
assert cache.get("fresh") == "yes"
assert cache.get("stale") is None
def test_stats(cache: Cache) -> None:
cache.set("a", 1, ttl_seconds=300)
cache.set("b", 2, ttl_seconds=300)
stats = cache.stats()
assert stats["total_entries"] == 2
assert stats["active_entries"] == 2
def test_complex_values(cache: Cache) -> None:
data = {"nested": {"list": [1, 2, 3], "bool": True, "null": None}}
cache.set("complex", data, ttl_seconds=60)
assert cache.get("complex") == data
+218
View File
@@ -0,0 +1,218 @@
"""Tests for source modules — uses respx to mock HTTP calls."""
import json
from pathlib import Path
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
from world_intel_mcp.cache import Cache
from world_intel_mcp.circuit_breaker import CircuitBreaker
from world_intel_mcp.fetcher import Fetcher
@pytest.fixture
def cache(tmp_path: Path) -> Cache:
return Cache(db_path=tmp_path / "test_cache.db")
@pytest.fixture
def fetcher(cache: Cache) -> Fetcher:
breaker = CircuitBreaker()
return Fetcher(cache=cache, breaker=breaker, default_timeout=5.0)
# ---------------------------------------------------------------------------
# Markets
# ---------------------------------------------------------------------------
@respx.mock
@pytest.mark.asyncio
async def test_fetch_market_quotes(fetcher: Fetcher) -> None:
from world_intel_mcp.sources.markets import fetch_market_quotes
# Mock Yahoo Finance v8 chart response for ^GSPC
chart_response = {
"chart": {
"result": [{
"meta": {
"symbol": "^GSPC",
"regularMarketPrice": 5123.45,
"regularMarketChangePercent": 0.42,
"currency": "USD",
}
}]
}
}
respx.get("https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC").mock(
return_value=httpx.Response(200, json=chart_response)
)
result = await fetch_market_quotes(fetcher, symbols=["^GSPC"])
assert "quotes" in result
assert len(result["quotes"]) == 1
assert result["quotes"][0]["symbol"] == "^GSPC"
assert result["quotes"][0]["price"] == 5123.45
assert result["source"] == "yahoo-finance"
@respx.mock
@pytest.mark.asyncio
async def test_fetch_crypto_quotes(fetcher: Fetcher) -> None:
from world_intel_mcp.sources.markets import fetch_crypto_quotes
coins = [
{
"id": "bitcoin",
"symbol": "btc",
"name": "Bitcoin",
"current_price": 98000,
"market_cap": 1900000000000,
"price_change_percentage_24h": 2.5,
"sparkline_in_7d": {"price": [95000, 96000, 97000, 98000]},
}
]
respx.get("https://api.coingecko.com/api/v3/coins/markets").mock(
return_value=httpx.Response(200, json=coins)
)
result = await fetch_crypto_quotes(fetcher, limit=5)
assert "coins" in result
assert len(result["coins"]) == 1
assert result["coins"][0]["symbol"] == "btc"
assert result["source"] == "coingecko"
# ---------------------------------------------------------------------------
# Seismology
# ---------------------------------------------------------------------------
@respx.mock
@pytest.mark.asyncio
async def test_fetch_earthquakes(fetcher: Fetcher) -> None:
from world_intel_mcp.sources.seismology import fetch_earthquakes
geojson = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": "us7000abc1",
"properties": {
"mag": 5.2,
"place": "100km SSW of Somewhere",
"time": 1708700000000,
"tsunami": 0,
"felt": 15,
"alert": "green",
"url": "https://earthquake.usgs.gov/earthquakes/eventpage/us7000abc1",
},
"geometry": {
"type": "Point",
"coordinates": [-120.5, 35.2, 10.0],
},
}
],
}
respx.get("https://earthquake.usgs.gov/fdsnws/event/1/query").mock(
return_value=httpx.Response(200, json=geojson)
)
result = await fetch_earthquakes(fetcher, min_magnitude=4.0, hours=24)
assert result["count"] == 1
eq = result["earthquakes"][0]
assert eq["magnitude"] == 5.2
assert eq["id"] == "us7000abc1"
assert eq["depth_km"] == 10.0
assert eq["latitude"] == 35.2
assert eq["longitude"] == -120.5
assert result["source"] == "usgs"
# ---------------------------------------------------------------------------
# Wildfire
# ---------------------------------------------------------------------------
@respx.mock
@pytest.mark.asyncio
async def test_fetch_wildfires_no_api_key(fetcher: Fetcher) -> None:
from world_intel_mcp.sources.wildfire import fetch_wildfires
with patch.dict("os.environ", {}, clear=False):
# Remove the key if present
import os
os.environ.pop("NASA_FIRMS_API_KEY", None)
result = await fetch_wildfires(fetcher, api_key=None)
assert "error" in result
@respx.mock
@pytest.mark.asyncio
async def test_fetch_wildfires_single_region(fetcher: Fetcher) -> None:
from world_intel_mcp.sources.wildfire import fetch_wildfires
csv_data = (
"latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,confidence,version,bright_ti5,frp,daynight\n"
"34.5,-118.2,350.0,0.5,0.5,2024-02-23,1200,N,high,2.0,290.0,45.0,D\n"
"34.6,-118.3,360.0,0.5,0.5,2024-02-23,1200,N,high,2.0,295.0,55.0,D\n"
"34.5,-118.2,340.0,0.5,0.5,2024-02-23,1200,N,low,2.0,285.0,30.0,D\n"
)
respx.get(url__regex=r".*firms\.modaps\.eosdis\.nasa\.gov.*").mock(
return_value=httpx.Response(200, text=csv_data)
)
result = await fetch_wildfires(fetcher, region="north_america", api_key="testkey")
assert "fires_by_region" in result
na = result["fires_by_region"]["north_america"]
assert na["count"] == 2 # only 2 high-confidence
assert result["total_fires"] == 2
# ---------------------------------------------------------------------------
# Economic
# ---------------------------------------------------------------------------
@respx.mock
@pytest.mark.asyncio
async def test_fetch_fred_series_no_key(fetcher: Fetcher) -> None:
from world_intel_mcp.sources.economic import fetch_fred_series
import os
os.environ.pop("FRED_API_KEY", None)
result = await fetch_fred_series(fetcher, series_id="UNRATE", api_key=None)
assert "error" in result
@respx.mock
@pytest.mark.asyncio
async def test_fetch_world_bank_indicators(fetcher: Fetcher) -> None:
from world_intel_mcp.sources.economic import fetch_world_bank_indicators
wb_response = [
{"page": 1, "pages": 1, "per_page": 5, "total": 2},
[
{"indicator": {"id": "NY.GDP.MKTP.CD", "value": "GDP"}, "date": "2023", "value": 25000000000000},
{"indicator": {"id": "NY.GDP.MKTP.CD", "value": "GDP"}, "date": "2022", "value": 24000000000000},
],
]
respx.get(url__regex=r".*api\.worldbank\.org.*").mock(
return_value=httpx.Response(200, json=wb_response)
)
result = await fetch_world_bank_indicators(fetcher, country="USA", indicators=["NY.GDP.MKTP.CD"])
assert "indicators" in result
assert len(result["indicators"]) == 1
assert result["indicators"][0]["id"] == "NY.GDP.MKTP.CD"
assert result["source"] == "world-bank"