diff --git a/CLAUDE.md b/CLAUDE.md index 9faef39..972f1cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What This Is -World Intelligence MCP Server — 81 tools across 30 domains providing real-time global intelligence from free public APIs. Serves three interfaces: MCP stdio (for Claude Code/Cursor), a live Starlette dashboard with SSE, and a Click CLI with Rich output. Python 3.11+, built with hatchling. +World Intelligence MCP Server — 80 tools across 30 domains providing real-time global intelligence from free public APIs. Serves three interfaces: MCP stdio (for Claude Code/Cursor), a live Starlette dashboard with SSE, and a Click CLI with Rich output. Python 3.11+, built with hatchling. ## Commands @@ -23,7 +23,6 @@ pytest src/world_intel_mcp/tests/test_sources.py::test_fetch_market_quotes -v # # CLI intel markets # stock indices intel earthquakes --min-mag 5.0 # USGS quakes -intel report daily # generate HTML report intel status # cache + circuit breaker health # Dashboard (requires [dashboard] extra) @@ -52,8 +51,6 @@ dashboard/app.py (SSE) ─┘ **Analysis modules** (`analysis/*.py`): Cross-domain intelligence that consumes outputs from multiple sources. Includes signal aggregation, instability indexing, NLP (entity extraction, classification, clustering, spike detection via Welford's algorithm), and strategic synthesis. -**Reports** (`reports/*.py`): Jinja2-templated HTML/Markdown reports (daily brief, country dossier, threat landscape). `generator.py` orchestrates parallel source fetches. Output dir defaults to `$STORAGE_BASE/reports/intel` or `INTEL_REPORT_DIR` env var. - **Static config** (`config/*.py`): Curated datasets — 22 intel hotspots, 70+ military bases, 40 ports, 24 pipelines, 24 nuclear facilities, 34 undersea cables, 48 AI datacenters, 27 spaceports, 27 mineral deposits, 82 stock exchanges, 105 major cities, 28 world leaders, 36 APT groups. **Dashboard** (`dashboard/`): Self-contained Starlette app with a single `index.html` template (no frontend build step). SSE endpoint streams all domains in parallel via `asyncio.gather()`, refreshes every 30 seconds. Loads `.env` from project root on startup. diff --git a/ROADMAP.md b/ROADMAP.md index fcd8db9..34ae9dc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,8 +1,8 @@ # World Intel MCP — Feature Parity Roadmap **Benchmark**: [koala73/worldmonitor](https://github.com/koala73/worldmonitor) -**Updated**: 2026-02-24 -**Current tools**: 68 (67 intel + 1 status) +**Updated**: 2026-02-26 +**Current tools**: 80 (79 intel + 1 status) --- @@ -16,9 +16,9 @@ --- -## 1. Data Sources — Complete Inventory (59 intel tools) +## 1. Data Sources — Complete Inventory -### Markets & Economics (10 tools) +### Markets & Economics (11 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| | `intel_market_quotes` | `list-market-quotes` | :white_check_mark: | @@ -31,13 +31,16 @@ | `intel_energy_prices` | `get-energy-prices` | :white_check_mark: | | `intel_fred_series` | `get-fred-series` | :white_check_mark: | | `intel_world_bank_indicators` | `list-world-bank-indicators` | :white_check_mark: | +| `intel_country_stocks` | Country main index ticker | :white_check_mark: | -### Natural Disasters & Climate (3 tools) +### Natural Disasters & Climate (5 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| | `intel_earthquakes` | `list-earthquakes` | :white_check_mark: | | `intel_wildfires` | `list-fire-detections` | :white_check_mark: | | `intel_climate_anomalies` | `list-climate-anomalies` | :white_check_mark: | +| `intel_environmental_events` | NASA EONET events | :white_check_mark: | +| `intel_disaster_alerts` | GDACS global disaster alerts | :white_check_mark: | ### Conflict & Security (4 tools) | Tool | WM Equivalent | Status | @@ -47,15 +50,17 @@ | `intel_unrest_events` | ACLED protests + GDELT dedup | :white_check_mark: | | `intel_cyber_threats` | `list-cyber-threats` | :white_check_mark: | -### Military & Defense (6 tools) +### Military & Defense (8 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| | `intel_military_flights` | `list-military-flights` | :white_check_mark: | | `intel_theater_posture` | `get-theater-posture` | :white_check_mark: | | `intel_aircraft_details` | `get-aircraft-details` | :white_check_mark: | +| `intel_aircraft_batch` | Batch ICAO24 lookup | :white_check_mark: | | `intel_vessel_snapshot` | `get-vessel-snapshot` | :white_check_mark: | | `intel_military_surge` | `military-surge.ts` | :white_check_mark: | | `intel_military_bases` | Static dataset (70 bases) | :white_check_mark: | +| `intel_usni_fleet` | USNI Fleet Tracker weekly disposition | :white_check_mark: | ### Infrastructure & Maritime (6 tools) | Tool | WM Equivalent | Status | @@ -78,7 +83,7 @@ ### News & Information (4 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| -| `intel_news_feed` | 80+ RSS feeds, 4-tier sources | :white_check_mark: | +| `intel_news_feed` | 90+ RSS feeds, 4-tier sources | :white_check_mark: | | `intel_trending_keywords` | trending-keywords service | :white_check_mark: | | `intel_gdelt_search` | `search-gdelt-documents` | :white_check_mark: | | `intel_ai_releases` | AI model/paper tracker | :white_check_mark: | @@ -90,7 +95,7 @@ | `intel_airport_delays` | `list-airport-delays` | :white_check_mark: | | `intel_shipping_index` | Yahoo Finance shipping ETFs | :white_check_mark: | -### Analysis & Intelligence (11 tools) +### Analysis & Intelligence (9 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| | `intel_risk_scores` | `get-risk-scores` | :white_check_mark: | @@ -102,17 +107,13 @@ | `intel_hotspot_escalation` | `hotspot-escalation.ts` | :white_check_mark: | | `intel_alert_digest` | Cross-domain alert synthesis | :white_check_mark: | | `intel_weekly_trends` | Temporal trend analysis | :white_check_mark: | -| `intel_daily_brief` | — | :white_check_mark: | -| `intel_threat_landscape` | `StrategicPosturePanel` | :white_check_mark: | -### Country & Geopolitical (5 tools) +### Country & Geopolitical (3 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| | `intel_country_brief` | `get-country-intel-brief` | :white_check_mark: | -| `intel_country_dossier` | CountryBriefPage | :white_check_mark: | | `intel_election_calendar` | Election proximity risk | :white_check_mark: | | `intel_sanctions_search` | OFAC SDN search | :white_check_mark: | -| `intel_nuclear_facilities` | Static dataset (24 facilities) | :white_check_mark: | ### Strategic Synthesis (4 tools) | Tool | WM Equivalent | Status | @@ -122,6 +123,18 @@ | `intel_fleet_report` | Naval fleet activity report | :white_check_mark: | | `intel_population_exposure` | Population near active events | :white_check_mark: | +### Tech & Science (3 tools) +| Tool | WM Equivalent | Status | +|------|---------------|--------| +| `intel_hacker_news` | Top HN stories (Firebase API) | :white_check_mark: | +| `intel_trending_repos` | GitHub trending repos | :white_check_mark: | +| `intel_arxiv_papers` | Recent AI/ML papers | :white_check_mark: | + +### Government (1 tool) +| Tool | WM Equivalent | Status | +|------|---------------|--------| +| `intel_usa_spending` | USAspending.gov federal data | :white_check_mark: | + ### Specialist (3 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| @@ -129,6 +142,14 @@ | `intel_nuclear_monitor` | USGS seismic near test sites | :white_check_mark: | | `intel_service_status` | Cloudflare/AWS/Azure/GCP | :white_check_mark: | +### NLP Intelligence (4 tools) +| Tool | WM Equivalent | Status | +|------|---------------|--------| +| `intel_extract_entities` | Regex NER (28 leaders, 41 orgs, 36 APTs) | :white_check_mark: | +| `intel_classify_event` | Keyword threat classification (14 categories) | :white_check_mark: | +| `intel_news_clusters` | Jaccard similarity clustering | :white_check_mark: | +| `intel_keyword_spikes` | Welford's algorithm spike detection | :white_check_mark: | + ### System (1 tool) | Tool | Purpose | Status | |------|---------|--------| @@ -150,17 +171,17 @@ | Nuclear test sites | 5 sites with monitoring | config/countries.py | :white_check_mark: | | Countries config | 22 nations with risk baselines | config/countries.py | :white_check_mark: | | Major cities | 105 cities (pop > 2M, 1B coverage) | config/population.py | :white_check_mark: | -| Undersea cables | Cable routes with landing points | — | :red_circle: | -| AI datacenters | Major clusters globally | — | :red_circle: | -| Spaceports | Launch facilities worldwide | — | :red_circle: | -| Critical minerals | Strategic mineral locations | — | :red_circle: | -| Stock exchanges | 92 global exchanges | — | :red_circle: | +| Undersea cables | 34 cables with landing points | `intel_undersea_cables` | :white_check_mark: | +| AI datacenters | 48 global clusters | `intel_ai_datacenters` | :white_check_mark: | +| Spaceports | 27 launch facilities | `intel_spaceports` | :white_check_mark: | +| Critical minerals | 27 deposit types | `intel_critical_minerals` | :white_check_mark: | +| Stock exchanges | 82 global exchanges | `intel_stock_exchanges` | :white_check_mark: | --- ## 3. RSS Feed Coverage -Expanded from 20 to **80+ feeds** across **15+ categories** with 4-tier source ranking (wire/major/specialty/aggregator) and propaganda risk labels. +Expanded to **90+ feeds** across **16 categories** with 4-tier source ranking (wire/major/specialty/aggregator) and propaganda risk labels. | Category | Count | Status | |----------|-------|--------| @@ -178,65 +199,53 @@ Expanded from 20 to **80+ feeds** across **15+ categories** with 4-tier source r | Health (WHO/ProMED) | 3+ | :white_check_mark: | | Space weather | 2+ | :white_check_mark: | | AI/ML releases | 3+ | :white_check_mark: | -| Latin America | 0 | :red_circle: | -| Multilingual feeds | 0 | :red_circle: | +| Latin America | 8+ | :white_check_mark: | +| Multilingual (ES/FR/DE) | 7+ | :white_check_mark: | --- -## 4. What's Still Missing +## 4. Dashboard -### Data Sources (P2-P3) -| # | Feature | Priority | Effort | -|---|---------|----------|--------| -| 1 | **Country stock index lookup** — ticker for any country's main index | P2 | S | -| 2 | **Aircraft details batch** — batch lookup by multiple ICAO24 codes | P3 | S | -| 3 | **USNI fleet tracker** — US Navy fleet disposition from USNI News | P2 | M | -| 4 | **Wingbits ADS-B** — crowd-sourced ADS-B coverage | P3 | S | -| 5 | ~~**Population exposure**~~ — :white_check_mark: `intel_population_exposure` | — | — | -| 6 | **Hacker News items** — top HN stories | P3 | S | -| 7 | **Trending repos** — GitHub trending repos | P3 | S | -| 8 | **arXiv papers** — recent AI/ML papers | P3 | S | -| 9 | **PizzInt indicator** — pizza delivery patterns as OSINT proxy | P3 | S | +Live Starlette app with SSE streaming at `intel-dashboard --port 8501`. -### Analysis Layers (P2-P3) -| # | Feature | Priority | Effort | -|---|---------|----------|--------| -| 10 | ~~**Strategic Posture Assessment**~~ — :white_check_mark: `intel_strategic_posture` | — | — | -| 11 | ~~**World Brief**~~ — :white_check_mark: `intel_world_brief` (structured, data-driven) | — | — | -| 12 | **USA Spending Tracker** — Federal contract data from USAspending.gov | P3 | S | +| Feature | Status | +|---------|--------| +| SSE streaming (30s refresh) | :white_check_mark: 37 data streams | +| Leaflet map (14 layers + 6 static) | :white_check_mark: | +| 12 expandable drawer sections | :white_check_mark: | +| USNI Fleet Tracker drawer | :white_check_mark: | +| Data freshness monitoring drawer | :white_check_mark: | +| Per-source circuit breaker health | :white_check_mark: | +| AI situation brief (Ollama-powered) | :white_check_mark: | +| Static HTML reports | Removed (dashboard replaces) | -### Static Datasets (P3) -| # | Feature | Priority | Effort | -|---|---------|----------|--------| -| 17 | **Undersea cable routes** — landing points (we have health, not routes) | P2 | M | -| 18 | **AI datacenters** — 111 major clusters globally | P3 | S | -| 19 | **Spaceports** — launch facilities worldwide | P3 | S | -| 20 | **Critical mineral deposits** — strategic mineral locations | P3 | S | -| 21 | **Stock exchanges** — 92 global exchanges with coordinates | P3 | M | +--- -### System Architecture (P2) -| # | Feature | Priority | Effort | -|---|---------|----------|--------| -| 22 | **Redis caching backend** — persistent TTL cache (currently in-memory) | P1 | M | -| 23 | **Circuit breaker per-source** — configurable thresholds per API | P2 | S | -| 24 | **Data freshness monitoring** — per-source staleness tracking | P2 | S | +## 5. System Architecture + +| Feature | Status | Notes | +|---------|--------|-------| +| SQLite WAL-mode cache | :white_check_mark: | Persistent TTL, stale fallback | +| Per-source circuit breaker | :white_check_mark: | Configurable thresholds | +| Data freshness monitoring | :white_check_mark: | Per-source staleness in dashboard | +| Per-coro timeout (45s) | :white_check_mark: | No single source blocks dashboard | --- ## Completed Phases -### Phase 1-4: Foundation (0 -> 39 tools) -Core data sources: markets, crypto, macro, earthquakes, wildfires, ACLED, UCDP, humanitarian, military flights, theater posture, aircraft, internet outages, cable health, nav warnings, climate, prediction markets, displacement, airport delays, cyber threats, news feeds, GDELT, trending keywords, country briefs, risk scores, instability index, signal convergence, daily brief, country dossier, threat landscape. +### Phase 1-4: Foundation (0 -> 36 tools) +Core data sources: markets, crypto, macro, earthquakes, wildfires, ACLED, UCDP, humanitarian, military flights, theater posture, aircraft, internet outages, cable health, nav warnings, climate, prediction markets, displacement, airport delays, cyber threats, news feeds, GDELT, trending keywords, country briefs, risk scores, instability index, signal convergence. -### Phase 5: Core Analysis Engine (+3 = 42 tools) +### Phase 5: Core Analysis Engine (+3 = 39 tools) `intel_focal_points`, `intel_signal_summary`, `intel_temporal_anomalies` Countries config with 22 nations, intel hotspots, conflict zones, strategic waterways. CII v2 upgraded with multi-signal weighted blend. Welford's online algorithm for temporal baseline anomaly detection. -### Phase 6: Military & Infrastructure Intelligence (+6 = 48 tools) +### Phase 6: Military & Infrastructure Intelligence (+6 = 45 tools) `intel_vessel_snapshot`, `intel_military_surge`, `intel_cascade_analysis`, `intel_hotspot_escalation`, `intel_commodity_quotes`, `intel_unrest_events` AIS vessel tracking, military surge detection in 17 sensitive regions, infrastructure cascade analysis, hotspot escalation scoring for 22 locations. -### Phase 7: Domain Expansion (+10 = 58 tools) +### Phase 7: Domain Expansion (+10 = 55 tools) `intel_space_weather`, `intel_ai_releases`, `intel_disease_outbreaks`, `intel_sanctions_search`, `intel_election_calendar`, `intel_shipping_index`, `intel_social_signals`, `intel_nuclear_monitor`, `intel_alert_digest`, `intel_weekly_trends` 80+ RSS feeds with 4-tier source ranking. WHO/ProMED/CIDRAP health monitoring. OFAC sanctions search. Election proximity risk scoring. Reddit social signals. Nuclear test site seismic monitoring. Cross-domain alert digest. Temporal weekly trend analysis. @@ -252,19 +261,10 @@ Regex-based NER (28 leaders, 41 orgs, 25 companies, 36 APT groups, CVE extractio `intel_strategic_posture`, `intel_world_brief`, `intel_fleet_report`, `intel_population_exposure` Composite strategic posture assessment from 9 weighted domains (military, political, conflict, infrastructure, economic, cyber, health, climate, space). Structured world intelligence brief aggregating posture, focal points, news clusters, temporal anomalies, and keyword spikes. Naval fleet activity report combining theater posture, vessel snapshots, and surge detections. Population exposure analysis near active events using 105-city dataset (1B pop coverage). ---- +### Phase 11: Extended Data & Geospatial (+12 = 80 tools) +`intel_country_stocks`, `intel_aircraft_batch`, `intel_hacker_news`, `intel_trending_repos`, `intel_arxiv_papers`, `intel_usa_spending`, `intel_environmental_events`, `intel_disaster_alerts`, `intel_undersea_cables`, `intel_ai_datacenters`, `intel_spaceports`, `intel_critical_minerals`, `intel_stock_exchanges`, `intel_usni_fleet` -## Next Phase - -### Phase 11: Data Expansion -**Goal**: Fill remaining data gaps and static datasets. - -1. **Country stock index lookup** — ticker for any country's main stock index -2. **USNI fleet tracker** — US Navy fleet disposition scraping -3. **Hacker News** — top HN stories via public API -4. **Trending repos** — GitHub trending repositories - -New tools: `intel_country_stocks`, `intel_usni_fleet`, `intel_hacker_news`, `intel_trending_repos` +Static datasets completed: 34 undersea cables, 48 AI datacenters, 27 spaceports, 27 critical mineral deposits, 82 stock exchanges. USNI Fleet Tracker for Navy disposition. RSS feeds expanded to 90+ across 16 categories (added Latin America 8+, multilingual ES/FR/DE 7+). Data freshness monitoring added to dashboard. Static HTML report generation removed (live dashboard replaces). --- @@ -272,10 +272,10 @@ New tools: `intel_country_stocks`, `intel_usni_fleet`, `intel_hacker_news`, `int | Category | Have | Benchmark | Coverage | |----------|------|-----------|----------| -| Data source tools | 68 | 42 | **162%** | +| Data source tools | 80 | 42 | **190%** | | Analysis engines | 19 | 15 | **127%** | -| Static datasets | 10 | 12 | 83% | -| RSS feeds | 80+ | 150+ | 53% | -| Strategic synthesis | Strategic posture + world brief + fleet report + population exposure | Dashboard-only | **Exceeds** | +| Static datasets | 15 | 12 | **125%** | +| RSS feeds | 90+ | 150+ | 60% | +| Strategic synthesis | Posture + brief + fleet + exposure + USNI | Dashboard-only | **Exceeds** | -**Bottom line**: 68 tools across 27 domains, exceeding WorldMonitor benchmark by 62% in tool count and 27% in analysis engines. Strategic synthesis layer complete — composite risk assessment, structured intelligence briefs, fleet reporting, and population exposure analysis. Remaining gaps: RSS feed breadth (80+ vs 150+), static datasets (10 vs 12), a few niche data sources. +**Bottom line**: 80 tools across 30 domains, exceeding WorldMonitor benchmark by 90% in tool count, 27% in analysis engines, and 25% in static datasets. All phases 1-11 complete. Live Starlette dashboard with 37 SSE streams, 14 map layers, and data freshness monitoring. No remaining critical gaps — only niche sources (Wingbits requires API key, PizzInt has no public API). diff --git a/src/world_intel_mcp/cli.py b/src/world_intel_mcp/cli.py index 66add0d..20bec9f 100644 --- a/src/world_intel_mcp/cli.py +++ b/src/world_intel_mcp/cli.py @@ -19,7 +19,6 @@ from .cache import Cache from .circuit_breaker import CircuitBreaker from .fetcher import Fetcher from .sources import markets, economic, seismology, wildfire, conflict, military, infrastructure, maritime, climate, news, intelligence, prediction, displacement, aviation, cyber -from .reports import generator as report_gen console = Console() @@ -859,60 +858,6 @@ def instability(ctx: click.Context, country_code: str | None) -> None: console.print(table) -# --------------------------------------------------------------------------- -# Reports -# --------------------------------------------------------------------------- - -@main.group() -def report() -> None: - """Generate intelligence reports (HTML).""" - - -main.add_command(report) - - -@report.command(name="daily") -@click.option("--output-dir", "-o", default=None, help="Output directory") -def report_daily(output_dir: str | None) -> None: - """Generate daily intelligence brief (HTML).""" - console.print("[bold]Generating daily brief...[/bold]") - result = _run(report_gen.generate_daily_brief(output_dir=output_dir)) - console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") - summary = result.get("summary", {}) - console.print(f" Quotes: {summary.get('market_quotes', 0)} | " - f"Conflicts: {summary.get('conflict_events', 0)} | " - f"Threats: {summary.get('cyber_threats', 0)} | " - f"Quakes: {summary.get('earthquakes', 0)}") - - -@report.command(name="threat") -@click.option("--output-dir", "-o", default=None, help="Output directory") -def report_threat(output_dir: str | None) -> None: - """Generate threat landscape report (HTML).""" - console.print("[bold]Generating threat landscape...[/bold]") - result = _run(report_gen.generate_threat_landscape(output_dir=output_dir)) - console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") - - -@report.command(name="market") -@click.option("--output-dir", "-o", default=None, help="Output directory") -def report_market(output_dir: str | None) -> None: - """Generate market overview report (HTML).""" - console.print("[bold]Generating market overview...[/bold]") - result = _run(report_gen.generate_market_overview(output_dir=output_dir)) - console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") - - -@report.command(name="dossier") -@click.argument("country_code") -@click.option("--output-dir", "-o", default=None, help="Output directory") -def report_dossier(country_code: str, output_dir: str | None) -> None: - """Generate country dossier report (HTML).""" - console.print(f"[bold]Generating dossier for {country_code}...[/bold]") - result = _run(report_gen.generate_country_dossier( - country_code=country_code, output_dir=output_dir, - )) - console.print(f"[green]Report saved:[/green] {result.get('file_path', '?')}") # --------------------------------------------------------------------------- diff --git a/src/world_intel_mcp/dashboard/app.py b/src/world_intel_mcp/dashboard/app.py index 7e53822..03a6274 100644 --- a/src/world_intel_mcp/dashboard/app.py +++ b/src/world_intel_mcp/dashboard/app.py @@ -52,6 +52,7 @@ from world_intel_mcp.analysis.posture import fetch_strategic_posture from world_intel_mcp.analysis.exposure import fetch_population_exposure from world_intel_mcp.analysis.situation import fetch_situation_brief from world_intel_mcp.sources.fleet import fetch_fleet_report +from world_intel_mcp.sources.usni_fleet import fetch_usni_fleet from world_intel_mcp.config.countries import INTEL_HOTSPOTS, STRATEGIC_WATERWAYS from world_intel_mcp.config.geospatial import MILITARY_BASES, STRATEGIC_PORTS, PIPELINES, NUCLEAR_FACILITIES from world_intel_mcp.sources.infrastructure import CABLE_CORRIDORS @@ -120,6 +121,7 @@ async def _fetch_overview() -> dict: "service_status": service_status.fetch_service_status(fetcher), "strategic_posture": fetch_strategic_posture(fetcher), "fleet_report": fetch_fleet_report(fetcher), + "usni_fleet": fetch_usni_fleet(fetcher), "population_exposure": fetch_population_exposure(fetcher), "domestic_flights": aviation.fetch_domestic_flights(fetcher), "traffic_flow": traffic.fetch_traffic_flow(fetcher), @@ -205,6 +207,7 @@ async def _fetch_overview() -> dict: # Attach source health + timestamp result["source_health"] = _breaker.status() if _breaker else {} result["cache_stats"] = _cache.stats() if _cache else {} + result["cache_freshness"] = _cache.freshness() if _cache else {} result["timestamp"] = datetime.now(timezone.utc).isoformat() return result @@ -281,51 +284,10 @@ async def api_health(request): async def api_report_pdf(request): - """Generate a PDF daily brief report. - - Renders the daily_brief.html template with live data, then converts - to PDF via weasyprint. Requires ``pip install world-intel-mcp[pdf]``. - """ - try: - from weasyprint import HTML as WeasyHTML - except ImportError: - return JSONResponse( - {"error": "weasyprint not installed — run: pip install world-intel-mcp[pdf]"}, - status_code=501, - ) - - from world_intel_mcp.reports.html_report import render_template - - data = await _fetch_overview() - - context = { - "title": "Daily Intelligence Brief", - "generated_at": data.get("timestamp", ""), - "market_quotes": data.get("market_quotes", {}), - "crypto_quotes": data.get("crypto_quotes", {}), - "macro_signals": data.get("macro_signals", {}), - "earthquakes": data.get("earthquakes", {}), - "cyber_threats": data.get("cyber_threats", {}), - "news_feed": data.get("news_feed", {}), - "military_flights": data.get("military_flights", {}), - "internet_outages": data.get("internet_outages", {}), - "climate_anomalies": data.get("climate_anomalies", {}), - "displacement": data.get("displacement", {}), - "risk_scores": data.get("risk_scores", {}), - "alert_digest": data.get("alert_digest", {}), - } - - html_str = render_template("daily_brief.html", context) - pdf_bytes = WeasyHTML(string=html_str).write_pdf() - - now_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") - return Response( - content=pdf_bytes, - media_type="application/pdf", - headers={ - "Content-Disposition": f'attachment; filename="intel-brief-{now_str}.pdf"', - "Access-Control-Allow-Origin": "*", - }, + """PDF report generation (removed — use the live dashboard instead).""" + return JSONResponse( + {"error": "PDF reports removed — use the live dashboard at /"}, + status_code=410, ) diff --git a/src/world_intel_mcp/dashboard/index.html b/src/world_intel_mcp/dashboard/index.html index 2ec5ad3..32d44a9 100644 --- a/src/world_intel_mcp/dashboard/index.html +++ b/src/world_intel_mcp/dashboard/index.html @@ -2461,6 +2461,49 @@ function updateDrawer(data) { }); } + // ── USNI FLEET TRACKER ── + if (data.usni_fleet && !data.usni_fleet.error && data.usni_fleet.ship_count > 0) { + var uf = data.usni_fleet; + h += '
USNI FLEET TRACKER
'; + h += '
' + esc(uf.report_title || '') + '
'; + if (uf.force_totals && uf.force_totals.battle_force) { + var bf = uf.force_totals.battle_force; + var dep = uf.force_totals.deployed || {}; + var uw = uf.force_totals.underway || {}; + h += '
'; + h += '
' + bf.total + '
Battle Force
'; + h += '
' + (dep.total || '?') + '
Deployed
'; + h += '
' + (uw.total || '?') + '
Underway
'; + h += '
'; + } + if (uf.region_breakdown) { + var regions = Object.entries(uf.region_breakdown).sort(function(a,b){return b[1]-a[1];}); + regions.forEach(function(r) { + h += '
' + esc(r[0]) + ' ' + r[1] + ' ships
'; + }); + } + (uf.ships || []).slice(0, 15).forEach(function(s) { + var cls = s.type === 'Aircraft Carrier' ? 'crit' : 'bright'; + h += '
' + esc(s.name) + ' (' + esc(s.hull_number) + ') ' + esc(s.region) + '
'; + }); + } + + // ── DATA FRESHNESS ── + if (data.cache_freshness && Object.keys(data.cache_freshness).length > 0) { + var cf = data.cache_freshness; + var sources = Object.entries(cf).sort(function(a,b) { return a[1].last_updated_s_ago - b[1].last_updated_s_ago; }); + var staleCount = sources.filter(function(s) { return s[1].is_stale; }).length; + h += '
DATA FRESHNESS
'; + h += '
' + sources.length + ' sources tracked, ' + staleCount + ' stale
'; + sources.forEach(function(s) { + var name = s[0], info = s[1]; + var ageMin = Math.round(info.last_updated_s_ago / 60); + var cls = info.is_stale ? 'crit' : ageMin > 10 ? 'warn' : 'dim'; + var label = ageMin < 1 ? '<1m' : ageMin + 'm'; + h += '
' + esc(name) + ' ' + label + (info.is_stale ? ' STALE' : '') + '
'; + }); + } + // ── AI SITUATION BRIEF ── if (data.situation_brief && !data.situation_brief.error && data.situation_brief.brief) { var sb = data.situation_brief; diff --git a/src/world_intel_mcp/reports/__init__.py b/src/world_intel_mcp/reports/__init__.py deleted file mode 100644 index 5353a30..0000000 --- a/src/world_intel_mcp/reports/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Report generation for world-intel-mcp.""" diff --git a/src/world_intel_mcp/reports/generator.py b/src/world_intel_mcp/reports/generator.py deleted file mode 100644 index e9807fe..0000000 --- a/src/world_intel_mcp/reports/generator.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Report orchestrator for world-intel-mcp. - -Gathers data from multiple intelligence sources in parallel and renders -HTML or Markdown reports via Jinja2 templates. -""" - -import asyncio -import logging -import os -from datetime import datetime, timezone -from pathlib import Path - -from ..cache import Cache -from ..circuit_breaker import CircuitBreaker -from ..fetcher import Fetcher -from ..sources import ( - markets, economic, seismology, wildfire, conflict, military, - infrastructure, maritime, climate, news, intelligence, - prediction, displacement, aviation, cyber, -) - -logger = logging.getLogger("world-intel-mcp.reports.generator") - -# Default output directory -_DEFAULT_OUTPUT_DIR = os.environ.get( - "INTEL_REPORT_DIR", - os.path.join(os.environ.get("STORAGE_BASE", "/tmp"), "reports", "intel"), -) - - -def _ensure_output_dir(path: str | None = None) -> Path: - """Create output directory if needed and return Path.""" - output_dir = Path(path or _DEFAULT_OUTPUT_DIR) - output_dir.mkdir(parents=True, exist_ok=True) - return output_dir - - -async def generate_daily_brief(output_dir: str | None = None) -> dict: - """Generate a daily intelligence brief HTML report. - - Gathers: market quotes, macro signals, conflict events, cyber threats, - earthquakes, wildfires, prediction markets, trending keywords. - Renders to HTML and returns the file path + summary. - """ - cache = Cache() - breaker = CircuitBreaker() - fetcher = Fetcher(cache=cache, breaker=breaker) - - try: - # Gather all data in parallel - ( - market_data, - macro_data, - conflict_data, - cyber_data, - quake_data, - fire_data, - predict_data, - keyword_data, - ) = await asyncio.gather( - markets.fetch_market_quotes(fetcher), - markets.fetch_macro_signals(fetcher), - conflict.fetch_acled_events(fetcher, days=1, limit=50), - cyber.fetch_cyber_threats(fetcher, limit=30), - seismology.fetch_earthquakes(fetcher, min_magnitude=4.5, hours=24), - wildfire.fetch_wildfires(fetcher), - prediction.fetch_prediction_markets(fetcher, limit=10), - news.fetch_trending_keywords(fetcher, min_count=3), - ) - - now = datetime.now(timezone.utc) - context = { - "title": "Daily Intelligence Brief", - "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), - "market_summary": { - "quotes": market_data.get("quotes", []), - "macro_signals": macro_data.get("signals", {}), - }, - "conflict_summary": { - "events": conflict_data.get("events", []), - "count": conflict_data.get("count", 0), - }, - "cyber_summary": { - "threats": cyber_data.get("threats", []), - "by_severity": cyber_data.get("by_severity", {}), - }, - "natural_summary": { - "earthquakes": quake_data.get("earthquakes", []), - "fire_count": fire_data.get("total_fires", 0), - }, - "prediction_highlights": predict_data.get("markets", []), - "trending_keywords": keyword_data.get("keywords", []), - } - - # Render HTML - from .html_report import render_template - html = render_template("daily_brief.html", context) - - # Write to file - out_dir = _ensure_output_dir(output_dir) - filename = f"daily_brief_{now.strftime('%Y%m%d_%H%M%S')}.html" - filepath = out_dir / filename - filepath.write_text(html, encoding="utf-8") - - logger.info("Daily brief generated: %s", filepath) - - return { - "report_type": "daily_brief", - "file_path": str(filepath), - "generated_at": context["generated_at"], - "summary": { - "market_quotes": len(context["market_summary"]["quotes"]), - "conflict_events": context["conflict_summary"]["count"], - "cyber_threats": len(context["cyber_summary"]["threats"]), - "earthquakes": len(context["natural_summary"]["earthquakes"]), - "predictions": len(context["prediction_highlights"]), - "keywords": len(context["trending_keywords"]), - }, - } - finally: - await fetcher.close() - - -async def generate_country_dossier( - country_code: str, - output_dir: str | None = None, -) -> dict: - """Generate a country dossier HTML report.""" - cache = Cache() - breaker = CircuitBreaker() - fetcher = Fetcher(cache=cache, breaker=breaker) - - try: - ( - brief_data, - instability_data, - conflict_data, - displacement_data, - ) = await asyncio.gather( - intelligence.fetch_country_brief(fetcher, country_code=country_code), - intelligence.fetch_instability_index(fetcher, country_code=country_code), - conflict.fetch_acled_events(fetcher, country=country_code, days=30, limit=50), - displacement.fetch_displacement_summary(fetcher), - ) - - now = datetime.now(timezone.utc) - context = { - "title": f"Country Dossier: {country_code}", - "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), - "country_code": country_code, - "brief": brief_data.get("brief", ""), - "instability": { - "instability_index": instability_data.get("instability_index", 0), - "components": instability_data.get("components", {}), - "risk_level": instability_data.get("risk_level", "unknown"), - }, - "conflict_events": conflict_data.get("events", []), - "displacement": { - "by_origin": displacement_data.get("by_origin", []), - "global_totals": displacement_data.get("global_totals", {}), - }, - "economic": { - "gdp": brief_data.get("data", {}).get("gdp", []), - "inflation": brief_data.get("data", {}).get("inflation", []), - }, - } - - from .html_report import render_template - html = render_template("country_dossier.html", context) - - out_dir = _ensure_output_dir(output_dir) - filename = f"dossier_{country_code}_{now.strftime('%Y%m%d_%H%M%S')}.html" - filepath = out_dir / filename - filepath.write_text(html, encoding="utf-8") - - logger.info("Country dossier generated: %s", filepath) - - return { - "report_type": "country_dossier", - "country_code": country_code, - "file_path": str(filepath), - "generated_at": context["generated_at"], - } - finally: - await fetcher.close() - - -async def generate_threat_landscape(output_dir: str | None = None) -> dict: - """Generate a threat landscape HTML report.""" - cache = Cache() - breaker = CircuitBreaker() - fetcher = Fetcher(cache=cache, breaker=breaker) - - try: - ( - cyber_data, - conflict_data, - military_data, - cable_data, - outage_data, - ) = await asyncio.gather( - cyber.fetch_cyber_threats(fetcher, limit=50), - conflict.fetch_acled_events(fetcher, days=7, limit=100), - military.fetch_theater_posture(fetcher), - infrastructure.fetch_cable_health(fetcher), - infrastructure.fetch_internet_outages(fetcher), - ) - - now = datetime.now(timezone.utc) - context = { - "title": "Threat Landscape Report", - "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), - "cyber_threats": { - "threats": cyber_data.get("threats", []), - "by_severity": cyber_data.get("by_severity", {}), - "by_type": cyber_data.get("by_type", {}), - }, - "conflict_events": conflict_data.get("events", []), - "military_activity": { - "theaters": military_data.get("theaters", {}), - "total_military_aircraft": military_data.get("total_military_aircraft", 0), - }, - "cable_health": { - "corridors": cable_data.get("corridors", {}), - }, - "outages": { - "outages": outage_data.get("outages", []), - "ongoing_count": outage_data.get("ongoing_count", 0), - }, - } - - from .html_report import render_template - html = render_template("threat_landscape.html", context) - - out_dir = _ensure_output_dir(output_dir) - filename = f"threat_landscape_{now.strftime('%Y%m%d_%H%M%S')}.html" - filepath = out_dir / filename - filepath.write_text(html, encoding="utf-8") - - logger.info("Threat landscape generated: %s", filepath) - - return { - "report_type": "threat_landscape", - "file_path": str(filepath), - "generated_at": context["generated_at"], - } - finally: - await fetcher.close() - - -async def generate_market_overview(output_dir: str | None = None) -> dict: - """Generate a market overview HTML report.""" - cache = Cache() - breaker = CircuitBreaker() - fetcher = Fetcher(cache=cache, breaker=breaker) - - try: - ( - quote_data, - crypto_data, - macro_data, - sector_data, - etf_data, - ) = await asyncio.gather( - markets.fetch_market_quotes(fetcher), - markets.fetch_crypto_quotes(fetcher, limit=20), - markets.fetch_macro_signals(fetcher), - markets.fetch_sector_heatmap(fetcher), - markets.fetch_etf_flows(fetcher), - ) - - now = datetime.now(timezone.utc) - context = { - "title": "Market Overview", - "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), - "quotes": quote_data.get("quotes", []), - "crypto": crypto_data.get("coins", []), - "macro_signals": macro_data.get("signals", {}), - "sector_heatmap": sector_data.get("sectors", []), - "etf_flows": etf_data, - } - - from .html_report import render_template - html = render_template("market_overview.html", context) - - out_dir = _ensure_output_dir(output_dir) - filename = f"market_overview_{now.strftime('%Y%m%d_%H%M%S')}.html" - filepath = out_dir / filename - filepath.write_text(html, encoding="utf-8") - - logger.info("Market overview generated: %s", filepath) - - return { - "report_type": "market_overview", - "file_path": str(filepath), - "generated_at": context["generated_at"], - } - finally: - await fetcher.close() diff --git a/src/world_intel_mcp/reports/html_report.py b/src/world_intel_mcp/reports/html_report.py deleted file mode 100644 index f673289..0000000 --- a/src/world_intel_mcp/reports/html_report.py +++ /dev/null @@ -1,35 +0,0 @@ -"""HTML report renderer using Jinja2 templates. - -Loads templates from the ``templates/`` subdirectory and renders them -with the provided context data. -""" - -import logging -from pathlib import Path - -from jinja2 import Environment, FileSystemLoader, select_autoescape - -logger = logging.getLogger("world-intel-mcp.reports.html_report") - -_TEMPLATE_DIR = Path(__file__).parent / "templates" - -_env = Environment( - loader=FileSystemLoader(str(_TEMPLATE_DIR)), - autoescape=select_autoescape(["html"]), - trim_blocks=True, - lstrip_blocks=True, -) - - -def render_template(template_name: str, context: dict) -> str: - """Render a Jinja2 HTML template with the given context. - - Args: - template_name: Name of the template file in ``templates/``. - context: Dict of variables to pass to the template. - - Returns: - Rendered HTML string. - """ - template = _env.get_template(template_name) - return template.render(**context) diff --git a/src/world_intel_mcp/reports/markdown_report.py b/src/world_intel_mcp/reports/markdown_report.py deleted file mode 100644 index a8b0453..0000000 --- a/src/world_intel_mcp/reports/markdown_report.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Markdown report generator for world-intel-mcp. - -Generates Markdown reports with optional Mermaid diagrams. -""" - -import logging -from datetime import datetime, timezone - -logger = logging.getLogger("world-intel-mcp.reports.markdown_report") - - -def generate_daily_brief_md( - market_summary: dict, - conflict_summary: dict, - cyber_summary: dict, - natural_summary: dict, - prediction_highlights: list, - trending_keywords: list, -) -> str: - """Generate a daily intelligence brief in Markdown format.""" - now = datetime.now(timezone.utc) - lines = [ - f"# Daily Intelligence Brief", - f"*Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}*", - "", - "## Markets", - ] - - quotes = market_summary.get("quotes", []) - if quotes: - lines.append("| Symbol | Price | Change |") - lines.append("|--------|------:|-------:|") - for q in quotes[:8]: - chg = q.get("change_pct") or 0 - lines.append(f"| {q.get('symbol', '?')} | {q.get('price', 0):,.2f} | {chg:+.2f}% |") - lines.append("") - - # Conflict - events = conflict_summary.get("events", []) - lines.append(f"## Conflict ({conflict_summary.get('count', 0)} events)") - if events: - lines.append("| Date | Type | Country | Fatalities |") - lines.append("|------|------|---------|----------:|") - for e in events[:10]: - lines.append( - f"| {(e.get('event_date') or '')[:10]} " - f"| {e.get('event_type', '')} " - f"| {e.get('country', '')} " - f"| {e.get('fatalities', 0)} |" - ) - lines.append("") - - # Cyber - by_sev = cyber_summary.get("by_severity", {}) - lines.append(f"## Cyber Threats") - lines.append(f"Critical: {by_sev.get('critical', 0)} | " - f"High: {by_sev.get('high', 0)} | " - f"Medium: {by_sev.get('medium', 0)}") - lines.append("") - - # Natural - quakes = natural_summary.get("earthquakes", []) - lines.append(f"## Natural Events") - lines.append(f"Earthquakes: {len(quakes)} | Fires: {natural_summary.get('fire_count', 0)}") - if quakes: - lines.append("") - lines.append("| Mag | Location | Depth |") - lines.append("|----:|----------|------:|") - for q in quakes[:5]: - lines.append(f"| {q.get('magnitude', 0):.1f} | {(q.get('place') or '')[:40]} | {q.get('depth_km', 0):.0f}km |") - lines.append("") - - # Predictions - if prediction_highlights: - lines.append("## Prediction Markets") - for p in prediction_highlights[:5]: - yes = (p.get("yes_probability", 0) or 0) * 100 - lines.append(f"- **{(p.get('question') or '')[:60]}** — YES: {yes:.0f}% ({p.get('sentiment', '')})") - lines.append("") - - # Trending - if trending_keywords: - lines.append("## Trending Keywords") - kw_str = ", ".join(f"**{k['word']}** ({k['count']})" for k in trending_keywords[:15]) - lines.append(kw_str) - lines.append("") - - lines.append(f"---\n*Phoenix AGI System — World Intelligence*") - return "\n".join(lines) - - -def generate_threat_landscape_md( - cyber_threats: dict, - conflict_events: list, - military_activity: dict, - cable_health: dict, - outages: dict, -) -> str: - """Generate a threat landscape report in Markdown with Mermaid diagram.""" - now = datetime.now(timezone.utc) - lines = [ - "# Threat Landscape Report", - f"*Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}*", - "", - ] - - # Mermaid threat overview - by_sev = cyber_threats.get("by_severity", {}) - lines.append("## Threat Overview") - lines.append("```mermaid") - lines.append("pie title Threat Severity Distribution") - for level in ["critical", "high", "medium", "low"]: - count = by_sev.get(level, 0) - if count > 0: - lines.append(f' "{level.title()}" : {count}') - lines.append("```") - lines.append("") - - # Cyber section - lines.append(f"## Cyber Threats ({len(cyber_threats.get('threats', []))})") - for t in cyber_threats.get("threats", [])[:10]: - lines.append(f"- [{t.get('severity', '').upper()}] **{(t.get('indicator') or '')[:40]}** — {t.get('threat', '')} (via {t.get('source_feed', '')})") - lines.append("") - - # Military - theaters = military_activity.get("theaters", {}) - total = military_activity.get("total_military_aircraft", 0) - lines.append(f"## Military Activity ({total} aircraft)") - for name, info in theaters.items(): - count = info.get("count", 0) - lines.append(f"- **{name.replace('_', ' ').title()}**: {count} aircraft") - lines.append("") - - # Conflict events count - conflict_count = len(conflict_events) - if conflict_count: - lines.append(f"## Active Conflicts ({conflict_count} events, 7 days)") - for e in conflict_events[:10]: - lines.append( - f"- [{(e.get('event_date') or '')[:10]}] **{e.get('country', '')}** — " - f"{e.get('event_type', '')} ({e.get('fatalities', 0)} fatalities)" - ) - lines.append("") - - # Infrastructure - corridors = cable_health.get("corridors", {}) - if corridors: - lines.append("## Cable Health") - status_map = {0: "Clear", 1: "Advisory", 2: "At Risk", 3: "Disrupted"} - for name, info in corridors.items(): - score = info.get("status_score", 0) - lines.append(f"- **{name.replace('_', ' ').title()}**: {status_map.get(score, '?')}") - lines.append("") - - # Outages - ongoing = outages.get("ongoing_count", 0) - lines.append(f"## Internet Outages ({ongoing} ongoing)") - lines.append("") - - lines.append(f"---\n*Phoenix AGI System — Threat Intelligence*") - return "\n".join(lines) diff --git a/src/world_intel_mcp/reports/templates/country_dossier.html b/src/world_intel_mcp/reports/templates/country_dossier.html deleted file mode 100644 index 244c496..0000000 --- a/src/world_intel_mcp/reports/templates/country_dossier.html +++ /dev/null @@ -1,270 +0,0 @@ - - - - - - {{ title }} — {{ generated_at[:10] }} - - - - -
-

{{ title }}

-

Country Code: {{ country_code }} | Generated: {{ generated_at }} | Phoenix AGI System

- - -

Instability Assessment

- {% if instability %} -
-
-

Instability Index

-
{{ "%.1f"|format(instability.instability_index or 0) }}
-
- {{ instability.risk_level|default('Unknown') }} -
-
-
-

Instability Components

-
- -
-
-
- {% else %} -

No instability data available for this country.

- {% endif %} - - -

Intelligence Brief

- {% if brief %} -
{{ brief }}
- {% else %} -

No brief generated.

- {% endif %} - - -

Conflict Events

- {% if conflict_events %} -
- - - {% for e in conflict_events[:15] %} - - - - - - - - {% endfor %} -
DateTypeSub-TypeLocationFatalities
{{ (e.event_date or '')[:10] }}{{ e.event_type|default('') }}{{ e.sub_event_type|default('') }}{{ e.location|default('') }}{{ e.fatalities|default(0) }}
-
- {% else %} -

No conflict events reported for this country.

- {% endif %} - - -

Displacement & Refugees

- {% if displacement %} -
-
-

Displacement by Origin

- {% if displacement.by_origin %} - - - {% for d in displacement.by_origin[:10] %} - - - - - - - {% endfor %} -
YearRefugeesIDPsAsylum Seekers
{{ d.year|default('') }}{{ "{:,}".format(d.refugees|default(0)|int) }}{{ "{:,}".format(d.idps|default(0)|int) }}{{ "{:,}".format(d.asylum_seekers|default(0)|int) }}
- {% else %} -

No displacement data by origin.

- {% endif %} -
- {% if displacement.global_totals %} -
-

Global Context

- - - {% for key, val in displacement.global_totals.items() %} - - - - - {% endfor %} -
MetricTotal
{{ key }}{{ "{:,}".format(val|int) if val is not none else 'N/A' }}
-
- {% endif %} -
- {% else %} -

No displacement data available.

- {% endif %} - - -

Economic Indicators

- {% if economic %} -
-
-

GDP Trend

- {% if economic.gdp %} -
- -
- {% else %} -

No GDP data available.

- {% endif %} -
-
-

Inflation Trend

- {% if economic.inflation %} - - - {% for entry in economic.inflation[:10] %} - - - - - {% endfor %} -
YearInflation (%)
{{ entry.year|default(entry.date|default('')) }}{{ "%.1f"|format(entry.value or 0) }}%
- {% else %} -

No inflation data available.

- {% endif %} -
-
- {% else %} -

No economic data available.

- {% endif %} - - -
- - - - diff --git a/src/world_intel_mcp/reports/templates/daily_brief.html b/src/world_intel_mcp/reports/templates/daily_brief.html deleted file mode 100644 index 828441b..0000000 --- a/src/world_intel_mcp/reports/templates/daily_brief.html +++ /dev/null @@ -1,186 +0,0 @@ - - - - - - {{ title }} — {{ generated_at[:10] }} - - - - -
-

{{ title }}

-

Generated: {{ generated_at }} | Phoenix AGI System

- - -

Markets

- {% if market_summary %} -
-
-

Index Quotes

- {% if market_summary.quotes %} - - - {% for q in market_summary.quotes[:8] %} - - - - - - {% endfor %} -
SymbolPriceChange
{{ q.symbol|default('—') }}{{ "%.2f"|format(q.price or 0) }}{{ "%+.2f"|format(q.change_pct or 0) }}%
- {% else %} -

No quote data available.

- {% endif %} -
-
-

Macro Signals

- {% if market_summary.macro_signals %} - - - {% for name, info in market_summary.macro_signals.items() %} - - - - - {% endfor %} -
SignalValue
{{ name }}{% if info is mapping %}{{ info.value|default(info.price|default('?')) }}{% elif info is not none %}{{ info }}{% else %}N/A{% endif %}
- {% else %} -

No macro signals available.

- {% endif %} -
-
- {% else %} -

Market data unavailable.

- {% endif %} - - -

Conflict & Security

-
-
-

Recent Events ({{ conflict_summary.count|default(0) }})

- {% if conflict_summary and conflict_summary.events %} - - - {% for e in conflict_summary.events[:10] %} - - - - - - - {% endfor %} -
DateTypeCountryFatalities
{{ (e.event_date or '')[:10] }}{{ e.event_type|default('') }}{{ e.country|default('') }}{{ e.fatalities|default(0) }}
- {% else %} -

No conflict events reported.

- {% endif %} -
-
-

Cyber Threats

- {% if cyber_summary and cyber_summary.threats %} -

- Critical: {{ cyber_summary.by_severity.critical|default(0) }} | - High: {{ cyber_summary.by_severity.high|default(0) }} | - Medium: {{ cyber_summary.by_severity.medium|default(0) }} -

- - - {% for t in cyber_summary.threats[:8] %} - - - - - - {% endfor %} -
SeverityIndicatorThreat
{{ t.severity|default('unknown') }}{{ (t.indicator or '')[:30] }}{{ (t.threat or '')[:30] }}
- {% else %} -

No cyber threat data available.

- {% endif %} -
-
- - -

Natural Events

-
-
-

Earthquakes

- {% if natural_summary and natural_summary.earthquakes %} - - - {% for q in natural_summary.earthquakes[:8] %} - - - - - - {% endfor %} -
MagLocationDepth
{{ "%.1f"|format(q.magnitude or 0) }}{{ (q.place or '')[:40] }}{{ "%.0f"|format(q.depth_km or 0) }}km
- {% else %} -

No earthquake data available.

- {% endif %} -
-
-

Wildfires

-

{{ natural_summary.fire_count|default(0) }} high-confidence fires detected

-
-
- - -

Signals

-
-
-

Prediction Markets

- {% if prediction_highlights %} - - - {% for p in prediction_highlights[:8] %} - - - - - - {% endfor %} -
QuestionYESSentiment
{{ (p.question or '')[:40] }}{{ "%.0f"|format((p.yes_probability or 0) * 100) }}%{{ p.sentiment|default('') }}
- {% else %} -

No prediction data available.

- {% endif %} -
-
-

Trending Keywords

- {% if trending_keywords %} - {% for kw in trending_keywords[:20] %} - {{ kw.word }} ({{ kw.count }}) - {% endfor %} - {% else %} -

No trending keyword data.

- {% endif %} -
-
- - -
- - diff --git a/src/world_intel_mcp/reports/templates/market_overview.html b/src/world_intel_mcp/reports/templates/market_overview.html deleted file mode 100644 index 61cfc47..0000000 --- a/src/world_intel_mcp/reports/templates/market_overview.html +++ /dev/null @@ -1,277 +0,0 @@ - - - - - - {{ title }} — {{ generated_at[:10] }} - - - - -
-

{{ title }}

-

Generated: {{ generated_at }} | Phoenix AGI System

- - -

Equity & Index Quotes

- {% if quotes %} -
- - - {% for q in quotes %} - - - - - - - - - {% endfor %} -
SymbolNamePriceChange% ChangeVolume
{{ q.symbol|default('—') }}{{ q.name|default(q.shortName|default('')) }}{{ "%.2f"|format(q.price or q.regularMarketPrice or 0) }} - {{ "%+.2f"|format(q.change or q.regularMarketChange or 0) }} - - {{ "%+.2f"|format(q.change_pct or q.regularMarketChangePercent or 0) }}% - {{ "{:,}".format((q.volume or q.regularMarketVolume or 0)|int) }}
-
- {% else %} -

No equity quote data available.

- {% endif %} - - -

Sector Performance

- {% if sector_heatmap %} -
-
-

Sector Heatmap

-
- {% for s in sector_heatmap %} - {% set pct = s.change_pct if s.change_pct is not none else 0 %} - {% if pct >= 2 %} - {% set bg = '#166534' %} - {% elif pct >= 0.5 %} - {% set bg = '#14532d' %} - {% elif pct >= 0 %} - {% set bg = '#1a2e1a' %} - {% elif pct >= -0.5 %} - {% set bg = '#2e1a1a' %} - {% elif pct >= -2 %} - {% set bg = '#7f1d1d' %} - {% else %} - {% set bg = '#991b1b' %} - {% endif %} -
- {{ s.symbol|default(s.name|default('?')) }}
{{ "%+.1f"|format(pct) }}% -
- {% endfor %} -
- -

Sector Bar Chart

-
- -
-
-
- {% else %} -

No sector data available.

- {% endif %} - - -

Cryptocurrency

- {% if crypto %} -
- - - {% for c in crypto[:15] %} - - - - - - - - {% endfor %} -
CoinPrice (USD)24h ChangeMarket Cap24h Volume
{{ c.symbol|default(c.id|default(''))|upper }}{% if c.name %} {{ c.name }}{% endif %}{{ "${:,.2f}".format(c.current_price or c.price or 0) }} - {{ "%+.2f"|format(c.price_change_percentage_24h or c.change_24h or 0) }}% - - {% set mcap = c.market_cap or 0 %} - {% if mcap >= 1e12 %}${{ "%.1f"|format(mcap / 1e12) }}T - {% elif mcap >= 1e9 %}${{ "%.1f"|format(mcap / 1e9) }}B - {% elif mcap >= 1e6 %}${{ "%.1f"|format(mcap / 1e6) }}M - {% elif mcap > 0 %}${{ "{:,.0f}".format(mcap) }} - {% else %}—{% endif %} - - {% set vol = c.total_volume or c.volume_24h or 0 %} - {% if vol >= 1e9 %}${{ "%.1f"|format(vol / 1e9) }}B - {% elif vol >= 1e6 %}${{ "%.1f"|format(vol / 1e6) }}M - {% elif vol > 0 %}${{ "{:,.0f}".format(vol) }} - {% else %}—{% endif %} -
-
- {% else %} -

No cryptocurrency data available.

- {% endif %} - - -

Macro Signals

- {% if macro_signals %} -
- {% for name, info in macro_signals.items() %} -
-

{{ name }}

- {% if info is mapping %} - - {% for key, val in info.items() %} - - - - - {% endfor %} -
{{ key.replace('_', ' ')|title }} - {% if val is number %} - {% if val >= 1e9 %}{{ "%.2f"|format(val / 1e9) }}B - {% elif val >= 1e6 %}{{ "%.2f"|format(val / 1e6) }}M - {% else %}{{ "%.4f"|format(val) if val < 1 and val > -1 else "%.2f"|format(val) }}{% endif %} - {% elif val is not none %}{{ val }} - {% else %}N/A{% endif %} -
- {% elif info is not none %} -

{{ info }}

- {% else %} -

N/A

- {% endif %} -
- {% endfor %} -
- {% else %} -

No macro signal data available.

- {% endif %} - - -

ETF Flows

- {% if etf_flows %} -
- {% if etf_flows is mapping %} - - - {% for name, data in etf_flows.items() %} - - - {% if data is mapping %} - - - {% else %} - - - {% endif %} - - {% endfor %} -
ETF / CategoryFlowDetails
{{ name }} - {% set flow = data.flow or data.net_flow or 0 %} - {% if flow >= 1e9 %}${{ "%+.1f"|format(flow / 1e9) }}B - {% elif flow >= 1e6 %}${{ "%+.1f"|format(flow / 1e6) }}M - {% else %}{{ "%+.2f"|format(flow) }}{% endif %} - {{ data.description|default(data.period|default('')) }}{{ data }}
- {% elif etf_flows is iterable and etf_flows is not string %} - - - {% for item in etf_flows %} - - - - - {% endfor %} -
ETFFlow
{{ item.name|default(item.symbol|default('')) }}{{ item.flow|default(item.value|default('')) }}
- {% else %} -

{{ etf_flows }}

- {% endif %} -
- {% else %} -

No ETF flow data available.

- {% endif %} - - -
- - - - diff --git a/src/world_intel_mcp/reports/templates/threat_landscape.html b/src/world_intel_mcp/reports/templates/threat_landscape.html deleted file mode 100644 index f84de26..0000000 --- a/src/world_intel_mcp/reports/templates/threat_landscape.html +++ /dev/null @@ -1,265 +0,0 @@ - - - - - - {{ title }} — {{ generated_at[:10] }} - - - - -
-

{{ title }}

-

Generated: {{ generated_at }} | Phoenix AGI System

- - -

Threat Overview

-
-
-
{{ cyber_threats.by_severity.critical|default(0) if cyber_threats else 0 }}
-
Critical
-
-
-
{{ cyber_threats.by_severity.high|default(0) if cyber_threats else 0 }}
-
High
-
-
-
{{ cyber_threats.by_severity.medium|default(0) if cyber_threats else 0 }}
-
Medium
-
-
-
{{ (cyber_threats.threats|length) if cyber_threats and cyber_threats.threats else 0 }}
-
Total Indicators
-
-
-
{{ outages.ongoing_count|default(0) if outages else 0 }}
-
Active Outages
-
-
- - -

Cyber Threats

- {% if cyber_threats and cyber_threats.threats %} -
-
-

Severity Distribution

-
- -
-
-
-

Threat Types

- {% if cyber_threats.by_type %} - - - {% for type_name, count in cyber_threats.by_type.items() %} - - - - - {% endfor %} -
TypeCount
{{ type_name }}{{ count }}
- {% else %} -

No threat type breakdown available.

- {% endif %} -
-
- -
-

Threat Indicators

- - - {% for t in cyber_threats.threats[:20] %} - - - - - - - - {% endfor %} -
SeverityIndicatorTypeThreatFirst Seen
{{ t.severity|default('unknown') }}{{ (t.indicator or '')[:40] }}{{ t.type|default(t.indicator_type|default('')) }}{{ (t.threat or '')[:30] }}{{ (t.first_seen or '')[:10] }}
-
- {% else %} -

No cyber threat data available.

- {% endif %} - - -

Conflict Events

- {% if conflict_events %} -
- - - {% for e in conflict_events[:15] %} - - - - - - - - {% endfor %} -
DateTypeCountryLocationFatalities
{{ (e.event_date or '')[:10] }}{{ e.event_type|default('') }}{{ e.country|default('') }}{{ e.location|default('') }}{{ e.fatalities|default(0) }}
-
- {% else %} -

No conflict events reported.

- {% endif %} - - -

Military Activity

- {% if military_activity %} -
- {% if military_activity.theaters %} - {% for theater_name, theater_data in military_activity.theaters.items() %} -
-

{{ theater_name }}

- {% if theater_data is mapping %} - - - {% for key, val in theater_data.items() %} - - - - - {% endfor %} -
MetricValue
{{ key.replace('_', ' ')|title }}{{ val }}
- {% else %} -

{{ theater_data }}

- {% endif %} -
- {% endfor %} - {% endif %} -
-

Summary

-
{{ military_activity.total_military_aircraft|default(0) }}
-
Military Aircraft Tracked
-
-
- {% else %} -

No military activity data available.

- {% endif %} - - -

Submarine Cable Health

- {% if cable_health and cable_health.corridors %} -
- {% for corridor_name, corridor_data in cable_health.corridors.items() %} -
-

{{ corridor_name }}

- {% if corridor_data is mapping %} - - {% for key, val in corridor_data.items() %} - - - - - {% endfor %} -
{{ key.replace('_', ' ')|title }}{{ val }}
- {% else %} -

{{ corridor_data }}

- {% endif %} -
- {% endfor %} -
- {% else %} -

No cable health data available.

- {% endif %} - - -

Infrastructure Outages

- {% if outages and outages.outages %} -
- - - {% for o in outages.outages[:15] %} - - - - - - - - {% endfor %} -
ServiceStatusRegionSinceImpact
{{ o.service|default(o.name|default('')) }}{{ o.status|default('unknown') }}{{ o.region|default('') }}{{ (o.since or o.started or '')[:16] }}{{ o.impact|default('') }}
-
- {% else %} -

No active outages reported.

- {% endif %} - - -
- - - - diff --git a/src/world_intel_mcp/server.py b/src/world_intel_mcp/server.py index 06059a9..60827b6 100644 --- a/src/world_intel_mcp/server.py +++ b/src/world_intel_mcp/server.py @@ -10,7 +10,7 @@ conflict, military flights, infrastructure, and more. Phase 1: Markets, Economic, Seismology, Wildfire (14 tools). Phase 2: Conflict, Military, Infrastructure, Maritime, Climate (+10 = 24 tools). Phase 3: News, Intelligence, Prediction, Displacement, Aviation, Cyber (+9 = 33 tools). -Phase 4: Reports — daily brief, country dossier, threat landscape (+3 = 36 tools). +Phase 4: (reports removed — use live dashboard instead). Phase 5: Analysis — focal points, signal summary, temporal anomalies, CII v2 (+3 = 39 tools). Phase 6: Military & infrastructure intelligence (+6 = 45 tools). Phase 7: Health, sanctions, elections, shipping, social, nuclear, alerts, trends (+10 = 55 tools). @@ -36,8 +36,7 @@ from mcp.types import Tool, TextContent from .cache import Cache from .circuit_breaker import CircuitBreaker from .fetcher import Fetcher -from .sources import markets, economic, seismology, wildfire, conflict, military, infrastructure, maritime, climate, news, intelligence, prediction, displacement, aviation, cyber, space_weather, ai_watch, health, sanctions, elections, shipping, social, nuclear, service_status, geospatial, hacker_news, github_trending, arxiv_papers, usa_spending, environmental -from .reports import generator as report_gen +from .sources import markets, economic, seismology, wildfire, conflict, military, infrastructure, maritime, climate, news, intelligence, prediction, displacement, aviation, cyber, space_weather, ai_watch, health, sanctions, elections, shipping, social, nuclear, service_status, geospatial, hacker_news, github_trending, arxiv_papers, usa_spending, environmental, usni_fleet logging.basicConfig( level=os.environ.get("WORLD_INTEL_LOG_LEVEL", "INFO"), @@ -455,39 +454,6 @@ TOOLS: list[Tool] = [ }, }, ), - # --- Reports (3 tools) --- - Tool( - name="intel_daily_brief", - description="Generate a daily intelligence brief HTML report (markets, conflict, cyber, natural, predictions, trending). Returns file path.", - inputSchema={ - "type": "object", - "properties": { - "output_dir": {"type": "string", "description": "Custom output directory (default: $STORAGE_BASE/reports/intel/)"}, - }, - }, - ), - Tool( - name="intel_country_dossier", - description="Generate a full country dossier HTML report (brief, instability index, conflict, displacement, economic). Returns file path.", - inputSchema={ - "type": "object", - "properties": { - "country_code": {"type": "string", "description": "ISO country code (e.g., UKR, SYR, MMR)"}, - "output_dir": {"type": "string", "description": "Custom output directory"}, - }, - "required": ["country_code"], - }, - ), - Tool( - name="intel_threat_landscape", - description="Generate a threat landscape HTML report (cyber threats, conflict, military, cable health, outages). Returns file path.", - inputSchema={ - "type": "object", - "properties": { - "output_dir": {"type": "string", "description": "Custom output directory"}, - }, - }, - ), # --- Space Weather (1 tool) --- Tool( name="intel_space_weather", @@ -854,6 +820,12 @@ TOOLS: list[Tool] = [ }, }, ), + # --- USNI Fleet (1 tool) --- + Tool( + name="intel_usni_fleet", + description="US Navy fleet disposition from USNI News Fleet Tracker. Extracts ships, hull numbers, carrier strike groups, regional deployment, and force totals from the latest weekly report.", + inputSchema={"type": "object", "properties": {}}, + ), # --- Environmental (2 tools) --- Tool( name="intel_environmental_events", @@ -1104,16 +1076,6 @@ async def _dispatch(name: str, arguments: dict[str, Any]) -> Any: from .analysis.alerts import fetch_weekly_trends return await fetch_weekly_trends(fetcher) - # Reports - case "intel_daily_brief": - return await report_gen.generate_daily_brief(output_dir=arguments.get("output_dir")) - case "intel_country_dossier": - return await report_gen.generate_country_dossier( - country_code=arguments["country_code"], - output_dir=arguments.get("output_dir"), - ) - case "intel_threat_landscape": - return await report_gen.generate_threat_landscape(output_dir=arguments.get("output_dir")) # Service Status case "intel_service_status": @@ -1239,6 +1201,10 @@ async def _dispatch(name: str, arguments: dict[str, Any]) -> Any: limit=arguments.get("limit", 25), ) + # USNI Fleet + case "intel_usni_fleet": + return await usni_fleet.fetch_usni_fleet(fetcher) + # Environmental case "intel_environmental_events": return await environmental.fetch_environmental_events( diff --git a/src/world_intel_mcp/sources/news.py b/src/world_intel_mcp/sources/news.py index 41e5f77..3ab24ff 100644 --- a/src/world_intel_mcp/sources/news.py +++ b/src/world_intel_mcp/sources/news.py @@ -99,6 +99,18 @@ _RSS_FEEDS: dict[str, list[tuple[str, str]]] = { ("Americas Quarterly", "https://www.americasquarterly.org/feed/"), ("Buenos Aires Times", "https://www.batimes.com.ar/feed"), ("Tico Times", "https://ticotimes.net/feed"), + ("InSight Crime", "https://insightcrime.org/feed/"), + ("Brazil Reports", "https://brazilian.report/feed/"), + ("Mexico News Daily", "https://mexiconewsdaily.com/feed/"), + ], + "multilingual": [ + ("BBC Mundo", "https://feeds.bbci.co.uk/mundo/rss.xml"), + ("DW Español", "https://rss.dw.com/rss/es/top_news/rss-es-top"), + ("DW Deutsch", "https://rss.dw.com/rss/de/top_news/rss-de-top"), + ("France24 Français", "https://www.france24.com/fr/rss"), + ("RFI Français", "https://www.rfi.fr/fr/rss"), + ("UN News Español", "https://news.un.org/feed/subscribe/es/news/all/rss.xml"), + ("UN News Français", "https://news.un.org/feed/subscribe/fr/news/all/rss.xml"), ], "energy": [ ("Oil Price", "https://oilprice.com/rss/main"), @@ -151,6 +163,16 @@ SOURCE_TIERS: dict[str, str] = { "ReliefWeb": "intl_org", "Lowy Interpreter": "think_tank", "Dialogo Americas": "specialty", + "InSight Crime": "specialty", + "Brazil Reports": "specialty", + "Mexico News Daily": "specialty", + "BBC Mundo": "major", + "DW Español": "major", + "DW Deutsch": "major", + "France24 Français": "major", + "RFI Français": "major", + "UN News Español": "government", + "UN News Français": "government", "Nikkei Asia": "major", "The National UAE": "major", "Zero Hedge": "aggregator", diff --git a/src/world_intel_mcp/sources/usni_fleet.py b/src/world_intel_mcp/sources/usni_fleet.py new file mode 100644 index 0000000..eaff3c9 --- /dev/null +++ b/src/world_intel_mcp/sources/usni_fleet.py @@ -0,0 +1,275 @@ +"""USNI News Fleet and Marine Tracker source for world-intel-mcp. + +Parses the USNI Fleet Tracker RSS feed for US Navy fleet disposition, +extracting carrier strike groups, ship deployments, and force posture. +No API key required — uses the public RSS category feed. +""" + +import logging +import re +from datetime import datetime, timezone + +from ..fetcher import Fetcher + +try: + import feedparser +except ImportError: + feedparser = None # type: ignore[assignment] + +logger = logging.getLogger("world-intel-mcp.sources.usni_fleet") + +_FEED_URL = "https://news.usni.org/category/fleet-tracker/feed" + +# Regex patterns for extracting fleet data from article content +_SHIP_PATTERN = re.compile( + r"USS\s+([\w\s.]+?)\s*\(((?:CVN|DDG|CG|LHD|LHA|LPD|LSD|SSN|SSBN|SSGN|FFG|MCM|PC|AS|ESB|ESD|EPF|AFSB|T-AO|T-AKE|T-ESB|WAGB|LCS)-?\d+)\)", + re.IGNORECASE, +) +_USCG_PATTERN = re.compile( + r"USCGC\s+([\w\s.]+?)\s*\((WAGB|WMSL|WPC|WPB|WLB)-?\d+\)", + re.IGNORECASE, +) +_CSG_PATTERN = re.compile( + r"Carrier Strike Group\s+(\d+|[A-Z]+)", + re.IGNORECASE, +) +_ESG_PATTERN = re.compile( + r"(?:Expeditionary Strike Group|Amphibious Ready Group)\s+(\d+|[A-Z]+)", + re.IGNORECASE, +) +_BATTLE_FORCE_PATTERN = re.compile( + r"(\d+)\s+ships?\s*\((\d+)\s+USS,\s*(\d+)\s+USNS\)", + re.IGNORECASE, +) +_DEPLOYED_PATTERN = re.compile( + r"(\d+)\s+deployed\s*\((\d+)\s+USS,\s*(\d+)\s+USNS\)", + re.IGNORECASE, +) +_UNDERWAY_PATTERN = re.compile( + r"(\d+)\s+underway\s*\((\d+)\s+deployed,\s*(\d+)\s+local\)", + re.IGNORECASE, +) + +# Region keywords for location classification +_REGION_KEYWORDS = { + "Arabian Sea": "CENTCOM", + "Persian Gulf": "CENTCOM", + "Red Sea": "CENTCOM", + "Gulf of Oman": "CENTCOM", + "Gulf of Aden": "CENTCOM", + "Mediterranean": "EUCOM", + "Atlantic": "EUCOM", + "North Sea": "EUCOM", + "Baltic": "EUCOM", + "Caribbean": "SOUTHCOM", + "Pacific": "INDOPACOM", + "Philippine Sea": "INDOPACOM", + "South China Sea": "INDOPACOM", + "East China Sea": "INDOPACOM", + "Western Pacific": "INDOPACOM", + "Japan": "INDOPACOM", + "Yokosuka": "INDOPACOM", + "Guam": "INDOPACOM", + "Indian Ocean": "INDOPACOM", + "Antarctica": "OTHER", + "Arctic": "NORTHCOM", + "San Diego": "HOMEPORT", + "Norfolk": "HOMEPORT", + "Mayport": "HOMEPORT", + "Bremerton": "HOMEPORT", +} + + +def _classify_region(text: str) -> str: + """Classify a text snippet into a combatant command region.""" + for keyword, region in _REGION_KEYWORDS.items(): + if keyword.lower() in text.lower(): + return region + return "UNKNOWN" + + +def _extract_fleet_data(content: str) -> dict: + """Extract structured fleet disposition from article HTML/text content.""" + ships = [] + strike_groups = [] + + # Extract USS ships + for match in _SHIP_PATTERN.finditer(content): + name = match.group(1).strip() + hull = match.group(2).strip() + # Find surrounding context for region classification + start = max(0, match.start() - 200) + end = min(len(content), match.end() + 200) + context = content[start:end] + region = _classify_region(context) + + ship_type = hull.split("-")[0] if "-" in hull else hull[:3] + type_labels = { + "CVN": "Aircraft Carrier", + "DDG": "Destroyer", + "CG": "Cruiser", + "LHD": "Amphibious Assault Ship", + "LHA": "Amphibious Assault Ship", + "LPD": "Amphibious Transport Dock", + "LSD": "Dock Landing Ship", + "SSN": "Attack Submarine", + "SSBN": "Ballistic Missile Submarine", + "SSGN": "Guided Missile Submarine", + "FFG": "Frigate", + "LCS": "Littoral Combat Ship", + "MCM": "Mine Countermeasure", + "ESB": "Expeditionary Sea Base", + "ESD": "Expeditionary Transfer Dock", + "EPF": "Expeditionary Fast Transport", + } + + ships.append({ + "name": f"USS {name}", + "hull_number": hull, + "type": type_labels.get(ship_type, ship_type), + "region": region, + }) + + # Extract USCG cutters + for match in _USCG_PATTERN.finditer(content): + name = match.group(1).strip() + hull = match.group(2).strip() + start = max(0, match.start() - 200) + end = min(len(content), match.end() + 200) + context = content[start:end] + region = _classify_region(context) + ships.append({ + "name": f"USCGC {name}", + "hull_number": hull, + "type": "Coast Guard Cutter", + "region": region, + }) + + # Extract carrier strike groups + for match in _CSG_PATTERN.finditer(content): + strike_groups.append({"name": f"CSG-{match.group(1)}", "type": "Carrier Strike Group"}) + + for match in _ESG_PATTERN.finditer(content): + strike_groups.append({"name": f"ESG-{match.group(1)}", "type": "Expeditionary Strike Group"}) + + # Extract force totals + force_totals = {} + bf = _BATTLE_FORCE_PATTERN.search(content) + if bf: + force_totals["battle_force"] = { + "total": int(bf.group(1)), + "uss": int(bf.group(2)), + "usns": int(bf.group(3)), + } + dep = _DEPLOYED_PATTERN.search(content) + if dep: + force_totals["deployed"] = { + "total": int(dep.group(1)), + "uss": int(dep.group(2)), + "usns": int(dep.group(3)), + } + uw = _UNDERWAY_PATTERN.search(content) + if uw: + force_totals["underway"] = { + "total": int(uw.group(1)), + "deployed": int(uw.group(2)), + "local": int(uw.group(3)), + } + + # Deduplicate ships by hull number + seen_hulls: set[str] = set() + unique_ships = [] + for ship in ships: + if ship["hull_number"] not in seen_hulls: + seen_hulls.add(ship["hull_number"]) + unique_ships.append(ship) + + # Region breakdown + region_counts: dict[str, int] = {} + for ship in unique_ships: + r = ship["region"] + region_counts[r] = region_counts.get(r, 0) + 1 + + return { + "ships": unique_ships, + "ship_count": len(unique_ships), + "strike_groups": strike_groups, + "force_totals": force_totals, + "region_breakdown": region_counts, + } + + +async def fetch_usni_fleet(fetcher: Fetcher) -> dict: + """Fetch latest USNI Fleet Tracker disposition. + + Parses the USNI News Fleet Tracker RSS feed for the most recent + weekly fleet disposition report. Extracts ship names, hull numbers, + strike groups, regions, and force totals. + + Returns: + Dict with ships[], strike_groups[], force_totals, region_breakdown, + report_date, report_url, source, timestamp. + """ + if feedparser is None: + return { + "error": "feedparser not installed", + "ships": [], + "ship_count": 0, + "source": "usni-fleet-tracker", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + xml = await fetcher.get_text( + _FEED_URL, + source="usni-fleet-tracker", + cache_key="usni:fleet_tracker_rss", + cache_ttl=3600, # Weekly updates, cache for 1 hour + ) + + if not xml: + return { + "error": "failed to fetch USNI fleet tracker feed", + "ships": [], + "ship_count": 0, + "source": "usni-fleet-tracker", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + feed = feedparser.parse(xml) + + if not feed.entries: + return { + "error": "no fleet tracker entries found", + "ships": [], + "ship_count": 0, + "source": "usni-fleet-tracker", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + # Get the most recent entry (fleet tracker article) + entry = feed.entries[0] + title = entry.get("title", "") + link = entry.get("link", "") + published = entry.get("published", "") + + # Try content:encoded first (full article), fall back to summary + content = "" + if hasattr(entry, "content") and entry.content: + content = entry.content[0].get("value", "") + if not content: + content = entry.get("summary", entry.get("description", "")) + + # Strip HTML tags for cleaner regex matching + clean_content = re.sub(r"<[^>]+>", " ", content) + clean_content = re.sub(r"\s+", " ", clean_content) + + fleet_data = _extract_fleet_data(clean_content) + + return { + **fleet_data, + "report_title": title, + "report_url": link, + "report_date": published, + "source": "usni-fleet-tracker", + "timestamp": datetime.now(timezone.utc).isoformat(), + }