From 4528a6ed69f97ab93f91111ae814871bc1bc6a1a Mon Sep 17 00:00:00 2001 From: Marc Shade Date: Thu, 4 Jun 2026 11:18:09 -0400 Subject: [PATCH] Fix dashboard startup and space weather parsing --- CLAUDE.md | 2 +- README.md | 11 ++-- ROADMAP.md | 28 ++++++--- src/world_intel_mcp/cache.py | 31 ++++++++- src/world_intel_mcp/dashboard/app.py | 38 ++++++++++- src/world_intel_mcp/server.py | 2 + src/world_intel_mcp/sources/space_weather.py | 59 +++++++++++------ src/world_intel_mcp/tests/test_cache.py | 33 ++++++++++ src/world_intel_mcp/tests/test_dashboard.py | 29 +++++++++ src/world_intel_mcp/tests/test_sources.py | 66 ++++++++++++++++++++ 10 files changed, 260 insertions(+), 39 deletions(-) create mode 100644 src/world_intel_mcp/tests/test_dashboard.py diff --git a/CLAUDE.md b/CLAUDE.md index 089508a..a88007a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What This Is -World Intelligence MCP Server — 110 tools across 30+ domains providing real-time global intelligence from free public APIs. Serves four interfaces: MCP stdio (for Claude Code/Cursor), a live Starlette dashboard with SSE, a Click CLI with Rich output, and a collector daemon for 24/7 vector store population. Python 3.11+, built with hatchling. +World Intelligence MCP Server — 113 tools across 30+ domains providing real-time global intelligence from free public APIs. Serves four interfaces: MCP stdio (for Claude Code/Cursor), a live Starlette dashboard with SSE, a Click CLI with Rich output, and a collector daemon for 24/7 vector store population. Python 3.11+, built with hatchling. ## Commands diff --git a/README.md b/README.md index f5e6d3f..3ace263 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Python 3.11+](https://img.shields.io/badge/Python-3.11%2B-green)](https://python.org) [![License](https://img.shields.io/badge/License-MIT-yellow)](LICENSE) -Real-time global intelligence across **30+ domains** with **109 MCP tools**, a live ops-center dashboard, a CLI, and a **Qdrant vector store** for enterprise-grade semantic search across accumulated intelligence. All data comes from free, public APIs — no paid subscriptions required. +Real-time global intelligence across **30+ domains** with **113 MCP tools**, a live ops-center dashboard, a CLI, and a **Qdrant vector store** for enterprise-grade semantic search across accumulated intelligence. All data comes from free, public APIs — no paid subscriptions required. Built for AI agents that need world awareness: market conditions, geopolitical risk, military posture, supply chain disruptions, cyber threats, and more — all queryable via the Model Context Protocol. The vector store enables natural language queries like *"military activity near Taiwan"* or *"cyber threats targeting healthcare"* across all historical data. @@ -23,7 +23,7 @@ Built for AI agents that need world awareness: market conditions, geopolitical r | **SEC Filings** | 3 | SEC EDGAR (full-text search, company filings, 8-K material events) | | **Company Enrichment** | 1 | Yahoo Finance + GDELT + SEC + GitHub (composite profile) | | **Macro Composite** | 1 | Weighted 6-signal market verdict (Fear&Greed, VIX, sectors, DXY, BTC, yields) | -| **Economic Indicators** | 3 | EIA energy, FRED macro, World Bank | +| **Economic Indicators** | 6 | AAA fuel prices, EIA energy, FRED macro, World Bank | | **Central Banks** | 1 | 8 central bank policy rates | | **BTC Technicals** | 1 | SMA 50/200, golden/death cross, Mayer Multiple | | **Natural Disasters** | 2 | USGS earthquakes, NASA FIRMS wildfires | @@ -57,7 +57,7 @@ Built for AI agents that need world awareness: market conditions, geopolitical r | **Cross-Domain Analytics** | 3 | Correlation, domain summary, trend detection | | **Reports** | 1 | PDF/HTML multi-domain intelligence reports | -**Total: 110 tools** across 30+ intelligence domains. +**Total: 113 tools** across 30+ intelligence domains. --- @@ -194,9 +194,12 @@ collector.py (daemon) ─┘ |------|-------------| | `intel_macro_composite` | Weighted market score (0-100) with verdict: RISK_ON to STRONG_CAUTION | -### Economic (3) +### Economic (6) | Tool | Description | |------|-------------| +| `intel_gas_prices` | Daily US retail gasoline, diesel, and E85 prices from AAA | +| `intel_residential_natgas` | US residential natural gas prices from EIA | +| `intel_electricity_rates` | US electricity retail rates by sector/state from EIA | | `intel_energy_prices` | Brent/WTI crude oil and natural gas from EIA | | `intel_fred_series` | FRED economic data (GDP, CPI, unemployment, rates) | | `intel_world_bank_indicators` | World Bank development indicators by country | diff --git a/ROADMAP.md b/ROADMAP.md index 013bf66..d69aea1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,8 +1,8 @@ # World Intel MCP — Feature Parity Roadmap **Benchmark**: [koala73/worldmonitor](https://github.com/koala73/worldmonitor) -**Updated**: 2026-03-08 -**Current tools**: 110 (109 intel + 1 status) +**Updated**: 2026-06-04 +**Current tools**: 113 (112 intel + 1 status) --- @@ -20,11 +20,11 @@ | Area | Finding | Status | Action | |------|---------|--------|--------| -| MCP tool parity | 110 tools declared in `TOOLS`; 110 routed in `_dispatch()` | :white_check_mark: | Keep as an invariant | +| MCP tool parity | 113 tools declared in `TOOLS`; 113 routed in `_dispatch()` | :white_check_mark: | Keep as an invariant | | Optional vector runtime | Missing `qdrant-client` / `fastembed` previously surfaced as runtime failures | :white_check_mark: Fixed | Vector features now degrade cleanly and report availability | | Base-environment test run | `pytest -q` fails collection without dev extras because `respx` is not installed | :yellow_circle: | Run `pip install -e ".[dev]"` before full-suite validation | -| Core verification | 127 infrastructure/report/vector tests pass in the base environment | :white_check_mark: | `test_reports.py`, `test_cache.py`, `test_analysis.py`, `test_vector_store.py` | -| Documentation drift | Prior roadmap documented 89 tools while the codebase exposed 110 | :white_check_mark: Updated below | Keep roadmap synced with phase increments | +| Core verification | 226 non-smoke tests pass with dev extras installed | :white_check_mark: | Full default `pytest` run | +| Documentation drift | Prior roadmap documented 89/110 tools while the codebase now exposes 113 | :white_check_mark: Updated below | Keep roadmap synced with phase increments | | Maintainability | `src/world_intel_mcp/server.py` is ~2.5k lines and remains the main refactor target | :yellow_circle: | Split tool registry and dispatch by domain | ### Implemented Addendum Missing From Prior Roadmap @@ -69,7 +69,7 @@ ## 1. Data Sources — Complete Inventory -### Markets & Economics (13 tools) +### Markets & Economics (16 tools) | Tool | WM Equivalent | Status | |------|---------------|--------| @@ -80,6 +80,9 @@ | `intel_sector_heatmap` | `get-sector-summary` | :white_check_mark: | | `intel_macro_signals` | `get-macro-signals` | :white_check_mark: | | `intel_commodity_quotes` | `list-commodity-quotes` | :white_check_mark: | +| `intel_gas_prices` | AAA retail fuel prices | :white_check_mark: | +| `intel_residential_natgas` | EIA residential natural gas prices | :white_check_mark: | +| `intel_electricity_rates` | EIA electricity rates by sector/state | :white_check_mark: | | `intel_energy_prices` | `get-energy-prices` | :white_check_mark: | | `intel_fred_series` | `get-fred-series` | :white_check_mark: | | `intel_world_bank_indicators` | `list-world-bank-indicators` | :white_check_mark: | @@ -422,17 +425,22 @@ Added historical cross-category correlation, stored-data summarization, and rece Added PDF/HTML intelligence report generation over the existing multi-domain data collection stack. PDF output remains optional behind `.[pdf]`, with HTML fallback available when WeasyPrint is not installed. +### Phase 18: Consumer Energy Signals (+3 = 113 tools) +`intel_gas_prices`, `intel_residential_natgas`, `intel_electricity_rates` + +Added retail fuel, residential natural gas, and electricity-rate tools to round out consumer energy monitoring alongside existing EIA crude, gas, FRED, and World Bank economic signals. + --- ## Summary | Category | Current | Notes | |----------|---------|-------| -| Total MCP tools | 110 | 109 intelligence tools + `intel_status` | -| Tool parity | 110 / 110 | `TOOLS` and `_dispatch()` are aligned | +| Total MCP tools | 113 | 112 intelligence tools + `intel_status` | +| Tool parity | 113 / 113 | `TOOLS` and `_dispatch()` are aligned | | Static datasets | 18 | Bases, ports, pipelines, nuclear, cables, datacenters, spaceports, minerals, exchanges, trade routes, cloud regions, financial centers | | RSS feeds | 119 | 24 categories | -| Tests in repo | 344 | 11 test files; full suite requires `.[dev]` | +| Tests in repo | 244 | 226 non-smoke tests + 18 live smoke tests; full suite requires `.[dev]` | | Primary remaining gap | Architecture | `server.py` monolith remains the main refactor target | -**Bottom line**: 110 tools across 30+ domains, with the roadmap now aligned to the live MCP registry. The main remaining gaps are full-environment test bootstrapping (`.[dev]`) and continued modularization of the monolithic `server.py` tool registry/dispatcher. +**Bottom line**: 113 tools across 30+ domains, with the roadmap now aligned to the live MCP registry. The main remaining gaps are full-environment test bootstrapping (`.[dev]`) and continued modularization of the monolithic `server.py` tool registry/dispatcher. diff --git a/src/world_intel_mcp/cache.py b/src/world_intel_mcp/cache.py index cd0f4d3..d465280 100644 --- a/src/world_intel_mcp/cache.py +++ b/src/world_intel_mcp/cache.py @@ -6,7 +6,9 @@ No external deps — just stdlib sqlite3. import json import logging +import os import sqlite3 +import tempfile import time from pathlib import Path from typing import Any @@ -16,16 +18,39 @@ logger = logging.getLogger("world-intel-mcp.cache") _DEFAULT_DB = Path.home() / ".cache" / "world-intel-mcp" / "cache.db" +def _default_db_path() -> Path: + if env_path := os.environ.get("WORLD_INTEL_CACHE_DB"): + return Path(env_path).expanduser() + if xdg_cache := os.environ.get("XDG_CACHE_HOME"): + return Path(xdg_cache).expanduser() / "world-intel-mcp" / "cache.db" + return _DEFAULT_DB + + class Cache: """SQLite-backed TTL cache.""" def __init__(self, db_path: Path | None = None): - self.db_path = db_path or _DEFAULT_DB - self.db_path.parent.mkdir(parents=True, exist_ok=True) + explicit_path = db_path is not None + self.db_path = Path(db_path) if db_path is not None else _default_db_path() self._conn: sqlite3.Connection | None = None - self._init_db() + try: + self._init_db() + except (OSError, sqlite3.OperationalError) as exc: + if explicit_path: + raise + fallback = Path(tempfile.gettempdir()) / "world-intel-mcp" / "cache.db" + logger.warning( + "Cache unavailable at %s: %s; falling back to %s", + self.db_path, + exc, + fallback, + ) + self.close() + self.db_path = fallback + self._init_db() def _init_db(self) -> None: + self.db_path.parent.mkdir(parents=True, exist_ok=True) conn = self._get_conn() conn.execute(""" CREATE TABLE IF NOT EXISTS cache ( diff --git a/src/world_intel_mcp/dashboard/app.py b/src/world_intel_mcp/dashboard/app.py index fc68f66..f89f16f 100644 --- a/src/world_intel_mcp/dashboard/app.py +++ b/src/world_intel_mcp/dashboard/app.py @@ -648,10 +648,42 @@ app = Starlette( ) -def run(host: str = "127.0.0.1", port: int = 8501) -> None: +def _parse_run_args(argv: list[str] | None = None) -> tuple[str, int]: + """Parse dashboard CLI args for the console script and module runner.""" + import argparse + import os + + default_port = int( + os.environ.get( + "WORLD_INTEL_DASHBOARD_PORT", + os.environ.get("PORT", "8501"), + ) + ) + parser = argparse.ArgumentParser(description="Run the World Intelligence dashboard") + parser.add_argument( + "--host", + default=os.environ.get("WORLD_INTEL_DASHBOARD_HOST", "127.0.0.1"), + help="Interface to bind (default: 127.0.0.1)", + ) + parser.add_argument( + "--port", + type=int, + default=default_port, + help="Port to bind (default: 8501, or WORLD_INTEL_DASHBOARD_PORT/PORT)", + ) + args = parser.parse_args(argv) + return args.host, args.port + + +def run(host: str | None = None, port: int | None = None) -> None: """Launch the dashboard server.""" import uvicorn + if host is None or port is None: + parsed_host, parsed_port = _parse_run_args() + host = host or parsed_host + port = parsed_port if port is None else port + logger.info("Starting Intelligence Dashboard on http://%s:%d", host, port) uvicorn.run( app, @@ -660,3 +692,7 @@ def run(host: str = "127.0.0.1", port: int = 8501) -> None: log_level="info", access_log=False, ) + + +if __name__ == "__main__": + run() diff --git a/src/world_intel_mcp/server.py b/src/world_intel_mcp/server.py index 3929100..3998f1c 100644 --- a/src/world_intel_mcp/server.py +++ b/src/world_intel_mcp/server.py @@ -32,6 +32,8 @@ Phase 17: Cross-domain analytics — cross-domain correlation, domain summary, t (+3 = 109 tools). Historical analysis and early warning from accumulated vector data. Phase 18: PDF/HTML intelligence reports (+1 = 110 tools). WeasyPrint-based multi-section report generation covering 18 intelligence domains in parallel. +Phase 19: Consumer energy signals (+3 = 113 tools). Retail fuel, residential natural gas, + and electricity rates round out consumer energy monitoring. """ import asyncio diff --git a/src/world_intel_mcp/sources/space_weather.py b/src/world_intel_mcp/sources/space_weather.py index 0f851f0..8b23f4f 100644 --- a/src/world_intel_mcp/sources/space_weather.py +++ b/src/world_intel_mcp/sources/space_weather.py @@ -81,6 +81,21 @@ def _classify_xray(flux: float) -> str: return "A" +def _parse_kp_row(row) -> tuple[str | None, float] | None: + """Parse NOAA Kp rows from either current dict or legacy list payloads.""" + try: + if isinstance(row, dict): + raw_kp = row.get("Kp", row.get("kp")) + if raw_kp is None: + return None + return row.get("time_tag") or row.get("time"), float(raw_kp) + if isinstance(row, list) and len(row) >= 2: + return row[0], float(row[1]) + except (ValueError, TypeError): + return None + return None + + # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- @@ -135,31 +150,35 @@ async def fetch_space_weather(fetcher: Fetcher) -> dict: # --- Kp index --- if kp_data and isinstance(kp_data, list) and len(kp_data) > 1: - # First row is header, rest are data [time_tag, Kp, ...] - try: - # Get most recent Kp reading - latest = kp_data[-1] - kp_val = float(latest[1]) + # NOAA has served both legacy list rows [time_tag, Kp, ...] and + # current dict rows {"time_tag": "...", "Kp": ...}; tolerate both. + latest = next( + ( + parsed + for parsed in (_parse_kp_row(row) for row in reversed(kp_data)) + if parsed is not None + ), + None, + ) + if latest is not None: + _, kp_val = latest result["current_kp"] = kp_val result["kp_level"] = _classify_kp(kp_val) - # Last 8 readings (24 hours of 3-hourly data) - recent = [] - for row in kp_data[-9:-1]: # skip header - if isinstance(row, list) and len(row) >= 2: - try: - recent.append({ - "time": row[0], - "kp": float(row[1]), - }) - except (ValueError, TypeError, IndexError): - pass - result["kp_recent"] = recent - except (ValueError, TypeError, IndexError) as exc: - logger.warning("Failed to parse Kp data: %s", exc) + recent = [] + for row in kp_data[-8:]: + parsed = _parse_kp_row(row) + if parsed is None: + continue + row_time, kp_val = parsed + recent.append({ + "time": row_time, + "kp": kp_val, + }) + result["kp_recent"] = recent # --- X-ray flux (flare activity) --- - if flare_data and isinstance(flare_data, list) and len(flare_data) > 1: + if flare_data and isinstance(flare_data, list): try: # Last entry has the most recent flux reading latest_flare = flare_data[-1] diff --git a/src/world_intel_mcp/tests/test_cache.py b/src/world_intel_mcp/tests/test_cache.py index 5c97652..c155e8c 100644 --- a/src/world_intel_mcp/tests/test_cache.py +++ b/src/world_intel_mcp/tests/test_cache.py @@ -1,5 +1,6 @@ """Tests for SQLite TTL cache.""" +import tempfile import time from pathlib import Path @@ -64,3 +65,35 @@ def test_complex_values(cache: Cache) -> None: data = {"nested": {"list": [1, 2, 3], "bool": True, "null": None}} cache.set("complex", data, ttl_seconds=60) assert cache.get("complex") == data + + +def test_default_path_honors_env_override( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + db_path = tmp_path / "custom-cache.db" + monkeypatch.setenv("WORLD_INTEL_CACHE_DB", str(db_path)) + + cache = Cache() + try: + assert cache.db_path == db_path + cache.set("env", "ok", ttl_seconds=60) + assert cache.get("env") == "ok" + finally: + cache.close() + + +def test_default_path_falls_back_when_unavailable( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + bad_path = tmp_path / "cache-dir" + bad_path.mkdir() + monkeypatch.setenv("WORLD_INTEL_CACHE_DB", str(bad_path)) + + cache = Cache() + try: + expected = Path(tempfile.gettempdir()) / "world-intel-mcp" / "cache.db" + assert cache.db_path == expected + cache.set("fallback", "ok", ttl_seconds=60) + assert cache.get("fallback") == "ok" + finally: + cache.close() diff --git a/src/world_intel_mcp/tests/test_dashboard.py b/src/world_intel_mcp/tests/test_dashboard.py new file mode 100644 index 0000000..c347239 --- /dev/null +++ b/src/world_intel_mcp/tests/test_dashboard.py @@ -0,0 +1,29 @@ +"""Tests for dashboard runner configuration.""" + +import pytest + +from world_intel_mcp.dashboard.app import _parse_run_args + + +def test_parse_run_args_port() -> None: + host, port = _parse_run_args(["--port", "8765"]) + + assert host == "127.0.0.1" + assert port == 8765 + + +def test_parse_run_args_host_and_port() -> None: + host, port = _parse_run_args(["--host", "0.0.0.0", "--port", "9000"]) + + assert host == "0.0.0.0" + assert port == 9000 + + +def test_parse_run_args_env_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("WORLD_INTEL_DASHBOARD_HOST", "0.0.0.0") + monkeypatch.setenv("WORLD_INTEL_DASHBOARD_PORT", "7777") + + host, port = _parse_run_args([]) + + assert host == "0.0.0.0" + assert port == 7777 diff --git a/src/world_intel_mcp/tests/test_sources.py b/src/world_intel_mcp/tests/test_sources.py index d082cbd..8d68324 100644 --- a/src/world_intel_mcp/tests/test_sources.py +++ b/src/world_intel_mcp/tests/test_sources.py @@ -1176,6 +1176,72 @@ async def test_fetch_central_bank_rates_with_fred(fetcher: Fetcher) -> None: os.environ.pop("FRED_API_KEY", None) +# --------------------------------------------------------------------------- +# Space Weather +# --------------------------------------------------------------------------- + + +@respx.mock +@pytest.mark.asyncio +async def test_fetch_space_weather_current_kp_dict_payload(fetcher: Fetcher) -> None: + from world_intel_mcp.sources.space_weather import ( + _ALERTS_URL, + _FLARE_URL, + _KP_URL, + fetch_space_weather, + ) + + respx.get(_KP_URL).mock( + return_value=httpx.Response( + 200, + json=[ + { + "time_tag": "2026-06-04T03:00:00", + "Kp": 4.67, + "a_running": 17, + "station_count": 8, + }, + { + "time_tag": "2026-06-04T06:00:00", + "Kp": 5.0, + "a_running": 20, + "station_count": 8, + }, + ], + ) + ) + respx.get(_FLARE_URL).mock( + return_value=httpx.Response( + 200, + json=[{"time_tag": "2026-06-04T06:00:00Z", "flux": 1.2e-5}], + ) + ) + respx.get(_ALERTS_URL).mock( + return_value=httpx.Response( + 200, + json=[ + { + "issue_datetime": "2026-06-04T06:10:00Z", + "message": "Geomagnetic storm conditions observed", + "product_id": "WATA50", + } + ], + ) + ) + + result = await fetch_space_weather(fetcher) + + assert result["source"] == "noaa-swpc" + assert result["current_kp"] == 5.0 + assert result["kp_level"] == "G1 Minor" + assert result["kp_recent"][-1] == { + "time": "2026-06-04T06:00:00", + "kp": 5.0, + } + assert result["latest_flare_class"] == "M1.2" + assert result["alerts"][0]["product_id"] == "WATA50" + + # --------------------------------------------------------------------------- # USNI Fleet Tracker # ---------------------------------------------------------------------------