Improve scan terminal freshness and AMSC overlay consistency

tests/test_cache_warmer_service.py

@@ -1,6 +1,11 @@
 from datetime import datetime, timezone
 
-from web.cache_warmer_service import CacheWarmer, build_default_cache_warmer, build_priority_city_batch
+from web.cache_warmer_service import (
+    CacheWarmer,
+    build_default_cache_warmer,
+    build_priority_city_batch,
+    warmer_tick_sec,
+)
 
 
 def test_priority_city_batch_prefers_local_active_hours_over_night():
@@ -82,6 +87,20 @@ def test_default_cache_warmer_enqueues_city_refresh_without_direct_panel_refresh
     ]
 
 
+def test_default_cache_warmer_uses_faster_city_refresh_defaults(monkeypatch):
+    monkeypatch.delenv("POLYWEATHER_WARMER_CITY_BATCH_SIZE", raising=False)
+    monkeypatch.delenv("POLYWEATHER_WARMER_CITY_INTERVAL_SEC", raising=False)
+    monkeypatch.delenv("POLYWEATHER_WARMER_SCAN_INTERVAL_SEC", raising=False)
+    monkeypatch.delenv("POLYWEATHER_WARMER_TICK_SEC", raising=False)
+
+    warmer = build_default_cache_warmer()
+
+    assert warmer.scan_interval_sec == 120
+    assert warmer.city_batch_size == 16
+    assert warmer.city_interval_sec == 30
+    assert warmer_tick_sec() == 30
+
+
 def test_cache_warmer_skips_work_when_intervals_are_not_due():
     now_ts = datetime(2026, 6, 14, 12, 0, tzinfo=timezone.utc).timestamp()
     scan_calls = []

tests/test_canonical_engine.py

@@ -98,6 +98,12 @@ def test_canonical_engine_builds_realtime_event_from_canonical():
             "freshness_sec": 300,
             "freshness_status": "fresh",
             "confidence": 0.9,
+            "max_so_far": 28.6,
+            "signed_gap": 1.2,
+            "gap_to_target": -1.2,
+            "touch_distance": 0.0,
+            "edge": 0.07,
+            "edge_percent": 7.0,
         }
     )
 
@@ -112,3 +118,9 @@ def test_canonical_engine_builds_realtime_event_from_canonical():
     assert event["payload"]["unit"] == "celsius"
     assert event["payload"]["freshness_status"] == "fresh"
     assert event["payload"]["confidence"] == 0.9
+    assert event["payload"]["max_so_far"] == 28.6
+    assert event["payload"]["signed_gap"] == 1.2
+    assert event["payload"]["gap_to_target"] == -1.2
+    assert event["payload"]["touch_distance"] == 0.0
+    assert event["payload"]["edge"] == 0.07
+    assert event["payload"]["edge_percent"] == 7.0

tests/test_deployment_runtime_config.py

@@ -127,10 +127,10 @@ def test_docker_compose_isolates_collector_from_web_and_bot_services():
     assert "POLYWEATHER_OBSERVATION_COLLECTOR_AMSC_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_AMSC_SEC:-60}" in collector_block
     assert "POLYWEATHER_OBSERVATION_COLLECTOR_CACHE_REFRESH_WORKERS: ${POLYWEATHER_OBSERVATION_COLLECTOR_CACHE_REFRESH_WORKERS:-2}" in collector_block
     assert "POLYWEATHER_WARMER_ENABLED: ${POLYWEATHER_WARMER_ENABLED:-true}" in warmer_block
-    assert "POLYWEATHER_WARMER_TICK_SEC: ${POLYWEATHER_WARMER_TICK_SEC:-60}" in warmer_block
-    assert "POLYWEATHER_WARMER_SCAN_INTERVAL_SEC: ${POLYWEATHER_WARMER_SCAN_INTERVAL_SEC:-300}" in warmer_block
-    assert "POLYWEATHER_WARMER_CITY_INTERVAL_SEC: ${POLYWEATHER_WARMER_CITY_INTERVAL_SEC:-60}" in warmer_block
-    assert "POLYWEATHER_WARMER_CITY_BATCH_SIZE: ${POLYWEATHER_WARMER_CITY_BATCH_SIZE:-8}" in warmer_block
+    assert "POLYWEATHER_WARMER_TICK_SEC: ${POLYWEATHER_WARMER_TICK_SEC:-30}" in warmer_block
+    assert "POLYWEATHER_WARMER_SCAN_INTERVAL_SEC: ${POLYWEATHER_WARMER_SCAN_INTERVAL_SEC:-120}" in warmer_block
+    assert "POLYWEATHER_WARMER_CITY_INTERVAL_SEC: ${POLYWEATHER_WARMER_CITY_INTERVAL_SEC:-30}" in warmer_block
+    assert "POLYWEATHER_WARMER_CITY_BATCH_SIZE: ${POLYWEATHER_WARMER_CITY_BATCH_SIZE:-16}" in warmer_block
     assert "cpus: ${POLYWEATHER_WARMER_CPUS:-0.75}" in warmer_block
     assert "TELEGRAM_AIRPORT_PUSH_INTERVAL_SEC: ${POLYWEATHER_BOT_AIRPORT_PUSH_INTERVAL_SEC:-60}" in bot_block
     assert "POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC:-300}" in collector_block

tests/test_latest_observation_overlay.py

@@ -0,0 +1,48 @@
+from web.services.latest_observation_overlay import overlay_latest_amsc_observation
+
+
+def test_overlay_replaces_amos_when_old_local_time_string_looks_later_than_new_utc():
+    class FakeDB:
+        def get_latest_raw_observation(self, source, city):
+            assert (source, city) == ("amsc_awos", "qingdao")
+            return {
+                "observed_at": "2026-06-14T17:23:00+00:00",
+                "fetched_at": "2026-06-14T17:23:30+00:00",
+                "station_code": "ZSQD",
+                "station_name": "Qingdao Jiaodong",
+                "payload": {
+                    "source": "amsc_awos",
+                    "source_label": "AMSC AWOS Qingdao Jiaodong (ZSQD)",
+                    "icao": "ZSQD",
+                    "temp_c": 18.2,
+                    "observation_time": "2026-06-14T17:23:00+00:00",
+                    "observation_time_local": "2026-06-15 01:23:00",
+                    "runway_obs": {
+                        "point_temperatures": [
+                            {
+                                "runway": "17L/35R",
+                                "tdz_temp": 18.0,
+                                "mid_temp": 18.2,
+                                "end_temp": 18.1,
+                            }
+                        ],
+                    },
+                },
+            }
+
+    stale_payload = {
+        "name": "qingdao",
+        "temp_symbol": "C",
+        "amos": {
+            "source": "amsc_awos",
+            "temp_c": 21.0,
+            "observation_time": "2026-06-14T10:46:00+00:00",
+            "observation_time_local": "2026-06-14 18:46:00",
+        },
+    }
+
+    result = overlay_latest_amsc_observation(FakeDB(), "qingdao", stale_payload)
+
+    assert result["amos"]["observation_time"] == "2026-06-14T17:23:00+00:00"
+    assert result["amos"]["observation_time_local"] == "2026-06-15 01:23:00"
+    assert result["amos"]["runway_obs"]["point_temperatures"][0]["runway"] == "17L/35R"

tests/test_observation_collector.py

@@ -132,6 +132,25 @@ def test_observation_collector_run_due_once_collects_without_panel_cache_refresh
     assert calls == [("qingdao", False), ("qingdao", False)]
 
 
+def test_observation_collector_default_cache_refresh_workers_is_two(monkeypatch):
+    from web.observation_collector_service import ObservationCollector
+
+    monkeypatch.delenv(
+        "POLYWEATHER_OBSERVATION_COLLECTOR_CACHE_REFRESH_WORKERS",
+        raising=False,
+    )
+    collector = ObservationCollector(
+        weather=object(),
+        profiles=[],
+        cache_refresher=lambda _city: None,
+    )
+    try:
+        assert collector._cache_refresh_executor is not None
+        assert collector._cache_refresh_executor._max_workers == 2
+    finally:
+        collector._cache_refresh_executor.shutdown(wait=False)
+
+
 def test_raw_observation_store_records_latest_observation(tmp_path):
     from src.database.db_manager import DBManager
 

tests/test_realtime_patch_schema.py

@@ -66,6 +66,12 @@ def test_v1_patch_payload_is_accepted_and_normalized():
             "obs_time": "2026-05-26T07:01:00Z",
             "payload": {
                 "temp": 29.4,
+                "max_so_far": 30.1,
+                "signed_gap": 0.6,
+                "gap_to_target": -0.6,
+                "touch_distance": 0,
+                "edge": 0.04,
+                "edge_percent": 4.0,
                 "station_code": "46692",
                 "runway_points": [{"runway": "05/23", "temp": 30.2}],
             },
@@ -75,6 +81,12 @@ def test_v1_patch_payload_is_accepted_and_normalized():
     assert event["city"] == "taipei"
     assert event["source"] == "cwa"
     assert event["payload"]["temp"] == 29.4
+    assert event["payload"]["max_so_far"] == 30.1
+    assert event["payload"]["signed_gap"] == 0.6
+    assert event["payload"]["gap_to_target"] == -0.6
+    assert event["payload"]["touch_distance"] == 0
+    assert event["payload"]["edge"] == 0.04
+    assert event["payload"]["edge_percent"] == 4.0
     assert event["payload"]["runway_points"][0]["temp"] == 30.2
 
 

tests/test_refresh_policy.py

@@ -10,7 +10,7 @@ from src.utils.refresh_policy import (
 def test_refresh_policy_cadences_are_layered():
     assert OBSERVATION_REFRESH_SEC == 60
     assert METAR_POLL_TTL_SEC == 300
-    assert SCAN_ROWS_REFRESH_SEC == 300
+    assert SCAN_ROWS_REFRESH_SEC == 120
     assert MARKET_OVERVIEW_TTL_SEC == 600
     assert MODEL_CACHE_TTL_SEC == 1800
 

tests/test_scan_terminal_modules.py

@@ -3,6 +3,7 @@ from web import scan_terminal_cache
 from web import scan_terminal_service
 from web.scan_terminal_metar_gate import _apply_metar_gate_to_row
 from web.scan_terminal_payloads import (
+    build_scan_terminal_incremental_payload,
     build_failed_scan_terminal_payload,
     build_scan_terminal_snapshot_id,
     build_stale_scan_terminal_payload,
@@ -76,6 +77,146 @@ def test_scan_terminal_failure_state_preserves_redis_success_payload(monkeypatch
     assert entry["last_error"] == "timeout"
 
 
+def test_scan_terminal_cache_keeps_previous_success_snapshot_for_diffs(monkeypatch):
+    fake_redis = _FakeRedis()
+    monkeypatch.setenv("POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_ENABLED", "true")
+    monkeypatch.setattr(scan_terminal_cache, "_get_redis_client", lambda: fake_redis)
+    scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
+
+    filters = {"scan_mode": "tradable", "limit": 9}
+    scan_terminal_cache.set_cached_scan_terminal_payload(
+        filters,
+        {
+            "snapshot_id": "scan-old",
+            "generated_at": "2026-06-01T00:00:00Z",
+            "rows": [{"id": "row-1", "edge_percent": 3}],
+        },
+    )
+    scan_terminal_cache.set_cached_scan_terminal_payload(
+        filters,
+        {
+            "snapshot_id": "scan-new",
+            "generated_at": "2026-06-01T00:01:00Z",
+            "rows": [{"id": "row-1", "edge_percent": 4}],
+        },
+    )
+
+    scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
+    entry = scan_terminal_cache.get_scan_terminal_cache_entry(filters)
+
+    assert entry["success_payload"]["snapshot_id"] == "scan-new"
+    assert entry["previous_success_payload"]["snapshot_id"] == "scan-old"
+
+
+def test_build_scan_terminal_incremental_payload_returns_not_modified():
+    filters = {"scan_mode": "tradable", "limit": 2}
+    current = {
+        "generated_at": "2026-06-01T00:01:00Z",
+        "snapshot_id": "scan-current",
+        "status": "ready",
+        "stale": False,
+        "filters": filters,
+        "summary": {"candidate_total": 2},
+        "top_signal": {"id": "row-1"},
+        "rows": [{"id": "row-1"}, {"id": "row-2"}],
+    }
+
+    payload = build_scan_terminal_incremental_payload(
+        filters=filters,
+        current_payload=current,
+        since_snapshot_id="scan-current",
+        base_payload=current,
+    )
+
+    assert payload["status"] == "not_modified"
+    assert payload["rows"] == []
+    assert payload["summary"] == current["summary"]
+    assert payload["top_signal"] == current["top_signal"]
+    assert payload["diff"] == {
+        "mode": "not_modified",
+        "base_snapshot_id": "scan-current",
+        "snapshot_id": "scan-current",
+        "rows_changed": [],
+        "removed_row_ids": [],
+    }
+
+
+def test_build_scan_terminal_incremental_payload_returns_changed_row_delta():
+    filters = {"scan_mode": "tradable", "limit": 3}
+    base = {
+        "generated_at": "2026-06-01T00:00:00Z",
+        "snapshot_id": "scan-old",
+        "status": "ready",
+        "stale": False,
+        "filters": filters,
+        "summary": {"candidate_total": 2},
+        "top_signal": {"id": "row-1"},
+        "rows": [
+            {"id": "row-1", "rank": 1, "edge_percent": 3},
+            {"id": "row-removed", "rank": 2, "edge_percent": 2},
+        ],
+    }
+    current = {
+        "generated_at": "2026-06-01T00:01:00Z",
+        "snapshot_id": "scan-new",
+        "status": "ready",
+        "stale": False,
+        "filters": filters,
+        "summary": {"candidate_total": 2},
+        "top_signal": {"id": "row-added"},
+        "rows": [
+            {"id": "row-1", "rank": 1, "edge_percent": 4},
+            {"id": "row-added", "rank": 2, "edge_percent": 5},
+        ],
+    }
+
+    payload = build_scan_terminal_incremental_payload(
+        filters=filters,
+        current_payload=current,
+        since_snapshot_id="scan-old",
+        base_payload=base,
+    )
+
+    assert payload["status"] == "ready"
+    assert payload["rows"] == []
+    assert payload["snapshot_id"] == "scan-new"
+    assert payload["diff"]["mode"] == "row_delta"
+    assert payload["diff"]["base_snapshot_id"] == "scan-old"
+    assert payload["diff"]["snapshot_id"] == "scan-new"
+    assert {row["id"] for row in payload["diff"]["rows_changed"]} == {
+        "row-1",
+        "row-added",
+    }
+    assert payload["diff"]["removed_row_ids"] == ["row-removed"]
+
+
+def test_build_scan_terminal_incremental_payload_falls_back_to_full_without_base():
+    filters = {"scan_mode": "tradable", "limit": 2}
+    current = {
+        "generated_at": "2026-06-01T00:01:00Z",
+        "snapshot_id": "scan-new",
+        "status": "ready",
+        "stale": False,
+        "filters": filters,
+        "summary": {"candidate_total": 1},
+        "top_signal": None,
+        "rows": [{"id": "row-1", "edge_percent": 4}],
+    }
+
+    payload = build_scan_terminal_incremental_payload(
+        filters=filters,
+        current_payload=current,
+        since_snapshot_id="scan-old",
+        base_payload=None,
+    )
+
+    assert payload["status"] == "ready"
+    assert payload["rows"] == current["rows"]
+    assert payload["diff"]["mode"] == "full"
+    assert payload["diff"]["base_snapshot_id"] == "scan-old"
+    assert payload["diff"]["snapshot_id"] == "scan-new"
+
+
 def test_scan_terminal_prewarm_covers_default_api_limit():
     limits = {filters["limit"] for filters in _scan_terminal_prewarm_filters()}
 

tests/test_web_observability.py

@@ -3851,9 +3851,17 @@ def test_scan_terminal_endpoint_forwards_filters(monkeypatch):
 
     captured = {}
 
-    def _fake_build_scan_terminal_payload(filters, *, force_refresh=False):
+    def _fake_build_scan_terminal_payload(
+        filters,
+        *,
+        force_refresh=False,
+        diff=False,
+        since_snapshot_id=None,
+    ):
         captured["filters"] = dict(filters)
         captured["force_refresh"] = force_refresh
+        captured["diff"] = diff
+        captured["since_snapshot_id"] = since_snapshot_id
         return {
             "generated_at": "2026-04-23T00:00:00Z",
             "filters": filters,
@@ -3876,12 +3884,15 @@ def test_scan_terminal_endpoint_forwards_filters(monkeypatch):
     response = client.get(
         "/api/scan/terminal?scan_mode=trend&min_price=0.1&max_price=0.8&min_edge_pct=3"
         "&min_liquidity=700&high_liquidity_only=true&market_type=all&time_range=week&limit=12&force_refresh=true"
+        "&diff=true&since_snapshot_id=scan-old"
     )
 
     assert response.status_code == 200
     payload = response.json()
     assert payload["summary"]["recommended_count"] == 1
     assert captured["force_refresh"] is True
+    assert captured["diff"] is True
+    assert captured["since_snapshot_id"] == "scan-old"
     assert captured["filters"]["scan_mode"] == "trend"
     assert captured["filters"]["market_type"] == "all"
     assert captured["filters"]["time_range"] == "week"