feat(ingestor): enrich market metadata with gamma-api volume/liquidity (#107)

The CLOB API does not expose 24h volume or order-book liquidity, so the
size_anomaly detector currently has no real ratio to compare a trade
against and falls back to the niche-base 0.2 confidence floor. That makes
the volume_impact / book_impact thresholds essentially dead code.

This change adds a small client for the public gamma-api markets endpoint
and merges its volume24hr / liquidityNum snapshot into MarketMetadata
during the existing periodic sync. The detector can now compute real
volume and book impact ratios.

Notes on the gamma client:
- gamma-api enforces a server-side max of 100 markets per page and caps
  `offset` around 10000. The client paginates with bounded concurrency,
  sorted by `volume24hr desc`, so the most-traded markets (which is where
  size anomalies actually matter) are always covered. Markets beyond that
  window have negligible recent volume and the niche path handles them
  fine without a ratio.
- Failures are swallowed: a degraded gamma endpoint must not stop CLOB
  metadata from being cached, since size_anomaly + the niche path remain
  functional with `daily_volume=None`.

MarketMetadata gains three optional Decimal fields (daily_volume,
weekly_volume, liquidity); to_dict / from_dict round-trip is preserved
and older cache entries without these keys deserialize cleanly.

Tests: 12 new tests for GammaClient (parsing, single-page, short-page
stop, offset-cap clean stop, retry, malformed responses); existing
metadata_sync tests updated to inject a mocked GammaClient so they don't
hit the real network.

Co-authored-by: schrodinger01 <schrodinger01@users.noreply.github.com>
This commit is contained in:
Patrick Selamy
2026-06-14 15:17:46 -04:00
committed by GitHub
co-authored by schrodinger01
parent 5afbb35ee9
commit e003e4ba47
5 changed files with 580 additions and 31 deletions
+249
View File
@@ -0,0 +1,249 @@
"""Tests for the gamma-api client."""
from __future__ import annotations
from decimal import Decimal
import httpx
import pytest
from polymarket_insider_tracker.ingestor import gamma_client as gamma_module
from polymarket_insider_tracker.ingestor.gamma_client import (
GammaClient,
GammaClientError,
GammaMarketStats,
_parse_market,
)
class TestParseMarket:
def test_parses_full_payload(self) -> None:
raw = {
"conditionId": "0xabc",
"volume24hr": "12345.67",
"volume1wk": "100000",
"volume1mo": "500000",
"volumeNum": "999999.5",
"liquidityNum": "42000",
}
stats = _parse_market(raw)
assert stats is not None
assert stats.condition_id == "0xabc"
assert stats.daily_volume == Decimal("12345.67")
assert stats.weekly_volume == Decimal("100000")
assert stats.monthly_volume == Decimal("500000")
assert stats.total_volume == Decimal("999999.5")
assert stats.liquidity == Decimal("42000")
def test_falls_back_to_alternative_keys(self) -> None:
raw = {
"conditionId": "0x1",
"volume24hr": "1",
"volume": "777",
"liquidity": "55",
}
stats = _parse_market(raw)
assert stats is not None
assert stats.total_volume == Decimal("777")
assert stats.liquidity == Decimal("55")
def test_handles_missing_numeric_fields(self) -> None:
stats = _parse_market({"conditionId": "0x2"})
assert stats is not None
assert stats.daily_volume is None
assert stats.liquidity is None
def test_drops_garbage_decimals(self) -> None:
stats = _parse_market(
{"conditionId": "0x3", "volume24hr": "not-a-number", "liquidityNum": ""}
)
assert stats is not None
assert stats.daily_volume is None
assert stats.liquidity is None
def test_rejects_missing_condition_id(self) -> None:
assert _parse_market({"volume24hr": "1"}) is None
assert _parse_market({"conditionId": ""}) is None
assert _parse_market({"conditionId": 123}) is None # type: ignore[arg-type]
def _make_client(
_handler: httpx.MockTransport,
*,
page_limit: int = 100,
max_pages: int = 5,
page_concurrency: int = 5,
max_retries: int = 1,
) -> GammaClient:
"""Build a GammaClient that constructs httpx.AsyncClient with the given transport.
GammaClient creates its own AsyncClient inside `get_active_market_stats`,
so we monkeypatch the AsyncClient factory in the module to inject the mock
transport.
"""
return GammaClient(
page_limit=page_limit,
max_pages=max_pages,
page_concurrency=page_concurrency,
max_retries=max_retries,
retry_base_delay_seconds=0.0,
)
@pytest.fixture
def patch_async_client(monkeypatch: pytest.MonkeyPatch):
"""Replace the AsyncClient used by gamma_client with one bound to a MockTransport."""
def _apply(handler: httpx.MockTransport) -> None:
original = gamma_module.httpx.AsyncClient
def factory(*args: object, **kwargs: object) -> httpx.AsyncClient:
kwargs["transport"] = handler # type: ignore[index]
return original(*args, **kwargs) # type: ignore[arg-type]
monkeypatch.setattr(gamma_module.httpx, "AsyncClient", factory)
return _apply
@pytest.mark.asyncio
async def test_get_active_market_stats_single_page(patch_async_client) -> None:
page_one = [
{"conditionId": "0xa", "volume24hr": "100", "liquidityNum": "10"},
{"conditionId": "0xb", "volume24hr": "200", "liquidityNum": "20"},
]
calls: list[dict[str, str]] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(dict(request.url.params))
offset = int(request.url.params.get("offset", "0"))
if offset == 0:
return httpx.Response(200, json=page_one)
return httpx.Response(200, json=[])
transport = httpx.MockTransport(handler)
patch_async_client(transport)
client = _make_client(transport, page_limit=2, max_pages=3)
result = await client.get_active_market_stats()
assert set(result.keys()) == {"0xa", "0xb"}
assert isinstance(result["0xa"], GammaMarketStats)
assert result["0xa"].daily_volume == Decimal("100")
assert calls[0]["limit"] == "2"
assert calls[0]["order"] == "volume24hr"
assert calls[0]["ascending"] == "false"
@pytest.mark.asyncio
async def test_get_active_market_stats_short_page_stops(patch_async_client) -> None:
"""A page shorter than page_limit signals end-of-data after a small empty streak."""
page_zero = [{"conditionId": f"0x{i}", "volume24hr": str(i)} for i in range(5)]
page_one_short = [{"conditionId": "0xshort", "volume24hr": "1"}]
def handler(request: httpx.Request) -> httpx.Response:
offset = int(request.url.params.get("offset", "0"))
if offset == 0:
return httpx.Response(200, json=page_zero)
if offset == 5:
return httpx.Response(200, json=page_one_short)
return httpx.Response(200, json=[])
transport = httpx.MockTransport(handler)
patch_async_client(transport)
client = _make_client(transport, page_limit=5, max_pages=10)
result = await client.get_active_market_stats()
assert "0xshort" in result
assert len(result) == 6
@pytest.mark.asyncio
async def test_get_active_market_stats_offset_cap_clean_stop(
patch_async_client,
) -> None:
"""Gamma rejects offsets past its hard cap; that error is swallowed cleanly."""
def handler(request: httpx.Request) -> httpx.Response:
offset = int(request.url.params.get("offset", "0"))
if offset == 0:
return httpx.Response(200, json=[{"conditionId": "0xa", "volume24hr": "1"}])
return httpx.Response(400, json={"error": "offset too large"})
transport = httpx.MockTransport(handler)
patch_async_client(transport)
client = _make_client(transport, page_limit=1, max_pages=4, max_retries=1)
result = await client.get_active_market_stats()
assert "0xa" in result
@pytest.mark.asyncio
async def test_get_with_retry_recovers_after_transient_error(
patch_async_client,
) -> None:
"""Transient HTTP errors retry up to max_retries before giving up."""
state = {"attempts": 0}
def handler(request: httpx.Request) -> httpx.Response:
offset = int(request.url.params.get("offset", "0"))
if offset == 0:
state["attempts"] += 1
if state["attempts"] < 2:
return httpx.Response(503, json={"error": "transient"})
return httpx.Response(200, json=[{"conditionId": "0xrecover", "volume24hr": "1"}])
return httpx.Response(200, json=[])
transport = httpx.MockTransport(handler)
patch_async_client(transport)
client = _make_client(transport, page_limit=1, max_pages=2, max_retries=3)
result = await client.get_active_market_stats()
assert "0xrecover" in result
assert state["attempts"] == 2
@pytest.mark.asyncio
async def test_unexpected_response_shape_is_handled(patch_async_client) -> None:
"""A non-list payload becomes a clean stop, not a crash."""
def handler(_request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"unexpected": "shape"})
transport = httpx.MockTransport(handler)
patch_async_client(transport)
client = _make_client(transport, page_limit=1, max_pages=2, max_retries=1)
result = await client.get_active_market_stats()
assert result == {}
@pytest.mark.asyncio
async def test_skips_non_dict_entries(patch_async_client) -> None:
"""Defensive: server returning mixed-type list items shouldn't crash."""
def handler(request: httpx.Request) -> httpx.Response:
offset = int(request.url.params.get("offset", "0"))
if offset == 0:
return httpx.Response(
200,
json=[
{"conditionId": "0xa", "volume24hr": "1"},
"garbage",
None,
42,
],
)
return httpx.Response(200, json=[])
transport = httpx.MockTransport(handler)
patch_async_client(transport)
client = _make_client(transport, page_limit=4, max_pages=2)
result = await client.get_active_market_stats()
assert list(result.keys()) == ["0xa"]
def test_gamma_client_error_inherits_exception() -> None:
assert issubclass(GammaClientError, Exception)
+67 -26
View File
@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from polymarket_insider_tracker.ingestor.clob_client import ClobClient
from polymarket_insider_tracker.ingestor.gamma_client import GammaClient
from polymarket_insider_tracker.ingestor.metadata_sync import (
DEFAULT_CACHE_TTL_SECONDS,
DEFAULT_REDIS_KEY_PREFIX,
@@ -72,6 +73,18 @@ def mock_clob(sample_market: Market) -> MagicMock:
return clob
@pytest.fixture
def mock_gamma() -> MagicMock:
"""Create a mock GammaClient that returns empty volume stats.
Without this, MarketMetadataSync would instantiate a default
GammaClient and hit the real gamma-api over HTTP during unit tests.
"""
gamma = MagicMock(spec=GammaClient)
gamma.get_active_market_stats = AsyncMock(return_value={})
return gamma
class TestDeriveCategory:
"""Tests for the derive_category function."""
@@ -195,9 +208,9 @@ class TestSyncStats:
class TestMarketMetadataSync:
"""Tests for the MarketMetadataSync class."""
def test_init(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
def test_init(self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock) -> None:
"""Test initialization."""
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
assert sync.state == SyncState.STOPPED
assert sync.stats.total_syncs == 0
@@ -205,11 +218,14 @@ class TestMarketMetadataSync:
assert sync._cache_ttl == DEFAULT_CACHE_TTL_SECONDS
assert sync._key_prefix == DEFAULT_REDIS_KEY_PREFIX
def test_init_custom_config(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
def test_init_custom_config(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test initialization with custom config."""
sync = MarketMetadataSync(
redis=mock_redis,
clob_client=mock_clob,
gamma_client=mock_gamma,
sync_interval_seconds=60,
cache_ttl_seconds=120,
key_prefix="custom:",
@@ -220,9 +236,11 @@ class TestMarketMetadataSync:
assert sync._key_prefix == "custom:"
@pytest.mark.asyncio
async def test_start_stop(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_start_stop(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test starting and stopping the sync service."""
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
# Start
await sync.start()
@@ -236,10 +254,10 @@ class TestMarketMetadataSync:
@pytest.mark.asyncio
async def test_start_performs_initial_sync(
self, mock_redis: AsyncMock, mock_clob: MagicMock
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test that start performs an initial sync."""
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.start()
@@ -252,10 +270,12 @@ class TestMarketMetadataSync:
await sync.stop()
@pytest.mark.asyncio
async def test_start_failure(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_start_failure(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test start failure handling."""
mock_clob.get_markets.side_effect = Exception("API error")
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
with pytest.raises(MetadataSyncError, match="initial sync failed"):
await sync.start()
@@ -268,6 +288,7 @@ class TestMarketMetadataSync:
self,
mock_redis: AsyncMock,
mock_clob: MagicMock,
mock_gamma: MagicMock,
sample_metadata: MarketMetadata,
) -> None:
"""Test get_market with cache hit."""
@@ -275,7 +296,7 @@ class TestMarketMetadataSync:
cached_data = json.dumps(sample_metadata.to_dict())
mock_redis.get = AsyncMock(return_value=cached_data)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.start()
result = await sync.get_market("cond123")
@@ -288,12 +309,14 @@ class TestMarketMetadataSync:
await sync.stop()
@pytest.mark.asyncio
async def test_get_market_cache_miss(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_get_market_cache_miss(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test get_market with cache miss."""
# Setup cache miss
mock_redis.get = AsyncMock(return_value=None)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.start()
result = await sync.get_market("cond123")
@@ -308,12 +331,14 @@ class TestMarketMetadataSync:
await sync.stop()
@pytest.mark.asyncio
async def test_get_market_not_found(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_get_market_not_found(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test get_market when market doesn't exist."""
mock_redis.get = AsyncMock(return_value=None)
mock_clob.get_market.return_value = None
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.start()
result = await sync.get_market("nonexistent")
@@ -323,9 +348,11 @@ class TestMarketMetadataSync:
await sync.stop()
@pytest.mark.asyncio
async def test_invalidate_market(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_invalidate_market(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test cache invalidation."""
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.start()
result = await sync.invalidate_market("cond123")
@@ -336,9 +363,11 @@ class TestMarketMetadataSync:
await sync.stop()
@pytest.mark.asyncio
async def test_force_sync(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_force_sync(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test forced sync."""
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.start()
# Initial sync
@@ -353,7 +382,9 @@ class TestMarketMetadataSync:
await sync.stop()
@pytest.mark.asyncio
async def test_state_change_callback(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_state_change_callback(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test state change callback."""
states: list[SyncState] = []
@@ -363,6 +394,7 @@ class TestMarketMetadataSync:
sync = MarketMetadataSync(
redis=mock_redis,
clob_client=mock_clob,
gamma_client=mock_gamma,
on_state_change=on_state_change,
)
@@ -377,7 +409,7 @@ class TestMarketMetadataSync:
@pytest.mark.asyncio
async def test_sync_complete_callback(
self, mock_redis: AsyncMock, mock_clob: MagicMock
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test sync complete callback."""
sync_stats: list[SyncStats] = []
@@ -388,6 +420,7 @@ class TestMarketMetadataSync:
sync = MarketMetadataSync(
redis=mock_redis,
clob_client=mock_clob,
gamma_client=mock_gamma,
on_sync_complete=on_sync_complete,
)
@@ -401,7 +434,11 @@ class TestMarketMetadataSync:
@pytest.mark.asyncio
async def test_get_markets_by_category(
self, mock_redis: AsyncMock, mock_clob: MagicMock, sample_metadata: MarketMetadata
self,
mock_redis: AsyncMock,
mock_clob: MagicMock,
mock_gamma: MagicMock,
sample_metadata: MarketMetadata,
) -> None:
"""Test getting markets by category."""
# Setup scan to return keys
@@ -412,7 +449,7 @@ class TestMarketMetadataSync:
cached_data = json.dumps(sample_metadata.to_dict())
mock_redis.get = AsyncMock(return_value=cached_data)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
# Don't start to avoid initial sync complexity
sync._state = SyncState.IDLE
@@ -422,9 +459,11 @@ class TestMarketMetadataSync:
assert results[0].category == "crypto"
@pytest.mark.asyncio
async def test_cannot_start_twice(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_cannot_start_twice(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test that starting twice doesn't double-start."""
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.start()
await sync.start() # Should be a no-op
@@ -434,9 +473,11 @@ class TestMarketMetadataSync:
await sync.stop()
@pytest.mark.asyncio
async def test_stop_when_stopped(self, mock_redis: AsyncMock, mock_clob: MagicMock) -> None:
async def test_stop_when_stopped(
self, mock_redis: AsyncMock, mock_clob: MagicMock, mock_gamma: MagicMock
) -> None:
"""Test stopping when already stopped."""
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob)
sync = MarketMetadataSync(redis=mock_redis, clob_client=mock_clob, gamma_client=mock_gamma)
await sync.stop() # Should be a no-op