fix(profiler): cap default lookback at 80k blocks + early-break on pruned
Field-test of the chunking fix on a public Polygon RPC (publicnode)
revealed a second wall behind the first: after a chunk request lands
outside the provider's archive horizon, every subsequent chunk fails
with the same error:
{'code': -32701, 'message': 'History has been pruned for this block.
To remove restrictions, order a dedicated full node here: ...'}
publicnode empirically retains roughly the most recent 100_000 blocks
(~55 hours) of log history. Surveying other public free-tier RPCs:
drpc.org — archive, but rejects ranges >= ~1_000 blocks
llamarpc — empty responses on archive ranges
ankr — now requires API key
blockpi/onfin — block-range limits 50–500
1rpc.io/matic — limited to 50 blocks
Two changes to make funding traces actually return data on a public
RPC instead of swallowing 140 pruned-history warnings per wallet:
1. Lower DEFAULT_MAX_LOOKBACK_BLOCKS from 1_300_000 to 80_000. Fresh
wallets — the population this signal exists to flag — are by
definition new, so a ~44 hour window covers their entire funding
history. Older wallets lose archive coverage on free RPCs but
they're not what the fresh-wallet signal scores on anyway.
2. Detect pruned-history errors by message substring and short-circuit
the chunk walk. Walking further back is futile once we're past the
cutoff; bailing early avoids burning RPC quota on chunks that are
guaranteed to fail.
Both knobs remain constructor parameters — deployments behind a paid
archive node can dial DEFAULT_MAX_LOOKBACK_BLOCKS back up.
Two new tests:
- test_get_transfer_logs_breaks_on_pruned_history: pruned error on
chunk #2 must keep chunk #3 from ever being issued
- test_get_transfer_logs_default_lookback_fits_pruned_horizon:
regression guard pinning the default at <= 100_000 so a future
refactor doesn't silently re-introduce the unusable default
This commit is contained in:
@@ -33,8 +33,34 @@ TRANSFER_EVENT_SIGNATURE = AsyncWeb3.keccak(text="Transfer(address,address,uint2
|
||||
# llamarpc) cap the range at 10_000 blocks per call; pick a window slightly
|
||||
# under the cap so off-by-one differences between providers don't trip us up.
|
||||
DEFAULT_CHUNK_SIZE_BLOCKS = 9_000
|
||||
# Polygon block time is ~2.0s, so 1.3M blocks covers roughly 30 days.
|
||||
DEFAULT_MAX_LOOKBACK_BLOCKS = 1_300_000
|
||||
# Polygon block time is ~2.0s. publicnode (the most common free RPC) prunes
|
||||
# log history aggressively — empirically only ~100k blocks (~55 hours) are
|
||||
# served before requests start returning "History has been pruned". We default
|
||||
# to 80k blocks (~44 hours), which is more than enough for fresh-wallet
|
||||
# funding traces (those wallets are by definition new) and fits comfortably
|
||||
# inside what most public providers retain.
|
||||
DEFAULT_MAX_LOOKBACK_BLOCKS = 80_000
|
||||
|
||||
# Substrings that, when present in an RPC error, indicate the chunk we just
|
||||
# asked for is outside the provider's archive horizon. Walking further back
|
||||
# is futile, so we stop the trace early instead of hammering every chunk.
|
||||
_PRUNED_HISTORY_MARKERS: tuple[str, ...] = (
|
||||
"history has been pruned",
|
||||
"missing trie node",
|
||||
"older than",
|
||||
)
|
||||
|
||||
|
||||
def _is_pruned_history_error(err: BaseException) -> bool:
|
||||
"""Return True if the RPC error indicates pruned history.
|
||||
|
||||
Public Polygon nodes only retain a recent slice of log history. When we
|
||||
walk back through that slice in chunks and hit the cutoff, every further
|
||||
chunk will fail with the same message — so we stop early instead of
|
||||
burning quota on guaranteed failures.
|
||||
"""
|
||||
text = str(err).lower()
|
||||
return any(marker in text for marker in _PRUNED_HISTORY_MARKERS)
|
||||
|
||||
|
||||
class FundingTracer:
|
||||
@@ -70,7 +96,8 @@ class FundingTracer:
|
||||
chunk_size_blocks: Block window size per eth_getLogs call. Public
|
||||
Polygon RPCs cap at 10_000 blocks; default leaves a safety margin.
|
||||
max_lookback_blocks: How far back to scan when caller passes
|
||||
``from_block=0``. Default ~30 days at 2s block time.
|
||||
``from_block=0``. Default ~44 hours at 2s block time, which
|
||||
fits inside the pruned-history horizon of most public RPCs.
|
||||
"""
|
||||
self.polygon_client = polygon_client
|
||||
self.entity_registry = entity_registry or EntityRegistry()
|
||||
@@ -232,6 +259,11 @@ class FundingTracer:
|
||||
once ``limit`` matches are collected. Walking oldest-first preserves
|
||||
the "first transfer" semantics expected by the funding chain tracer.
|
||||
|
||||
If a chunk comes back with a "history has been pruned" style error
|
||||
the rest of the walk is short-circuited — every subsequent chunk
|
||||
would hit the same archive cutoff and there's no point burning quota
|
||||
on guaranteed failures.
|
||||
|
||||
Args:
|
||||
to_address: Filter by recipient address.
|
||||
token_address: ERC20 token contract address.
|
||||
@@ -270,6 +302,18 @@ class FundingTracer:
|
||||
to_block=chunk_end,
|
||||
)
|
||||
except Exception as e:
|
||||
if _is_pruned_history_error(e):
|
||||
# The provider has dropped this slice of history. Walking
|
||||
# further back will hit the same wall on every chunk;
|
||||
# stop now and return what we already have.
|
||||
logger.info(
|
||||
"eth_getLogs chunk %d-%d outside archive horizon for %s; "
|
||||
"stopping trace",
|
||||
chunk_start,
|
||||
chunk_end,
|
||||
to_address,
|
||||
)
|
||||
break
|
||||
logger.warning(
|
||||
"eth_getLogs chunk %d-%d failed for %s: %s",
|
||||
chunk_start,
|
||||
|
||||
@@ -519,6 +519,74 @@ class TestGetTransferLogs:
|
||||
assert call_args["fromBlock"] == 0
|
||||
assert call_args["toBlock"] == 5_000
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_transfer_logs_breaks_on_pruned_history(
|
||||
self,
|
||||
funding_tracer: FundingTracer,
|
||||
mock_polygon_client: MagicMock,
|
||||
) -> None:
|
||||
"""A pruned-history error must short-circuit the whole walk.
|
||||
|
||||
Public Polygon RPCs prune log history. Once we walk past the cutoff,
|
||||
every subsequent chunk will raise the same error — keep walking and
|
||||
we just burn quota on guaranteed failures. The first such error must
|
||||
end the walk and return whatever we already collected.
|
||||
"""
|
||||
mock_w3 = MagicMock()
|
||||
good_log = MagicMock()
|
||||
|
||||
responses: list[Any] = [
|
||||
[good_log],
|
||||
RuntimeError(
|
||||
"{'code': -32701, 'message': 'History has been pruned for "
|
||||
"this block. To remove restrictions, order a dedicated full "
|
||||
"node here: https://www.allnodes.com/pol/host'}"
|
||||
),
|
||||
# If the early-break logic is missing, this third chunk would
|
||||
# also be requested. The test asserts it isn't.
|
||||
[MagicMock()],
|
||||
]
|
||||
|
||||
async def fake_get_logs(_params: dict[str, Any]) -> list[Any]:
|
||||
outcome = responses.pop(0)
|
||||
if isinstance(outcome, BaseException):
|
||||
raise outcome
|
||||
return outcome
|
||||
|
||||
mock_w3.eth.get_logs = AsyncMock(side_effect=fake_get_logs)
|
||||
mock_polygon_client._w3 = mock_w3
|
||||
|
||||
# 3 chunks total. The pruned error fires on chunk #2; chunk #3 must
|
||||
# never be issued.
|
||||
result = await funding_tracer._get_transfer_logs(
|
||||
to_address=TEST_WALLET,
|
||||
token_address=USDC_BRIDGED,
|
||||
from_block=1_000_000,
|
||||
to_block=1_027_000,
|
||||
)
|
||||
|
||||
assert result == [dict(good_log)]
|
||||
assert mock_w3.eth.get_logs.call_count == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_transfer_logs_default_lookback_fits_pruned_horizon(
|
||||
self,
|
||||
funding_tracer: FundingTracer,
|
||||
mock_polygon_client: MagicMock,
|
||||
) -> None:
|
||||
"""Default ``max_lookback_blocks`` must stay inside what public RPCs serve.
|
||||
|
||||
publicnode prunes after ~100k blocks. If we default to 1.3M, every
|
||||
funding trace blows through the archive horizon and produces nothing
|
||||
but pruned-history warnings. Pin the default at <= 100k as a
|
||||
regression guard.
|
||||
"""
|
||||
from polymarket_insider_tracker.profiler.funding import (
|
||||
DEFAULT_MAX_LOOKBACK_BLOCKS,
|
||||
)
|
||||
|
||||
assert DEFAULT_MAX_LOOKBACK_BLOCKS <= 100_000
|
||||
|
||||
|
||||
class TestLogToFundingTransfer:
|
||||
"""Tests for _log_to_funding_transfer method."""
|
||||
|
||||
Reference in New Issue
Block a user