From ff1e23a0d36b4218cfa0f9969e4cf8c23293b947 Mon Sep 17 00:00:00 2001 From: schrodinger01 Date: Fri, 22 May 2026 17:08:13 +0800 Subject: [PATCH] fix(profiler): cap default lookback at 80k blocks + early-break on pruned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field-test of the chunking fix on a public Polygon RPC (publicnode) revealed a second wall behind the first: after a chunk request lands outside the provider's archive horizon, every subsequent chunk fails with the same error: {'code': -32701, 'message': 'History has been pruned for this block. To remove restrictions, order a dedicated full node here: ...'} publicnode empirically retains roughly the most recent 100_000 blocks (~55 hours) of log history. Surveying other public free-tier RPCs: drpc.org — archive, but rejects ranges >= ~1_000 blocks llamarpc — empty responses on archive ranges ankr — now requires API key blockpi/onfin — block-range limits 50–500 1rpc.io/matic — limited to 50 blocks Two changes to make funding traces actually return data on a public RPC instead of swallowing 140 pruned-history warnings per wallet: 1. Lower DEFAULT_MAX_LOOKBACK_BLOCKS from 1_300_000 to 80_000. Fresh wallets — the population this signal exists to flag — are by definition new, so a ~44 hour window covers their entire funding history. Older wallets lose archive coverage on free RPCs but they're not what the fresh-wallet signal scores on anyway. 2. Detect pruned-history errors by message substring and short-circuit the chunk walk. Walking further back is futile once we're past the cutoff; bailing early avoids burning RPC quota on chunks that are guaranteed to fail. Both knobs remain constructor parameters — deployments behind a paid archive node can dial DEFAULT_MAX_LOOKBACK_BLOCKS back up. Two new tests: - test_get_transfer_logs_breaks_on_pruned_history: pruned error on chunk #2 must keep chunk #3 from ever being issued - test_get_transfer_logs_default_lookback_fits_pruned_horizon: regression guard pinning the default at <= 100_000 so a future refactor doesn't silently re-introduce the unusable default --- .../profiler/funding.py | 50 +++++++++++++- tests/profiler/test_funding.py | 68 +++++++++++++++++++ 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/polymarket_insider_tracker/profiler/funding.py b/src/polymarket_insider_tracker/profiler/funding.py index 03d64f7..34213e0 100644 --- a/src/polymarket_insider_tracker/profiler/funding.py +++ b/src/polymarket_insider_tracker/profiler/funding.py @@ -33,8 +33,34 @@ TRANSFER_EVENT_SIGNATURE = AsyncWeb3.keccak(text="Transfer(address,address,uint2 # llamarpc) cap the range at 10_000 blocks per call; pick a window slightly # under the cap so off-by-one differences between providers don't trip us up. DEFAULT_CHUNK_SIZE_BLOCKS = 9_000 -# Polygon block time is ~2.0s, so 1.3M blocks covers roughly 30 days. -DEFAULT_MAX_LOOKBACK_BLOCKS = 1_300_000 +# Polygon block time is ~2.0s. publicnode (the most common free RPC) prunes +# log history aggressively — empirically only ~100k blocks (~55 hours) are +# served before requests start returning "History has been pruned". We default +# to 80k blocks (~44 hours), which is more than enough for fresh-wallet +# funding traces (those wallets are by definition new) and fits comfortably +# inside what most public providers retain. +DEFAULT_MAX_LOOKBACK_BLOCKS = 80_000 + +# Substrings that, when present in an RPC error, indicate the chunk we just +# asked for is outside the provider's archive horizon. Walking further back +# is futile, so we stop the trace early instead of hammering every chunk. +_PRUNED_HISTORY_MARKERS: tuple[str, ...] = ( + "history has been pruned", + "missing trie node", + "older than", +) + + +def _is_pruned_history_error(err: BaseException) -> bool: + """Return True if the RPC error indicates pruned history. + + Public Polygon nodes only retain a recent slice of log history. When we + walk back through that slice in chunks and hit the cutoff, every further + chunk will fail with the same message — so we stop early instead of + burning quota on guaranteed failures. + """ + text = str(err).lower() + return any(marker in text for marker in _PRUNED_HISTORY_MARKERS) class FundingTracer: @@ -70,7 +96,8 @@ class FundingTracer: chunk_size_blocks: Block window size per eth_getLogs call. Public Polygon RPCs cap at 10_000 blocks; default leaves a safety margin. max_lookback_blocks: How far back to scan when caller passes - ``from_block=0``. Default ~30 days at 2s block time. + ``from_block=0``. Default ~44 hours at 2s block time, which + fits inside the pruned-history horizon of most public RPCs. """ self.polygon_client = polygon_client self.entity_registry = entity_registry or EntityRegistry() @@ -232,6 +259,11 @@ class FundingTracer: once ``limit`` matches are collected. Walking oldest-first preserves the "first transfer" semantics expected by the funding chain tracer. + If a chunk comes back with a "history has been pruned" style error + the rest of the walk is short-circuited — every subsequent chunk + would hit the same archive cutoff and there's no point burning quota + on guaranteed failures. + Args: to_address: Filter by recipient address. token_address: ERC20 token contract address. @@ -270,6 +302,18 @@ class FundingTracer: to_block=chunk_end, ) except Exception as e: + if _is_pruned_history_error(e): + # The provider has dropped this slice of history. Walking + # further back will hit the same wall on every chunk; + # stop now and return what we already have. + logger.info( + "eth_getLogs chunk %d-%d outside archive horizon for %s; " + "stopping trace", + chunk_start, + chunk_end, + to_address, + ) + break logger.warning( "eth_getLogs chunk %d-%d failed for %s: %s", chunk_start, diff --git a/tests/profiler/test_funding.py b/tests/profiler/test_funding.py index 79f942c..312cc65 100644 --- a/tests/profiler/test_funding.py +++ b/tests/profiler/test_funding.py @@ -519,6 +519,74 @@ class TestGetTransferLogs: assert call_args["fromBlock"] == 0 assert call_args["toBlock"] == 5_000 + @pytest.mark.asyncio + async def test_get_transfer_logs_breaks_on_pruned_history( + self, + funding_tracer: FundingTracer, + mock_polygon_client: MagicMock, + ) -> None: + """A pruned-history error must short-circuit the whole walk. + + Public Polygon RPCs prune log history. Once we walk past the cutoff, + every subsequent chunk will raise the same error — keep walking and + we just burn quota on guaranteed failures. The first such error must + end the walk and return whatever we already collected. + """ + mock_w3 = MagicMock() + good_log = MagicMock() + + responses: list[Any] = [ + [good_log], + RuntimeError( + "{'code': -32701, 'message': 'History has been pruned for " + "this block. To remove restrictions, order a dedicated full " + "node here: https://www.allnodes.com/pol/host'}" + ), + # If the early-break logic is missing, this third chunk would + # also be requested. The test asserts it isn't. + [MagicMock()], + ] + + async def fake_get_logs(_params: dict[str, Any]) -> list[Any]: + outcome = responses.pop(0) + if isinstance(outcome, BaseException): + raise outcome + return outcome + + mock_w3.eth.get_logs = AsyncMock(side_effect=fake_get_logs) + mock_polygon_client._w3 = mock_w3 + + # 3 chunks total. The pruned error fires on chunk #2; chunk #3 must + # never be issued. + result = await funding_tracer._get_transfer_logs( + to_address=TEST_WALLET, + token_address=USDC_BRIDGED, + from_block=1_000_000, + to_block=1_027_000, + ) + + assert result == [dict(good_log)] + assert mock_w3.eth.get_logs.call_count == 2 + + @pytest.mark.asyncio + async def test_get_transfer_logs_default_lookback_fits_pruned_horizon( + self, + funding_tracer: FundingTracer, + mock_polygon_client: MagicMock, + ) -> None: + """Default ``max_lookback_blocks`` must stay inside what public RPCs serve. + + publicnode prunes after ~100k blocks. If we default to 1.3M, every + funding trace blows through the archive horizon and produces nothing + but pruned-history warnings. Pin the default at <= 100k as a + regression guard. + """ + from polymarket_insider_tracker.profiler.funding import ( + DEFAULT_MAX_LOOKBACK_BLOCKS, + ) + + assert DEFAULT_MAX_LOOKBACK_BLOCKS <= 100_000 + class TestLogToFundingTransfer: """Tests for _log_to_funding_transfer method."""