mirror of
https://github.com/labrinyang/lp-terminal.git
synced 2026-08-13 20:58:04 +00:00
LP TERMINAL — terminal-style LP frontend for Robinhood Chain
Static SPA over UP33 (ve(3,3)) + the official Uniswap v2/v3 deployments: browse and add liquidity across all three, manage CL/v2 positions, one-token ZAP adds, aggregator vs native swaps, and LP-based limit orders. Ships with a zero-dependency pool indexer for uniswap discovery. Browser-wallet signing only; no key material anywhere.
This commit is contained in:
labrinyang
+188
@@ -0,0 +1,188 @@
|
||||
// Read-only HTTP API. Response shapes mirror the frontend's PoolsData /
|
||||
// PoolStat structures so the POOLS tab maps rows 1:1 (bigints travel as
|
||||
// strings). Served same-origin in production (nginx /api → this) and through
|
||||
// the vite dev/preview proxy locally.
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
|
||||
import { PORT, log, now } from './config'
|
||||
import { db, kvGet, poolCounts } from './store'
|
||||
|
||||
const JSONH = { 'content-type': 'application/json; charset=utf-8' }
|
||||
|
||||
type Params = URLSearchParams
|
||||
|
||||
const PROTOS = new Set(['univ2', 'univ3'])
|
||||
const HEX40 = /^0x[0-9a-f]{40}$/
|
||||
|
||||
function poolsWhere(params: Params): { where: string; args: (string | number)[] } {
|
||||
const clauses: string[] = []
|
||||
const args: (string | number)[] = []
|
||||
|
||||
const proto = (params.get('proto') ?? '')
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => PROTOS.has(s))
|
||||
if (proto.length) {
|
||||
clauses.push(`p.proto IN (${proto.map(() => '?').join(',')})`)
|
||||
args.push(...proto)
|
||||
}
|
||||
|
||||
const minTvl = Number(params.get('min_tvl'))
|
||||
if (Number.isFinite(minTvl) && minTvl > 0) {
|
||||
clauses.push('s.tvl_usd >= ?')
|
||||
args.push(minTvl)
|
||||
}
|
||||
|
||||
const q = (params.get('q') ?? '').trim().toLowerCase()
|
||||
if (q) {
|
||||
if (HEX40.test(q)) {
|
||||
clauses.push('(p.address = ? OR p.token0 = ? OR p.token1 = ?)')
|
||||
args.push(q, q, q)
|
||||
} else if (q.includes('/')) {
|
||||
// pair search: "weth/usdg" — both sides must match (either orientation)
|
||||
const [a, b] = q.split('/', 2).map((s) => s.trim())
|
||||
const side = `SELECT address FROM tokens WHERE symbol LIKE ?`
|
||||
clauses.push(
|
||||
`((p.token0 IN (${side}) AND p.token1 IN (${side})) OR (p.token0 IN (${side}) AND p.token1 IN (${side})))`,
|
||||
)
|
||||
args.push(a + '%', b + '%', b + '%', a + '%')
|
||||
} else {
|
||||
const side = `SELECT address FROM tokens WHERE symbol LIKE ?`
|
||||
clauses.push(`(p.token0 IN (${side}) OR p.token1 IN (${side}))`)
|
||||
args.push(q + '%', q + '%')
|
||||
}
|
||||
}
|
||||
return { where: clauses.length ? 'WHERE ' + clauses.join(' AND ') : '', args }
|
||||
}
|
||||
|
||||
const ORDER: Record<string, string> = {
|
||||
tvl: 'ORDER BY (s.tvl_usd IS NULL), s.tvl_usd DESC',
|
||||
vol: 'ORDER BY (st.vol24h_usd IS NULL), st.vol24h_usd DESC',
|
||||
created: 'ORDER BY (p.created_block IS NULL), p.created_block DESC, p.pair_index DESC',
|
||||
}
|
||||
|
||||
type PoolOut = Record<string, unknown>
|
||||
|
||||
function getPools(params: Params) {
|
||||
const { where, args } = poolsWhere(params)
|
||||
const order = ORDER[params.get('sort') ?? 'tvl'] ?? ORDER.tvl
|
||||
const limit = Math.min(Math.max(Number(params.get('limit')) || 100, 1), 500)
|
||||
const offset = Math.min(Math.max(Number(params.get('offset')) || 0, 0), 20_000)
|
||||
|
||||
const base = `FROM pools p LEFT JOIN pool_state s ON s.address = p.address LEFT JOIN pool_stats st ON st.address = p.address ${where}`
|
||||
const count = (db.prepare(`SELECT COUNT(*) AS n ${base}`).get(...args) as { n: number }).n
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT p.address, p.proto, p.token0, p.token1, p.fee_ppm, p.tick_spacing, p.created_block,
|
||||
s.sqrt_price, s.tick, s.liquidity, s.reserve0, s.reserve1, s.total_supply,
|
||||
s.tvl_usd, s.tvl_approx, s.updated AS state_updated,
|
||||
st.vol24h_usd, st.txns24h, st.liq_usd, st.source AS stats_source
|
||||
${base} ${order} LIMIT ? OFFSET ?`,
|
||||
)
|
||||
.all(...args, limit, offset) as Record<string, unknown>[]
|
||||
|
||||
const tokenAddrs = new Set<string>()
|
||||
const pools: PoolOut[] = rows.map((r) => {
|
||||
tokenAddrs.add(r.token0 as string)
|
||||
tokenAddrs.add(r.token1 as string)
|
||||
return {
|
||||
proto: r.proto,
|
||||
address: r.address,
|
||||
token0: r.token0,
|
||||
token1: r.token1,
|
||||
feePpm: r.fee_ppm,
|
||||
tickSpacing: r.tick_spacing,
|
||||
createdBlock: r.created_block,
|
||||
sqrtPriceX96: r.sqrt_price,
|
||||
tick: r.tick,
|
||||
liquidity: r.liquidity,
|
||||
reserve0: r.reserve0 ?? '0',
|
||||
reserve1: r.reserve1 ?? '0',
|
||||
totalSupply: r.total_supply,
|
||||
tvlUsd: r.tvl_usd,
|
||||
tvlApprox: r.tvl_approx === 1,
|
||||
vol24hUsd: r.vol24h_usd,
|
||||
txns24h: r.txns24h,
|
||||
gtLiqUsd: r.liq_usd,
|
||||
statsSource: r.stats_source,
|
||||
stateUpdated: r.state_updated,
|
||||
}
|
||||
})
|
||||
|
||||
const tokens: Record<string, unknown> = {}
|
||||
if (tokenAddrs.size) {
|
||||
const list = [...tokenAddrs]
|
||||
const trs = db
|
||||
.prepare(`SELECT address, symbol, decimals, price_usd FROM tokens WHERE address IN (${list.map(() => '?').join(',')})`)
|
||||
.all(...list) as { address: string; symbol: string; decimals: number; price_usd: number | null }[]
|
||||
for (const t of trs) tokens[t.address] = { address: t.address, symbol: t.symbol, decimals: t.decimals, priceUsd: t.price_usd }
|
||||
}
|
||||
|
||||
const totals = Object.fromEntries(poolCounts().map((c) => [c.proto, c.n]))
|
||||
return { ready: kvGet('ready') === '1', asof: now(), totals, count, pools, tokens }
|
||||
}
|
||||
|
||||
function getTokens(params: Params) {
|
||||
const q = (params.get('q') ?? '').trim().toLowerCase()
|
||||
if (!q) return { tokens: [] }
|
||||
const rows = HEX40.test(q)
|
||||
? db.prepare('SELECT address, symbol, decimals, price_usd FROM tokens WHERE address = ?').all(q)
|
||||
: db
|
||||
.prepare(
|
||||
`SELECT t.address, t.symbol, t.decimals, t.price_usd,
|
||||
(SELECT COUNT(*) FROM pools p WHERE p.token0 = t.address OR p.token1 = t.address) AS pools
|
||||
FROM tokens t WHERE t.symbol LIKE ? ORDER BY pools DESC LIMIT 20`,
|
||||
)
|
||||
.all(q + '%')
|
||||
return { tokens: rows }
|
||||
}
|
||||
|
||||
function getHealth() {
|
||||
const totals = Object.fromEntries(poolCounts().map((c) => [c.proto, c.n]))
|
||||
const tokens = (db.prepare('SELECT COUNT(*) AS n FROM tokens').get() as { n: number }).n
|
||||
const priced = (db.prepare('SELECT COUNT(*) AS n FROM tokens WHERE price_usd > 0').get() as { n: number }).n
|
||||
const tvl = (db.prepare('SELECT COUNT(*) AS n FROM pool_state WHERE tvl_usd IS NOT NULL').get() as { n: number }).n
|
||||
return {
|
||||
ready: kvGet('ready') === '1',
|
||||
asof: now(),
|
||||
pools: totals,
|
||||
tokens,
|
||||
pricedTokens: priced,
|
||||
tvlPools: tvl,
|
||||
v3Cursor: Number(kvGet('v3_cursor') ?? 0),
|
||||
v2Count: Number(kvGet('v2_count') ?? 0),
|
||||
rssMb: Math.round(process.memoryUsage.rss() / 1e6),
|
||||
}
|
||||
}
|
||||
|
||||
export function startApi(): void {
|
||||
const srv = createServer((req: IncomingMessage, res: ServerResponse) => {
|
||||
const started = Date.now()
|
||||
try {
|
||||
const url = new URL(req.url ?? '/', 'http://indexer')
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405, JSONH)
|
||||
res.end('{"error":"GET only"}')
|
||||
return
|
||||
}
|
||||
let body: unknown
|
||||
let cache = 'public, max-age=10'
|
||||
if (url.pathname === '/api/pools') body = getPools(url.searchParams)
|
||||
else if (url.pathname === '/api/tokens') body = getTokens(url.searchParams)
|
||||
else if (url.pathname === '/api/health') {
|
||||
body = getHealth()
|
||||
cache = 'no-store'
|
||||
} else {
|
||||
res.writeHead(404, JSONH)
|
||||
res.end('{"error":"not found"}')
|
||||
return
|
||||
}
|
||||
res.writeHead(200, { ...JSONH, 'cache-control': cache })
|
||||
res.end(JSON.stringify(body))
|
||||
if (Date.now() - started > 500) log(`[api] slow ${url.pathname} ${Date.now() - started}ms`)
|
||||
} catch (e) {
|
||||
res.writeHead(500, JSONH)
|
||||
res.end(JSON.stringify({ error: String(e) }))
|
||||
}
|
||||
})
|
||||
srv.listen(PORT, () => log(`[api] listening on :${PORT}`))
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Pool catalog — the authoritative list, built ONLY from the official
|
||||
// factories themselves (events / enumeration). Third-party APIs never admit a
|
||||
// pool here, they only enrich pools that already exist (spoofing is therefore
|
||||
// structurally impossible: a fork pool is simply never in the table).
|
||||
//
|
||||
// univ3: PoolCreated logs. Backfill via Blockscout's etherscan-style getLogs
|
||||
// (no block-range cap, 1000/page, fromBlock-cursor pagination —
|
||||
// measured ~22 pages for the full 21,979-pool history), then a plain
|
||||
// RPC getLogs tail in ≤9k-block windows (Alchemy caps at 10k).
|
||||
// univ2: the factory keeps an allPairs array — enumeration IS the catalog.
|
||||
// Backfill and tail are the same code path: read allPairsLength,
|
||||
// fetch any indices we haven't seen.
|
||||
import { parseAbiItem, toEventSelector } from 'viem'
|
||||
import { uniV2FactoryAbi, uniV2PairAbi } from '../src/abi'
|
||||
import { BLOCKSCOUT, log, sleep, UNI } from './config'
|
||||
import { mc, ok, pc } from './rpc'
|
||||
import { insertPool, kvGet, kvSet, tx } from './store'
|
||||
|
||||
const POOL_CREATED = parseAbiItem(
|
||||
'event PoolCreated(address indexed token0, address indexed token1, uint24 indexed fee, int24 tickSpacing, address pool)',
|
||||
)
|
||||
const POOL_CREATED_TOPIC = toEventSelector(POOL_CREATED)
|
||||
|
||||
const hexInt = (x: string | number) => (typeof x === 'number' ? x : parseInt(x, 16))
|
||||
const addrOfTopic = (t: string) => ('0x' + t.slice(-40)).toLowerCase()
|
||||
|
||||
async function bsJson(url: string): Promise<Record<string, unknown>> {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
try {
|
||||
const r = await fetch(url, {
|
||||
headers: { accept: 'application/json', 'user-agent': 'up33-lp-indexer/0.1' },
|
||||
})
|
||||
const text = await r.text()
|
||||
if (text.trim()) return JSON.parse(text)
|
||||
} catch {
|
||||
/* retry below */
|
||||
}
|
||||
await sleep(1_200 * (i + 1))
|
||||
}
|
||||
throw new Error('blockscout: no response after retries')
|
||||
}
|
||||
|
||||
type BsLog = { topics: string[]; data: string; blockNumber: string }
|
||||
|
||||
/**
|
||||
* One-time full-history PoolCreated scan (resumable via the v3_cursor kv).
|
||||
* Primary source is Blockscout (no range cap, ~22 pages for full history);
|
||||
* if it flakes persistently the scan falls back to windowed RPC getLogs from
|
||||
* the same cursor — slower (~1.2k windows) but unconditionally available.
|
||||
*/
|
||||
export async function backfillV3(): Promise<number> {
|
||||
if (kvGet('v3_backfilled')) return 0
|
||||
let cursor = Number(kvGet('v3_cursor') ?? 0)
|
||||
let added = 0
|
||||
let flakes = 0
|
||||
// INDEXER_BACKFILL=rpc skips Blockscout entirely — useful when it throttles
|
||||
// (observed 2026-07-16: page pace degraded 1min → 6min mid-backfill). With a
|
||||
// private RPC the windowed scan is deterministic (~770 windows for full
|
||||
// history) and resumes from the same cursor.
|
||||
if (process.env.INDEXER_BACKFILL === 'rpc') {
|
||||
log(`[catalog] v3 backfill via RPC windows from blk ${cursor} (INDEXER_BACKFILL=rpc)`)
|
||||
const head = Number(await pc.getBlockNumber())
|
||||
added = (await scanV3Windows(cursor, head)).length
|
||||
kvSet('v3_cursor', String(head))
|
||||
kvSet('v3_backfilled', '1')
|
||||
return added
|
||||
}
|
||||
for (;;) {
|
||||
const j = await bsJson(
|
||||
`${BLOCKSCOUT}/api?module=logs&action=getLogs&fromBlock=${cursor}&toBlock=latest&address=${UNI.V3_FACTORY}&topic0=${POOL_CREATED_TOPIC}`,
|
||||
).catch(() => ({ status: '0', message: 'no response' }) as Record<string, unknown>)
|
||||
if (j.status !== '1') {
|
||||
if (/no records/i.test(String(j.message))) break
|
||||
if (++flakes >= 6) {
|
||||
// Blockscout is down/unhappy — finish the remaining range over RPC
|
||||
log(`[catalog] blockscout flaking ("${j.message}") — RPC-window fallback from blk ${cursor}`)
|
||||
const head = Number(await pc.getBlockNumber())
|
||||
added += (await scanV3Windows(cursor, head)).length
|
||||
kvSet('v3_cursor', String(head))
|
||||
break
|
||||
}
|
||||
await sleep(2_000 * flakes)
|
||||
continue
|
||||
}
|
||||
flakes = 0
|
||||
const logs = j.result as BsLog[]
|
||||
tx(() => {
|
||||
for (const l of logs) {
|
||||
// topics: [sig, token0, token1, fee]; data: [tickSpacing:int24, pool:address]
|
||||
const tickSpacing = Number(BigInt.asIntN(24, BigInt('0x' + l.data.slice(2, 66))))
|
||||
if (
|
||||
insertPool({
|
||||
address: addrOfTopic(l.data.slice(66, 130)),
|
||||
proto: 'univ3',
|
||||
token0: addrOfTopic(l.topics[1]),
|
||||
token1: addrOfTopic(l.topics[2]),
|
||||
feePpm: hexInt(l.topics[3]),
|
||||
tickSpacing,
|
||||
createdBlock: hexInt(l.blockNumber),
|
||||
})
|
||||
)
|
||||
added++
|
||||
}
|
||||
})
|
||||
const last = hexInt(logs[logs.length - 1].blockNumber)
|
||||
kvSet('v3_cursor', String(last))
|
||||
log(`[catalog] v3 backfill +${added} pools (cursor blk ${last})`)
|
||||
if (logs.length < 1000) break
|
||||
cursor = last // overlap the last block; PK dedupes
|
||||
await sleep(300)
|
||||
}
|
||||
kvSet('v3_backfilled', '1')
|
||||
return added
|
||||
}
|
||||
|
||||
/** windowed RPC getLogs scan (≤9k blocks per request — under Alchemy's 10k cap) */
|
||||
async function scanV3Windows(from: number, to: number): Promise<string[]> {
|
||||
const fresh: string[] = []
|
||||
let logged = 0
|
||||
for (let lo = from; lo <= to; lo += 9_001) {
|
||||
const hi = Math.min(lo + 9_000, to)
|
||||
const logs = await pc.getLogs({
|
||||
address: UNI.V3_FACTORY,
|
||||
event: POOL_CREATED,
|
||||
fromBlock: BigInt(lo),
|
||||
toBlock: BigInt(hi),
|
||||
})
|
||||
for (const l of logs) {
|
||||
const a = l.args
|
||||
if (!a.pool || !a.token0 || !a.token1 || a.fee === undefined || a.tickSpacing === undefined) continue
|
||||
if (
|
||||
insertPool({
|
||||
address: a.pool.toLowerCase(),
|
||||
proto: 'univ3',
|
||||
token0: a.token0,
|
||||
token1: a.token1,
|
||||
feePpm: a.fee,
|
||||
tickSpacing: a.tickSpacing,
|
||||
createdBlock: Number(l.blockNumber),
|
||||
})
|
||||
)
|
||||
fresh.push(a.pool.toLowerCase())
|
||||
}
|
||||
if (to - from > 100_000 && ++logged % 100 === 0)
|
||||
log(`[catalog] rpc scan blk ${hi}/${to} (+${fresh.length})`)
|
||||
}
|
||||
return fresh
|
||||
}
|
||||
|
||||
/** RPC tail from the stored cursor to head; returns newly added pool addresses */
|
||||
export async function tailV3(): Promise<string[]> {
|
||||
const head = Number(await pc.getBlockNumber())
|
||||
const from = Math.max(0, Number(kvGet('v3_cursor') ?? head - 2_000) - 120) // ~12s overlap
|
||||
const fresh = await scanV3Windows(from, head)
|
||||
kvSet('v3_cursor', String(head))
|
||||
return fresh
|
||||
}
|
||||
|
||||
/**
|
||||
* univ2 catalog sync (backfill == tail): fetch allPairs indices we haven't
|
||||
* seen yet. The cursor only advances past indices that fully resolved, so a
|
||||
* partial multicall failure is retried on the next tick.
|
||||
*/
|
||||
export async function syncV2(): Promise<string[]> {
|
||||
const count = Number(
|
||||
await pc.readContract({ abi: uniV2FactoryAbi, address: UNI.V2_FACTORY, functionName: 'allPairsLength' }),
|
||||
)
|
||||
let known = Number(kvGet('v2_count') ?? 0)
|
||||
if (count <= known) return []
|
||||
const fresh: string[] = []
|
||||
while (known < count) {
|
||||
const n = Math.min(2_000, count - known) // 2k pairs per round = 5 + 10 aggregates
|
||||
const idx = Array.from({ length: n }, (_, i) => known + i)
|
||||
const pairRes = await mc(
|
||||
idx.map((i) => ({ abi: uniV2FactoryAbi, address: UNI.V2_FACTORY, functionName: 'allPairs', args: [BigInt(i)] })),
|
||||
)
|
||||
const pairs: string[] = []
|
||||
for (const r of pairRes) {
|
||||
const a = ok<string>(r)
|
||||
if (!a) break // stop at first failure — cursor advances only past successes
|
||||
pairs.push(a)
|
||||
}
|
||||
if (!pairs.length) break
|
||||
const tokRes = await mc(
|
||||
pairs.flatMap((p) => [
|
||||
{ abi: uniV2PairAbi, address: p as `0x${string}`, functionName: 'token0' },
|
||||
{ abi: uniV2PairAbi, address: p as `0x${string}`, functionName: 'token1' },
|
||||
]),
|
||||
)
|
||||
let done = 0
|
||||
tx(() => {
|
||||
for (let i = 0; i < pairs.length; i++) {
|
||||
const t0 = ok<string>(tokRes[i * 2])
|
||||
const t1 = ok<string>(tokRes[i * 2 + 1])
|
||||
if (!t0 || !t1) break
|
||||
if (
|
||||
insertPool({
|
||||
address: pairs[i].toLowerCase(),
|
||||
proto: 'univ2',
|
||||
token0: t0,
|
||||
token1: t1,
|
||||
feePpm: 3_000, // vanilla v2: fixed 0.30%
|
||||
pairIndex: known + i,
|
||||
})
|
||||
)
|
||||
fresh.push(pairs[i].toLowerCase())
|
||||
done++
|
||||
}
|
||||
})
|
||||
if (!done) break
|
||||
known += done
|
||||
kvSet('v2_count', String(known))
|
||||
if (known < count) log(`[catalog] v2 sync ${known}/${count}`)
|
||||
}
|
||||
return fresh
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
// Indexer constants + tuning. Contract addresses come from the shared frontend
|
||||
// config — src/config/addresses.ts and src/abi are pure modules and load fine
|
||||
// under node/tsx. src/config/env.ts does NOT (import.meta.env is vite-only),
|
||||
// which is why the public RPC is duplicated here instead of imported.
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
export { ADDR, UNI } from '../src/config/addresses'
|
||||
|
||||
export const PUBLIC_RPC = 'https://rpc.mainnet.chain.robinhood.com'
|
||||
export const BLOCKSCOUT = 'https://robinhoodchain.blockscout.com'
|
||||
export const GT = 'https://api.geckoterminal.com/api/v2'
|
||||
|
||||
export const PORT = Number(process.env.INDEXER_PORT || 8787)
|
||||
export const DB_PATH =
|
||||
process.env.INDEXER_DB || fileURLToPath(new URL('./data/index.db', import.meta.url))
|
||||
|
||||
export const TUNE = {
|
||||
tailMs: 10_000, // factory tail + v2 allPairsLength poll
|
||||
hotSweepMs: 60_000, // state refresh for hot pools
|
||||
fullSweepMs: 3_600_000, // state refresh for ACTIVE pools (≥$100 TVL or <48h old)
|
||||
censusMs: 21_600_000, // 6h full-catalog dust census (~114k pools and growing)
|
||||
statsMs: 300_000, // GeckoTerminal enrichment cycle
|
||||
gtPaceMs: 2_600, // ≥2.6s between GT calls (free tier: 30/min)
|
||||
batch: 400, // calls per multicall aggregate
|
||||
batchGapMs: 40, // pause between aggregates (gentle on the RPC)
|
||||
hotTvlUsd: 10_000, // pools at/above this TVL refresh every hotSweepMs
|
||||
minDepthUsd: 300, // min priced-side USD depth to propagate a price through a pool
|
||||
gtFreshSecs: 1_800, // GT prices younger than this are never overwritten by propagation
|
||||
}
|
||||
|
||||
/** repo-root .env `RPC` (SECRET — never log/print it). Fallback: key-free public RPC. */
|
||||
export function rpcUrl(): string {
|
||||
const env = process.env.RPC?.trim()
|
||||
if (env) return env
|
||||
try {
|
||||
const text = readFileSync(new URL('../.env', import.meta.url), 'utf8')
|
||||
const m = text.match(/^\s*RPC\s*=\s*(\S+)\s*$/m)
|
||||
if (m) return m[1]
|
||||
} catch {
|
||||
/* no repo .env — public RPC below */
|
||||
}
|
||||
return PUBLIC_RPC
|
||||
}
|
||||
|
||||
export const now = () => Math.floor(Date.now() / 1000)
|
||||
|
||||
/** terminal-style timestamped log line */
|
||||
export const log = (...a: unknown[]) =>
|
||||
console.log(new Date().toISOString().slice(11, 19), ...a)
|
||||
|
||||
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
// UP33 LP-terminal pool indexer — catalog (factory events/enumeration) +
|
||||
// on-chain state sweeps + GT enrichment, served over a tiny read-only API.
|
||||
// Run: `npm run indexer` (tsx). Data lives in indexer/data/index.db (SQLite).
|
||||
//
|
||||
// Boot: backfill → token meta → full state sweep → GT cycle → reprice → ready.
|
||||
// Loops: tail 10s · hot sweep 60s · full sweep 60min · GT stats 5min.
|
||||
// The API starts listening immediately; `ready:false` in responses tells the
|
||||
// frontend to keep using its client-side fallback until the first pass lands.
|
||||
import { log, PORT, TUNE } from './config'
|
||||
import { usingPrivateRpc } from './rpc'
|
||||
import { backfillV3, syncV2, tailV3 } from './catalog'
|
||||
import { computeTvlFor, ensureTokenMeta, reprice, sweepState } from './state'
|
||||
import { gtCycle } from './stats'
|
||||
import { activeAddrs, allPoolAddrs, db, hotAddrs, kvGet, kvSet, poolCounts } from './store'
|
||||
import { startApi } from './api'
|
||||
|
||||
/** setTimeout-chained loop — never overlaps itself, logs failures and keeps going */
|
||||
function loop(name: string, ms: number, fn: () => Promise<void>): void {
|
||||
const tick = async () => {
|
||||
try {
|
||||
await fn()
|
||||
} catch (e) {
|
||||
log(`[${name}] error:`, String(e).slice(0, 200))
|
||||
}
|
||||
setTimeout(tick, ms)
|
||||
}
|
||||
setTimeout(tick, ms)
|
||||
}
|
||||
|
||||
const timed = async <T,>(fn: () => Promise<T>): Promise<[T, number]> => {
|
||||
const t0 = Date.now()
|
||||
const r = await fn()
|
||||
return [r, Date.now() - t0]
|
||||
}
|
||||
|
||||
async function boot(): Promise<void> {
|
||||
log('up33 lp-indexer starting —', usingPrivateRpc ? 'rpc: private (.env)' : 'rpc: public')
|
||||
startApi()
|
||||
|
||||
const [addedV3, msV3] = await timed(backfillV3)
|
||||
if (addedV3 > 0 || !kvGet('v3_boot_logged')) {
|
||||
log(`[catalog] univ3 backfill done: +${addedV3} pools (${(msV3 / 1000).toFixed(0)}s)`)
|
||||
kvSet('v3_boot_logged', '1')
|
||||
}
|
||||
const [freshV2, msV2] = await timed(syncV2)
|
||||
if (freshV2.length) log(`[catalog] univ2 sync: +${freshV2.length} pairs (${(msV2 / 1000).toFixed(0)}s)`)
|
||||
log('[catalog]', poolCounts().map((c) => `${c.proto}=${c.n}`).join(' '))
|
||||
|
||||
const [metaN, msMeta] = await timed(ensureTokenMeta)
|
||||
if (metaN) log(`[tokens] metadata fetched for ${metaN} tokens (${(msMeta / 1000).toFixed(0)}s)`)
|
||||
|
||||
const all = allPoolAddrs()
|
||||
const [, msSweep] = await timed(() => sweepState(all))
|
||||
log(`[sweep] full ${all.length} pools (${(msSweep / 1000).toFixed(0)}s)`)
|
||||
|
||||
await gtCycle().catch((e) => log('[stats] gt cycle failed:', String(e).slice(0, 120)))
|
||||
const pr = reprice()
|
||||
log(`[price] ${pr.priced} tokens priced · tvl on ${pr.tvlPools} pools`)
|
||||
|
||||
kvSet('ready', '1')
|
||||
log(`READY — http://localhost:${PORT}/api/health`)
|
||||
|
||||
loop('tail', TUNE.tailMs, async () => {
|
||||
const fresh = [...(await tailV3()), ...(await syncV2())]
|
||||
if (fresh.length) {
|
||||
log(`[tail] ${fresh.length} new pools`)
|
||||
await ensureTokenMeta()
|
||||
await sweepState(fresh)
|
||||
computeTvlFor(fresh)
|
||||
}
|
||||
})
|
||||
loop('hot', TUNE.hotSweepMs, async () => {
|
||||
const hot = hotAddrs()
|
||||
await sweepState(hot)
|
||||
computeTvlFor(hot)
|
||||
})
|
||||
loop('active', TUNE.fullSweepMs, async () => {
|
||||
const addrs = activeAddrs()
|
||||
const [, ms] = await timed(() => sweepState(addrs))
|
||||
const p = reprice()
|
||||
log(`[sweep] active ${addrs.length} pools (${(ms / 1000).toFixed(0)}s) · ${p.priced} tokens priced · tvl on ${p.tvlPools}`)
|
||||
})
|
||||
loop('census', TUNE.censusMs, async () => {
|
||||
const addrs = allPoolAddrs()
|
||||
const [, ms] = await timed(() => sweepState(addrs))
|
||||
const p = reprice()
|
||||
log(`[sweep] census ${addrs.length} pools (${(ms / 1000).toFixed(0)}s) · tvl on ${p.tvlPools}`)
|
||||
})
|
||||
loop('stats', TUNE.statsMs, async () => {
|
||||
await gtCycle()
|
||||
reprice()
|
||||
})
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
log('shutting down')
|
||||
db.close()
|
||||
process.exit(0)
|
||||
})
|
||||
process.on('SIGTERM', () => {
|
||||
db.close()
|
||||
process.exit(0)
|
||||
})
|
||||
|
||||
boot().catch((e) => {
|
||||
log('FATAL boot:', e)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -0,0 +1,76 @@
|
||||
import { createPublicClient, defineChain, http, type PublicClient } from 'viem'
|
||||
import { log, PUBLIC_RPC, TUNE, rpcUrl, sleep } from './config'
|
||||
|
||||
// duplicated from src/config/chain.ts — that module imports src/config/env.ts
|
||||
// (import.meta.env, vite-only) so it can't be loaded under node
|
||||
const robinhood = defineChain({
|
||||
id: 4663,
|
||||
name: 'Robinhood Chain',
|
||||
nativeCurrency: { name: 'Ether', symbol: 'ETH', decimals: 18 },
|
||||
rpcUrls: { default: { http: [PUBLIC_RPC] } },
|
||||
contracts: { multicall3: { address: '0xcA11bde05977b3631167028862bE2a173976CA11' } },
|
||||
})
|
||||
|
||||
const url = rpcUrl()
|
||||
export const usingPrivateRpc = url !== PUBLIC_RPC
|
||||
// timeout is deliberately tight: a healthy 400-call aggregate answers in 2-4s
|
||||
// (measured 2026-07-16); a stalled attempt should fail fast and retry, not
|
||||
// pin the whole boot for 30s. Bad chunks degrade to sub-chunks in mc().
|
||||
export const pc: PublicClient = createPublicClient({
|
||||
chain: robinhood,
|
||||
transport: http(url, { timeout: 10_000, retryCount: 2, retryDelay: 400 }),
|
||||
})
|
||||
|
||||
/** error text safe to log — the RPC url (secret) is redacted */
|
||||
const redact = (e: unknown) =>
|
||||
String(e instanceof Error ? `${e.name}: ${e.message.split('\n')[0]}` : e)
|
||||
.replaceAll(url, '<rpc>')
|
||||
.slice(0, 120)
|
||||
|
||||
// loose call shape — abi fragments come from parseAbi, results are narrowed by ok<T>()
|
||||
export type Call = { abi: unknown; address: `0x${string}`; functionName: string; args?: unknown[] }
|
||||
export type McRes = { status: 'success' | 'failure'; result?: unknown }
|
||||
|
||||
const agg = async (chunk: Call[]): Promise<McRes[]> =>
|
||||
(await pc.multicall({ contracts: chunk as never, batchSize: 250_000 })) as McRes[]
|
||||
|
||||
/**
|
||||
* Chunked multicall: fixed TUNE.batch calls per aggregate3 (batchSize is set
|
||||
* high so viem never sub-chunks by calldata bytes), allowFailure semantics,
|
||||
* gentle pacing between chunks. A failing chunk is retried once, then split
|
||||
* into 100-call sub-chunks so one bad call can only take 100 results down
|
||||
* with it — mc() never throws, it returns per-call failures instead.
|
||||
*/
|
||||
export async function mc(calls: Call[]): Promise<McRes[]> {
|
||||
const out: McRes[] = []
|
||||
for (let i = 0; i < calls.length; i += TUNE.batch) {
|
||||
const chunk = calls.slice(i, i + TUNE.batch)
|
||||
const t0 = Date.now()
|
||||
try {
|
||||
out.push(...(await agg(chunk)))
|
||||
} catch (e) {
|
||||
log('[rpc] chunk failed, retrying:', redact(e))
|
||||
await sleep(600)
|
||||
try {
|
||||
out.push(...(await agg(chunk)))
|
||||
} catch {
|
||||
for (let j = 0; j < chunk.length; j += 100) {
|
||||
const part = chunk.slice(j, j + 100)
|
||||
try {
|
||||
out.push(...(await agg(part)))
|
||||
} catch (e2) {
|
||||
log(`[rpc] dropped ${part.length}-call sub-chunk:`, redact(e2))
|
||||
out.push(...part.map(() => ({ status: 'failure' as const })))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const ms = Date.now() - t0
|
||||
if (ms > 8_000) log(`[rpc] slow chunk: ${ms}ms (${chunk.length} calls)`)
|
||||
if (i + TUNE.batch < calls.length) await sleep(TUNE.batchGapMs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export const ok = <T,>(r?: McRes): T | undefined =>
|
||||
r && r.status === 'success' ? (r.result as T) : undefined
|
||||
@@ -0,0 +1,232 @@
|
||||
// On-chain state sweeps + USD pricing.
|
||||
//
|
||||
// State per pool (multicall):
|
||||
// univ3: slot0 + liquidity + erc20 balanceOf(token0/1) — balances (not L)
|
||||
// are the TVL basis, matching how GT/dexscreener report "reserve".
|
||||
// univ2: getReserves + totalSupply.
|
||||
//
|
||||
// Pricing is a waterfall: GeckoTerminal token prices are ground truth while
|
||||
// fresh (stats.ts seeds them, depth = pool reserve/2); everything else comes
|
||||
// from anchor propagation — a token gets priced through the deepest pool that
|
||||
// pairs it against an already-priced token, requiring ≥ TUNE.minDepthUsd of
|
||||
// priced-side depth so dust pools can't set prices. TVL then = sum of priced
|
||||
// sides (single-priced-side pools: 2× that side, flagged approximate).
|
||||
import { erc20Abi, formatUnits } from 'viem'
|
||||
import { uniV2PairAbi, uniV3PoolAbi } from '../src/abi'
|
||||
import { ADDR, TUNE, log, now } from './config'
|
||||
import { mc, ok, type Call } from './rpc'
|
||||
import {
|
||||
allTokens,
|
||||
db,
|
||||
missingMetaTokens,
|
||||
setTokenPrice,
|
||||
setTvl,
|
||||
tx,
|
||||
upsertState,
|
||||
upsertTokenMeta,
|
||||
type PoolRow,
|
||||
} from './store'
|
||||
|
||||
const printable = (s: unknown): string | null => {
|
||||
if (typeof s !== 'string') return null
|
||||
const t = s.replace(/[^\x20-\x7e]/g, '').trim()
|
||||
return t ? t.slice(0, 24) : null
|
||||
}
|
||||
|
||||
/** fetch symbol/decimals for catalog tokens we haven't met yet (10k/slice) */
|
||||
export async function ensureTokenMeta(): Promise<number> {
|
||||
const all = missingMetaTokens()
|
||||
for (let i = 0; i < all.length; i += 10_000) {
|
||||
const missing = all.slice(i, i + 10_000)
|
||||
const res = await mc(
|
||||
missing.flatMap((t) => [
|
||||
{ abi: erc20Abi, address: t as `0x${string}`, functionName: 'symbol' },
|
||||
{ abi: erc20Abi, address: t as `0x${string}`, functionName: 'decimals' },
|
||||
]),
|
||||
)
|
||||
tx(() => {
|
||||
missing.forEach((t, j) => {
|
||||
const sym = printable(ok<string>(res[j * 2]))
|
||||
const dec = ok<number>(res[j * 2 + 1])
|
||||
upsertTokenMeta(t, sym ?? t.slice(0, 6) + '…', dec ?? 18, sym !== null && dec !== undefined)
|
||||
})
|
||||
})
|
||||
}
|
||||
return all.length
|
||||
}
|
||||
|
||||
const poolRowsQ = (addrs: string[]): PoolRow[] => {
|
||||
const out: PoolRow[] = []
|
||||
const q = db.prepare('SELECT address, proto, token0, token1, fee_ppm, tick_spacing FROM pools WHERE address = ?')
|
||||
for (const a of addrs) {
|
||||
const r = q.get(a.toLowerCase()) as PoolRow | undefined
|
||||
if (r) out.push(r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** refresh raw on-chain state for the given pools (memory-bounded slices) */
|
||||
export async function sweepState(addrs: string[]): Promise<number> {
|
||||
let done = 0
|
||||
for (let i = 0; i < addrs.length; i += 5_000) {
|
||||
done += await sweepSlice(addrs.slice(i, i + 5_000))
|
||||
}
|
||||
return done
|
||||
}
|
||||
|
||||
async function sweepSlice(addrs: string[]): Promise<number> {
|
||||
if (!addrs.length) return 0
|
||||
const rows = poolRowsQ(addrs)
|
||||
const calls: Call[] = []
|
||||
for (const p of rows) {
|
||||
const a = p.address as `0x${string}`
|
||||
if (p.proto === 'univ3')
|
||||
calls.push(
|
||||
{ abi: uniV3PoolAbi, address: a, functionName: 'slot0' },
|
||||
{ abi: uniV3PoolAbi, address: a, functionName: 'liquidity' },
|
||||
{ abi: erc20Abi, address: p.token0 as `0x${string}`, functionName: 'balanceOf', args: [a] },
|
||||
{ abi: erc20Abi, address: p.token1 as `0x${string}`, functionName: 'balanceOf', args: [a] },
|
||||
)
|
||||
else
|
||||
calls.push(
|
||||
{ abi: uniV2PairAbi, address: a, functionName: 'getReserves' },
|
||||
{ abi: uniV2PairAbi, address: a, functionName: 'totalSupply' },
|
||||
)
|
||||
}
|
||||
const res = await mc(calls)
|
||||
let i = 0
|
||||
tx(() => {
|
||||
for (const p of rows) {
|
||||
if (p.proto === 'univ3') {
|
||||
const s0 = ok<readonly [bigint, number, ...unknown[]]>(res[i++])
|
||||
const liq = ok<bigint>(res[i++])
|
||||
const b0 = ok<bigint>(res[i++])
|
||||
const b1 = ok<bigint>(res[i++])
|
||||
if (!s0) continue
|
||||
upsertState(p.address, {
|
||||
sqrtPrice: s0[0],
|
||||
tick: s0[1],
|
||||
liquidity: liq ?? 0n,
|
||||
reserve0: b0 ?? 0n,
|
||||
reserve1: b1 ?? 0n,
|
||||
})
|
||||
} else {
|
||||
const rs = ok<readonly [bigint, bigint, number]>(res[i++])
|
||||
const ts = ok<bigint>(res[i++])
|
||||
if (!rs) continue
|
||||
upsertState(p.address, { reserve0: rs[0], reserve1: rs[1], totalSupply: ts ?? 0n })
|
||||
}
|
||||
}
|
||||
})
|
||||
return rows.length
|
||||
}
|
||||
|
||||
type PriceEntry = { usd: number; depth: number; src: string; updated: number }
|
||||
|
||||
const loadPrices = (): Map<string, PriceEntry> => {
|
||||
const m = new Map<string, PriceEntry>()
|
||||
for (const t of allTokens())
|
||||
if (t.price_usd != null && t.price_usd > 0)
|
||||
m.set(t.address, { usd: t.price_usd, depth: t.price_depth_usd, src: t.price_src ?? '?', updated: t.price_updated ?? 0 })
|
||||
return m
|
||||
}
|
||||
|
||||
type StateRow = {
|
||||
address: string
|
||||
proto: string
|
||||
token0: string
|
||||
token1: string
|
||||
reserve0: string
|
||||
reserve1: string
|
||||
}
|
||||
const statesQ = () =>
|
||||
db
|
||||
.prepare(
|
||||
`SELECT p.address, p.proto, p.token0, p.token1, s.reserve0, s.reserve1
|
||||
FROM pools p JOIN pool_state s ON s.address = p.address`,
|
||||
)
|
||||
.all() as StateRow[]
|
||||
|
||||
/**
|
||||
* Full pricing pass: propagate USD prices from GT/anchor seeds through pools,
|
||||
* then recompute every pool's TVL. Pure JS over in-memory rows (~35k pools),
|
||||
* runs after full sweeps and after each GT cycle.
|
||||
*/
|
||||
export function reprice(): { priced: number; tvlPools: number } {
|
||||
const decs = new Map(allTokens().map((t) => [t.address, t.decimals]))
|
||||
const prices = loadPrices()
|
||||
// bootstrap anchor before the first GT cycle: USDG ≈ $1 (GT overwrites it)
|
||||
if (!prices.has(ADDR.USDG.toLowerCase()))
|
||||
prices.set(ADDR.USDG.toLowerCase(), { usd: 1, depth: 1, src: 'anchor', updated: now() })
|
||||
|
||||
const states = statesQ()
|
||||
const human = (raw: string, addr: string) => Number(formatUnits(BigInt(raw), decs.get(addr) ?? 18))
|
||||
const gtFresh = (e: PriceEntry) => e.src === 'gt' && now() - e.updated < TUNE.gtFreshSecs
|
||||
|
||||
const dirty = new Map<string, PriceEntry>()
|
||||
for (let round = 0; round < 3; round++) {
|
||||
let changed = 0
|
||||
for (const s of states) {
|
||||
const b0 = human(s.reserve0, s.token0)
|
||||
const b1 = human(s.reserve1, s.token1)
|
||||
for (const [known, other, kb, ob] of [
|
||||
[s.token0, s.token1, b0, b1],
|
||||
[s.token1, s.token0, b1, b0],
|
||||
] as const) {
|
||||
const kp = prices.get(known)
|
||||
if (!kp || ob <= 0) continue
|
||||
const depth = kb * kp.usd
|
||||
if (depth < TUNE.minDepthUsd) continue
|
||||
const existing = prices.get(other)
|
||||
if (existing && (gtFresh(existing) || existing.depth >= depth)) continue
|
||||
const e: PriceEntry = { usd: depth / ob, depth, src: 'pool', updated: now() }
|
||||
prices.set(other, e)
|
||||
dirty.set(other, e)
|
||||
changed++
|
||||
}
|
||||
}
|
||||
if (!changed) break
|
||||
}
|
||||
|
||||
let tvlPools = 0
|
||||
tx(() => {
|
||||
for (const [addr, e] of dirty) setTokenPrice(addr, e.usd, e.depth, e.src)
|
||||
for (const s of states) {
|
||||
const p0 = prices.get(s.token0)
|
||||
const p1 = prices.get(s.token1)
|
||||
const u0 = p0 ? human(s.reserve0, s.token0) * p0.usd : null
|
||||
const u1 = p1 ? human(s.reserve1, s.token1) * p1.usd : null
|
||||
const tvl = u0 != null && u1 != null ? u0 + u1 : u0 != null ? u0 * 2 : u1 != null ? u1 * 2 : null
|
||||
setTvl(s.address, tvl, tvl != null && (u0 == null || u1 == null))
|
||||
if (tvl != null) tvlPools++
|
||||
}
|
||||
})
|
||||
return { priced: prices.size, tvlPools }
|
||||
}
|
||||
|
||||
/** cheap TVL refresh for a few pools using already-stored prices (no propagation) */
|
||||
export function computeTvlFor(addrs: string[]): void {
|
||||
if (!addrs.length) return
|
||||
const decs = new Map(allTokens().map((t) => [t.address, t.decimals]))
|
||||
const prices = loadPrices()
|
||||
const q = db.prepare(
|
||||
`SELECT p.address, p.proto, p.token0, p.token1, s.reserve0, s.reserve1
|
||||
FROM pools p JOIN pool_state s ON s.address = p.address WHERE p.address = ?`,
|
||||
)
|
||||
tx(() => {
|
||||
for (const a of addrs) {
|
||||
const s = q.get(a.toLowerCase()) as StateRow | undefined
|
||||
if (!s) continue
|
||||
const human = (raw: string, addr: string) => Number(formatUnits(BigInt(raw), decs.get(addr) ?? 18))
|
||||
const p0 = prices.get(s.token0)
|
||||
const p1 = prices.get(s.token1)
|
||||
const u0 = p0 ? human(s.reserve0, s.token0) * p0.usd : null
|
||||
const u1 = p1 ? human(s.reserve1, s.token1) * p1.usd : null
|
||||
const tvl = u0 != null && u1 != null ? u0 + u1 : u0 != null ? u0 * 2 : u1 != null ? u1 * 2 : null
|
||||
setTvl(s.address, tvl, tvl != null && (u0 == null || u1 == null))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export const sweepLog = (label: string, n: number, ms: number) =>
|
||||
log(`[sweep] ${label} ${n} pools in ${(ms / 1000).toFixed(1)}s`)
|
||||
@@ -0,0 +1,91 @@
|
||||
// GeckoTerminal enrichment — volume/liquidity/txn stats + token USD price
|
||||
// seeds for the pricing waterfall. GT fully covers this chain's Uniswap
|
||||
// deployments (network `robinhood`, per-dex top lists) but each list is capped
|
||||
// at 10 pages × 20 = top 200 — the long tail keeps chain-derived TVL only.
|
||||
// Free tier is 30 calls/min: calls are paced ≥ TUNE.gtPaceMs apart and the
|
||||
// whole cycle (≤30 calls) runs every TUNE.statsMs.
|
||||
//
|
||||
// NOTE: GT has no UP33 dex entry — UP33 pool stats stay on the frontend's
|
||||
// existing dexscreener path; this indexer only serves the Uniswap catalog.
|
||||
import { GT, TUNE, log, sleep } from './config'
|
||||
import { poolRow, setTokenPrice, upsertStats } from './store'
|
||||
|
||||
const LISTS = [
|
||||
{ path: '/networks/robinhood/pools', label: 'network' },
|
||||
{ path: '/networks/robinhood/dexes/uniswap-v2-robinhood/pools', label: 'uni-v2' },
|
||||
{ path: '/networks/robinhood/dexes/uniswap-v3-robinhood/pools', label: 'uni-v3' },
|
||||
]
|
||||
|
||||
type GtPool = {
|
||||
attributes?: {
|
||||
address?: string
|
||||
reserve_in_usd?: string
|
||||
volume_usd?: { h24?: string }
|
||||
transactions?: { h24?: { buys?: number; sells?: number } }
|
||||
base_token_price_usd?: string
|
||||
quote_token_price_usd?: string
|
||||
}
|
||||
relationships?: {
|
||||
base_token?: { data?: { id?: string } }
|
||||
quote_token?: { data?: { id?: string } }
|
||||
}
|
||||
}
|
||||
|
||||
let lastCall = 0
|
||||
async function gtJson(url: string): Promise<{ data?: GtPool[] } | null> {
|
||||
const wait = lastCall + TUNE.gtPaceMs - Date.now()
|
||||
if (wait > 0) await sleep(wait)
|
||||
lastCall = Date.now()
|
||||
try {
|
||||
const r = await fetch(url, { headers: { accept: 'application/json', 'user-agent': 'up33-lp-indexer/0.1' } })
|
||||
if (!r.ok) return null
|
||||
return (await r.json()) as { data?: GtPool[] }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const num = (x: unknown): number | null => {
|
||||
const n = Number(x)
|
||||
return Number.isFinite(n) ? n : null
|
||||
}
|
||||
const tokenOfId = (id?: string): string | null =>
|
||||
id?.startsWith('robinhood_0x') ? id.slice('robinhood_'.length).toLowerCase() : null
|
||||
|
||||
function ingest(p: GtPool): boolean {
|
||||
const a = p.attributes
|
||||
const addr = a?.address?.toLowerCase()
|
||||
if (!a || !addr || !poolRow(addr)) return false // catalog is the gate — unknown pools are ignored
|
||||
const reserve = num(a.reserve_in_usd)
|
||||
const h24 = a.transactions?.h24
|
||||
const txns = h24 ? (h24.buys ?? 0) + (h24.sells ?? 0) : null
|
||||
upsertStats(addr, num(a.volume_usd?.h24), txns, reserve, 'geckoterminal')
|
||||
// token price seeds: ground truth while fresh; depth = half the pool's reserve
|
||||
const depth = (reserve ?? 0) / 2
|
||||
if (depth > 0) {
|
||||
const base = tokenOfId(p.relationships?.base_token?.data?.id)
|
||||
const quote = tokenOfId(p.relationships?.quote_token?.data?.id)
|
||||
const bp = num(a.base_token_price_usd)
|
||||
const qp = num(a.quote_token_price_usd)
|
||||
if (base && bp && bp > 0) setTokenPrice(base, bp, depth, 'gt')
|
||||
if (quote && qp && qp > 0) setTokenPrice(quote, qp, depth, 'gt')
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/** one enrichment cycle over the three GT top lists */
|
||||
export async function gtCycle(): Promise<void> {
|
||||
let matched = 0
|
||||
let seen = 0
|
||||
for (const list of LISTS) {
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const j = await gtJson(`${GT}${list.path}?page=${page}`)
|
||||
const items = j?.data
|
||||
if (!items?.length) break
|
||||
seen += items.length
|
||||
for (const it of items) if (ingest(it)) matched++
|
||||
if (items.length < 20) break
|
||||
}
|
||||
}
|
||||
log(`[stats] gt cycle: ${matched}/${seen} list entries matched catalog`)
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
// SQLite store (node:sqlite — built into node ≥22.13, zero dependencies).
|
||||
// bigints are stored as TEXT and travel as strings through the API; REAL
|
||||
// columns are display/ranking data only, never used to build transactions.
|
||||
import { mkdirSync } from 'node:fs'
|
||||
import { dirname } from 'node:path'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import { DB_PATH, now } from './config'
|
||||
|
||||
mkdirSync(dirname(DB_PATH), { recursive: true })
|
||||
export const db = new DatabaseSync(DB_PATH)
|
||||
|
||||
db.exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pools (
|
||||
address TEXT PRIMARY KEY, -- lowercase
|
||||
proto TEXT NOT NULL, -- 'univ2' | 'univ3'
|
||||
token0 TEXT NOT NULL, -- lowercase
|
||||
token1 TEXT NOT NULL,
|
||||
fee_ppm INTEGER NOT NULL, -- univ2 fixed 3000 (0.30%)
|
||||
tick_spacing INTEGER, -- univ3 only
|
||||
created_block INTEGER, -- univ3 only (from PoolCreated)
|
||||
pair_index INTEGER, -- univ2 only (allPairs index)
|
||||
added_ts INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_pools_t0 ON pools(token0);
|
||||
CREATE INDEX IF NOT EXISTS idx_pools_t1 ON pools(token1);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tokens (
|
||||
address TEXT PRIMARY KEY,
|
||||
symbol TEXT NOT NULL DEFAULT '?',
|
||||
decimals INTEGER NOT NULL DEFAULT 18,
|
||||
meta_ok INTEGER NOT NULL DEFAULT 0, -- 0 = symbol/decimals defaulted (call reverted)
|
||||
price_usd REAL,
|
||||
price_depth_usd REAL NOT NULL DEFAULT 0, -- USD depth backing the price (bigger wins)
|
||||
price_src TEXT, -- 'gt' | 'pool' | 'anchor'
|
||||
price_updated INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pool_state (
|
||||
address TEXT PRIMARY KEY,
|
||||
sqrt_price TEXT, -- univ3
|
||||
tick INTEGER, -- univ3
|
||||
liquidity TEXT, -- univ3 in-range L
|
||||
reserve0 TEXT NOT NULL DEFAULT '0', -- univ2: reserves; univ3: erc20 balances (TVL basis)
|
||||
reserve1 TEXT NOT NULL DEFAULT '0',
|
||||
total_supply TEXT, -- univ2 LP supply
|
||||
tvl_usd REAL,
|
||||
tvl_approx INTEGER NOT NULL DEFAULT 0, -- 1 = only one side priced (tvl = 2× that side)
|
||||
updated INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_state_tvl ON pool_state(tvl_usd);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pool_stats (
|
||||
address TEXT PRIMARY KEY,
|
||||
vol24h_usd REAL,
|
||||
txns24h INTEGER,
|
||||
liq_usd REAL, -- GT's own reserve figure (cross-check; tvl_usd is chain-derived)
|
||||
source TEXT NOT NULL,
|
||||
updated INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS kv (k TEXT PRIMARY KEY, v TEXT NOT NULL);
|
||||
`)
|
||||
|
||||
// ---- kv ----
|
||||
const kvGetQ = db.prepare('SELECT v FROM kv WHERE k = ?')
|
||||
const kvSetQ = db.prepare('INSERT INTO kv (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v')
|
||||
export const kvGet = (k: string): string | undefined => (kvGetQ.get(k) as { v: string } | undefined)?.v
|
||||
export const kvSet = (k: string, v: string) => void kvSetQ.run(k, v)
|
||||
|
||||
// ---- pools ----
|
||||
const insPoolQ = db.prepare(`
|
||||
INSERT OR IGNORE INTO pools (address, proto, token0, token1, fee_ppm, tick_spacing, created_block, pair_index, added_ts)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
/** returns true when the pool is new */
|
||||
export function insertPool(p: {
|
||||
address: string
|
||||
proto: 'univ2' | 'univ3'
|
||||
token0: string
|
||||
token1: string
|
||||
feePpm: number
|
||||
tickSpacing?: number
|
||||
createdBlock?: number
|
||||
pairIndex?: number
|
||||
}): boolean {
|
||||
const r = insPoolQ.run(
|
||||
p.address.toLowerCase(),
|
||||
p.proto,
|
||||
p.token0.toLowerCase(),
|
||||
p.token1.toLowerCase(),
|
||||
p.feePpm,
|
||||
p.tickSpacing ?? null,
|
||||
p.createdBlock ?? null,
|
||||
p.pairIndex ?? null,
|
||||
now(),
|
||||
)
|
||||
return Number(r.changes) > 0
|
||||
}
|
||||
|
||||
export type PoolRow = {
|
||||
address: string
|
||||
proto: 'univ2' | 'univ3'
|
||||
token0: string
|
||||
token1: string
|
||||
fee_ppm: number
|
||||
tick_spacing: number | null
|
||||
}
|
||||
const poolsByAddrQ = db.prepare('SELECT address, proto, token0, token1, fee_ppm, tick_spacing FROM pools WHERE address = ?')
|
||||
export const poolRow = (addr: string) => poolsByAddrQ.get(addr.toLowerCase()) as PoolRow | undefined
|
||||
export const allPoolAddrs = (): string[] =>
|
||||
(db.prepare('SELECT address FROM pools').all() as { address: string }[]).map((r) => r.address)
|
||||
export const poolCounts = () =>
|
||||
db.prepare(`SELECT proto, COUNT(*) AS n FROM pools GROUP BY proto`).all() as { proto: string; n: number }[]
|
||||
|
||||
// ---- tokens ----
|
||||
const insTokenQ = db.prepare(`
|
||||
INSERT INTO tokens (address, symbol, decimals, meta_ok) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(address) DO UPDATE SET symbol = excluded.symbol, decimals = excluded.decimals, meta_ok = excluded.meta_ok`)
|
||||
export const upsertTokenMeta = (addr: string, symbol: string, decimals: number, metaOk: boolean) =>
|
||||
void insTokenQ.run(addr.toLowerCase(), symbol, decimals, metaOk ? 1 : 0)
|
||||
|
||||
const priceQ = db.prepare(`
|
||||
INSERT INTO tokens (address, price_usd, price_depth_usd, price_src, price_updated) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(address) DO UPDATE SET price_usd = excluded.price_usd, price_depth_usd = excluded.price_depth_usd,
|
||||
price_src = excluded.price_src, price_updated = excluded.price_updated`)
|
||||
export const setTokenPrice = (addr: string, usd: number, depthUsd: number, src: string) =>
|
||||
void priceQ.run(addr.toLowerCase(), usd, depthUsd, src, now())
|
||||
|
||||
export type TokenRow = {
|
||||
address: string
|
||||
symbol: string
|
||||
decimals: number
|
||||
meta_ok: number
|
||||
price_usd: number | null
|
||||
price_depth_usd: number
|
||||
price_src: string | null
|
||||
price_updated: number | null
|
||||
}
|
||||
export const allTokens = () => db.prepare('SELECT * FROM tokens').all() as TokenRow[]
|
||||
export const missingMetaTokens = (): string[] =>
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT DISTINCT u.addr FROM (SELECT token0 AS addr FROM pools UNION SELECT token1 FROM pools) u
|
||||
LEFT JOIN tokens t ON t.address = u.addr WHERE t.address IS NULL`,
|
||||
)
|
||||
.all() as { addr: string }[]
|
||||
).map((r) => r.addr)
|
||||
|
||||
// ---- pool_state ----
|
||||
const upStateQ = db.prepare(`
|
||||
INSERT INTO pool_state (address, sqrt_price, tick, liquidity, reserve0, reserve1, total_supply, updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(address) DO UPDATE SET sqrt_price = excluded.sqrt_price, tick = excluded.tick,
|
||||
liquidity = excluded.liquidity, reserve0 = excluded.reserve0, reserve1 = excluded.reserve1,
|
||||
total_supply = excluded.total_supply, updated = excluded.updated`)
|
||||
export const upsertState = (
|
||||
addr: string,
|
||||
s: { sqrtPrice?: bigint; tick?: number; liquidity?: bigint; reserve0: bigint; reserve1: bigint; totalSupply?: bigint },
|
||||
) =>
|
||||
void upStateQ.run(
|
||||
addr.toLowerCase(),
|
||||
s.sqrtPrice !== undefined ? String(s.sqrtPrice) : null,
|
||||
s.tick ?? null,
|
||||
s.liquidity !== undefined ? String(s.liquidity) : null,
|
||||
String(s.reserve0),
|
||||
String(s.reserve1),
|
||||
s.totalSupply !== undefined ? String(s.totalSupply) : null,
|
||||
now(),
|
||||
)
|
||||
|
||||
const tvlQ = db.prepare('UPDATE pool_state SET tvl_usd = ?, tvl_approx = ? WHERE address = ?')
|
||||
export const setTvl = (addr: string, tvl: number | null, approx: boolean) =>
|
||||
void tvlQ.run(tvl, approx ? 1 : 0, addr.toLowerCase())
|
||||
|
||||
// ---- pool_stats ----
|
||||
const upStatsQ = db.prepare(`
|
||||
INSERT INTO pool_stats (address, vol24h_usd, txns24h, liq_usd, source, updated) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(address) DO UPDATE SET vol24h_usd = excluded.vol24h_usd, txns24h = excluded.txns24h,
|
||||
liq_usd = excluded.liq_usd, source = excluded.source, updated = excluded.updated`)
|
||||
export const upsertStats = (addr: string, vol24h: number | null, txns24h: number | null, liqUsd: number | null, source: string) =>
|
||||
void upStatsQ.run(addr.toLowerCase(), vol24h, txns24h, liqUsd, source, now())
|
||||
|
||||
/** hot set: real TVL, or GT-visible activity, or freshly created */
|
||||
export const hotAddrs = (): string[] =>
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT address FROM pool_state WHERE tvl_usd >= ?
|
||||
UNION SELECT address FROM pool_stats WHERE vol24h_usd > 0
|
||||
UNION SELECT address FROM pools WHERE added_ts > ?`,
|
||||
)
|
||||
.all(10_000, now() - 3_600) as { address: string }[]
|
||||
).map((r) => r.address)
|
||||
|
||||
/**
|
||||
* active set for the hourly sweep: anything that ever showed ≥$100 TVL plus
|
||||
* everything younger than 48h. The launchpads mint ~20k dust pools/day — the
|
||||
* 6-hourly census (allPoolAddrs) keeps their state honest, the hourly sweep
|
||||
* stays bounded by real liquidity instead of catalog size.
|
||||
*/
|
||||
export const activeAddrs = (): string[] =>
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT address FROM pool_state WHERE tvl_usd >= ?
|
||||
UNION SELECT address FROM pools WHERE added_ts > ?`,
|
||||
)
|
||||
.all(100, now() - 172_800) as { address: string }[]
|
||||
).map((r) => r.address)
|
||||
|
||||
export const tx = (fn: () => void) => {
|
||||
db.exec('BEGIN')
|
||||
try {
|
||||
fn()
|
||||
db.exec('COMMIT')
|
||||
} catch (e) {
|
||||
db.exec('ROLLBACK')
|
||||
throw e
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user