32 lines
1.2 KiB
Bash
32 lines
1.2 KiB
Bash
|
#!/bin/bash
|
||
|
|
# Massive ingest of the wallet data we still need, from the FAST subgraph
|
||
|
|
# (accounts + market_positions). The CLOB token map is already sufficient
|
||
|
|
# (1.27M tokens covering every resolved condition), so it's NOT re-run here —
|
||
|
|
# run `python3 clob_tokens.py` separately if you want to refresh it.
|
||
|
|
#
|
||
|
|
# Both steps are parallel (16 id-shards) and resumable (per-shard cursors), so
|
||
|
|
# killing and re-running continues from the last checkpoint.
|
||
|
|
#
|
||
|
|
# nohup ./run_full.sh > run_full.log 2>&1 < /dev/null & disown
|
||
|
|
# tail -f run_full.log
|
||
|
|
set -u
|
||
|
|
cd "$(dirname "$0")"
|
||
|
|
|
||
|
|
until python3 -c "import duckdb;duckdb.connect('pmkt.duckdb',read_only=True).close()" 2>/dev/null; do
|
||
|
|
sleep 5
|
||
|
|
done
|
||
|
|
|
||
|
|
echo "[run] $(date '+%F %T') 1/2 accounts (parallel, resumable) …"
|
||
|
|
python3 ingest.py -p accounts
|
||
|
|
|
||
|
|
echo "[run] $(date '+%F %T') 2/2 market_positions (parallel, resumable, the big one) …"
|
||
|
|
python3 ingest.py -p market_positions
|
||
|
|
|
||
|
|
echo "[run] $(date '+%F %T') DONE"
|
||
|
|
python3 - <<'PY'
|
||
|
|
import duckdb
|
||
|
|
c = duckdb.connect("pmkt.duckdb", read_only=True)
|
||
|
|
for t in ("conditions", "market_data", "accounts", "market_positions"):
|
||
|
|
print(f" {t:18} {c.execute('select count(*) from '+t).fetchone()[0]:,}")
|
||
|
|
PY
|