From 303ff0a1635e158b89cee0f789bc74975b25d41b Mon Sep 17 00:00:00 2001 From: kingchenc Date: Sat, 23 May 2026 00:40:27 +0200 Subject: [PATCH] examples(node): add a fetch_btcusdt script using built-in fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node had no sibling for the Rust and Python `fetch_btcusdt` data-generators — adding it closes the "fetch (data-gen)" cell for the last remaining row of the cross-language example matrix where the pattern makes sense. * examples/node/fetch_btcusdt.js — uses Node 18+'s built-in global `fetch` (no npm dependencies); same pagination logic as the Rust and Python siblings (paginate backwards via `endTime`, drop the in-progress bucket, sort and trim to the configured target). Applies the same OHLC validity check the Rust `Candle::new` constructor enforces so a malformed kline is skipped rather than written. * JavaScript's `String(v)` already gives the shortest round-trip representation and strips the `.0` suffix for whole-number floats, so the CSV output is byte-for-byte identical to what the Rust and Python fetchers produce on the same Binance snapshot. Verified by running it and `git diff`-ing against the checked-in dataset: every row older than the run is unchanged; only the most recent ~24 hours drift because the market kept moving. examples/README.md gains the new row. --- examples/README.md | 1 + examples/node/fetch_btcusdt.js | 163 +++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 examples/node/fetch_btcusdt.js diff --git a/examples/README.md b/examples/README.md index 9085d097..48c868ac 100644 --- a/examples/README.md +++ b/examples/README.md @@ -47,6 +47,7 @@ cd ../../examples/node && npm install # links wickra + installs `ws` | `multi_timeframe.js` | Roll a 1-minute CSV up to 5m / 15m / 1h / 4h / 1d and print indicators per timeframe. | `node multi_timeframe.js [path/to/1m.csv]` | | `parallel_assets.js` | Serial vs `worker_threads` pool over a synthetic panel, with speedup. | `node parallel_assets.js --assets 200 --bars 5000` | | `live_trading.js` | Live Binance feed → RSI / MACD / Bollinger → signals. | `node live_trading.js --symbol BTCUSDT --interval 1m` | +| `fetch_btcusdt.js` | Download real BTCUSDT klines from the Binance REST API into `examples/data/` (built-in `fetch`, Node 18+). | `node fetch_btcusdt.js` | ## WebAssembly — `examples/wasm/` diff --git a/examples/node/fetch_btcusdt.js b/examples/node/fetch_btcusdt.js new file mode 100644 index 00000000..9a9fae6a --- /dev/null +++ b/examples/node/fetch_btcusdt.js @@ -0,0 +1,163 @@ +// Download real BTCUSDT spot candles from the Binance REST API and write +// them as CSV datasets under `examples/data/`. +// +// The Node counterpart of `examples/rust/src/bin/fetch_btcusdt.rs` and +// `examples/python/fetch_btcusdt.py` — same pagination, same validity +// check, same output layout. Uses Node's built-in global `fetch` and +// `node:fs` only; zero npm dependencies beyond the bundled `wickra` +// loader (which this script does not even import — it just talks HTTP +// and writes CSV). +// +// Requires Node 18 or newer (`fetch` is built-in there). +// +// Run with: +// node fetch_btcusdt.js + +const fs = require('node:fs'); +const path = require('node:path'); + +// Binance Spot REST endpoint for historical klines. +const KLINES_URL = 'https://api.binance.com/api/v3/klines'; +const SYMBOL = 'BTCUSDT'; +// Binance caps a single klines response at 1000 rows. +const PAGE_LIMIT = 1000; +const REQUEST_PAUSE_MS = 200; + +/** @type {{interval: string, file: string, target: number}[]} */ +const DATASETS = [ + { interval: '1m', file: 'btcusdt-1m.csv', target: 50_000 }, + { interval: '5m', file: 'btcusdt-5m.csv', target: 10_000 }, + { interval: '15m', file: 'btcusdt-15m.csv', target: 10_000 }, + { interval: '1h', file: 'btcusdt-1h.csv', target: 10_000 }, + { interval: '12h', file: 'btcusdt-12h.csv', target: 5_000 }, + { interval: '1d', file: 'btcusdt-1d.csv', target: 5_000 }, + // The monthly file is `btcusdt-1month.csv`, not `btcusdt-1M.csv`, so it + // does not collide with `btcusdt-1m.csv` on case-insensitive filesystems. + { interval: '1M', file: 'btcusdt-1month.csv', target: 5_000 }, +]; + +// Parse one Binance kline array. Returns null for any row that is malformed +// or fails the same OHLC validity check `Candle::new` applies in the Rust +// core. +function parseKline(raw) { + if (!Array.isArray(raw) || raw.length < 7) return null; + const openTime = Number(raw[0]); + const closeTime = Number(raw[6]); + const o = Number(raw[1]); + const h = Number(raw[2]); + const l = Number(raw[3]); + const c = Number(raw[4]); + const v = Number(raw[5]); + if (![openTime, closeTime, o, h, l, c, v].every(Number.isFinite)) return null; + if (h < l || h < o || h < c || l > o || l > c || v < 0) return null; + return { openTime, closeTime, o, h, l, c, v }; +} + +async function fetchPage(interval, endTime) { + const params = new URLSearchParams({ + symbol: SYMBOL, + interval, + limit: String(PAGE_LIMIT), + }); + if (endTime !== null) params.set('endTime', String(endTime)); + const url = `${KLINES_URL}?${params}`; + + const resp = await fetch(url, { signal: AbortSignal.timeout(30000) }); + if (!resp.ok) { + throw new Error(`Binance returned HTTP ${resp.status} for ${url}`); + } + const data = await resp.json(); + if (!Array.isArray(data)) { + const sample = JSON.stringify(data).slice(0, 160); + throw new Error(`expected a JSON array from Binance, got: ${sample}`); + } + return data; +} + +async function collect(interval, target, nowMs) { + // Keyed by open time so the map sorts chronologically and dedupes the + // small overlap that can occur between adjacent pages. + const byOpen = new Map(); + let endTime = null; + let pages = 0; + + while (true) { + const page = await fetchPage(interval, endTime); + pages += 1; + if (page.length === 0) break; + + let oldest = Number.MAX_SAFE_INTEGER; + for (const raw of page) { + const k = parseKline(raw); + if (!k) continue; + if (k.openTime < oldest) oldest = k.openTime; + // Keep only fully closed candles — drop the in-progress bucket. + if (k.closeTime < nowMs) byOpen.set(k.openTime, k); + } + process.stderr.write( + `\r ${interval}: collected ${byOpen.size} candles over ${pages} page(s)…`, + ); + + if (byOpen.size >= target || page.length < PAGE_LIMIT) break; + if (oldest === Number.MAX_SAFE_INTEGER) { + throw new Error(`Binance page for ${interval} held no parseable klines`); + } + endTime = oldest - 1; + await new Promise((r) => setTimeout(r, REQUEST_PAUSE_MS)); + } + process.stderr.write('\n'); + + const sorted = [...byOpen.values()].sort((a, b) => a.openTime - b.openTime); + return sorted.length > target ? sorted.slice(sorted.length - target) : sorted; +} + +// JavaScript's `String(v)` already produces the shortest round-trip +// representation for finite numbers, and (unlike Python's repr) it already +// strips the `.0` suffix for whole-number floats — `String(77552.0)` is +// `'77552'`. The CSVs the Rust, Python and Node fetchers write are +// byte-for-byte identical on the same Binance snapshot. +function fmtNumber(v) { + return String(v); +} + +function writeCsv(filePath, candles) { + const out = ['timestamp,open,high,low,close,volume']; + for (const k of candles) { + out.push( + `${k.openTime},${fmtNumber(k.o)},${fmtNumber(k.h)},${fmtNumber(k.l)},${fmtNumber(k.c)},${fmtNumber(k.v)}`, + ); + } + fs.writeFileSync(filePath, out.join('\n') + '\n'); +} + +async function main() { + if (typeof fetch !== 'function') { + console.error( + 'This example needs Node 18+ for the global fetch() function.', + ); + process.exit(1); + } + + const nowMs = Date.now(); + const dataDir = path.join(__dirname, '..', 'data'); + fs.mkdirSync(dataDir, { recursive: true }); + console.log(`Fetching ${SYMBOL} klines from Binance into ${dataDir}`); + + for (const ds of DATASETS) { + const candles = await collect(ds.interval, ds.target, nowMs); + if (candles.length === 0) { + console.error(`error: Binance returned no closed candles for ${ds.interval}`); + process.exit(1); + } + writeCsv(path.join(dataDir, ds.file), candles); + console.log( + ` ${ds.interval.padStart(3)} ${String(candles.length).padStart(6)} candles -> examples/data/${ds.file}`, + ); + } + console.log(`Done — ${DATASETS.length} datasets written.`); +} + +main().catch((err) => { + console.error(`error: ${err.message || err}`); + process.exit(1); +});