examples: add real BTCUSDT candle datasets from Binance
Add seven OHLCV datasets under crates/wickra/examples/data/, one per timeframe (1m/5m/15m/1h/12h/1d/1month), holding real BTCUSDT spot klines fetched from the Binance REST API. The new fetch_btcusdt example regenerates them: it paginates the klines endpoint through the system curl, parses with serde_json, validates every candle via Candle::new and keeps only fully closed buckets. The indicator benchmarks now run against the 1m dataset instead of a synthetic series, and a new example_data integration test checks that every file parses and carries evenly spaced, monotonic timestamps. The monthly file is named btcusdt-1month.csv rather than btcusdt-1M.csv so it does not collide with btcusdt-1m.csv on case-insensitive filesystems (Windows, default macOS).
This commit is contained in:
kingchenc
Generated
+1
@@ -1974,6 +1974,7 @@ dependencies = [
|
||||
"approx",
|
||||
"criterion",
|
||||
"proptest",
|
||||
"serde_json",
|
||||
"wickra-core",
|
||||
"wickra-data",
|
||||
]
|
||||
|
||||
@@ -180,7 +180,8 @@ wickra/
|
||||
├── crates/
|
||||
│ ├── wickra-core/ core engine + all 71 indicators
|
||||
│ ├── wickra/ top-level facade crate (publishes on crates.io)
|
||||
│ │ + benches/ and examples/backtest.rs
|
||||
│ │ + benches/, examples/ (backtest, fetch_btcusdt)
|
||||
│ │ and examples/data/ real BTCUSDT datasets
|
||||
│ └── wickra-data/ CSV reader, tick aggregator, live exchange feeds
|
||||
│ + examples/live_binance.rs
|
||||
├── bindings/
|
||||
|
||||
@@ -32,6 +32,9 @@ approx = { workspace = true }
|
||||
criterion = { workspace = true }
|
||||
proptest = { workspace = true }
|
||||
wickra-data = { path = "../wickra-data" }
|
||||
# Only the `fetch_btcusdt` example parses Binance REST JSON; a dev-dependency
|
||||
# never reaches a downstream consumer's dependency tree.
|
||||
serde_json = "1"
|
||||
|
||||
[[bench]]
|
||||
name = "indicators"
|
||||
|
||||
@@ -5,54 +5,52 @@
|
||||
//! cargo bench -p wickra
|
||||
//! ```
|
||||
//!
|
||||
//! Each benchmark feeds a deterministic synthetic price series through both the
|
||||
//! streaming (`update` loop) and batch APIs of an indicator. Sizes cover small
|
||||
//! (1 000), medium (10 000), and large (100 000) workloads.
|
||||
//! Each benchmark feeds real BTCUSDT 1-minute candles — read from the
|
||||
//! checked-in dataset `examples/data/btcusdt-1m.csv` — through both the
|
||||
//! streaming (`update` loop) and batch APIs of an indicator. Sizes cover
|
||||
//! small (1 000), medium (10 000), and large (50 000) workloads, taken as
|
||||
//! prefixes of that dataset.
|
||||
//!
|
||||
//! Regenerate the dataset with:
|
||||
//! ```text
|
||||
//! cargo run -p wickra --example fetch_btcusdt
|
||||
//! ```
|
||||
|
||||
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
|
||||
use wickra::{
|
||||
Atr, BatchExt, BollingerBands, Candle, Ema, Indicator, MacdIndicator, Obv, Rsi, Sma,
|
||||
Stochastic, Wma,
|
||||
};
|
||||
use wickra_data::csv::CandleReader;
|
||||
|
||||
/// Deterministic synthetic price series of length `n`.
|
||||
fn price_series(n: usize) -> Vec<f64> {
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let t = i as f64;
|
||||
100.0 + (t * 0.013).sin() * 12.0 + (t * 0.071).cos() * 4.0 + (t * 0.003).sin() * 30.0
|
||||
})
|
||||
.collect()
|
||||
/// Workload sizes, in candles. Each is taken as a prefix of the dataset.
|
||||
const SIZES: &[usize] = &[1_000, 10_000, 50_000];
|
||||
|
||||
/// Load the checked-in BTCUSDT 1-minute candle dataset.
|
||||
fn load_candles() -> Vec<Candle> {
|
||||
let path = concat!(env!("CARGO_MANIFEST_DIR"), "/examples/data/btcusdt-1m.csv");
|
||||
let mut reader = CandleReader::open(path).unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"could not open the benchmark dataset {path}: {e}\n\
|
||||
generate it with `cargo run -p wickra --example fetch_btcusdt`"
|
||||
)
|
||||
});
|
||||
reader
|
||||
.read_all()
|
||||
.expect("the benchmark dataset is valid OHLCV")
|
||||
}
|
||||
|
||||
/// Synthetic OHLC candle series.
|
||||
fn candle_series(n: usize) -> Vec<Candle> {
|
||||
let closes = price_series(n);
|
||||
closes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, c)| {
|
||||
let t = i as f64;
|
||||
let spread = 0.5 + (t * 0.05).sin().abs();
|
||||
// Benchmark synthetic data: i originates from a usize counter capped at 100_000,
|
||||
// well within i64::MAX. The wrap-around lint does not apply here.
|
||||
#[allow(clippy::cast_possible_wrap)]
|
||||
let ts = i as i64;
|
||||
Candle::new_unchecked(*c, c + spread, c - spread, *c, 1_000.0, ts)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bench_scalar<I, F>(c: &mut Criterion, name: &str, sizes: &[usize], make: F)
|
||||
fn bench_scalar<I, F>(c: &mut Criterion, name: &str, prices: &[f64], make: F)
|
||||
where
|
||||
F: Fn() -> I,
|
||||
I: Indicator<Input = f64, Output = f64> + BatchExt,
|
||||
{
|
||||
let mut group = c.benchmark_group(name);
|
||||
for &n in sizes {
|
||||
let series = price_series(n);
|
||||
for &n in SIZES {
|
||||
let n = n.min(prices.len());
|
||||
let series = &prices[..n];
|
||||
group.throughput(Throughput::Elements(n as u64));
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), &series, |b, prices| {
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), series, |b, prices| {
|
||||
b.iter(|| {
|
||||
let mut ind = make();
|
||||
for p in prices {
|
||||
@@ -60,7 +58,7 @@ where
|
||||
}
|
||||
});
|
||||
});
|
||||
group.bench_with_input(BenchmarkId::new("batch", n), &series, |b, prices| {
|
||||
group.bench_with_input(BenchmarkId::new("batch", n), series, |b, prices| {
|
||||
b.iter(|| {
|
||||
let mut ind = make();
|
||||
black_box(ind.batch(prices));
|
||||
@@ -70,12 +68,13 @@ where
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_macd(c: &mut Criterion, sizes: &[usize]) {
|
||||
fn bench_macd(c: &mut Criterion, prices: &[f64]) {
|
||||
let mut group = c.benchmark_group("macd");
|
||||
for &n in sizes {
|
||||
let series = price_series(n);
|
||||
for &n in SIZES {
|
||||
let n = n.min(prices.len());
|
||||
let series = &prices[..n];
|
||||
group.throughput(Throughput::Elements(n as u64));
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), &series, |b, prices| {
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), series, |b, prices| {
|
||||
b.iter(|| {
|
||||
let mut ind = MacdIndicator::classic();
|
||||
for p in prices {
|
||||
@@ -87,12 +86,13 @@ fn bench_macd(c: &mut Criterion, sizes: &[usize]) {
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_bollinger(c: &mut Criterion, sizes: &[usize]) {
|
||||
fn bench_bollinger(c: &mut Criterion, prices: &[f64]) {
|
||||
let mut group = c.benchmark_group("bollinger");
|
||||
for &n in sizes {
|
||||
let series = price_series(n);
|
||||
for &n in SIZES {
|
||||
let n = n.min(prices.len());
|
||||
let series = &prices[..n];
|
||||
group.throughput(Throughput::Elements(n as u64));
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), &series, |b, prices| {
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), series, |b, prices| {
|
||||
b.iter(|| {
|
||||
let mut ind = BollingerBands::classic();
|
||||
for p in prices {
|
||||
@@ -104,16 +104,17 @@ fn bench_bollinger(c: &mut Criterion, sizes: &[usize]) {
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_candle_input<I, F, O>(c: &mut Criterion, name: &str, sizes: &[usize], make: F)
|
||||
fn bench_candle_input<I, F, O>(c: &mut Criterion, name: &str, candles: &[Candle], make: F)
|
||||
where
|
||||
F: Fn() -> I,
|
||||
I: Indicator<Input = Candle, Output = O>,
|
||||
{
|
||||
let mut group = c.benchmark_group(name);
|
||||
for &n in sizes {
|
||||
let candles = candle_series(n);
|
||||
for &n in SIZES {
|
||||
let n = n.min(candles.len());
|
||||
let series = &candles[..n];
|
||||
group.throughput(Throughput::Elements(n as u64));
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), &candles, |b, candles| {
|
||||
group.bench_with_input(BenchmarkId::new("streaming", n), series, |b, candles| {
|
||||
b.iter(|| {
|
||||
let mut ind = make();
|
||||
for c in candles {
|
||||
@@ -126,16 +127,18 @@ where
|
||||
}
|
||||
|
||||
fn benches(c: &mut Criterion) {
|
||||
let sizes = [1_000_usize, 10_000, 100_000];
|
||||
bench_scalar(c, "sma", &sizes, || Sma::new(14).unwrap());
|
||||
bench_scalar(c, "ema", &sizes, || Ema::new(14).unwrap());
|
||||
bench_scalar(c, "wma", &sizes, || Wma::new(14).unwrap());
|
||||
bench_scalar(c, "rsi", &sizes, || Rsi::new(14).unwrap());
|
||||
bench_macd(c, &sizes);
|
||||
bench_bollinger(c, &sizes);
|
||||
bench_candle_input(c, "atr", &sizes, || Atr::new(14).unwrap());
|
||||
bench_candle_input(c, "stochastic", &sizes, Stochastic::classic);
|
||||
bench_candle_input(c, "obv", &sizes, Obv::new);
|
||||
let candles = load_candles();
|
||||
let closes: Vec<f64> = candles.iter().map(|c| c.close).collect();
|
||||
|
||||
bench_scalar(c, "sma", &closes, || Sma::new(14).unwrap());
|
||||
bench_scalar(c, "ema", &closes, || Ema::new(14).unwrap());
|
||||
bench_scalar(c, "wma", &closes, || Wma::new(14).unwrap());
|
||||
bench_scalar(c, "rsi", &closes, || Rsi::new(14).unwrap());
|
||||
bench_macd(c, &closes);
|
||||
bench_bollinger(c, &closes);
|
||||
bench_candle_input(c, "atr", &candles, || Atr::new(14).unwrap());
|
||||
bench_candle_input(c, "stochastic", &candles, Stochastic::classic);
|
||||
bench_candle_input(c, "obv", &candles, Obv::new);
|
||||
}
|
||||
|
||||
criterion_group!(name = wickra_benches; config = Criterion::default(); targets = benches);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,106 @@
|
||||
timestamp,open,high,low,close,volume
|
||||
1501545600000,4261.48,4745.42,3400,4724.89,10015.640272
|
||||
1504224000000,4689.89,4939.19,2817,4378.51,27634.18912
|
||||
1506816000000,4378.49,6498.01,4110,6463,41626.388463
|
||||
1509494400000,6463,11300.03,5325.01,9838.96,108487.978119
|
||||
1512086400000,9837,19798.68,9380,13716.36,408476.658399
|
||||
1514764800000,13715.65,17176.24,9035,10285.1,816675.564467
|
||||
1517443200000,10285.1,11786.01,6000.01,10326.76,1243940.85531
|
||||
1519862400000,10325.64,11710,6600.1,6923.91,1235326.31402
|
||||
1522540800000,6922,9759.82,6430,9246.01,1110964.015581
|
||||
1525132800000,9246.01,10020,7032.95,7485.01,914476.377885
|
||||
1527811200000,7485.01,7786.69,5750,6390.07,942249.765944
|
||||
1530403200000,6391.08,8491.77,6070,7730.93,1102510.436786
|
||||
1533081600000,7735.67,7750,5880,7011.21,1408159.816556
|
||||
1535760000000,7011.21,7410,6111,6626.57,1100653.423315
|
||||
1538352000000,6626.57,7680,6205,6371.93,629922.086219
|
||||
1541030400000,6369.52,6615.15,3652.66,4041.32,1210366.564567
|
||||
1543622400000,4041.27,4312.99,3156.26,3702.9,1591229.611862
|
||||
1546300800000,3701.23,4069.8,3349.92,3434.1,908244.14054
|
||||
1548979200000,3434.1,4198,3373.1,3813.69,861783.986727
|
||||
1551398400000,3814.26,4140,3670.69,4103.95,787190.48925
|
||||
1554076800000,4102.44,5600,4067,5320.81,1126961.315101
|
||||
1556668800000,5321.94,9074.26,5316.2,8555,1498410.025617
|
||||
1559347200000,8555,13970,7444.58,10854.1,1689489.647326
|
||||
1561939200000,10854.1,13147.08,9060,10080.53,1886176.065915
|
||||
1564617600000,10080.53,12330.7,9320,9587.47,1201961.576316
|
||||
1567296000000,9588.74,10905.87,7710,8289.34,1116856.475836
|
||||
1569888000000,8289.97,10370,7300,9140.85,1446763.024045
|
||||
1572566400000,9140.86,9513.68,6515,7541.89,1499118.774995
|
||||
1575158400000,7540.63,7750,6435,7195.23,1307033.020384
|
||||
1577836800000,7195.24,9578,6871.04,9352.89,1691323.137782
|
||||
1580515200000,9351.71,10500,8445,8523.61,1609726.154564
|
||||
1583020800000,8523.61,9188,3782.13,6410.44,3789768.913125
|
||||
1585699200000,6412.14,9460,6150.11,8620,2528373.691121
|
||||
1588291200000,8620,10067,8117,9448.27,2685340.078508
|
||||
1590969600000,9448.27,10380,8833,9138.55,1504745.517922
|
||||
1593561600000,9138.08,11444,8893.03,11335.46,1507827.21494
|
||||
1596240000000,11335.46,12468,10518.5,11649.51,1891193.007128
|
||||
1598918400000,11649.51,12050.85,9825,10776.59,1730389.160179
|
||||
1601510400000,10776.59,14100,10374,13791,1592634.419946
|
||||
1604188800000,13791,19863.16,13195.05,19695.87,2707064.911165
|
||||
1606780800000,19695.87,29300,17572.33,28923.63,2495281.856217
|
||||
1609459200000,28923.63,41950,28130,33092.98,3440864.750019
|
||||
1612137600000,33092.97,58352.8,32296.16,45135.66,2518242.148517
|
||||
1614556800000,45134.11,61844,44950.53,58740.55,2098808.027432
|
||||
1617235200000,58739.46,64854,46930,57694.27,1993468.938007
|
||||
1619827200000,57697.25,59500,30000,37253.81,3536245.256573
|
||||
1622505600000,37253.82,41330,28805,35045,2901775.305923
|
||||
1625097600000,35045,42448,29278,41461.83,1778463.264837
|
||||
1627776000000,41461.84,50500,37332.7,47100.89,1635402.874245
|
||||
1630454400000,47100.89,52920,39600,43824.1,1527799.510768
|
||||
1633046400000,43820.01,67000,43283.03,61299.8,1565556.292623
|
||||
1635724800000,61299.81,69000,53256.64,56950.56,1291900.105248
|
||||
1638316800000,56950.56,59053.55,42000.3,46216.93,1233745.524318
|
||||
1640995200000,46216.93,47990,32917.17,38466.9,1279407.465721
|
||||
1643673600000,38466.9,45821,34322.28,43160,1253514.21906
|
||||
1646092800000,43160,48189.84,37155,45510.34,1501398.79591
|
||||
1648771200000,45510.35,47444.11,37578.2,37630.8,1267655.68178
|
||||
1651363200000,37630.8,40023.77,26700,31801.04,2387839.680804
|
||||
1654041600000,31801.05,31982.97,17622,19942.21,2816058.473226
|
||||
1656633600000,19942.21,24668,18781,23293.32,4983278.5881
|
||||
1659312000000,23296.36,25211.32,19520,20050.02,5692462.41571
|
||||
1661990400000,20048.44,22799,18125.98,19422.61,9838930.53657
|
||||
1664582400000,19422.61,21085,18190,20490.74,7499121.81542
|
||||
1667260800000,20490.74,21480.65,15476,17163.64,9127693.509065
|
||||
1669852800000,17165.53,18387.95,16256.3,16542.4,5803833.88187
|
||||
1672531200000,16541.77,23960.54,16499.01,23125.13,7977028.87801
|
||||
1675209600000,23125.13,25250,21351.07,23141.57,8642691.27165
|
||||
1677628800000,23141.57,29184.68,19549.09,28465.36,9516189.35846
|
||||
1680307200000,28465.36,31000,26942.82,29233.21,1626745.5585
|
||||
1682899200000,29233.2,29820,25811.46,27210.35,1302000.49221
|
||||
1685577600000,27210.36,31431.94,24800,30472,1387207.48275
|
||||
1688169600000,30471.99,31804.2,28861.9,29232.25,925773.81731
|
||||
1690848000000,29232.26,30244,25166,25940.78,1025866.55023
|
||||
1693526400000,25940.77,27483.57,24901,26962.56,809329.04893
|
||||
1696118400000,26962.57,35280,26538.66,34639.77,1141403.6799
|
||||
1698796800000,34639.78,38450,34097.39,37723.96,1055690.59638
|
||||
1701388800000,37723.97,44700,37615.86,42283.58,1195409.976
|
||||
1704067200000,42283.58,48969.48,38555,42580,1403408.84978
|
||||
1706745600000,42580,64000,41884.28,61130.98,1206112.69545
|
||||
1709251200000,61130.99,73777,59005,71280.01,1706807.381342
|
||||
1711929600000,71280,72797.99,59191.6,60672,1201500.95852
|
||||
1714521600000,60672.01,71979,56552.82,67540.01,945031.04072
|
||||
1717200000000,67540.01,71997.02,58402,62772.01,696818.18818
|
||||
1719792000000,62772.01,70079.99,53485.93,64628,908004.33426
|
||||
1722470400000,64628.01,65659.78,49000,58973.99,1010291.47396
|
||||
1725148800000,58974,66498,52550,63327.59,734117.07575
|
||||
1727740800000,63327.6,73620.12,58946,70292.01,756010.86343
|
||||
1730419200000,70292.01,99588.01,66835,96407.99,1343559.242196
|
||||
1733011200000,96407.99,108353,90500,93576,1019450.065773
|
||||
1735689600000,93576,109588,89256.69,102429.56,864534.738322
|
||||
1738368000000,102429.56,102783.71,78258.52,84349.94,810850.1813
|
||||
1740787200000,84349.95,95000,76606,82550.01,845293.53101
|
||||
1743465600000,82550,95758.04,74508,94172,793597.00179
|
||||
1746057600000,94172,111980,93377,104591.88,642216.546125
|
||||
1748736000000,104591.88,110530.17,98200,107146.5,427546.46336
|
||||
1751328000000,107146.51,123218,105100.19,115764.08,484315.651017
|
||||
1754006400000,115764.07,124474,107350.1,108246.35,471366.942936
|
||||
1756684800000,108246.36,117900,107255,114048.93,374551.99407
|
||||
1759276800000,114048.94,126199.63,102000,109608.01,720300.285006
|
||||
1761955200000,109608.01,111250.01,80600,90360,784853.66178
|
||||
1764547200000,90360.01,94588.99,83822.76,87648.22,491084.34878
|
||||
1767225600000,87648.21,97924.49,75719.9,78741.09,491752.73296
|
||||
1769904000000,78741.1,79424,60000,66973.26,837887.67719
|
||||
1772323200000,66973.26,76000,65000,68284.48,705639.88404
|
||||
1775001600000,68284.49,79485.66,65712.12,76346.57,473256.46982
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,253 @@
|
||||
//! Rust example: download real BTCUSDT spot candles from the Binance REST API
|
||||
//! and write them as CSV datasets under `examples/data/`.
|
||||
//!
|
||||
//! These are the datasets the indicator benchmarks (`benches/indicators.rs`)
|
||||
//! and the `example_data` integration test run against. Re-run this example
|
||||
//! to refresh them with the latest market history:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run -p wickra --example fetch_btcusdt
|
||||
//! ```
|
||||
//!
|
||||
//! HTTPS is handled by shelling out to the system `curl` — shipped with
|
||||
//! Windows 10+, macOS, and every Linux distribution — so this example adds no
|
||||
//! HTTP or TLS dependency to the crate (`serde_json` is the only extra, and
|
||||
//! only as a dev-dependency). Each Binance klines response is capped at 1000
|
||||
//! rows, so larger datasets are paginated backwards through `endTime`. Only
|
||||
//! fully closed candles are kept; the still-forming candle of the current
|
||||
//! bucket is dropped so the datasets stay reproducible.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::error::Error;
|
||||
use std::fmt::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde_json::Value as Json;
|
||||
use wickra::Candle;
|
||||
|
||||
/// Binance Spot REST endpoint for historical klines.
|
||||
const KLINES_URL: &str = "https://api.binance.com/api/v3/klines";
|
||||
/// Trading pair to download.
|
||||
const SYMBOL: &str = "BTCUSDT";
|
||||
/// Binance caps a single klines response at 1000 rows.
|
||||
const PAGE_LIMIT: usize = 1000;
|
||||
/// Courtesy pause between paginated requests to stay well under the rate limit.
|
||||
const REQUEST_PAUSE: Duration = Duration::from_millis(200);
|
||||
|
||||
/// One dataset to produce: the Binance interval code, the output file name,
|
||||
/// and how many of the most-recent *closed* candles to collect.
|
||||
struct Dataset {
|
||||
interval: &'static str,
|
||||
file: &'static str,
|
||||
target: usize,
|
||||
}
|
||||
|
||||
/// The seven datasets, one per timeframe. `12h`/`1d`/`1M` simply collect all
|
||||
/// the history Binance offers — it is shorter than their `target`.
|
||||
///
|
||||
/// The monthly file is named `btcusdt-1month.csv`, not `btcusdt-1M.csv`: on
|
||||
/// case-insensitive filesystems (Windows, default macOS) the latter would
|
||||
/// collide with `btcusdt-1m.csv` and one dataset would silently overwrite the
|
||||
/// other.
|
||||
const DATASETS: &[Dataset] = &[
|
||||
Dataset {
|
||||
interval: "1m",
|
||||
file: "btcusdt-1m.csv",
|
||||
target: 50_000,
|
||||
},
|
||||
Dataset {
|
||||
interval: "5m",
|
||||
file: "btcusdt-5m.csv",
|
||||
target: 10_000,
|
||||
},
|
||||
Dataset {
|
||||
interval: "15m",
|
||||
file: "btcusdt-15m.csv",
|
||||
target: 10_000,
|
||||
},
|
||||
Dataset {
|
||||
interval: "1h",
|
||||
file: "btcusdt-1h.csv",
|
||||
target: 10_000,
|
||||
},
|
||||
Dataset {
|
||||
interval: "12h",
|
||||
file: "btcusdt-12h.csv",
|
||||
target: 5_000,
|
||||
},
|
||||
Dataset {
|
||||
interval: "1d",
|
||||
file: "btcusdt-1d.csv",
|
||||
target: 5_000,
|
||||
},
|
||||
Dataset {
|
||||
interval: "1M",
|
||||
file: "btcusdt-1month.csv",
|
||||
target: 5_000,
|
||||
},
|
||||
];
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
// A single reference instant for the whole run: any candle whose bucket has
|
||||
// not closed by this point is treated as in-progress and dropped.
|
||||
let now_ms = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())
|
||||
.map_err(|_| "system clock is beyond the i64 millisecond range")?;
|
||||
|
||||
let data_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("examples")
|
||||
.join("data");
|
||||
std::fs::create_dir_all(&data_dir)?;
|
||||
|
||||
println!(
|
||||
"Fetching {SYMBOL} klines from Binance into {}",
|
||||
data_dir.display()
|
||||
);
|
||||
for ds in DATASETS {
|
||||
let candles = collect(ds.interval, ds.target, now_ms)?;
|
||||
if candles.is_empty() {
|
||||
return Err(format!(
|
||||
"Binance returned no closed candles for interval {}",
|
||||
ds.interval
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let path = data_dir.join(ds.file);
|
||||
write_csv(&path, &candles)?;
|
||||
println!(
|
||||
" {:>3} {:>6} candles -> examples/data/{}",
|
||||
ds.interval,
|
||||
candles.len(),
|
||||
ds.file
|
||||
);
|
||||
}
|
||||
println!("Done — {} datasets written.", DATASETS.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Paginate the Binance REST API backwards until `target` closed candles have
|
||||
/// been collected (or the exchange runs out of history), then return the most
|
||||
/// recent `target` of them in ascending time order.
|
||||
fn collect(interval: &str, target: usize, now_ms: i64) -> Result<Vec<Candle>, Box<dyn Error>> {
|
||||
// Keyed by open time: the map sorts chronologically and dedupes the small
|
||||
// overlap that can occur between adjacent pages.
|
||||
let mut by_open: BTreeMap<i64, Candle> = BTreeMap::new();
|
||||
let mut end_time: Option<i64> = None;
|
||||
let mut pages = 0usize;
|
||||
|
||||
loop {
|
||||
let page = fetch_page(interval, end_time)?;
|
||||
pages += 1;
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut oldest_open = i64::MAX;
|
||||
for raw in &page {
|
||||
let Some((open_time, close_time, candle)) = parse_kline(raw) else {
|
||||
continue;
|
||||
};
|
||||
oldest_open = oldest_open.min(open_time);
|
||||
// Keep only fully closed candles — drop the in-progress bucket.
|
||||
if close_time < now_ms {
|
||||
by_open.insert(open_time, candle);
|
||||
}
|
||||
}
|
||||
eprint!(
|
||||
"\r {interval}: collected {} candles over {pages} page(s)…",
|
||||
by_open.len()
|
||||
);
|
||||
|
||||
// Stop once enough is collected or Binance has no older history left.
|
||||
if by_open.len() >= target || page.len() < PAGE_LIMIT {
|
||||
break;
|
||||
}
|
||||
if oldest_open == i64::MAX {
|
||||
return Err(format!("Binance page for {interval} held no parseable klines").into());
|
||||
}
|
||||
// Step the window strictly before the oldest open time seen so far.
|
||||
end_time = Some(oldest_open - 1);
|
||||
std::thread::sleep(REQUEST_PAUSE);
|
||||
}
|
||||
eprintln!();
|
||||
|
||||
let mut candles: Vec<Candle> = by_open.into_values().collect();
|
||||
if candles.len() > target {
|
||||
// Trim the oldest surplus, keeping the most recent `target` candles.
|
||||
candles.drain(..candles.len() - target);
|
||||
}
|
||||
Ok(candles)
|
||||
}
|
||||
|
||||
/// Fetch one page of klines. `end_time`, when set, caps the open time so the
|
||||
/// caller can walk backwards through history.
|
||||
fn fetch_page(interval: &str, end_time: Option<i64>) -> Result<Vec<Json>, Box<dyn Error>> {
|
||||
let mut url = format!("{KLINES_URL}?symbol={SYMBOL}&interval={interval}&limit={PAGE_LIMIT}");
|
||||
if let Some(end) = end_time {
|
||||
write!(url, "&endTime={end}").expect("writing to a String never fails");
|
||||
}
|
||||
|
||||
let output = Command::new("curl")
|
||||
.args([
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--fail",
|
||||
"--max-time",
|
||||
"30",
|
||||
&url,
|
||||
])
|
||||
.output()
|
||||
.map_err(|e| format!("could not run `curl` (install it / put it on PATH): {e}"))?;
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(format!("curl failed for {url}: {}", stderr.trim()).into());
|
||||
}
|
||||
|
||||
let body = String::from_utf8(output.stdout)?;
|
||||
match serde_json::from_str(&body)? {
|
||||
Json::Array(rows) => Ok(rows),
|
||||
other => Err(format!("expected a JSON array of klines from Binance, got: {other}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse one raw Binance kline array into `(open_time, close_time, Candle)`.
|
||||
///
|
||||
/// A Binance kline is `[openTime, "open", "high", "low", "close", "volume",
|
||||
/// closeTime, …]`; the OHLCV fields arrive as decimal strings. Returns `None`
|
||||
/// for any row that is malformed or fails `Candle::new`'s OHLC validation.
|
||||
fn parse_kline(raw: &Json) -> Option<(i64, i64, Candle)> {
|
||||
let arr = raw.as_array()?;
|
||||
if arr.len() < 7 {
|
||||
return None;
|
||||
}
|
||||
let open_time = arr[0].as_i64()?;
|
||||
let close_time = arr[6].as_i64()?;
|
||||
let field = |i: usize| -> Option<f64> { arr[i].as_str()?.parse::<f64>().ok() };
|
||||
let candle = Candle::new(
|
||||
field(1)?,
|
||||
field(2)?,
|
||||
field(3)?,
|
||||
field(4)?,
|
||||
field(5)?,
|
||||
open_time,
|
||||
)
|
||||
.ok()?;
|
||||
Some((open_time, close_time, candle))
|
||||
}
|
||||
|
||||
/// Write candles as a standard `timestamp,open,high,low,close,volume` CSV that
|
||||
/// [`wickra_data::csv::CandleReader`] can read back.
|
||||
fn write_csv(path: &Path, candles: &[Candle]) -> Result<(), Box<dyn Error>> {
|
||||
let mut out = String::with_capacity(candles.len() * 80 + 32);
|
||||
out.push_str("timestamp,open,high,low,close,volume\n");
|
||||
for c in candles {
|
||||
writeln!(
|
||||
out,
|
||||
"{},{},{},{},{},{}",
|
||||
c.timestamp, c.open, c.high, c.low, c.close, c.volume
|
||||
)?;
|
||||
}
|
||||
std::fs::write(path, out)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
//! Integration test: the checked-in BTCUSDT example datasets parse cleanly,
|
||||
//! hold enough rows, and carry strictly increasing — and, for the fixed
|
||||
//! timeframes, evenly spaced — timestamps.
|
||||
//!
|
||||
//! The datasets live in `examples/data/` and are produced by the
|
||||
//! `fetch_btcusdt` example. Regenerate them with:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run -p wickra --example fetch_btcusdt
|
||||
//! ```
|
||||
|
||||
use wickra_data::csv::CandleReader;
|
||||
|
||||
/// `(file name, minimum row count, expected step in ms)`. The step is `None`
|
||||
/// for the monthly file, whose buckets are 28–31 days and thus uneven.
|
||||
const DATASETS: &[(&str, usize, Option<i64>)] = &[
|
||||
("btcusdt-1m.csv", 50_000, Some(60_000)),
|
||||
("btcusdt-5m.csv", 10_000, Some(300_000)),
|
||||
("btcusdt-15m.csv", 10_000, Some(900_000)),
|
||||
("btcusdt-1h.csv", 10_000, Some(3_600_000)),
|
||||
("btcusdt-12h.csv", 5_000, Some(43_200_000)),
|
||||
// 1d and 1month collect all the history Binance offers, which grows over
|
||||
// time — assert a lower bound rather than an exact count.
|
||||
("btcusdt-1d.csv", 3_000, Some(86_400_000)),
|
||||
("btcusdt-1month.csv", 100, None),
|
||||
];
|
||||
|
||||
fn dataset_path(file: &str) -> String {
|
||||
format!("{}/examples/data/{file}", env!("CARGO_MANIFEST_DIR"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_dataset_parses_and_is_well_formed() {
|
||||
for &(file, min_rows, step) in DATASETS {
|
||||
let path = dataset_path(file);
|
||||
let mut reader =
|
||||
CandleReader::open(&path).unwrap_or_else(|e| panic!("{file}: cannot open {path}: {e}"));
|
||||
// `read_all` validates every row through `Candle::new`, so a successful
|
||||
// read already proves each OHLC tuple is finite and internally
|
||||
// consistent (high >= low, etc.).
|
||||
let candles = reader
|
||||
.read_all()
|
||||
.unwrap_or_else(|e| panic!("{file}: invalid OHLCV row: {e}"));
|
||||
|
||||
assert!(
|
||||
candles.len() >= min_rows,
|
||||
"{file}: expected at least {min_rows} rows, got {}",
|
||||
candles.len()
|
||||
);
|
||||
|
||||
for pair in candles.windows(2) {
|
||||
let (prev, next) = (pair[0], pair[1]);
|
||||
assert!(
|
||||
next.timestamp > prev.timestamp,
|
||||
"{file}: timestamps must strictly increase, saw {} then {}",
|
||||
prev.timestamp,
|
||||
next.timestamp
|
||||
);
|
||||
if let Some(step) = step {
|
||||
assert_eq!(
|
||||
next.timestamp - prev.timestamp,
|
||||
step,
|
||||
"{file}: a fixed timeframe must be evenly spaced by {step} ms"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -152,6 +152,37 @@ A runnable example lives at `crates/wickra-data/examples/live_binance.rs`:
|
||||
cargo run -p wickra-data --example live_binance --features live-binance
|
||||
```
|
||||
|
||||
## Example datasets
|
||||
|
||||
The repository ships seven ready-to-use OHLCV datasets under
|
||||
`crates/wickra/examples/data/`, one per timeframe, holding real Binance
|
||||
**BTCUSDT** spot candles in the standard `timestamp,open,high,low,close,volume`
|
||||
layout the `CandleReader` reads. The timestamp is each candle's open time in
|
||||
milliseconds.
|
||||
|
||||
| File | Timeframe | Rows |
|
||||
| --- | --- | --- |
|
||||
| `btcusdt-1m.csv` | 1 minute | 50 000 |
|
||||
| `btcusdt-5m.csv` | 5 minutes | 10 000 |
|
||||
| `btcusdt-15m.csv` | 15 minutes | 10 000 |
|
||||
| `btcusdt-1h.csv` | 1 hour | 10 000 |
|
||||
| `btcusdt-12h.csv` | 12 hours | 5 000 |
|
||||
| `btcusdt-1d.csv` | 1 day | full history |
|
||||
| `btcusdt-1month.csv` | 1 month | full history |
|
||||
|
||||
The monthly file is named `btcusdt-1month.csv` rather than `btcusdt-1M.csv` so
|
||||
it does not collide with `btcusdt-1m.csv` on case-insensitive filesystems. The
|
||||
indicator benchmarks and the `example_data` integration test both run against
|
||||
these files.
|
||||
|
||||
Regenerate them with the latest market history — this downloads from the
|
||||
Binance REST API and needs the system `curl` (shipped with Windows 10+, macOS
|
||||
and Linux):
|
||||
|
||||
```bash
|
||||
cargo run -p wickra --example fetch_btcusdt
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Quickstart: Rust](Quickstart-Rust.md) — the core indicator API.
|
||||
|
||||
Reference in New Issue
Block a user