first commit

This commit is contained in:
2026-07-12 04:18:40 +08:00
commit 0d735e43c2
555 changed files with 131932 additions and 0 deletions
+69
View File
@@ -0,0 +1,69 @@
[package]
name = "wickra-data"
description = "Data sources for Wickra: CSV readers, tick-to-candle aggregator, and live exchange feeds."
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
readme.workspace = true
keywords.workspace = true
categories.workspace = true
documentation = "https://docs.rs/wickra-data"
# Render the docs on docs.rs with every feature enabled so the optional
# live-binance feed is documented (otherwise it is hidden behind its feature).
[package.metadata.docs.rs]
all-features = true
[lints]
workspace = true
[dependencies]
# Direct path+version (not the workspace inheritance) so `default-features = false`
# is honoured: depending on wickra-data must NOT force wickra-core's `parallel`
# (rayon) feature on — the WASM binding needs a rayon-free build and the data
# layer never uses the parallel batch path. Native bindings re-enable `parallel`
# through their own wickra-core dependency (cargo unifies the features).
wickra-core = { path = "../wickra-core", version = "0.9.9", default-features = false }
thiserror = { workspace = true }
csv = "1.3"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
# Async / live feeds are opt-in: only pulled when a `live-*` feature is requested.
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "net", "time", "io-util"], optional = true }
tokio-tungstenite = { version = "0.29", optional = true, features = ["native-tls"] }
futures-util = { version = "0.3", optional = true }
url = { version = "2", optional = true }
# Blocking HTTP client for the historical REST kline fetcher. ureq 2.x with
# native-tls verifies against the OS trust store (SChannel / Security.framework /
# OpenSSL) — the same backend tokio-tungstenite uses above — so it ships no
# bundled CA roots. (ureq 3.x hard-depends on `webpki-root-certs` regardless of
# TLS backend, which is CDLA-Permissive-2.0 and dead weight under native-tls.)
ureq = { version = "2", default-features = false, features = ["native-tls"], optional = true }
# Direct dependency so the agent can be built with a native-tls connector (ureq
# 2.x does not auto-configure native-tls). Already in the tree via tokio-tungstenite.
native-tls = { version = "0.2", optional = true }
[features]
default = []
# Each exchange is gated so users only pay for the WS/REST stack they actually
# want. `live-binance` covers both the live WebSocket feed and the historical
# REST kline fetcher.
live-binance = ["dep:tokio", "dep:tokio-tungstenite", "dep:futures-util", "dep:url", "dep:ureq", "dep:native-tls"]
# `live-binance` with a statically built OpenSSL instead of the system one. The
# native-tls stack (tokio-tungstenite + ureq, unified on the same `native-tls`
# crate) links `openssl-sys`, which needs OpenSSL at build time. The manylinux
# and musllinux wheel containers do not provide it — manylinux lacks the headers
# and the musllinux build cross-compiles against a musl sysroot that has no
# OpenSSL at all — so the Linux wheels are built with this feature, which
# compiles OpenSSL from source and links it statically. No-op on macOS/Windows,
# where native-tls uses Security.framework / SChannel and never pulls openssl-sys.
vendored-tls = ["live-binance", "native-tls/vendored"]
[dev-dependencies]
approx = { workspace = true }
tempfile = "3"
+614
View File
@@ -0,0 +1,614 @@
//! Roll trade ticks up into candles of an arbitrary timeframe.
use crate::error::{Error, Result};
use wickra_core::{Candle, Tick};
/// Hard cap on the number of placeholder candles a single
/// [`TickAggregator::push`] call may emit when gap-fill is enabled. One
/// million minute-candles is roughly 1.9 years of contiguous one-minute bars
/// — orders of magnitude beyond any realistic missing-data window in
/// production while still keeping the resulting `Vec<Candle>` to well under
/// 50 MB. Any larger gap is treated as malformed input rather than allowed
/// to OOM the process.
pub const MAX_GAP_FILL_CANDLES: i64 = 1_000_000;
/// A candle bucket size measured in the same unit as the tick timestamps.
///
/// Wickra is unit-agnostic about timestamps: choose whichever makes sense for
/// your source (milliseconds for Binance trade events, microseconds for IB,
/// seconds for daily bars).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Timeframe {
bucket: i64,
}
impl Timeframe {
/// Construct a timeframe with the given bucket size in the chosen unit.
///
/// # Errors
/// Returns [`Error::InvalidTimeframe`] if `bucket <= 0`.
pub fn new(bucket: i64) -> Result<Self> {
if bucket <= 0 {
return Err(Error::InvalidTimeframe(format!(
"bucket size must be positive, got {bucket}"
)));
}
Ok(Self { bucket })
}
/// Convenience: build a millisecond timeframe.
pub fn millis(ms: i64) -> Result<Self> {
Self::new(ms)
}
/// Convenience: build a seconds-resolution timeframe.
pub fn seconds(s: i64) -> Result<Self> {
Self::new(s)
}
/// One-minute timeframe in milliseconds (`60_000`).
pub fn one_minute_ms() -> Self {
Self::new(60_000).expect("60_000 > 0")
}
/// Convenience: build a timeframe of `n` whole minutes, measured in
/// seconds — consistent with [`Timeframe::seconds`].
///
/// `minutes(5)` yields a bucket of `300`, for use with second-resolution
/// timestamps. For millisecond timestamps (Binance) multiply yourself or
/// use [`Timeframe::millis`].
///
/// # Errors
/// Returns [`Error::InvalidTimeframe`] if `n` is not positive or if
/// `n * 60` overflows `i64`.
///
/// ```
/// use wickra_data::aggregator::Timeframe;
/// assert_eq!(Timeframe::minutes(5)?.bucket(), 300);
/// # Ok::<(), wickra_data::Error>(())
/// ```
pub fn minutes(n: i64) -> Result<Self> {
let bucket = n
.checked_mul(60)
.ok_or_else(|| Error::InvalidTimeframe(format!("{n} minutes overflows i64 seconds")))?;
Self::new(bucket)
}
/// Convenience: build a timeframe of `n` whole hours, measured in seconds
/// (`hours(2)` → a bucket of `7_200`).
///
/// # Errors
/// Returns [`Error::InvalidTimeframe`] if `n` is not positive or if
/// `n * 3_600` overflows `i64`.
///
/// ```
/// use wickra_data::aggregator::Timeframe;
/// assert_eq!(Timeframe::hours(2)?.bucket(), 7_200);
/// # Ok::<(), wickra_data::Error>(())
/// ```
pub fn hours(n: i64) -> Result<Self> {
let bucket = n
.checked_mul(3_600)
.ok_or_else(|| Error::InvalidTimeframe(format!("{n} hours overflows i64 seconds")))?;
Self::new(bucket)
}
/// Convenience: build a timeframe of `n` whole days, measured in seconds
/// (`days(1)` → a bucket of `86_400`).
///
/// # Errors
/// Returns [`Error::InvalidTimeframe`] if `n` is not positive or if
/// `n * 86_400` overflows `i64`.
///
/// ```
/// use wickra_data::aggregator::Timeframe;
/// assert_eq!(Timeframe::days(1)?.bucket(), 86_400);
/// # Ok::<(), wickra_data::Error>(())
/// ```
pub fn days(n: i64) -> Result<Self> {
let bucket = n
.checked_mul(86_400)
.ok_or_else(|| Error::InvalidTimeframe(format!("{n} days overflows i64 seconds")))?;
Self::new(bucket)
}
/// Bucket size.
pub const fn bucket(self) -> i64 {
self.bucket
}
/// Floor a raw timestamp to this timeframe's bucket boundary.
///
/// For a timestamp within one bucket of [`i64::MIN`] the mathematically
/// exact boundary lies below `i64::MIN` and cannot be represented; in that
/// (practically unreachable) case the result saturates at `i64::MIN`
/// rather than overflowing and panicking in debug builds. `bucket` is
/// always positive, so `rem_euclid` itself cannot panic.
pub fn floor(self, ts: i64) -> i64 {
ts.saturating_sub(ts.rem_euclid(self.bucket))
}
}
/// Incrementally builds candles out of arriving ticks.
///
/// Each call to [`TickAggregator::push`] returns the candles that closed as a
/// result of the new tick — normally at most one. Use
/// [`TickAggregator::flush`] at the end of a stream to capture the final open
/// bar.
///
/// # Gaps
///
/// By default a tick that jumps across one or more empty buckets simply opens
/// the next non-empty bar — the skipped buckets produce no candle, so the
/// output series can have time holes. Enable [`TickAggregator::with_gap_fill`]
/// to instead emit a flat placeholder candle for every skipped bucket, giving
/// downstream indicators an unbroken, evenly spaced series. To bound memory
/// against an adversarial timestamp jump, gap-filling refuses to emit more
/// than [`MAX_GAP_FILL_CANDLES`] placeholders in a single step; a larger gap
/// surfaces as an `Error::Malformed` so the caller can decide how to handle
/// the discontinuity.
#[derive(Debug, Clone)]
pub struct TickAggregator {
timeframe: Timeframe,
open_bar: Option<OpenBar>,
fill_gaps: bool,
}
#[derive(Debug, Clone, Copy)]
struct OpenBar {
bucket_start: i64,
/// Timestamp of the most recently absorbed tick. Used to reject ticks that
/// arrive out of order *within* the current bucket — without it an older
/// tick would silently overwrite `close` with a stale price.
last_ts: i64,
open: f64,
high: f64,
low: f64,
close: f64,
volume: f64,
}
impl OpenBar {
fn from_tick(t: Tick, bucket_start: i64) -> Self {
Self {
bucket_start,
last_ts: t.timestamp,
open: t.price,
high: t.price,
low: t.price,
close: t.price,
volume: t.volume,
}
}
fn absorb(&mut self, t: Tick) {
if t.price > self.high {
self.high = t.price;
}
if t.price < self.low {
self.low = t.price;
}
self.close = t.price;
self.volume += t.volume;
self.last_ts = t.timestamp;
}
/// Finalise the bar into a validated [`Candle`].
///
/// # Errors
/// Returns [`Error::Core`] if the accumulated `volume` is no longer finite.
/// `volume` is summed across every absorbed tick, so an astronomically
/// long or large run can drift it to `inf`; emitting such a candle would
/// silently poison every downstream indicator, so it is surfaced instead.
/// The OHLC fields are finite and correctly ordered by construction, so
/// `Candle::new` only ever rejects this bar for a non-finite volume.
fn into_candle(self) -> Result<Candle> {
Candle::new(
self.open,
self.high,
self.low,
self.close,
self.volume,
self.bucket_start,
)
.map_err(Error::from)
}
}
impl TickAggregator {
/// Construct a new aggregator for the given timeframe.
pub fn new(timeframe: Timeframe) -> Self {
Self {
timeframe,
open_bar: None,
fill_gaps: false,
}
}
/// Enable or disable gap filling, returning the (re)configured aggregator.
///
/// When enabled, [`push`](Self::push) emits a flat candle
/// (`open == high == low == close`, `volume == 0`) for every bucket that is
/// skipped between two consecutive ticks. The flat candle's price is the
/// close of the bar that preceded the gap, so the series stays continuous.
#[must_use]
pub fn with_gap_fill(mut self, fill: bool) -> Self {
self.fill_gaps = fill;
self
}
/// Whether gap filling is enabled.
pub const fn fills_gaps(&self) -> bool {
self.fill_gaps
}
/// Push a tick. Returns every candle that closed as a result — an empty
/// vector while the open bar keeps growing, one candle when a bar boundary
/// is crossed, and (with gap filling enabled) additionally one flat candle
/// per skipped bucket.
///
/// # Errors
/// Returns [`Error::Malformed`] if `tick.timestamp` goes backwards — both
/// across buckets (older than the open bar's start) and within a bucket
/// (older than the last tick absorbed into it) — or if gap filling
/// overflows the timestamp range. Ticks sharing a timestamp are accepted.
pub fn push(&mut self, tick: Tick) -> Result<Vec<Candle>> {
let bucket = self.timeframe.floor(tick.timestamp);
if let Some(mut bar) = self.open_bar {
if bucket < bar.bucket_start {
return Err(Error::Malformed(format!(
"tick timestamp {} is older than the open bar start {}",
tick.timestamp, bar.bucket_start
)));
}
if bucket > bar.bucket_start {
// Close the previous bar and start a new one with this tick.
let closed = bar.into_candle()?;
let mut out = Vec::with_capacity(1);
out.push(closed);
if self.fill_gaps {
self.fill_between(closed, bucket, &mut out)?;
}
self.open_bar = Some(OpenBar::from_tick(tick, bucket));
return Ok(out);
}
// Same bucket: reject a tick that predates the last one absorbed,
// which would otherwise overwrite `close` with a stale price.
// Equal timestamps are allowed — several trades can share a
// millisecond.
if tick.timestamp < bar.last_ts {
return Err(Error::Malformed(format!(
"tick timestamp {} predates the last tick {} in the same bucket",
tick.timestamp, bar.last_ts
)));
}
bar.absorb(tick);
self.open_bar = Some(bar);
return Ok(Vec::new());
}
self.open_bar = Some(OpenBar::from_tick(tick, bucket));
Ok(Vec::new())
}
/// Append a flat placeholder candle for every empty bucket strictly between
/// the just-closed bar and the next bucket that received a tick.
///
/// Returns `Error::Malformed` when the gap would exceed
/// [`MAX_GAP_FILL_CANDLES`] — an adversarial timestamp jump (a clock-glitch
/// tick years in the future) must surface as a defined error, not as an
/// out-of-memory panic from allocating millions of placeholder candles.
fn fill_between(&self, prev: Candle, next_bucket: i64, out: &mut Vec<Candle>) -> Result<()> {
let step = self.timeframe.bucket();
let start = prev
.timestamp
.checked_add(step)
.ok_or_else(|| Error::Malformed("timestamp overflow while gap-filling".to_string()))?;
if start >= next_bucket {
return Ok(());
}
// Compute the gap size up-front so an adversarial timestamp delta
// is refused before we allocate. `step > 0` by `Timeframe::new`'s
// invariant, so the divisor is safe. Saturating the subtraction
// makes the arithmetic infallible; an overflowed-saturated span is
// still far above the cap so the limit check below catches it.
let span = next_bucket.saturating_sub(start);
let gap_count = span / step + i64::from(span % step != 0);
if gap_count > MAX_GAP_FILL_CANDLES {
return Err(Error::Malformed(format!(
"gap-fill between bucket {} and {next_bucket} would emit {gap_count} \
flat candles at step {step}, exceeding the {MAX_GAP_FILL_CANDLES} \
cap; reject the discontinuity instead of allocating",
prev.timestamp
)));
}
out.reserve(gap_count as usize);
// Bucket alignment guarantees start + (gap_count - 1) * step ≤
// next_bucket - step < i64::MAX, so iterating `gap_count` times
// with `saturating_add(step)` cannot reach i64::MAX inside the
// loop body. `prev.close` is finite (it came from a validated
// bar) and volume is exactly 0.0, so the OHLCV invariants hold
// by construction — skip re-validation via Candle::new_unchecked.
let mut t = start;
for _ in 0..gap_count {
out.push(Candle::new_unchecked(
prev.close, prev.close, prev.close, prev.close, 0.0, t,
));
t = t.saturating_add(step);
}
Ok(())
}
/// Drain the currently open bar (if any) and return it. Useful at the end of
/// a backtest or when shutting down a live aggregator.
///
/// # Errors
/// Returns an error if the open bar's accumulated volume is non-finite
/// (see the internal `OpenBar::into_candle`).
pub fn flush(&mut self) -> Result<Option<Candle>> {
self.open_bar.take().map(OpenBar::into_candle).transpose()
}
/// Configured timeframe.
pub const fn timeframe(&self) -> Timeframe {
self.timeframe
}
}
#[cfg(test)]
mod tests {
use super::*;
fn t(price: f64, ts: i64) -> Tick {
Tick::new(price, 1.0, ts).unwrap()
}
#[test]
fn timeframe_rejects_non_positive() {
assert!(Timeframe::new(0).is_err());
assert!(Timeframe::new(-1).is_err());
}
/// Cover the `Timeframe::millis`, `Timeframe::seconds`, and
/// `Timeframe::one_minute_ms` convenience constructors (lines 40-52).
/// All existing tests build Timeframes via `new` / `minutes` / `hours` /
/// `days`, never via the three thin convenience wrappers.
#[test]
fn timeframe_convenience_constructors() {
assert_eq!(Timeframe::millis(250).unwrap().bucket(), 250);
assert!(Timeframe::millis(0).is_err());
assert_eq!(Timeframe::seconds(30).unwrap().bucket(), 30);
assert!(Timeframe::seconds(-1).is_err());
// one_minute_ms is the infallible 60_000-ms shortcut.
assert_eq!(Timeframe::one_minute_ms().bucket(), 60_000);
}
/// Cover the `TickAggregator::timeframe` const accessor (lines 353-355).
/// Existing tests only inspect emitted candles, never query the
/// configured timeframe back out.
#[test]
fn aggregator_timeframe_getter() {
let tf = Timeframe::new(60).unwrap();
let agg = TickAggregator::new(tf);
assert_eq!(agg.timeframe().bucket(), 60);
}
#[test]
fn minute_hour_day_constructors_compute_seconds() {
assert_eq!(Timeframe::minutes(1).unwrap().bucket(), 60);
assert_eq!(Timeframe::minutes(5).unwrap().bucket(), 300);
assert_eq!(Timeframe::hours(1).unwrap().bucket(), 3_600);
assert_eq!(Timeframe::hours(4).unwrap().bucket(), 14_400);
assert_eq!(Timeframe::days(1).unwrap().bucket(), 86_400);
assert_eq!(Timeframe::days(7).unwrap().bucket(), 604_800);
}
#[test]
fn minute_hour_day_constructors_reject_non_positive() {
for n in [0, -1, -60] {
assert!(Timeframe::minutes(n).is_err());
assert!(Timeframe::hours(n).is_err());
assert!(Timeframe::days(n).is_err());
}
}
#[test]
fn minute_hour_day_constructors_reject_overflow() {
// `n * unit` overflows i64 long before `new`'s sign check runs.
assert!(matches!(
Timeframe::minutes(i64::MAX),
Err(Error::InvalidTimeframe(_))
));
assert!(matches!(
Timeframe::hours(i64::MAX),
Err(Error::InvalidTimeframe(_))
));
assert!(matches!(
Timeframe::days(i64::MAX),
Err(Error::InvalidTimeframe(_))
));
}
#[test]
fn floors_to_bucket_boundary() {
let tf = Timeframe::new(100).unwrap();
assert_eq!(tf.floor(0), 0);
assert_eq!(tf.floor(99), 0);
assert_eq!(tf.floor(100), 100);
assert_eq!(tf.floor(150), 100);
assert_eq!(tf.floor(250), 200);
// Negative timestamps still floor toward negative infinity.
assert_eq!(tf.floor(-1), -100);
assert_eq!(tf.floor(-100), -100);
assert_eq!(tf.floor(-101), -200);
}
#[test]
fn floor_saturates_instead_of_overflowing_at_min() {
let tf = Timeframe::new(100).unwrap();
// The exact boundary lies below i64::MIN — must not panic.
assert_eq!(tf.floor(i64::MIN), i64::MIN);
// i64::MAX must not overflow either (subtracting a non-negative).
let hi = tf.floor(i64::MAX);
assert!(hi > i64::MAX - 100 && hi % 100 == 0);
}
#[test]
fn aggregates_ticks_into_one_candle_within_bucket() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
assert!(agg.push(t(10.0, 0)).unwrap().is_empty());
assert!(agg.push(t(12.0, 15)).unwrap().is_empty());
assert!(agg.push(t(8.0, 30)).unwrap().is_empty());
assert!(agg.push(t(11.0, 50)).unwrap().is_empty());
let bar = agg.flush().unwrap().expect("open bar");
assert_eq!(bar.open, 10.0);
assert_eq!(bar.high, 12.0);
assert_eq!(bar.low, 8.0);
assert_eq!(bar.close, 11.0);
assert!((bar.volume - 4.0).abs() < 1e-12);
assert_eq!(bar.timestamp, 0);
}
#[test]
fn emits_candle_on_bucket_crossing() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
agg.push(t(10.0, 0)).unwrap();
agg.push(t(12.0, 30)).unwrap();
let closed = agg.push(t(15.0, 60)).unwrap();
assert_eq!(closed.len(), 1);
let closed = closed[0];
assert_eq!(closed.open, 10.0);
assert_eq!(closed.high, 12.0);
assert_eq!(closed.low, 10.0);
assert_eq!(closed.close, 12.0);
// The new tick at ts=60 opens the next bar.
let still_open = agg.flush().unwrap().unwrap();
assert_eq!(still_open.open, 15.0);
assert_eq!(still_open.timestamp, 60);
}
#[test]
fn rejects_out_of_order_ticks() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
agg.push(t(10.0, 100)).unwrap();
let err = agg.push(t(11.0, 30)).unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn rejects_same_bucket_out_of_order_tick() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
agg.push(t(10.0, 50)).unwrap();
// ts=10 is still bucket 0 but predates the tick at ts=50 — rejecting
// it prevents a stale price silently overwriting `close`.
let err = agg.push(t(99.0, 10)).unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
// The open bar is untouched: close is still the ts=50 price.
assert_eq!(agg.flush().unwrap().unwrap().close, 10.0);
}
#[test]
fn accepts_same_bucket_ticks_sharing_a_timestamp() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
agg.push(t(10.0, 20)).unwrap();
// Two trades in the same millisecond are legitimate.
agg.push(t(12.0, 20)).unwrap();
agg.push(t(11.0, 20)).unwrap();
let bar = agg.flush().unwrap().unwrap();
assert_eq!(bar.high, 12.0);
assert_eq!(bar.close, 11.0);
}
#[test]
fn flushes_a_non_finite_volume_as_an_error() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
// Two near-max volumes sum to +inf — the closed candle would carry a
// non-finite volume that poisons every downstream indicator.
agg.push(Tick::new(10.0, f64::MAX, 0).unwrap()).unwrap();
agg.push(Tick::new(10.0, f64::MAX, 1).unwrap()).unwrap();
let err = agg.flush().unwrap_err();
assert!(matches!(err, Error::Core(_)));
}
#[test]
fn skips_empty_buckets_without_gap_fill() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
assert!(!agg.fills_gaps());
agg.push(t(10.0, 0)).unwrap();
// Jump from bucket 0 straight to bucket 180 — buckets 60 and 120 empty.
let closed = agg.push(t(20.0, 200)).unwrap();
assert_eq!(closed.len(), 1, "only the real bar closes");
assert_eq!(closed[0].timestamp, 0);
}
#[test]
fn gap_fill_rejects_runaway_timestamp_jump() {
// An adversarial clock-glitch tick years in the future must surface
// as an Error::Malformed rather than allocating millions of flat
// candles and OOMing. Found by the `tick_aggregator` fuzz target.
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
agg.push(t(10.0, 0)).unwrap();
// Two-billion-second jump = ~63 years of minute bars = ~33 million
// candles, well above the 1_000_000 cap.
let err = agg.push(t(20.0, 2_000_000_000)).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("gap-fill") && msg.contains("cap"),
"expected a malformed-gap error, got: {msg}"
);
}
#[test]
fn gap_fill_at_the_cap_succeeds() {
// Exactly one million minute-buckets between the two ticks (one real
// bar + one million flat fillers + the third tick's open bar) — the
// limit is inclusive, so this must succeed.
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
agg.push(t(10.0, 0)).unwrap();
// bucket 0 closes; jump straight to bucket 60_000_060 (1_000_001 buckets
// away). fill_between emits 1_000_000 flat candles between them, then
// the new tick opens its own bucket. Output: 1 real bar + 1_000_000 fillers.
let out = agg.push(t(20.0, 60_000_060)).unwrap();
assert_eq!(out.len(), 1 + MAX_GAP_FILL_CANDLES as usize);
}
#[test]
fn gap_fill_emits_flat_candles_for_skipped_buckets() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
assert!(agg.fills_gaps());
agg.push(t(10.0, 0)).unwrap();
agg.push(t(13.0, 30)).unwrap(); // still bucket 0, close = 13.0
// Next tick lands in bucket 180 — buckets 60 and 120 are skipped.
let out = agg.push(t(20.0, 200)).unwrap();
assert_eq!(out.len(), 3, "real bar + two flat fillers");
let real = out[0];
assert_eq!(real.timestamp, 0);
assert_eq!(real.close, 13.0);
for (filler, ts) in out[1..].iter().zip([60, 120]) {
assert_eq!(filler.timestamp, ts);
assert_eq!(filler.open, 13.0);
assert_eq!(filler.high, 13.0);
assert_eq!(filler.low, 13.0);
assert_eq!(filler.close, 13.0);
assert_eq!(filler.volume, 0.0);
}
// The tick at ts=200 opens bucket 180.
assert_eq!(agg.flush().unwrap().unwrap().timestamp, 180);
}
#[test]
fn gap_fill_emits_nothing_extra_for_adjacent_buckets() {
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
agg.push(t(10.0, 0)).unwrap();
// Bucket 60 directly follows bucket 0 — no gap to fill.
let out = agg.push(t(11.0, 70)).unwrap();
assert_eq!(out.len(), 1);
assert_eq!(out[0].timestamp, 0);
}
}
+346
View File
@@ -0,0 +1,346 @@
//! Stream OHLCV candles out of a CSV file.
//!
//! The reader is generic over the column layout, but ships with a sensible
//! default ("timestamp,open,high,low,close,volume") that matches the standard
//! Binance / Yahoo Finance / kaggle dataset format.
//!
//! The reader is defensive about real-world files: a leading UTF-8 byte-order
//! mark is stripped, surrounding whitespace is trimmed from every field, and a
//! file whose header does not name the required columns is rejected with a
//! clear [`Error::Malformed`] instead of silently consuming its first data row.
use std::path::Path;
use serde::Deserialize;
use crate::error::{Error, Result};
use wickra_core::Candle;
/// Column names the default OHLCV layout requires. The CSV header must contain
/// every one of these (extra columns are ignored); matching is exact and
/// case-sensitive because the underlying `serde` deserialization maps header
/// names to [`DefaultRow`]'s fields by name.
const REQUIRED_COLUMNS: [&str; 6] = ["timestamp", "open", "high", "low", "close", "volume"];
/// Default OHLCV CSV row layout.
///
/// The timestamp is parsed as an `i64` (for example a Unix epoch). RFC3339 /
/// ISO8601 string timestamps are not handled by this layout; convert them to
/// integers before reading.
#[derive(Debug, Clone, Deserialize)]
pub struct DefaultRow {
pub timestamp: i64,
pub open: f64,
pub high: f64,
pub low: f64,
pub close: f64,
pub volume: f64,
}
impl DefaultRow {
fn into_candle(self) -> Result<Candle> {
Candle::new(
self.open,
self.high,
self.low,
self.close,
self.volume,
self.timestamp,
)
.map_err(Error::from)
}
}
/// A [`std::io::Read`] adapter that transparently skips a leading UTF-8
/// byte-order mark.
///
/// Spreadsheet exporters — Excel in particular — prefix CSV files with the
/// three-byte UTF-8 BOM `EF BB BF`. Left in place it becomes part of the first
/// header name (`\u{feff}timestamp`), which then fails to match the
/// `timestamp` column. This adapter drops the BOM before the CSV parser ever
/// sees it; files without a BOM pass through untouched.
#[derive(Debug)]
pub struct BomStripReader<R> {
inner: R,
/// Whether the leading bytes have been inspected for a BOM yet.
checked: bool,
/// Bytes read during BOM detection that turned out *not* to be a BOM and
/// must still be handed to the consumer.
leftover: Vec<u8>,
leftover_pos: usize,
}
impl<R: std::io::Read> BomStripReader<R> {
/// Wrap `inner`, stripping a leading UTF-8 BOM on the first read.
pub fn new(inner: R) -> Self {
Self {
inner,
checked: false,
leftover: Vec::new(),
leftover_pos: 0,
}
}
/// On the first read, consume up to three bytes and decide whether they
/// form a BOM. A BOM is discarded; anything else is buffered for replay.
fn check_bom(&mut self) -> std::io::Result<()> {
if self.checked {
return Ok(());
}
self.checked = true;
let mut probe = [0u8; 3];
let mut filled = 0;
while filled < probe.len() {
let n = self.inner.read(&mut probe[filled..])?;
if n == 0 {
break; // short source — fewer than 3 bytes total
}
filled += n;
}
if probe[..filled] != [0xEF, 0xBB, 0xBF] {
// Not a BOM (or a short file): replay every probed byte verbatim.
self.leftover.extend_from_slice(&probe[..filled]);
}
Ok(())
}
}
impl<R: std::io::Read> std::io::Read for BomStripReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.check_bom()?;
if self.leftover_pos < self.leftover.len() {
let n = (self.leftover.len() - self.leftover_pos).min(buf.len());
buf[..n].copy_from_slice(&self.leftover[self.leftover_pos..self.leftover_pos + n]);
self.leftover_pos += n;
return Ok(n);
}
self.inner.read(buf)
}
}
/// Validate that a CSV reader's header row names every required OHLCV column.
fn validate_headers<R: std::io::Read>(reader: &mut csv::Reader<R>) -> Result<()> {
let headers = reader.headers()?;
let present: Vec<String> = headers.iter().map(|h| h.trim().to_string()).collect();
let missing: Vec<&str> = REQUIRED_COLUMNS
.iter()
.copied()
.filter(|col| !present.iter().any(|h| h == col))
.collect();
if !missing.is_empty() {
return Err(Error::Malformed(format!(
"CSV header is missing required column(s) [{}]; found [{}] — \
the first line must be a header naming {}",
missing.join(", "),
present.join(", "),
REQUIRED_COLUMNS.join(",")
)));
}
Ok(())
}
/// Streaming OHLCV CSV reader.
#[derive(Debug)]
pub struct CandleReader<R: std::io::Read> {
reader: csv::Reader<R>,
}
impl<R: std::io::Read> CandleReader<R> {
/// Build a trimming CSV reader around `inner` and validate its header.
fn build(inner: R) -> Result<Self> {
let mut reader = csv::ReaderBuilder::new()
.has_headers(true)
.trim(csv::Trim::All)
.from_reader(inner);
validate_headers(&mut reader)?;
Ok(Self { reader })
}
}
impl CandleReader<BomStripReader<std::fs::File>> {
/// Open a CSV file at `path`.
///
/// The first line must be a header row naming the OHLCV columns; a leading
/// UTF-8 BOM and whitespace around values are tolerated.
///
/// # Errors
/// Returns [`Error::Io`] if the file cannot be opened and
/// [`Error::Malformed`] if the header does not contain every required
/// column (`timestamp,open,high,low,close,volume`).
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let file = std::fs::File::open(path)?;
Self::from_reader(file)
}
}
impl<R: std::io::Read> CandleReader<BomStripReader<R>> {
/// Build a reader from any [`std::io::Read`] source.
///
/// A leading UTF-8 BOM is stripped and the header is validated.
///
/// # Errors
/// Returns [`Error::Malformed`] if the header does not contain every
/// required column.
pub fn from_reader(inner: R) -> Result<Self> {
Self::build(BomStripReader::new(inner))
}
}
impl<R: std::io::Read> CandleReader<R> {
/// Wrap a pre-built [`csv::Reader`]; useful for testing or for non-default
/// reader configuration.
///
/// Unlike [`from_reader`](Self::from_reader) this does *not* strip a BOM —
/// the caller owns the reader's configuration — but the header is still
/// validated.
///
/// # Errors
/// Returns [`Error::Malformed`] if the header does not contain every
/// required column.
pub fn from_csv_reader(mut reader: csv::Reader<R>) -> Result<Self> {
validate_headers(&mut reader)?;
Ok(Self { reader })
}
/// Iterator over decoded candles.
pub fn candles(&mut self) -> impl Iterator<Item = Result<Candle>> + '_ {
self.reader.deserialize::<DefaultRow>().map(|row_res| {
let row = row_res?;
row.into_candle()
})
}
/// Read the entire stream into a `Vec<Candle>`. Convenient for backtests.
pub fn read_all(&mut self) -> Result<Vec<Candle>> {
self.candles().collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[test]
fn reads_well_formed_csv() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
writeln!(tmp, "timestamp,open,high,low,close,volume").unwrap();
writeln!(tmp, "1,10.0,11.0,9.0,10.5,100").unwrap();
writeln!(tmp, "2,10.5,11.5,10.0,11.0,150").unwrap();
writeln!(tmp, "3,11.0,12.0,10.5,11.5,200").unwrap();
tmp.flush().unwrap();
let mut r = CandleReader::open(tmp.path()).unwrap();
let candles = r.read_all().unwrap();
assert_eq!(candles.len(), 3);
assert_eq!(candles[0].open, 10.0);
assert_eq!(candles[2].close, 11.5);
assert_eq!(candles[1].timestamp, 2);
}
#[test]
fn rejects_invalid_ohlc() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
writeln!(tmp, "timestamp,open,high,low,close,volume").unwrap();
// high < low → core validation rejects it.
writeln!(tmp, "1,10.0,8.0,9.0,9.5,100").unwrap();
tmp.flush().unwrap();
let mut r = CandleReader::open(tmp.path()).unwrap();
let candles: Result<Vec<Candle>> = r.candles().collect();
assert!(candles.is_err());
}
#[test]
fn from_reader_works_on_in_memory_data() {
let data = "timestamp,open,high,low,close,volume\n1,1,2,0,1,10\n2,1,2,0,1,10\n";
let mut r = CandleReader::from_reader(data.as_bytes()).unwrap();
let v = r.read_all().unwrap();
assert_eq!(v.len(), 2);
}
#[test]
fn rejects_file_without_header() {
// No header row — the first line is data. Without validation the
// reader would silently swallow it as the header.
let data = "1,10.0,11.0,9.0,10.5,100\n2,10.5,11.5,10.0,11.0,150\n";
let err = CandleReader::from_reader(data.as_bytes()).unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn rejects_header_missing_a_column() {
// "volume" is absent.
let data = "timestamp,open,high,low,close\n1,10.0,11.0,9.0,10.5\n";
let err = CandleReader::from_reader(data.as_bytes()).unwrap_err();
// The error variant must be Malformed and the message must mention
// the missing column. Asserting directly (rather than match-and-
// panic-on-other) keeps the assertion's cold path branch-free for
// coverage and still pins the diagnostic.
assert!(
matches!(&err, Error::Malformed(msg) if msg.contains("volume")),
"expected Malformed mentioning 'volume', got {err:?}"
);
}
/// Cover `from_csv_reader` (lines 201-204): existing tests use
/// `from_reader` / `open`, which both construct the inner `csv::Reader`
/// internally. Callers that want non-default csv configuration must
/// build the reader themselves and pass it through `from_csv_reader`.
#[test]
fn from_csv_reader_accepts_a_prebuilt_reader() {
let data = "timestamp;open;high;low;close;volume\n1;10.0;11.0;9.0;10.5;100\n";
let inner = csv::ReaderBuilder::new()
.delimiter(b';')
.from_reader(data.as_bytes());
let mut r = CandleReader::from_csv_reader(inner).unwrap();
let candles = r.read_all().unwrap();
assert_eq!(candles.len(), 1);
assert_eq!(candles[0].close, 10.5);
}
#[test]
fn strips_leading_utf8_bom() {
// A BOM (\u{feff}) prefixes the header — Excel exports look like this.
let data = "\u{feff}timestamp,open,high,low,close,volume\n1,10.0,11.0,9.0,10.5,100\n";
let mut r = CandleReader::from_reader(data.as_bytes()).unwrap();
let v = r.read_all().unwrap();
assert_eq!(v.len(), 1);
assert_eq!(v[0].timestamp, 1);
assert_eq!(v[0].open, 10.0);
}
#[test]
fn tolerates_whitespace_around_fields() {
let data = " timestamp , open , high , low , close , volume \n\
1 , 10.0 , 11.0 , 9.0 , 10.5 , 100 \n";
let mut r = CandleReader::from_reader(data.as_bytes()).unwrap();
let v = r.read_all().unwrap();
assert_eq!(v.len(), 1);
assert_eq!(v[0].close, 10.5);
assert_eq!(v[0].volume, 100.0);
}
#[test]
fn bom_stripper_passes_through_non_bom_input() {
use std::io::Read;
let mut out = String::new();
BomStripReader::new("hello".as_bytes())
.read_to_string(&mut out)
.unwrap();
assert_eq!(out, "hello");
}
#[test]
fn bom_stripper_handles_short_input() {
use std::io::Read;
let mut out = Vec::new();
// Two bytes — shorter than a 3-byte BOM.
BomStripReader::new([0x41u8, 0x42u8].as_slice())
.read_to_end(&mut out)
.unwrap();
assert_eq!(out, vec![0x41, 0x42]);
}
}
+44
View File
@@ -0,0 +1,44 @@
//! Error types specific to the data sources.
use thiserror::Error;
/// Errors produced by the data layer.
#[derive(Debug, Error)]
pub enum Error {
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("CSV error: {0}")]
Csv(#[from] csv::Error),
#[error("invalid timeframe: {0}")]
InvalidTimeframe(String),
#[error("indicator-core error: {0}")]
Core(#[from] wickra_core::Error),
#[error("malformed payload: {0}")]
Malformed(String),
/// A live-feed read exceeded its deadline.
#[error("read timed out")]
Timeout,
#[cfg(feature = "live-binance")]
#[error("websocket error: {0}")]
WebSocket(#[from] tokio_tungstenite::tungstenite::Error),
#[cfg(feature = "live-binance")]
#[error("JSON decode error: {0}")]
Json(#[from] serde_json::Error),
/// Boxed because ureq 2.x's `Error::Status` carries a full `Response`, which
/// would otherwise make this the dominant enum variant (clippy
/// `large_enum_variant`).
#[cfg(feature = "live-binance")]
#[error("HTTP error: {0}")]
Http(#[from] Box<ureq::Error>),
}
/// Convenience alias for `Result<T, wickra_data::Error>`.
pub type Result<T> = core::result::Result<T, Error>;
+25
View File
@@ -0,0 +1,25 @@
//! `wickra-data`: offline and online data sources for the Wickra indicator engine.
//!
//! - [`csv`]: stream OHLCV bars out of CSV files without buffering the whole
//! history in memory.
//! - [`aggregator`]: roll trade ticks up into candles of arbitrary timeframes.
//! - [`resample`]: convert a stream of candles from one timeframe to a coarser one.
//! - [`live`] (feature `live-binance`): connect to exchange websockets for live
//! klines, or pull historical klines over REST — both yield typed candles
//! compatible with the rest of the crate.
#![cfg_attr(docsrs, feature(doc_cfg))]
// `tokio_tungstenite::Error` is large by itself (~200 B). Boxing every Err
// variant per clippy::result_large_err just shifts allocation pressure into
// the hot path. We accept the size because errors are rare in this crate.
#![allow(clippy::result_large_err)]
pub mod aggregator;
pub mod csv;
pub mod error;
pub mod resample;
#[cfg(feature = "live-binance")]
pub mod live;
pub use error::{Error, Result};
+5
View File
@@ -0,0 +1,5 @@
//! Live exchange feeds. Each adapter is feature-gated; the Binance adapter
//! lives behind the `live-binance` feature.
pub mod binance;
pub mod binance_rest;
+915
View File
@@ -0,0 +1,915 @@
//! Binance spot WebSocket kline feed.
//!
//! Subscribes to Binance's `<symbol>@kline_<interval>` stream and emits a
//! [`KlineEvent`] every time the server pushes a new tick. The event tells you
//! whether the current candle is still open or has just closed.
//!
//! Example (requires the `live-binance` feature):
//!
//! ```no_run
//! use wickra_data::live::binance::{BinanceKlineStream, Interval};
//! # async fn run() -> wickra_data::Result<()> {
//! let mut stream = BinanceKlineStream::connect(&["BTCUSDT".to_string()], Interval::OneMinute).await?;
//! while let Some(event) = stream.next_event().await? {
//! if event.is_closed {
//! println!("closed {} @ {}", event.symbol, event.candle.close);
//! }
//! }
//! # Ok(()) }
//! ```
use std::time::Duration;
use futures_util::SinkExt;
use futures_util::StreamExt;
use serde::Deserialize;
use tokio::net::TcpStream;
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::MaybeTlsStream;
use tokio_tungstenite::WebSocketStream;
use crate::error::{Error, Result};
use wickra_core::Candle;
/// Tunable knobs for a [`BinanceKlineStream`]. The defaults match Binance's
/// public production endpoint and are right for almost every caller; the
/// fields exist so an integration test or a Binance Testnet user can point
/// the stream at a different base URL and shrink the reconnect timing.
#[derive(Debug, Clone)]
pub struct BinanceConfig {
/// WebSocket endpoint **without** path (e.g. `"wss://stream.binance.com:9443"`).
/// The combined-stream path `/stream?streams=…` is appended internally.
pub base_url: String,
/// Maximum time to wait for the next inbound frame before treating the
/// connection as stalled. Binance pings roughly every 3 minutes, so a
/// healthy but quiet stream stays comfortably inside the 300 s default.
pub read_timeout: Duration,
/// Delay before the first reconnect attempt; doubles on each failure up
/// to [`Self::max_reconnect_backoff`].
pub initial_reconnect_delay: Duration,
/// Upper bound on the exponential reconnect backoff.
pub max_reconnect_backoff: Duration,
/// How many times [`BinanceKlineStream::next_event`] retries a dropped
/// connection before surfacing the last error. Must be `>= 1`.
pub max_reconnect_attempts: u32,
/// Upper bound on an inbound WebSocket message. Kline frames are tiny;
/// this only caps a pathological or hostile server from forcing an
/// unbounded allocation.
pub max_message_size: usize,
/// Upper bound on a single inbound WebSocket frame.
pub max_frame_size: usize,
}
impl Default for BinanceConfig {
fn default() -> Self {
Self {
base_url: "wss://stream.binance.com:9443".to_string(),
read_timeout: Duration::from_secs(300),
initial_reconnect_delay: Duration::from_secs(1),
max_reconnect_backoff: Duration::from_secs(30),
max_reconnect_attempts: 6,
max_message_size: 8 << 20,
max_frame_size: 2 << 20,
}
}
}
/// Supported Binance kline intervals. The `as_str` value matches Binance's
/// wire-format strings (`"1m"`, `"5m"`, `"1h"`, etc.).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Interval {
OneSecond,
OneMinute,
ThreeMinutes,
FiveMinutes,
FifteenMinutes,
ThirtyMinutes,
OneHour,
TwoHours,
FourHours,
SixHours,
EightHours,
TwelveHours,
OneDay,
ThreeDays,
OneWeek,
OneMonth,
}
impl Interval {
/// Wire-format string used in the stream name.
pub fn as_str(self) -> &'static str {
match self {
Self::OneSecond => "1s",
Self::OneMinute => "1m",
Self::ThreeMinutes => "3m",
Self::FiveMinutes => "5m",
Self::FifteenMinutes => "15m",
Self::ThirtyMinutes => "30m",
Self::OneHour => "1h",
Self::TwoHours => "2h",
Self::FourHours => "4h",
Self::SixHours => "6h",
Self::EightHours => "8h",
Self::TwelveHours => "12h",
Self::OneDay => "1d",
Self::ThreeDays => "3d",
Self::OneWeek => "1w",
Self::OneMonth => "1M",
}
}
}
/// One push from the Binance kline stream.
#[derive(Debug, Clone)]
pub struct KlineEvent {
/// Symbol in lowercase form as sent by Binance (e.g. `"btcusdt"`).
pub symbol: String,
/// Interval the candle belongs to.
pub interval: Interval,
/// Candle in its current state (may still be open).
pub candle: Candle,
/// Whether the candle has been closed by the server. Closed events are the
/// only ones safe to use for bar-completion logic.
pub is_closed: bool,
}
/// A live Binance kline stream.
#[derive(Debug)]
pub struct BinanceKlineStream {
socket: WebSocketStream<MaybeTlsStream<TcpStream>>,
/// Lowercased symbols the stream is subscribed to. Retained so the
/// connection can be rebuilt on a reconnect.
symbols: Vec<String>,
/// Interval requested at connect time. Used to tag every event.
interval: Interval,
/// `true` once the caller invoked [`close`](Self::close). A closed stream
/// is never polled or reconnected again.
closed: bool,
/// Timing / sizing knobs. Retained so reconnects honour the same config.
config: BinanceConfig,
}
/// Wire-format representation of an incoming Binance kline tick. Public so callers
/// can deserialize it themselves if they prefer.
#[derive(Debug, Clone, Deserialize)]
pub struct RawWsEnvelope {
/// Stream name, e.g. `"btcusdt@kline_1m"`.
pub stream: String,
pub data: RawKlinePayload,
}
#[derive(Debug, Clone, Deserialize)]
pub struct RawKlinePayload {
#[serde(rename = "e")]
pub event_type: String,
#[serde(rename = "E")]
pub event_time: i64,
#[serde(rename = "s")]
pub symbol: String,
#[serde(rename = "k")]
pub kline: RawKline,
}
#[derive(Debug, Clone, Deserialize)]
pub struct RawKline {
#[serde(rename = "t")]
pub open_time: i64,
#[serde(rename = "T")]
pub close_time: i64,
#[serde(rename = "s")]
pub symbol: String,
#[serde(rename = "i")]
pub interval: String,
#[serde(rename = "o")]
pub open: String,
#[serde(rename = "c")]
pub close: String,
#[serde(rename = "h")]
pub high: String,
#[serde(rename = "l")]
pub low: String,
#[serde(rename = "v")]
pub volume: String,
#[serde(rename = "x")]
pub is_closed: bool,
}
impl BinanceKlineStream {
/// Open a raw combined-stream WebSocket for the given (already-lowercased)
/// symbols.
async fn open(
symbols: &[String],
interval: Interval,
config: &BinanceConfig,
) -> Result<WebSocketStream<MaybeTlsStream<TcpStream>>> {
let streams: Vec<String> = symbols
.iter()
.map(|s| format!("{}@kline_{}", s, interval.as_str()))
.collect();
let url = format!("{}/stream?streams={}", config.base_url, streams.join("/"));
let url = url::Url::parse(&url).map_err(|e| Error::Malformed(e.to_string()))?;
// tokio-tungstenite 0.29's WebSocketConfig is #[non_exhaustive],
// so the only way to set fields is via the builder-style methods on
// a fresh `default()` value.
let ws_config = WebSocketConfig::default()
.max_message_size(Some(config.max_message_size))
.max_frame_size(Some(config.max_frame_size));
let (socket, _) =
tokio_tungstenite::connect_async_with_config(url.as_str(), Some(ws_config), false)
.await?;
Ok(socket)
}
/// Connect to Binance's combined-stream endpoint for one or more symbols.
///
/// Symbols may be passed in either case; they are lowercased to match
/// Binance's stream-name conventions. A dropped or stalled connection is
/// re-established transparently by [`next_event`](Self::next_event).
pub async fn connect(symbols: &[String], interval: Interval) -> Result<Self> {
Self::connect_with_config(symbols, interval, BinanceConfig::default()).await
}
/// Connect with a custom [`BinanceConfig`]. Useful for Binance Testnet
/// (`"wss://testnet.binance.vision"`) or for shrinking the reconnect
/// timing in integration tests.
pub async fn connect_with_config(
symbols: &[String],
interval: Interval,
config: BinanceConfig,
) -> Result<Self> {
if symbols.is_empty() {
return Err(Error::Malformed(
"BinanceKlineStream requires at least one symbol".into(),
));
}
let symbols: Vec<String> = symbols.iter().map(|s| s.to_lowercase()).collect();
let socket = Self::open(&symbols, interval, &config).await?;
Ok(Self {
socket,
symbols,
interval,
closed: false,
config,
})
}
/// Whether the caller has closed the stream. Once closed, every further
/// [`next_event`](Self::next_event) call yields `Ok(None)` immediately.
pub fn is_closed(&self) -> bool {
self.closed
}
/// Re-establish a dropped connection with exponential backoff. Returns the
/// last error if every attempt fails.
async fn reconnect(&mut self) -> Result<()> {
let mut delay = self.config.initial_reconnect_delay;
let mut last_err = None;
for _ in 0..self.config.max_reconnect_attempts {
tokio::time::sleep(delay).await;
match Self::open(&self.symbols, self.interval, &self.config).await {
Ok(socket) => {
self.socket = socket;
return Ok(());
}
Err(e) => {
last_err = Some(e);
delay = delay
.saturating_mul(2)
.min(self.config.max_reconnect_backoff);
}
}
}
Err(last_err.expect("max_reconnect_attempts is non-zero"))
}
/// Receive the next kline event. A dropped, errored or stalled connection
/// is re-established transparently (exponential backoff, up to
/// [`BinanceConfig::max_reconnect_attempts`]); an exhausted reconnect
/// surfaces as `Err`. `Ok(None)` is returned only after the caller has
/// [`close`](Self::close)d the stream.
pub async fn next_event(&mut self) -> Result<Option<KlineEvent>> {
if self.closed {
return Ok(None);
}
loop {
// A protocol error, a clean server close, or a read stall are all
// transient: reconnect with backoff and resume reading.
let Ok(Some(Ok(msg))) =
tokio::time::timeout(self.config.read_timeout, self.socket.next()).await
else {
self.reconnect().await?;
continue;
};
match msg {
Message::Text(text) => {
if let Some(event) = Self::parse_frame(&text, self.interval)? {
return Ok(Some(event));
}
// Non-kline frame (subscription ack / heartbeat / error):
// skip it and keep reading.
}
Message::Binary(bytes) => {
let text = String::from_utf8_lossy(&bytes);
if let Some(event) = Self::parse_frame(&text, self.interval)? {
return Ok(Some(event));
}
}
Message::Ping(payload) => {
// Best-effort Pong reply. If the write fails the
// connection is already dead — the next read will
// surface the error and trigger reconnect through
// the timeout/err arm above.
let _ = self.socket.send(Message::Pong(payload)).await;
}
Message::Pong(_) | Message::Frame(_) => {}
Message::Close(_) => {
self.reconnect().await?;
}
}
}
}
/// Parse one raw WebSocket text frame.
///
/// Returns `Ok(Some(event))` for a kline frame, `Ok(None)` for any other
/// frame (subscription acknowledgements, error objects, heartbeats), and
/// `Err` only when a frame that *is* a kline fails to decode.
fn parse_frame(text: &str, interval: Interval) -> Result<Option<KlineEvent>> {
let value: serde_json::Value = serde_json::from_str(text)?;
// Combined-stream kline frames carry `data.e == "kline"`. Everything
// else on the socket is control traffic that must not abort the feed.
let is_kline = value
.get("data")
.and_then(|d| d.get("e"))
.and_then(serde_json::Value::as_str)
== Some("kline");
if !is_kline {
return Ok(None);
}
let envelope: RawWsEnvelope = serde_json::from_value(value)?;
Ok(Some(envelope.into_event(interval)?))
}
/// Close the underlying socket cleanly and mark the stream closed. After
/// this, [`next_event`](Self::next_event) yields `Ok(None)` and never
/// reconnects.
pub async fn close(&mut self) -> Result<()> {
self.closed = true;
self.socket.close(None).await?;
Ok(())
}
}
impl RawWsEnvelope {
fn into_event(self, interval: Interval) -> Result<KlineEvent> {
let k = self.data.kline;
let open: f64 = k
.open
.parse()
.map_err(|_| Error::Malformed(format!("bad open '{}'", k.open)))?;
let high: f64 = k
.high
.parse()
.map_err(|_| Error::Malformed(format!("bad high '{}'", k.high)))?;
let low: f64 = k
.low
.parse()
.map_err(|_| Error::Malformed(format!("bad low '{}'", k.low)))?;
let close: f64 = k
.close
.parse()
.map_err(|_| Error::Malformed(format!("bad close '{}'", k.close)))?;
let volume: f64 = k
.volume
.parse()
.map_err(|_| Error::Malformed(format!("bad volume '{}'", k.volume)))?;
let candle = Candle::new(open, high, low, close, volume, k.open_time)?;
Ok(KlineEvent {
symbol: self.data.symbol.to_lowercase(),
interval,
candle,
is_closed: k.is_closed,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn interval_as_str_covers_every_variant() {
// Wire-format strings are part of Binance's public protocol — pin
// every mapping so a typo here is caught immediately.
let pairs: &[(Interval, &str)] = &[
(Interval::OneSecond, "1s"),
(Interval::OneMinute, "1m"),
(Interval::ThreeMinutes, "3m"),
(Interval::FiveMinutes, "5m"),
(Interval::FifteenMinutes, "15m"),
(Interval::ThirtyMinutes, "30m"),
(Interval::OneHour, "1h"),
(Interval::TwoHours, "2h"),
(Interval::FourHours, "4h"),
(Interval::SixHours, "6h"),
(Interval::EightHours, "8h"),
(Interval::TwelveHours, "12h"),
(Interval::OneDay, "1d"),
(Interval::ThreeDays, "3d"),
(Interval::OneWeek, "1w"),
(Interval::OneMonth, "1M"),
];
for (iv, expected) in pairs {
assert_eq!(iv.as_str(), *expected);
}
}
#[test]
fn binance_config_default_matches_production_endpoint() {
let cfg = BinanceConfig::default();
assert_eq!(cfg.base_url, "wss://stream.binance.com:9443");
assert_eq!(cfg.read_timeout, Duration::from_secs(300));
assert_eq!(cfg.initial_reconnect_delay, Duration::from_secs(1));
assert_eq!(cfg.max_reconnect_backoff, Duration::from_secs(30));
assert_eq!(cfg.max_reconnect_attempts, 6);
assert_eq!(cfg.max_message_size, 8 << 20);
assert_eq!(cfg.max_frame_size, 2 << 20);
}
#[tokio::test]
async fn connect_rejects_an_empty_symbol_list() {
let err = BinanceKlineStream::connect(&[], Interval::OneMinute)
.await
.unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn parses_real_binance_payload() {
// Sample event format from Binance's public docs (truncated).
let json = r#"{
"stream": "btcusdt@kline_1m",
"data": {
"e": "kline",
"E": 1700000000000,
"s": "BTCUSDT",
"k": {
"t": 1700000000000,
"T": 1700000059999,
"s": "BTCUSDT",
"i": "1m",
"f": 1,
"L": 100,
"o": "30000.0",
"c": "30050.0",
"h": "30100.0",
"l": "29950.0",
"v": "12.5",
"n": 50,
"x": false,
"q": "375000.0",
"V": "6.25",
"Q": "187500.0",
"B": "0"
}
}
}"#;
let env: RawWsEnvelope = serde_json::from_str(json).unwrap();
let evt = env.into_event(Interval::OneMinute).unwrap();
assert_eq!(evt.symbol, "btcusdt");
assert_eq!(evt.candle.open, 30_000.0);
assert_eq!(evt.candle.close, 30_050.0);
assert!(!evt.is_closed);
assert_eq!(evt.interval, Interval::OneMinute);
}
#[test]
fn rejects_non_parsable_numbers() {
let json = r#"{
"stream": "btcusdt@kline_1m",
"data": {
"e": "kline", "E": 0, "s": "BTCUSDT",
"k": {
"t": 0, "T": 0, "s": "BTCUSDT", "i": "1m",
"f": 0, "L": 0,
"o": "not-a-number", "c": "0", "h": "0", "l": "0",
"v": "0", "n": 0, "x": false, "q": "0", "V": "0", "Q": "0", "B": "0"
}
}
}"#;
let env: RawWsEnvelope = serde_json::from_str(json).unwrap();
let err = env.into_event(Interval::OneMinute).unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn skips_non_kline_frames() {
// Subscription acknowledgement: skipped, never an error.
let ack = r#"{"result":null,"id":1}"#;
assert!(BinanceKlineStream::parse_frame(ack, Interval::OneMinute)
.unwrap()
.is_none());
// Error object: also skipped.
let err = r#"{"error":{"code":2,"msg":"Invalid request"}}"#;
assert!(BinanceKlineStream::parse_frame(err, Interval::OneMinute)
.unwrap()
.is_none());
}
#[test]
fn parse_frame_decodes_a_kline() {
let json = r#"{
"stream": "btcusdt@kline_1m",
"data": {
"e": "kline", "E": 1700000000000, "s": "BTCUSDT",
"k": {
"t": 1700000000000, "T": 1700000059999, "s": "BTCUSDT", "i": "1m",
"f": 1, "L": 100, "o": "30000.0", "c": "30050.0", "h": "30100.0",
"l": "29950.0", "v": "12.5", "n": 50, "x": true,
"q": "375000.0", "V": "6.25", "Q": "187500.0", "B": "0"
}
}
}"#;
let event = BinanceKlineStream::parse_frame(json, Interval::OneMinute)
.unwrap()
.expect("a kline frame yields an event");
assert_eq!(event.symbol, "btcusdt");
assert!(event.is_closed);
}
// ====================================================================
// Mock WebSocket server: drives the async / reconnect / control-frame
// paths against a `127.0.0.1` listener instead of the real Binance
// endpoint. Each test gets its own port (`bind("127.0.0.1:0")`).
// ====================================================================
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
use tokio::net::TcpListener;
/// A kline JSON frame matching Binance's combined-stream envelope. Always
/// reports `is_closed = true` so the test can assert on the flag.
fn sample_kline_text() -> String {
r#"{"stream":"btcusdt@kline_1m","data":{"e":"kline","E":1700000000000,"s":"BTCUSDT","k":{"t":1700000000000,"T":1700000059999,"s":"BTCUSDT","i":"1m","f":1,"L":100,"o":"30000.0","c":"30050.0","h":"30100.0","l":"29950.0","v":"12.5","n":50,"x":true,"q":"375000.0","V":"6.25","Q":"187500.0","B":"0"}}}"#.to_string()
}
/// Test-tuned [`BinanceConfig`]: aim at the given mock and shrink every
/// timer so a failing reconnect loop completes in milliseconds.
fn test_config(base_url: String) -> BinanceConfig {
BinanceConfig {
base_url,
read_timeout: Duration::from_millis(200),
initial_reconnect_delay: Duration::from_millis(5),
max_reconnect_backoff: Duration::from_millis(10),
max_reconnect_attempts: 3,
..BinanceConfig::default()
}
}
/// Spawn a mock WS server that accepts one connection, drops the
/// listener (so any reconnect lands on a refused port), and then hands
/// the upgraded socket to `handler`. Every step `.unwrap()`s — a failure
/// here is a bug in the test scaffolding, not in the production code.
async fn one_shot_server<F, Fut>(handler: F) -> String
where
F: FnOnce(WebSocketStream<TcpStream>) -> Fut + Send + 'static,
Fut: std::future::Future<Output = ()> + Send + 'static,
{
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let base_url = format!("ws://{}", listener.local_addr().unwrap());
tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
drop(listener);
let ws = tokio_tungstenite::accept_async(stream).await.unwrap();
handler(ws).await;
});
base_url
}
/// Spawn a mock WS server that accepts exactly `n_accepts` connections
/// and invokes `handler` for each (with a zero-based index). Returns a
/// [`JoinHandle`](tokio::task::JoinHandle) the test can await so every
/// spawned handler is guaranteed to reach its closing brace before the
/// runtime is torn down.
async fn multi_shot_server<F, Fut>(
n_accepts: u32,
handler: F,
) -> (String, tokio::task::JoinHandle<()>)
where
F: Fn(u32, WebSocketStream<TcpStream>) -> Fut + Send + Sync + 'static,
Fut: std::future::Future<Output = ()> + Send + 'static,
{
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let base_url = format!("ws://{}", listener.local_addr().unwrap());
let handler = Arc::new(handler);
let h = tokio::spawn(async move {
let mut joins = Vec::with_capacity(n_accepts as usize);
for index in 0..n_accepts {
let (stream, _) = listener.accept().await.unwrap();
let handler = handler.clone();
joins.push(tokio::spawn(async move {
let ws = tokio_tungstenite::accept_async(stream).await.unwrap();
handler(index, ws).await;
}));
}
for j in joins {
j.await.unwrap();
}
});
(base_url, h)
}
#[tokio::test]
async fn next_event_decodes_a_text_kline_frame() {
let kline = sample_kline_text();
let base = one_shot_server(move |mut ws| async move {
let _ = ws.send(Message::Text(kline.into())).await;
while let Some(Ok(_)) = ws.next().await {}
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
assert!(!stream.is_closed());
let event = stream
.next_event()
.await
.unwrap()
.expect("server pushes a kline");
assert_eq!(event.symbol, "btcusdt");
assert!(event.is_closed);
}
#[tokio::test]
async fn next_event_decodes_a_binary_kline_frame() {
let kline = sample_kline_text();
let base = one_shot_server(move |mut ws| async move {
let bytes: Vec<u8> = kline.into_bytes();
let _ = ws.send(Message::Binary(bytes.into())).await;
while let Some(Ok(_)) = ws.next().await {}
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
let event = stream
.next_event()
.await
.unwrap()
.expect("server pushes a kline as Binary");
assert_eq!(event.symbol, "btcusdt");
}
#[tokio::test]
async fn next_event_replies_to_a_ping_with_a_pong() {
let kline = sample_kline_text();
let base = one_shot_server(move |mut ws| async move {
let _ = ws
.send(Message::Ping(b"binance-ping".as_slice().into()))
.await;
let _ = ws.send(Message::Text(kline.into())).await;
while let Some(Ok(_)) = ws.next().await {}
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
// If the client never replied to the Ping the server's drain would
// observe nothing — but for line coverage it's enough that the
// client received the Ping, sent a Pong, then read the next frame.
let event = stream
.next_event()
.await
.unwrap()
.expect("kline arrives right after the ping");
assert_eq!(event.symbol, "btcusdt");
}
#[tokio::test]
async fn next_event_skips_inbound_pong_frames() {
let kline = sample_kline_text();
let base = one_shot_server(move |mut ws| async move {
let _ = ws
.send(Message::Pong(b"unsolicited".as_slice().into()))
.await;
let _ = ws.send(Message::Text(kline.into())).await;
while let Some(Ok(_)) = ws.next().await {}
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
let event = stream
.next_event()
.await
.unwrap()
.expect("kline follows the ignored Pong");
assert_eq!(event.symbol, "btcusdt");
}
#[tokio::test]
async fn next_event_reconnects_after_a_server_close_frame() {
let kline = sample_kline_text();
let (base, server_done) = multi_shot_server(2, move |index, mut ws| {
let kline = kline.clone();
async move {
let msg = if index == 0 {
// First connection: send a clean Close so the client
// exercises the Message::Close reconnect path.
Message::Close(None)
} else {
Message::Text(kline.into())
};
let _ = ws.send(msg).await;
}
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
let event = stream
.next_event()
.await
.unwrap()
.expect("reconnect succeeds and the second connection serves a kline");
assert_eq!(event.symbol, "btcusdt");
// Wait for every spawned handler to reach its final state.
tokio::time::timeout(Duration::from_secs(1), server_done)
.await
.unwrap()
.unwrap();
}
#[tokio::test]
async fn next_event_reconnects_after_a_read_timeout() {
let kline = sample_kline_text();
let stall_token = Arc::new(AtomicU32::new(0));
let stall_token_h = stall_token.clone();
let (base, server_done) = multi_shot_server(2, move |index, mut ws| {
let kline = kline.clone();
let stall_token = stall_token_h.clone();
async move {
if index == 0 {
// First connection: never write anything. Outlast the
// client's 80 ms read_timeout but bounded so the
// handler still completes for the coverage check.
stall_token.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(250)).await;
} else {
let _ = ws.send(Message::Text(kline.into())).await;
}
}
})
.await;
let cfg = BinanceConfig {
read_timeout: Duration::from_millis(80),
..test_config(base)
};
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
cfg,
)
.await
.unwrap();
let event = stream
.next_event()
.await
.unwrap()
.expect("client times out, reconnects, and reads the kline");
assert_eq!(event.symbol, "btcusdt");
assert!(stall_token.load(Ordering::SeqCst) >= 1);
tokio::time::timeout(Duration::from_secs(1), server_done)
.await
.unwrap()
.unwrap();
}
#[tokio::test]
async fn next_event_yields_none_after_close() {
let base = one_shot_server(|mut ws| async move {
// Stay open until the client closes; this lets close() complete
// its handshake cleanly.
while let Some(Ok(_)) = ws.next().await {}
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
stream.close().await.unwrap();
assert!(stream.is_closed());
assert!(stream.next_event().await.unwrap().is_none());
}
#[tokio::test]
async fn next_event_surfaces_an_error_when_reconnect_attempts_are_exhausted() {
// After the first accept the listener is dropped (one_shot_server
// does this), so every reconnect attempt lands on a closed port.
let base = one_shot_server(|mut ws| async move {
let _ = ws.send(Message::Close(None)).await;
// Returning here also drops the socket, but the listener has
// already been released — the client's subsequent connects
// will be refused.
})
.await;
let cfg = BinanceConfig {
max_reconnect_attempts: 2,
initial_reconnect_delay: Duration::from_millis(1),
max_reconnect_backoff: Duration::from_millis(2),
..test_config(base)
};
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
cfg,
)
.await
.unwrap();
let err = stream
.next_event()
.await
.expect_err("reconnect attempts are exhausted");
// Either a WS-layer error or a Malformed error from URL parsing —
// we only care that the call surfaced as Err rather than panicked.
let _ = err;
}
#[tokio::test]
async fn next_event_skips_non_kline_frames_and_returns_the_next_kline() {
// Drives the loop through the Text- *and* Binary-arm "frame was
// not a kline, keep reading" fall-throughs before serving the
// actual kline.
let kline = sample_kline_text();
let base = one_shot_server(move |mut ws| async move {
let _ = ws
.send(Message::Text(r#"{"result":null,"id":1}"#.into()))
.await;
let _ = ws
.send(Message::Binary(b"{\"id\":2}".to_vec().into()))
.await;
let _ = ws.send(Message::Text(kline.into())).await;
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
let event = stream
.next_event()
.await
.unwrap()
.expect("kline arrives after the two skipped control frames");
assert_eq!(event.symbol, "btcusdt");
}
#[tokio::test]
async fn next_event_propagates_a_parse_error_from_a_malformed_kline() {
// A "kline" envelope whose open field is not a number — parse_frame
// identifies it as a kline, into_event then fails, and next_event
// bubbles the error rather than skipping the frame.
let bad = r#"{"stream":"btcusdt@kline_1m","data":{"e":"kline","E":0,"s":"BTCUSDT","k":{"t":0,"T":0,"s":"BTCUSDT","i":"1m","o":"not-a-number","c":"0","h":"0","l":"0","v":"0","x":false}}}"#.to_string();
let base = one_shot_server(move |mut ws| async move {
let _ = ws.send(Message::Text(bad.into())).await;
while let Some(Ok(_)) = ws.next().await {}
})
.await;
let mut stream = BinanceKlineStream::connect_with_config(
&["BTCUSDT".to_string()],
Interval::OneMinute,
test_config(base),
)
.await
.unwrap();
let err = stream.next_event().await.unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
}
+322
View File
@@ -0,0 +1,322 @@
//! Binance spot REST historical kline fetcher.
//!
//! Where [`super::binance`] streams *live* klines over a WebSocket, this module
//! pulls *historical* candles from Binance's public REST endpoint
//! (`GET /api/v3/klines`) with a single blocking request. It is the native,
//! dependency-free replacement for hand-rolled `urllib` / `jackson` / `jsonlite`
//! download helpers in every binding.
//!
//! Example (requires the `live-binance` feature):
//!
//! ```no_run
//! use wickra_data::live::binance::Interval;
//! use wickra_data::live::binance_rest::fetch_klines;
//! # fn run() -> wickra_data::Result<()> {
//! let candles = fetch_klines("BTCUSDT", Interval::OneHour, 500, None, None)?;
//! println!("got {} candles", candles.len());
//! # Ok(()) }
//! ```
use std::fmt::Write as _;
use serde::Deserialize;
use super::binance::Interval;
use crate::error::{Error, Result};
use wickra_core::Candle;
/// Binance allows at most 1000 klines per REST request.
const MAX_LIMIT: u16 = 1000;
/// Endpoint configuration for [`fetch_klines_with_config`]. The default points
/// at Binance's public production REST host; tests and Testnet users override
/// `base_url` to aim the request elsewhere.
#[derive(Debug, Clone)]
pub struct BinanceRestConfig {
/// REST host **without** path, e.g. `"https://api.binance.com"`. The
/// `/api/v3/klines` path and query string are appended internally.
pub base_url: String,
}
impl Default for BinanceRestConfig {
fn default() -> Self {
Self {
base_url: "https://api.binance.com".to_string(),
}
}
}
/// One row of Binance's `/api/v3/klines` response. The wire format is a fixed
/// 12-element JSON array of mixed types; only the first seven fields carry the
/// OHLCV candle, the rest are ignored. Deserializing positionally lets serde
/// reject a malformed shape before we touch the numbers.
#[derive(Debug, Deserialize)]
struct RawRestKline(
i64, // open time (ms)
String, // open
String, // high
String, // low
String, // close
String, // volume
serde::de::IgnoredAny, // close time
serde::de::IgnoredAny, // quote asset volume
serde::de::IgnoredAny, // number of trades
serde::de::IgnoredAny, // taker buy base volume
serde::de::IgnoredAny, // taker buy quote volume
serde::de::IgnoredAny, // unused
);
/// Parse one of the five OHLCV string fields into an `f64`, tagging the field
/// name on failure so a bad payload is diagnosable.
fn parse_field(raw: &str, field: &str) -> Result<f64> {
raw.parse()
.map_err(|_| Error::Malformed(format!("bad {field} '{raw}'")))
}
impl RawRestKline {
fn into_candle(self) -> Result<Candle> {
let open = parse_field(&self.1, "open")?;
let high = parse_field(&self.2, "high")?;
let low = parse_field(&self.3, "low")?;
let close = parse_field(&self.4, "close")?;
let volume = parse_field(&self.5, "volume")?;
Ok(Candle::new(open, high, low, close, volume, self.0)?)
}
}
/// Fetch historical klines from Binance's production endpoint.
///
/// `symbol` is upper-cased to match Binance's convention; `limit` must be in
/// `1..=1000`. `start_ms` / `end_ms` are optional inclusive Unix-millisecond
/// bounds (`None` lets Binance pick the most recent `limit` candles). Returns
/// the candles in ascending open-time order.
///
/// # Errors
/// Returns [`Error::Malformed`] for an out-of-range `limit` or an unparseable
/// price field, [`Error::Http`] for a transport or non-2xx response,
/// [`Error::Json`] for a malformed body, and [`Error::Core`] when a row's OHLC
/// values violate candle invariants.
pub fn fetch_klines(
symbol: &str,
interval: Interval,
limit: u16,
start_ms: Option<i64>,
end_ms: Option<i64>,
) -> Result<Vec<Candle>> {
fetch_klines_with_config(
symbol,
interval,
limit,
start_ms,
end_ms,
&BinanceRestConfig::default(),
)
}
/// Like [`fetch_klines`] but against a custom [`BinanceRestConfig`] (Testnet or
/// a local mock server).
///
/// # Errors
/// See [`fetch_klines`].
pub fn fetch_klines_with_config(
symbol: &str,
interval: Interval,
limit: u16,
start_ms: Option<i64>,
end_ms: Option<i64>,
config: &BinanceRestConfig,
) -> Result<Vec<Candle>> {
if limit == 0 || limit > MAX_LIMIT {
return Err(Error::Malformed(format!(
"limit must be in 1..={MAX_LIMIT}, got {limit}"
)));
}
let mut url = format!(
"{}/api/v3/klines?symbol={}&interval={}&limit={}",
config.base_url,
symbol.to_uppercase(),
interval.as_str(),
limit
);
if let Some(start) = start_ms {
let _ = write!(url, "&startTime={start}");
}
if let Some(end) = end_ms {
let _ = write!(url, "&endTime={end}");
}
// ureq 2.x does not auto-configure native-tls, so build an agent with an
// explicit native-tls connector (verifies against the OS trust store; no
// bundled CA roots). The connector is cheap and a one-shot fetch needs no
// pooling, so we build it per call.
let connector = native_tls::TlsConnector::new()
.map_err(|e| Error::Malformed(format!("native-tls init failed: {e}")))?;
let agent = ureq::AgentBuilder::new()
.tls_connector(std::sync::Arc::new(connector))
.build();
let body = agent.get(&url).call().map_err(Box::new)?.into_string()?;
let rows: Vec<RawRestKline> = serde_json::from_str(&body)?;
rows.into_iter().map(RawRestKline::into_candle).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::thread::JoinHandle;
/// A canonical single-row klines response (12-element array, OHLCV in the
/// first seven slots) — mirrors Binance's documented wire format.
fn sample_response() -> String {
r#"[[1700000000000,"30000.0","30100.0","29950.0","30050.0","12.5",1700000059999,"375000.0",50,"6.25","187500.0","0"]]"#.to_string()
}
/// Spawn a one-shot mock HTTP server: accept a single connection, read the
/// request, and reply `200 OK` with `body`. Returns the base URL plus a
/// [`JoinHandle`] the test joins so the handler thread always reaches its
/// closing brace (every step `.unwrap()`s — a failure is a test bug).
fn mock_http(status_line: &'static str, body: String) -> (String, JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
let handle = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf).unwrap();
let response = format!(
"{status_line}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
stream.write_all(response.as_bytes()).unwrap();
});
(format!("http://{addr}"), handle)
}
#[test]
fn fetch_parses_a_real_klines_response() {
let (base, handle) = mock_http("HTTP/1.1 200 OK", sample_response());
let candles = fetch_klines_with_config(
"btcusdt",
Interval::OneHour,
1,
Some(1_700_000_000_000),
Some(1_700_000_059_999),
&BinanceRestConfig { base_url: base },
)
.unwrap();
handle.join().unwrap();
assert_eq!(candles.len(), 1);
assert_eq!(candles[0].open, 30_000.0);
assert_eq!(candles[0].high, 30_100.0);
assert_eq!(candles[0].low, 29_950.0);
assert_eq!(candles[0].close, 30_050.0);
assert_eq!(candles[0].volume, 12.5);
assert_eq!(candles[0].timestamp, 1_700_000_000_000);
}
#[test]
fn fetch_handles_an_empty_result() {
let (base, handle) = mock_http("HTTP/1.1 200 OK", "[]".to_string());
let candles = fetch_klines_with_config(
"BTCUSDT",
Interval::OneMinute,
10,
None,
None,
&BinanceRestConfig { base_url: base },
)
.unwrap();
handle.join().unwrap();
assert!(candles.is_empty());
}
#[test]
fn fetch_rejects_a_zero_limit() {
let err = fetch_klines("BTCUSDT", Interval::OneHour, 0, None, None).unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn fetch_rejects_a_limit_above_the_cap() {
let err =
fetch_klines("BTCUSDT", Interval::OneHour, MAX_LIMIT + 1, None, None).unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn fetch_surfaces_a_transport_error_for_an_unreachable_host() {
// Port 1 is privileged and unbound — the connection is refused.
let err = fetch_klines_with_config(
"BTCUSDT",
Interval::OneHour,
1,
None,
None,
&BinanceRestConfig {
base_url: "http://127.0.0.1:1".to_string(),
},
)
.unwrap_err();
assert!(matches!(err, Error::Http(_)));
}
#[test]
fn fetch_surfaces_a_json_error_for_a_malformed_body() {
let (base, handle) = mock_http("HTTP/1.1 200 OK", "not json".to_string());
let err = fetch_klines_with_config(
"BTCUSDT",
Interval::OneHour,
1,
None,
None,
&BinanceRestConfig { base_url: base },
)
.unwrap_err();
handle.join().unwrap();
assert!(matches!(err, Error::Json(_)));
}
#[test]
fn fetch_rejects_an_unparseable_price_field() {
let body =
r#"[[1700000000000,"not-a-number","0","0","0","0",0,"0",0,"0","0","0"]]"#.to_string();
let (base, handle) = mock_http("HTTP/1.1 200 OK", body);
let err = fetch_klines_with_config(
"BTCUSDT",
Interval::OneHour,
1,
None,
None,
&BinanceRestConfig { base_url: base },
)
.unwrap_err();
handle.join().unwrap();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn fetch_rejects_a_row_that_violates_candle_invariants() {
// high (1) below low (100) — Candle::new rejects it.
let body = r#"[[1700000000000,"50","1","100","50","1",0,"0",0,"0","0","0"]]"#.to_string();
let (base, handle) = mock_http("HTTP/1.1 200 OK", body);
let err = fetch_klines_with_config(
"BTCUSDT",
Interval::OneHour,
1,
None,
None,
&BinanceRestConfig { base_url: base },
)
.unwrap_err();
handle.join().unwrap();
assert!(matches!(err, Error::Core(_)));
}
#[test]
fn rest_config_default_targets_production() {
assert_eq!(
BinanceRestConfig::default().base_url,
"https://api.binance.com"
);
}
}
+224
View File
@@ -0,0 +1,224 @@
//! Resample an existing candle stream from a finer timeframe to a coarser one.
use crate::aggregator::Timeframe;
use crate::error::{Error, Result};
use wickra_core::Candle;
/// Roll a stream of candles up to a coarser timeframe.
///
/// Used to derive 5m bars from a 1m feed, or 1h bars from 5m bars, without
/// touching the original tick stream. The output timeframe's bucket must be a
/// strict multiple of the input timeframe's bucket, but this is not enforced
/// — callers are responsible for picking sensible aggregations.
#[derive(Debug, Clone)]
pub struct Resampler {
timeframe: Timeframe,
open: Option<RolledBar>,
}
#[derive(Debug, Clone, Copy)]
struct RolledBar {
bucket_start: i64,
open: f64,
high: f64,
low: f64,
close: f64,
volume: f64,
}
impl RolledBar {
fn from_candle(c: Candle, bucket_start: i64) -> Self {
Self {
bucket_start,
open: c.open,
high: c.high,
low: c.low,
close: c.close,
volume: c.volume,
}
}
fn absorb(&mut self, c: Candle) {
if c.high > self.high {
self.high = c.high;
}
if c.low < self.low {
self.low = c.low;
}
self.close = c.close;
self.volume += c.volume;
}
/// Finalise the rolled bar into a validated [`Candle`].
///
/// # Errors
/// Returns [`Error::Core`] if the accumulated `volume` is no longer finite.
/// `volume` is summed across every absorbed candle, so a long or large run
/// can drift it to `inf`; emitting such a candle would silently poison
/// every downstream indicator, so it is surfaced instead. The OHLC fields
/// are finite and correctly ordered by construction.
fn into_candle(self) -> Result<Candle> {
Candle::new(
self.open,
self.high,
self.low,
self.close,
self.volume,
self.bucket_start,
)
.map_err(Error::from)
}
}
impl Resampler {
/// Build a resampler targeting the given output timeframe.
pub fn new(timeframe: Timeframe) -> Self {
Self {
timeframe,
open: None,
}
}
/// Push a finer-grained candle. Returns the coarser candle that just closed,
/// if any.
///
/// # Errors
/// Returns [`Error::Malformed`] if `candle.timestamp` falls into a bucket
/// strictly before the currently open bar — out-of-order candles are not
/// supported, matching [`crate::aggregator::TickAggregator::push`].
pub fn push(&mut self, candle: Candle) -> Result<Option<Candle>> {
let bucket = self.timeframe.floor(candle.timestamp);
match self.open {
Some(mut bar) if bucket == bar.bucket_start => {
bar.absorb(candle);
self.open = Some(bar);
Ok(None)
}
Some(bar) if bucket > bar.bucket_start => {
let closed = bar.into_candle()?;
self.open = Some(RolledBar::from_candle(candle, bucket));
Ok(Some(closed))
}
Some(bar) => Err(Error::Malformed(format!(
"candle timestamp {} is older than the open bar start {}",
candle.timestamp, bar.bucket_start
))),
None => {
self.open = Some(RolledBar::from_candle(candle, bucket));
Ok(None)
}
}
}
/// Flush the currently open coarser bar, if any.
///
/// # Errors
/// Returns an error if the open bar's accumulated volume is non-finite
/// (see the internal `RolledBar::into_candle`).
pub fn flush(&mut self) -> Result<Option<Candle>> {
self.open.take().map(RolledBar::into_candle).transpose()
}
}
/// Roll an entire iterator of candles into a `Vec` of coarser candles. The final
/// open bar (if any) is appended via [`Resampler::flush`].
pub fn resample_all<I>(timeframe: Timeframe, iter: I) -> Result<Vec<Candle>>
where
I: IntoIterator<Item = Result<Candle>>,
{
let mut r = Resampler::new(timeframe);
let mut out = Vec::new();
for c in iter {
let c = c?;
if let Some(closed) = r.push(c)? {
out.push(closed);
}
}
if let Some(last) = r.flush()? {
out.push(last);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
fn c(ts: i64, o: f64, h: f64, l: f64, cl: f64, v: f64) -> Candle {
Candle::new(o, h, l, cl, v, ts).unwrap()
}
#[test]
fn resamples_1m_to_5m() {
let tf = Timeframe::new(5).unwrap();
let one_m = vec![
c(0, 10.0, 11.0, 9.0, 10.5, 10.0),
c(1, 10.5, 12.0, 10.0, 11.5, 12.0),
c(2, 11.5, 13.0, 11.0, 12.5, 15.0),
c(3, 12.5, 12.8, 11.5, 12.0, 8.0),
c(4, 12.0, 12.2, 11.0, 11.5, 6.0),
c(5, 11.5, 11.9, 11.0, 11.5, 4.0),
];
let rolled = resample_all(tf, one_m.into_iter().map(Ok)).unwrap();
// First 5 candles share bucket 0 -> aggregate. Last candle opens bucket 5.
assert_eq!(rolled.len(), 2);
let a = rolled[0];
assert_eq!(a.open, 10.0);
assert_eq!(a.close, 11.5);
assert_eq!(a.high, 13.0);
assert_eq!(a.low, 9.0);
assert!((a.volume - 51.0).abs() < 1e-12);
let b = rolled[1];
assert_eq!(b.open, 11.5);
assert_eq!(b.timestamp, 5);
}
#[test]
fn rejects_out_of_order_candle() {
let mut r = Resampler::new(Timeframe::new(5).unwrap());
assert!(r.push(c(10, 10.0, 11.0, 9.0, 10.5, 1.0)).unwrap().is_none());
// A candle in an earlier bucket than the open bar is rejected.
let err = r.push(c(2, 10.0, 11.0, 9.0, 10.5, 1.0)).unwrap_err();
assert!(matches!(err, Error::Malformed(_)));
}
#[test]
fn same_bucket_candles_aggregate() {
let mut r = Resampler::new(Timeframe::new(5).unwrap());
assert!(r.push(c(0, 10.0, 11.0, 9.0, 10.5, 1.0)).unwrap().is_none());
assert!(r.push(c(3, 10.5, 12.0, 10.0, 11.0, 1.0)).unwrap().is_none());
let bar = r.flush().unwrap().unwrap();
assert_eq!(bar.high, 12.0);
assert_eq!(bar.low, 9.0);
}
#[test]
fn absorb_lowers_low_on_dipping_candle() {
// The first candle in the bucket sets low = 10.0; the second dips to
// 8.0 and must overwrite. Exercises the `c.low < self.low` branch in
// RolledBar::absorb that the other resampler tests never trigger
// because their follow-up candles always have a higher low.
let mut r = Resampler::new(Timeframe::new(5).unwrap());
r.push(c(0, 10.0, 11.0, 10.0, 10.5, 1.0)).unwrap();
r.push(c(1, 10.5, 11.5, 8.0, 9.0, 1.0)).unwrap();
let bar = r.flush().unwrap().unwrap();
assert_eq!(bar.low, 8.0);
assert_eq!(bar.high, 11.5);
}
#[test]
fn flushes_a_non_finite_volume_as_an_error() {
let mut r = Resampler::new(Timeframe::new(5).unwrap());
// Two near-max volumes in the same bucket sum to +inf.
assert!(r
.push(c(0, 10.0, 11.0, 9.0, 10.5, f64::MAX))
.unwrap()
.is_none());
assert!(r
.push(c(1, 10.0, 11.0, 9.0, 10.5, f64::MAX))
.unwrap()
.is_none());
let err = r.flush().unwrap_err();
assert!(matches!(err, Error::Core(_)));
}
}