first commit
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
[package]
|
||||
name = "wickra-data"
|
||||
description = "Data sources for Wickra: CSV readers, tick-to-candle aggregator, and live exchange feeds."
|
||||
version.workspace = true
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
homepage.workspace = true
|
||||
readme.workspace = true
|
||||
keywords.workspace = true
|
||||
categories.workspace = true
|
||||
documentation = "https://docs.rs/wickra-data"
|
||||
|
||||
# Render the docs on docs.rs with every feature enabled so the optional
|
||||
# live-binance feed is documented (otherwise it is hidden behind its feature).
|
||||
[package.metadata.docs.rs]
|
||||
all-features = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# Direct path+version (not the workspace inheritance) so `default-features = false`
|
||||
# is honoured: depending on wickra-data must NOT force wickra-core's `parallel`
|
||||
# (rayon) feature on — the WASM binding needs a rayon-free build and the data
|
||||
# layer never uses the parallel batch path. Native bindings re-enable `parallel`
|
||||
# through their own wickra-core dependency (cargo unifies the features).
|
||||
wickra-core = { path = "../wickra-core", version = "0.9.9", default-features = false }
|
||||
thiserror = { workspace = true }
|
||||
csv = "1.3"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
# Async / live feeds are opt-in: only pulled when a `live-*` feature is requested.
|
||||
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "net", "time", "io-util"], optional = true }
|
||||
tokio-tungstenite = { version = "0.29", optional = true, features = ["native-tls"] }
|
||||
futures-util = { version = "0.3", optional = true }
|
||||
url = { version = "2", optional = true }
|
||||
# Blocking HTTP client for the historical REST kline fetcher. ureq 2.x with
|
||||
# native-tls verifies against the OS trust store (SChannel / Security.framework /
|
||||
# OpenSSL) — the same backend tokio-tungstenite uses above — so it ships no
|
||||
# bundled CA roots. (ureq 3.x hard-depends on `webpki-root-certs` regardless of
|
||||
# TLS backend, which is CDLA-Permissive-2.0 and dead weight under native-tls.)
|
||||
ureq = { version = "2", default-features = false, features = ["native-tls"], optional = true }
|
||||
# Direct dependency so the agent can be built with a native-tls connector (ureq
|
||||
# 2.x does not auto-configure native-tls). Already in the tree via tokio-tungstenite.
|
||||
native-tls = { version = "0.2", optional = true }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Each exchange is gated so users only pay for the WS/REST stack they actually
|
||||
# want. `live-binance` covers both the live WebSocket feed and the historical
|
||||
# REST kline fetcher.
|
||||
live-binance = ["dep:tokio", "dep:tokio-tungstenite", "dep:futures-util", "dep:url", "dep:ureq", "dep:native-tls"]
|
||||
# `live-binance` with a statically built OpenSSL instead of the system one. The
|
||||
# native-tls stack (tokio-tungstenite + ureq, unified on the same `native-tls`
|
||||
# crate) links `openssl-sys`, which needs OpenSSL at build time. The manylinux
|
||||
# and musllinux wheel containers do not provide it — manylinux lacks the headers
|
||||
# and the musllinux build cross-compiles against a musl sysroot that has no
|
||||
# OpenSSL at all — so the Linux wheels are built with this feature, which
|
||||
# compiles OpenSSL from source and links it statically. No-op on macOS/Windows,
|
||||
# where native-tls uses Security.framework / SChannel and never pulls openssl-sys.
|
||||
vendored-tls = ["live-binance", "native-tls/vendored"]
|
||||
|
||||
[dev-dependencies]
|
||||
approx = { workspace = true }
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,614 @@
|
||||
//! Roll trade ticks up into candles of an arbitrary timeframe.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use wickra_core::{Candle, Tick};
|
||||
|
||||
/// Hard cap on the number of placeholder candles a single
|
||||
/// [`TickAggregator::push`] call may emit when gap-fill is enabled. One
|
||||
/// million minute-candles is roughly 1.9 years of contiguous one-minute bars
|
||||
/// — orders of magnitude beyond any realistic missing-data window in
|
||||
/// production while still keeping the resulting `Vec<Candle>` to well under
|
||||
/// 50 MB. Any larger gap is treated as malformed input rather than allowed
|
||||
/// to OOM the process.
|
||||
pub const MAX_GAP_FILL_CANDLES: i64 = 1_000_000;
|
||||
|
||||
/// A candle bucket size measured in the same unit as the tick timestamps.
|
||||
///
|
||||
/// Wickra is unit-agnostic about timestamps: choose whichever makes sense for
|
||||
/// your source (milliseconds for Binance trade events, microseconds for IB,
|
||||
/// seconds for daily bars).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Timeframe {
|
||||
bucket: i64,
|
||||
}
|
||||
|
||||
impl Timeframe {
|
||||
/// Construct a timeframe with the given bucket size in the chosen unit.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::InvalidTimeframe`] if `bucket <= 0`.
|
||||
pub fn new(bucket: i64) -> Result<Self> {
|
||||
if bucket <= 0 {
|
||||
return Err(Error::InvalidTimeframe(format!(
|
||||
"bucket size must be positive, got {bucket}"
|
||||
)));
|
||||
}
|
||||
Ok(Self { bucket })
|
||||
}
|
||||
|
||||
/// Convenience: build a millisecond timeframe.
|
||||
pub fn millis(ms: i64) -> Result<Self> {
|
||||
Self::new(ms)
|
||||
}
|
||||
|
||||
/// Convenience: build a seconds-resolution timeframe.
|
||||
pub fn seconds(s: i64) -> Result<Self> {
|
||||
Self::new(s)
|
||||
}
|
||||
|
||||
/// One-minute timeframe in milliseconds (`60_000`).
|
||||
pub fn one_minute_ms() -> Self {
|
||||
Self::new(60_000).expect("60_000 > 0")
|
||||
}
|
||||
|
||||
/// Convenience: build a timeframe of `n` whole minutes, measured in
|
||||
/// seconds — consistent with [`Timeframe::seconds`].
|
||||
///
|
||||
/// `minutes(5)` yields a bucket of `300`, for use with second-resolution
|
||||
/// timestamps. For millisecond timestamps (Binance) multiply yourself or
|
||||
/// use [`Timeframe::millis`].
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::InvalidTimeframe`] if `n` is not positive or if
|
||||
/// `n * 60` overflows `i64`.
|
||||
///
|
||||
/// ```
|
||||
/// use wickra_data::aggregator::Timeframe;
|
||||
/// assert_eq!(Timeframe::minutes(5)?.bucket(), 300);
|
||||
/// # Ok::<(), wickra_data::Error>(())
|
||||
/// ```
|
||||
pub fn minutes(n: i64) -> Result<Self> {
|
||||
let bucket = n
|
||||
.checked_mul(60)
|
||||
.ok_or_else(|| Error::InvalidTimeframe(format!("{n} minutes overflows i64 seconds")))?;
|
||||
Self::new(bucket)
|
||||
}
|
||||
|
||||
/// Convenience: build a timeframe of `n` whole hours, measured in seconds
|
||||
/// (`hours(2)` → a bucket of `7_200`).
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::InvalidTimeframe`] if `n` is not positive or if
|
||||
/// `n * 3_600` overflows `i64`.
|
||||
///
|
||||
/// ```
|
||||
/// use wickra_data::aggregator::Timeframe;
|
||||
/// assert_eq!(Timeframe::hours(2)?.bucket(), 7_200);
|
||||
/// # Ok::<(), wickra_data::Error>(())
|
||||
/// ```
|
||||
pub fn hours(n: i64) -> Result<Self> {
|
||||
let bucket = n
|
||||
.checked_mul(3_600)
|
||||
.ok_or_else(|| Error::InvalidTimeframe(format!("{n} hours overflows i64 seconds")))?;
|
||||
Self::new(bucket)
|
||||
}
|
||||
|
||||
/// Convenience: build a timeframe of `n` whole days, measured in seconds
|
||||
/// (`days(1)` → a bucket of `86_400`).
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::InvalidTimeframe`] if `n` is not positive or if
|
||||
/// `n * 86_400` overflows `i64`.
|
||||
///
|
||||
/// ```
|
||||
/// use wickra_data::aggregator::Timeframe;
|
||||
/// assert_eq!(Timeframe::days(1)?.bucket(), 86_400);
|
||||
/// # Ok::<(), wickra_data::Error>(())
|
||||
/// ```
|
||||
pub fn days(n: i64) -> Result<Self> {
|
||||
let bucket = n
|
||||
.checked_mul(86_400)
|
||||
.ok_or_else(|| Error::InvalidTimeframe(format!("{n} days overflows i64 seconds")))?;
|
||||
Self::new(bucket)
|
||||
}
|
||||
|
||||
/// Bucket size.
|
||||
pub const fn bucket(self) -> i64 {
|
||||
self.bucket
|
||||
}
|
||||
|
||||
/// Floor a raw timestamp to this timeframe's bucket boundary.
|
||||
///
|
||||
/// For a timestamp within one bucket of [`i64::MIN`] the mathematically
|
||||
/// exact boundary lies below `i64::MIN` and cannot be represented; in that
|
||||
/// (practically unreachable) case the result saturates at `i64::MIN`
|
||||
/// rather than overflowing and panicking in debug builds. `bucket` is
|
||||
/// always positive, so `rem_euclid` itself cannot panic.
|
||||
pub fn floor(self, ts: i64) -> i64 {
|
||||
ts.saturating_sub(ts.rem_euclid(self.bucket))
|
||||
}
|
||||
}
|
||||
|
||||
/// Incrementally builds candles out of arriving ticks.
|
||||
///
|
||||
/// Each call to [`TickAggregator::push`] returns the candles that closed as a
|
||||
/// result of the new tick — normally at most one. Use
|
||||
/// [`TickAggregator::flush`] at the end of a stream to capture the final open
|
||||
/// bar.
|
||||
///
|
||||
/// # Gaps
|
||||
///
|
||||
/// By default a tick that jumps across one or more empty buckets simply opens
|
||||
/// the next non-empty bar — the skipped buckets produce no candle, so the
|
||||
/// output series can have time holes. Enable [`TickAggregator::with_gap_fill`]
|
||||
/// to instead emit a flat placeholder candle for every skipped bucket, giving
|
||||
/// downstream indicators an unbroken, evenly spaced series. To bound memory
|
||||
/// against an adversarial timestamp jump, gap-filling refuses to emit more
|
||||
/// than [`MAX_GAP_FILL_CANDLES`] placeholders in a single step; a larger gap
|
||||
/// surfaces as an `Error::Malformed` so the caller can decide how to handle
|
||||
/// the discontinuity.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TickAggregator {
|
||||
timeframe: Timeframe,
|
||||
open_bar: Option<OpenBar>,
|
||||
fill_gaps: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct OpenBar {
|
||||
bucket_start: i64,
|
||||
/// Timestamp of the most recently absorbed tick. Used to reject ticks that
|
||||
/// arrive out of order *within* the current bucket — without it an older
|
||||
/// tick would silently overwrite `close` with a stale price.
|
||||
last_ts: i64,
|
||||
open: f64,
|
||||
high: f64,
|
||||
low: f64,
|
||||
close: f64,
|
||||
volume: f64,
|
||||
}
|
||||
|
||||
impl OpenBar {
|
||||
fn from_tick(t: Tick, bucket_start: i64) -> Self {
|
||||
Self {
|
||||
bucket_start,
|
||||
last_ts: t.timestamp,
|
||||
open: t.price,
|
||||
high: t.price,
|
||||
low: t.price,
|
||||
close: t.price,
|
||||
volume: t.volume,
|
||||
}
|
||||
}
|
||||
|
||||
fn absorb(&mut self, t: Tick) {
|
||||
if t.price > self.high {
|
||||
self.high = t.price;
|
||||
}
|
||||
if t.price < self.low {
|
||||
self.low = t.price;
|
||||
}
|
||||
self.close = t.price;
|
||||
self.volume += t.volume;
|
||||
self.last_ts = t.timestamp;
|
||||
}
|
||||
|
||||
/// Finalise the bar into a validated [`Candle`].
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Core`] if the accumulated `volume` is no longer finite.
|
||||
/// `volume` is summed across every absorbed tick, so an astronomically
|
||||
/// long or large run can drift it to `inf`; emitting such a candle would
|
||||
/// silently poison every downstream indicator, so it is surfaced instead.
|
||||
/// The OHLC fields are finite and correctly ordered by construction, so
|
||||
/// `Candle::new` only ever rejects this bar for a non-finite volume.
|
||||
fn into_candle(self) -> Result<Candle> {
|
||||
Candle::new(
|
||||
self.open,
|
||||
self.high,
|
||||
self.low,
|
||||
self.close,
|
||||
self.volume,
|
||||
self.bucket_start,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl TickAggregator {
|
||||
/// Construct a new aggregator for the given timeframe.
|
||||
pub fn new(timeframe: Timeframe) -> Self {
|
||||
Self {
|
||||
timeframe,
|
||||
open_bar: None,
|
||||
fill_gaps: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable or disable gap filling, returning the (re)configured aggregator.
|
||||
///
|
||||
/// When enabled, [`push`](Self::push) emits a flat candle
|
||||
/// (`open == high == low == close`, `volume == 0`) for every bucket that is
|
||||
/// skipped between two consecutive ticks. The flat candle's price is the
|
||||
/// close of the bar that preceded the gap, so the series stays continuous.
|
||||
#[must_use]
|
||||
pub fn with_gap_fill(mut self, fill: bool) -> Self {
|
||||
self.fill_gaps = fill;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether gap filling is enabled.
|
||||
pub const fn fills_gaps(&self) -> bool {
|
||||
self.fill_gaps
|
||||
}
|
||||
|
||||
/// Push a tick. Returns every candle that closed as a result — an empty
|
||||
/// vector while the open bar keeps growing, one candle when a bar boundary
|
||||
/// is crossed, and (with gap filling enabled) additionally one flat candle
|
||||
/// per skipped bucket.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Malformed`] if `tick.timestamp` goes backwards — both
|
||||
/// across buckets (older than the open bar's start) and within a bucket
|
||||
/// (older than the last tick absorbed into it) — or if gap filling
|
||||
/// overflows the timestamp range. Ticks sharing a timestamp are accepted.
|
||||
pub fn push(&mut self, tick: Tick) -> Result<Vec<Candle>> {
|
||||
let bucket = self.timeframe.floor(tick.timestamp);
|
||||
if let Some(mut bar) = self.open_bar {
|
||||
if bucket < bar.bucket_start {
|
||||
return Err(Error::Malformed(format!(
|
||||
"tick timestamp {} is older than the open bar start {}",
|
||||
tick.timestamp, bar.bucket_start
|
||||
)));
|
||||
}
|
||||
if bucket > bar.bucket_start {
|
||||
// Close the previous bar and start a new one with this tick.
|
||||
let closed = bar.into_candle()?;
|
||||
let mut out = Vec::with_capacity(1);
|
||||
out.push(closed);
|
||||
if self.fill_gaps {
|
||||
self.fill_between(closed, bucket, &mut out)?;
|
||||
}
|
||||
self.open_bar = Some(OpenBar::from_tick(tick, bucket));
|
||||
return Ok(out);
|
||||
}
|
||||
// Same bucket: reject a tick that predates the last one absorbed,
|
||||
// which would otherwise overwrite `close` with a stale price.
|
||||
// Equal timestamps are allowed — several trades can share a
|
||||
// millisecond.
|
||||
if tick.timestamp < bar.last_ts {
|
||||
return Err(Error::Malformed(format!(
|
||||
"tick timestamp {} predates the last tick {} in the same bucket",
|
||||
tick.timestamp, bar.last_ts
|
||||
)));
|
||||
}
|
||||
bar.absorb(tick);
|
||||
self.open_bar = Some(bar);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.open_bar = Some(OpenBar::from_tick(tick, bucket));
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
/// Append a flat placeholder candle for every empty bucket strictly between
|
||||
/// the just-closed bar and the next bucket that received a tick.
|
||||
///
|
||||
/// Returns `Error::Malformed` when the gap would exceed
|
||||
/// [`MAX_GAP_FILL_CANDLES`] — an adversarial timestamp jump (a clock-glitch
|
||||
/// tick years in the future) must surface as a defined error, not as an
|
||||
/// out-of-memory panic from allocating millions of placeholder candles.
|
||||
fn fill_between(&self, prev: Candle, next_bucket: i64, out: &mut Vec<Candle>) -> Result<()> {
|
||||
let step = self.timeframe.bucket();
|
||||
let start = prev
|
||||
.timestamp
|
||||
.checked_add(step)
|
||||
.ok_or_else(|| Error::Malformed("timestamp overflow while gap-filling".to_string()))?;
|
||||
if start >= next_bucket {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Compute the gap size up-front so an adversarial timestamp delta
|
||||
// is refused before we allocate. `step > 0` by `Timeframe::new`'s
|
||||
// invariant, so the divisor is safe. Saturating the subtraction
|
||||
// makes the arithmetic infallible; an overflowed-saturated span is
|
||||
// still far above the cap so the limit check below catches it.
|
||||
let span = next_bucket.saturating_sub(start);
|
||||
let gap_count = span / step + i64::from(span % step != 0);
|
||||
|
||||
if gap_count > MAX_GAP_FILL_CANDLES {
|
||||
return Err(Error::Malformed(format!(
|
||||
"gap-fill between bucket {} and {next_bucket} would emit {gap_count} \
|
||||
flat candles at step {step}, exceeding the {MAX_GAP_FILL_CANDLES} \
|
||||
cap; reject the discontinuity instead of allocating",
|
||||
prev.timestamp
|
||||
)));
|
||||
}
|
||||
|
||||
out.reserve(gap_count as usize);
|
||||
// Bucket alignment guarantees start + (gap_count - 1) * step ≤
|
||||
// next_bucket - step < i64::MAX, so iterating `gap_count` times
|
||||
// with `saturating_add(step)` cannot reach i64::MAX inside the
|
||||
// loop body. `prev.close` is finite (it came from a validated
|
||||
// bar) and volume is exactly 0.0, so the OHLCV invariants hold
|
||||
// by construction — skip re-validation via Candle::new_unchecked.
|
||||
let mut t = start;
|
||||
for _ in 0..gap_count {
|
||||
out.push(Candle::new_unchecked(
|
||||
prev.close, prev.close, prev.close, prev.close, 0.0, t,
|
||||
));
|
||||
t = t.saturating_add(step);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drain the currently open bar (if any) and return it. Useful at the end of
|
||||
/// a backtest or when shutting down a live aggregator.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the open bar's accumulated volume is non-finite
|
||||
/// (see the internal `OpenBar::into_candle`).
|
||||
pub fn flush(&mut self) -> Result<Option<Candle>> {
|
||||
self.open_bar.take().map(OpenBar::into_candle).transpose()
|
||||
}
|
||||
|
||||
/// Configured timeframe.
|
||||
pub const fn timeframe(&self) -> Timeframe {
|
||||
self.timeframe
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn t(price: f64, ts: i64) -> Tick {
|
||||
Tick::new(price, 1.0, ts).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeframe_rejects_non_positive() {
|
||||
assert!(Timeframe::new(0).is_err());
|
||||
assert!(Timeframe::new(-1).is_err());
|
||||
}
|
||||
|
||||
/// Cover the `Timeframe::millis`, `Timeframe::seconds`, and
|
||||
/// `Timeframe::one_minute_ms` convenience constructors (lines 40-52).
|
||||
/// All existing tests build Timeframes via `new` / `minutes` / `hours` /
|
||||
/// `days`, never via the three thin convenience wrappers.
|
||||
#[test]
|
||||
fn timeframe_convenience_constructors() {
|
||||
assert_eq!(Timeframe::millis(250).unwrap().bucket(), 250);
|
||||
assert!(Timeframe::millis(0).is_err());
|
||||
assert_eq!(Timeframe::seconds(30).unwrap().bucket(), 30);
|
||||
assert!(Timeframe::seconds(-1).is_err());
|
||||
// one_minute_ms is the infallible 60_000-ms shortcut.
|
||||
assert_eq!(Timeframe::one_minute_ms().bucket(), 60_000);
|
||||
}
|
||||
|
||||
/// Cover the `TickAggregator::timeframe` const accessor (lines 353-355).
|
||||
/// Existing tests only inspect emitted candles, never query the
|
||||
/// configured timeframe back out.
|
||||
#[test]
|
||||
fn aggregator_timeframe_getter() {
|
||||
let tf = Timeframe::new(60).unwrap();
|
||||
let agg = TickAggregator::new(tf);
|
||||
assert_eq!(agg.timeframe().bucket(), 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minute_hour_day_constructors_compute_seconds() {
|
||||
assert_eq!(Timeframe::minutes(1).unwrap().bucket(), 60);
|
||||
assert_eq!(Timeframe::minutes(5).unwrap().bucket(), 300);
|
||||
assert_eq!(Timeframe::hours(1).unwrap().bucket(), 3_600);
|
||||
assert_eq!(Timeframe::hours(4).unwrap().bucket(), 14_400);
|
||||
assert_eq!(Timeframe::days(1).unwrap().bucket(), 86_400);
|
||||
assert_eq!(Timeframe::days(7).unwrap().bucket(), 604_800);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minute_hour_day_constructors_reject_non_positive() {
|
||||
for n in [0, -1, -60] {
|
||||
assert!(Timeframe::minutes(n).is_err());
|
||||
assert!(Timeframe::hours(n).is_err());
|
||||
assert!(Timeframe::days(n).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minute_hour_day_constructors_reject_overflow() {
|
||||
// `n * unit` overflows i64 long before `new`'s sign check runs.
|
||||
assert!(matches!(
|
||||
Timeframe::minutes(i64::MAX),
|
||||
Err(Error::InvalidTimeframe(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
Timeframe::hours(i64::MAX),
|
||||
Err(Error::InvalidTimeframe(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
Timeframe::days(i64::MAX),
|
||||
Err(Error::InvalidTimeframe(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floors_to_bucket_boundary() {
|
||||
let tf = Timeframe::new(100).unwrap();
|
||||
assert_eq!(tf.floor(0), 0);
|
||||
assert_eq!(tf.floor(99), 0);
|
||||
assert_eq!(tf.floor(100), 100);
|
||||
assert_eq!(tf.floor(150), 100);
|
||||
assert_eq!(tf.floor(250), 200);
|
||||
// Negative timestamps still floor toward negative infinity.
|
||||
assert_eq!(tf.floor(-1), -100);
|
||||
assert_eq!(tf.floor(-100), -100);
|
||||
assert_eq!(tf.floor(-101), -200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floor_saturates_instead_of_overflowing_at_min() {
|
||||
let tf = Timeframe::new(100).unwrap();
|
||||
// The exact boundary lies below i64::MIN — must not panic.
|
||||
assert_eq!(tf.floor(i64::MIN), i64::MIN);
|
||||
// i64::MAX must not overflow either (subtracting a non-negative).
|
||||
let hi = tf.floor(i64::MAX);
|
||||
assert!(hi > i64::MAX - 100 && hi % 100 == 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregates_ticks_into_one_candle_within_bucket() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
|
||||
assert!(agg.push(t(10.0, 0)).unwrap().is_empty());
|
||||
assert!(agg.push(t(12.0, 15)).unwrap().is_empty());
|
||||
assert!(agg.push(t(8.0, 30)).unwrap().is_empty());
|
||||
assert!(agg.push(t(11.0, 50)).unwrap().is_empty());
|
||||
let bar = agg.flush().unwrap().expect("open bar");
|
||||
assert_eq!(bar.open, 10.0);
|
||||
assert_eq!(bar.high, 12.0);
|
||||
assert_eq!(bar.low, 8.0);
|
||||
assert_eq!(bar.close, 11.0);
|
||||
assert!((bar.volume - 4.0).abs() < 1e-12);
|
||||
assert_eq!(bar.timestamp, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_candle_on_bucket_crossing() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
|
||||
agg.push(t(10.0, 0)).unwrap();
|
||||
agg.push(t(12.0, 30)).unwrap();
|
||||
let closed = agg.push(t(15.0, 60)).unwrap();
|
||||
assert_eq!(closed.len(), 1);
|
||||
let closed = closed[0];
|
||||
assert_eq!(closed.open, 10.0);
|
||||
assert_eq!(closed.high, 12.0);
|
||||
assert_eq!(closed.low, 10.0);
|
||||
assert_eq!(closed.close, 12.0);
|
||||
|
||||
// The new tick at ts=60 opens the next bar.
|
||||
let still_open = agg.flush().unwrap().unwrap();
|
||||
assert_eq!(still_open.open, 15.0);
|
||||
assert_eq!(still_open.timestamp, 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_out_of_order_ticks() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
|
||||
agg.push(t(10.0, 100)).unwrap();
|
||||
let err = agg.push(t(11.0, 30)).unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_same_bucket_out_of_order_tick() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
|
||||
agg.push(t(10.0, 50)).unwrap();
|
||||
// ts=10 is still bucket 0 but predates the tick at ts=50 — rejecting
|
||||
// it prevents a stale price silently overwriting `close`.
|
||||
let err = agg.push(t(99.0, 10)).unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
// The open bar is untouched: close is still the ts=50 price.
|
||||
assert_eq!(agg.flush().unwrap().unwrap().close, 10.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_same_bucket_ticks_sharing_a_timestamp() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
|
||||
agg.push(t(10.0, 20)).unwrap();
|
||||
// Two trades in the same millisecond are legitimate.
|
||||
agg.push(t(12.0, 20)).unwrap();
|
||||
agg.push(t(11.0, 20)).unwrap();
|
||||
let bar = agg.flush().unwrap().unwrap();
|
||||
assert_eq!(bar.high, 12.0);
|
||||
assert_eq!(bar.close, 11.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flushes_a_non_finite_volume_as_an_error() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
|
||||
// Two near-max volumes sum to +inf — the closed candle would carry a
|
||||
// non-finite volume that poisons every downstream indicator.
|
||||
agg.push(Tick::new(10.0, f64::MAX, 0).unwrap()).unwrap();
|
||||
agg.push(Tick::new(10.0, f64::MAX, 1).unwrap()).unwrap();
|
||||
let err = agg.flush().unwrap_err();
|
||||
assert!(matches!(err, Error::Core(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_empty_buckets_without_gap_fill() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap());
|
||||
assert!(!agg.fills_gaps());
|
||||
agg.push(t(10.0, 0)).unwrap();
|
||||
// Jump from bucket 0 straight to bucket 180 — buckets 60 and 120 empty.
|
||||
let closed = agg.push(t(20.0, 200)).unwrap();
|
||||
assert_eq!(closed.len(), 1, "only the real bar closes");
|
||||
assert_eq!(closed[0].timestamp, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gap_fill_rejects_runaway_timestamp_jump() {
|
||||
// An adversarial clock-glitch tick years in the future must surface
|
||||
// as an Error::Malformed rather than allocating millions of flat
|
||||
// candles and OOMing. Found by the `tick_aggregator` fuzz target.
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
|
||||
agg.push(t(10.0, 0)).unwrap();
|
||||
// Two-billion-second jump = ~63 years of minute bars = ~33 million
|
||||
// candles, well above the 1_000_000 cap.
|
||||
let err = agg.push(t(20.0, 2_000_000_000)).unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("gap-fill") && msg.contains("cap"),
|
||||
"expected a malformed-gap error, got: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gap_fill_at_the_cap_succeeds() {
|
||||
// Exactly one million minute-buckets between the two ticks (one real
|
||||
// bar + one million flat fillers + the third tick's open bar) — the
|
||||
// limit is inclusive, so this must succeed.
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
|
||||
agg.push(t(10.0, 0)).unwrap();
|
||||
// bucket 0 closes; jump straight to bucket 60_000_060 (1_000_001 buckets
|
||||
// away). fill_between emits 1_000_000 flat candles between them, then
|
||||
// the new tick opens its own bucket. Output: 1 real bar + 1_000_000 fillers.
|
||||
let out = agg.push(t(20.0, 60_000_060)).unwrap();
|
||||
assert_eq!(out.len(), 1 + MAX_GAP_FILL_CANDLES as usize);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gap_fill_emits_flat_candles_for_skipped_buckets() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
|
||||
assert!(agg.fills_gaps());
|
||||
agg.push(t(10.0, 0)).unwrap();
|
||||
agg.push(t(13.0, 30)).unwrap(); // still bucket 0, close = 13.0
|
||||
// Next tick lands in bucket 180 — buckets 60 and 120 are skipped.
|
||||
let out = agg.push(t(20.0, 200)).unwrap();
|
||||
assert_eq!(out.len(), 3, "real bar + two flat fillers");
|
||||
|
||||
let real = out[0];
|
||||
assert_eq!(real.timestamp, 0);
|
||||
assert_eq!(real.close, 13.0);
|
||||
|
||||
for (filler, ts) in out[1..].iter().zip([60, 120]) {
|
||||
assert_eq!(filler.timestamp, ts);
|
||||
assert_eq!(filler.open, 13.0);
|
||||
assert_eq!(filler.high, 13.0);
|
||||
assert_eq!(filler.low, 13.0);
|
||||
assert_eq!(filler.close, 13.0);
|
||||
assert_eq!(filler.volume, 0.0);
|
||||
}
|
||||
|
||||
// The tick at ts=200 opens bucket 180.
|
||||
assert_eq!(agg.flush().unwrap().unwrap().timestamp, 180);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gap_fill_emits_nothing_extra_for_adjacent_buckets() {
|
||||
let mut agg = TickAggregator::new(Timeframe::new(60).unwrap()).with_gap_fill(true);
|
||||
agg.push(t(10.0, 0)).unwrap();
|
||||
// Bucket 60 directly follows bucket 0 — no gap to fill.
|
||||
let out = agg.push(t(11.0, 70)).unwrap();
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].timestamp, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
//! Stream OHLCV candles out of a CSV file.
|
||||
//!
|
||||
//! The reader is generic over the column layout, but ships with a sensible
|
||||
//! default ("timestamp,open,high,low,close,volume") that matches the standard
|
||||
//! Binance / Yahoo Finance / kaggle dataset format.
|
||||
//!
|
||||
//! The reader is defensive about real-world files: a leading UTF-8 byte-order
|
||||
//! mark is stripped, surrounding whitespace is trimmed from every field, and a
|
||||
//! file whose header does not name the required columns is rejected with a
|
||||
//! clear [`Error::Malformed`] instead of silently consuming its first data row.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use wickra_core::Candle;
|
||||
|
||||
/// Column names the default OHLCV layout requires. The CSV header must contain
|
||||
/// every one of these (extra columns are ignored); matching is exact and
|
||||
/// case-sensitive because the underlying `serde` deserialization maps header
|
||||
/// names to [`DefaultRow`]'s fields by name.
|
||||
const REQUIRED_COLUMNS: [&str; 6] = ["timestamp", "open", "high", "low", "close", "volume"];
|
||||
|
||||
/// Default OHLCV CSV row layout.
|
||||
///
|
||||
/// The timestamp is parsed as an `i64` (for example a Unix epoch). RFC3339 /
|
||||
/// ISO8601 string timestamps are not handled by this layout; convert them to
|
||||
/// integers before reading.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct DefaultRow {
|
||||
pub timestamp: i64,
|
||||
pub open: f64,
|
||||
pub high: f64,
|
||||
pub low: f64,
|
||||
pub close: f64,
|
||||
pub volume: f64,
|
||||
}
|
||||
|
||||
impl DefaultRow {
|
||||
fn into_candle(self) -> Result<Candle> {
|
||||
Candle::new(
|
||||
self.open,
|
||||
self.high,
|
||||
self.low,
|
||||
self.close,
|
||||
self.volume,
|
||||
self.timestamp,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`std::io::Read`] adapter that transparently skips a leading UTF-8
|
||||
/// byte-order mark.
|
||||
///
|
||||
/// Spreadsheet exporters — Excel in particular — prefix CSV files with the
|
||||
/// three-byte UTF-8 BOM `EF BB BF`. Left in place it becomes part of the first
|
||||
/// header name (`\u{feff}timestamp`), which then fails to match the
|
||||
/// `timestamp` column. This adapter drops the BOM before the CSV parser ever
|
||||
/// sees it; files without a BOM pass through untouched.
|
||||
#[derive(Debug)]
|
||||
pub struct BomStripReader<R> {
|
||||
inner: R,
|
||||
/// Whether the leading bytes have been inspected for a BOM yet.
|
||||
checked: bool,
|
||||
/// Bytes read during BOM detection that turned out *not* to be a BOM and
|
||||
/// must still be handed to the consumer.
|
||||
leftover: Vec<u8>,
|
||||
leftover_pos: usize,
|
||||
}
|
||||
|
||||
impl<R: std::io::Read> BomStripReader<R> {
|
||||
/// Wrap `inner`, stripping a leading UTF-8 BOM on the first read.
|
||||
pub fn new(inner: R) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
checked: false,
|
||||
leftover: Vec::new(),
|
||||
leftover_pos: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// On the first read, consume up to three bytes and decide whether they
|
||||
/// form a BOM. A BOM is discarded; anything else is buffered for replay.
|
||||
fn check_bom(&mut self) -> std::io::Result<()> {
|
||||
if self.checked {
|
||||
return Ok(());
|
||||
}
|
||||
self.checked = true;
|
||||
|
||||
let mut probe = [0u8; 3];
|
||||
let mut filled = 0;
|
||||
while filled < probe.len() {
|
||||
let n = self.inner.read(&mut probe[filled..])?;
|
||||
if n == 0 {
|
||||
break; // short source — fewer than 3 bytes total
|
||||
}
|
||||
filled += n;
|
||||
}
|
||||
|
||||
if probe[..filled] != [0xEF, 0xBB, 0xBF] {
|
||||
// Not a BOM (or a short file): replay every probed byte verbatim.
|
||||
self.leftover.extend_from_slice(&probe[..filled]);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: std::io::Read> std::io::Read for BomStripReader<R> {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
self.check_bom()?;
|
||||
if self.leftover_pos < self.leftover.len() {
|
||||
let n = (self.leftover.len() - self.leftover_pos).min(buf.len());
|
||||
buf[..n].copy_from_slice(&self.leftover[self.leftover_pos..self.leftover_pos + n]);
|
||||
self.leftover_pos += n;
|
||||
return Ok(n);
|
||||
}
|
||||
self.inner.read(buf)
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate that a CSV reader's header row names every required OHLCV column.
|
||||
fn validate_headers<R: std::io::Read>(reader: &mut csv::Reader<R>) -> Result<()> {
|
||||
let headers = reader.headers()?;
|
||||
let present: Vec<String> = headers.iter().map(|h| h.trim().to_string()).collect();
|
||||
let missing: Vec<&str> = REQUIRED_COLUMNS
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|col| !present.iter().any(|h| h == col))
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
return Err(Error::Malformed(format!(
|
||||
"CSV header is missing required column(s) [{}]; found [{}] — \
|
||||
the first line must be a header naming {}",
|
||||
missing.join(", "),
|
||||
present.join(", "),
|
||||
REQUIRED_COLUMNS.join(",")
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Streaming OHLCV CSV reader.
|
||||
#[derive(Debug)]
|
||||
pub struct CandleReader<R: std::io::Read> {
|
||||
reader: csv::Reader<R>,
|
||||
}
|
||||
|
||||
impl<R: std::io::Read> CandleReader<R> {
|
||||
/// Build a trimming CSV reader around `inner` and validate its header.
|
||||
fn build(inner: R) -> Result<Self> {
|
||||
let mut reader = csv::ReaderBuilder::new()
|
||||
.has_headers(true)
|
||||
.trim(csv::Trim::All)
|
||||
.from_reader(inner);
|
||||
validate_headers(&mut reader)?;
|
||||
Ok(Self { reader })
|
||||
}
|
||||
}
|
||||
|
||||
impl CandleReader<BomStripReader<std::fs::File>> {
|
||||
/// Open a CSV file at `path`.
|
||||
///
|
||||
/// The first line must be a header row naming the OHLCV columns; a leading
|
||||
/// UTF-8 BOM and whitespace around values are tolerated.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Io`] if the file cannot be opened and
|
||||
/// [`Error::Malformed`] if the header does not contain every required
|
||||
/// column (`timestamp,open,high,low,close,volume`).
|
||||
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
Self::from_reader(file)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: std::io::Read> CandleReader<BomStripReader<R>> {
|
||||
/// Build a reader from any [`std::io::Read`] source.
|
||||
///
|
||||
/// A leading UTF-8 BOM is stripped and the header is validated.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Malformed`] if the header does not contain every
|
||||
/// required column.
|
||||
pub fn from_reader(inner: R) -> Result<Self> {
|
||||
Self::build(BomStripReader::new(inner))
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: std::io::Read> CandleReader<R> {
|
||||
/// Wrap a pre-built [`csv::Reader`]; useful for testing or for non-default
|
||||
/// reader configuration.
|
||||
///
|
||||
/// Unlike [`from_reader`](Self::from_reader) this does *not* strip a BOM —
|
||||
/// the caller owns the reader's configuration — but the header is still
|
||||
/// validated.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Malformed`] if the header does not contain every
|
||||
/// required column.
|
||||
pub fn from_csv_reader(mut reader: csv::Reader<R>) -> Result<Self> {
|
||||
validate_headers(&mut reader)?;
|
||||
Ok(Self { reader })
|
||||
}
|
||||
|
||||
/// Iterator over decoded candles.
|
||||
pub fn candles(&mut self) -> impl Iterator<Item = Result<Candle>> + '_ {
|
||||
self.reader.deserialize::<DefaultRow>().map(|row_res| {
|
||||
let row = row_res?;
|
||||
row.into_candle()
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the entire stream into a `Vec<Candle>`. Convenient for backtests.
|
||||
pub fn read_all(&mut self) -> Result<Vec<Candle>> {
|
||||
self.candles().collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[test]
|
||||
fn reads_well_formed_csv() {
|
||||
let mut tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
writeln!(tmp, "timestamp,open,high,low,close,volume").unwrap();
|
||||
writeln!(tmp, "1,10.0,11.0,9.0,10.5,100").unwrap();
|
||||
writeln!(tmp, "2,10.5,11.5,10.0,11.0,150").unwrap();
|
||||
writeln!(tmp, "3,11.0,12.0,10.5,11.5,200").unwrap();
|
||||
tmp.flush().unwrap();
|
||||
|
||||
let mut r = CandleReader::open(tmp.path()).unwrap();
|
||||
let candles = r.read_all().unwrap();
|
||||
assert_eq!(candles.len(), 3);
|
||||
assert_eq!(candles[0].open, 10.0);
|
||||
assert_eq!(candles[2].close, 11.5);
|
||||
assert_eq!(candles[1].timestamp, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_ohlc() {
|
||||
let mut tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
writeln!(tmp, "timestamp,open,high,low,close,volume").unwrap();
|
||||
// high < low → core validation rejects it.
|
||||
writeln!(tmp, "1,10.0,8.0,9.0,9.5,100").unwrap();
|
||||
tmp.flush().unwrap();
|
||||
|
||||
let mut r = CandleReader::open(tmp.path()).unwrap();
|
||||
let candles: Result<Vec<Candle>> = r.candles().collect();
|
||||
assert!(candles.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_reader_works_on_in_memory_data() {
|
||||
let data = "timestamp,open,high,low,close,volume\n1,1,2,0,1,10\n2,1,2,0,1,10\n";
|
||||
let mut r = CandleReader::from_reader(data.as_bytes()).unwrap();
|
||||
let v = r.read_all().unwrap();
|
||||
assert_eq!(v.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_file_without_header() {
|
||||
// No header row — the first line is data. Without validation the
|
||||
// reader would silently swallow it as the header.
|
||||
let data = "1,10.0,11.0,9.0,10.5,100\n2,10.5,11.5,10.0,11.0,150\n";
|
||||
let err = CandleReader::from_reader(data.as_bytes()).unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_header_missing_a_column() {
|
||||
// "volume" is absent.
|
||||
let data = "timestamp,open,high,low,close\n1,10.0,11.0,9.0,10.5\n";
|
||||
let err = CandleReader::from_reader(data.as_bytes()).unwrap_err();
|
||||
// The error variant must be Malformed and the message must mention
|
||||
// the missing column. Asserting directly (rather than match-and-
|
||||
// panic-on-other) keeps the assertion's cold path branch-free for
|
||||
// coverage and still pins the diagnostic.
|
||||
assert!(
|
||||
matches!(&err, Error::Malformed(msg) if msg.contains("volume")),
|
||||
"expected Malformed mentioning 'volume', got {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Cover `from_csv_reader` (lines 201-204): existing tests use
|
||||
/// `from_reader` / `open`, which both construct the inner `csv::Reader`
|
||||
/// internally. Callers that want non-default csv configuration must
|
||||
/// build the reader themselves and pass it through `from_csv_reader`.
|
||||
#[test]
|
||||
fn from_csv_reader_accepts_a_prebuilt_reader() {
|
||||
let data = "timestamp;open;high;low;close;volume\n1;10.0;11.0;9.0;10.5;100\n";
|
||||
let inner = csv::ReaderBuilder::new()
|
||||
.delimiter(b';')
|
||||
.from_reader(data.as_bytes());
|
||||
let mut r = CandleReader::from_csv_reader(inner).unwrap();
|
||||
let candles = r.read_all().unwrap();
|
||||
assert_eq!(candles.len(), 1);
|
||||
assert_eq!(candles[0].close, 10.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_leading_utf8_bom() {
|
||||
// A BOM (\u{feff}) prefixes the header — Excel exports look like this.
|
||||
let data = "\u{feff}timestamp,open,high,low,close,volume\n1,10.0,11.0,9.0,10.5,100\n";
|
||||
let mut r = CandleReader::from_reader(data.as_bytes()).unwrap();
|
||||
let v = r.read_all().unwrap();
|
||||
assert_eq!(v.len(), 1);
|
||||
assert_eq!(v[0].timestamp, 1);
|
||||
assert_eq!(v[0].open, 10.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerates_whitespace_around_fields() {
|
||||
let data = " timestamp , open , high , low , close , volume \n\
|
||||
1 , 10.0 , 11.0 , 9.0 , 10.5 , 100 \n";
|
||||
let mut r = CandleReader::from_reader(data.as_bytes()).unwrap();
|
||||
let v = r.read_all().unwrap();
|
||||
assert_eq!(v.len(), 1);
|
||||
assert_eq!(v[0].close, 10.5);
|
||||
assert_eq!(v[0].volume, 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bom_stripper_passes_through_non_bom_input() {
|
||||
use std::io::Read;
|
||||
let mut out = String::new();
|
||||
BomStripReader::new("hello".as_bytes())
|
||||
.read_to_string(&mut out)
|
||||
.unwrap();
|
||||
assert_eq!(out, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bom_stripper_handles_short_input() {
|
||||
use std::io::Read;
|
||||
let mut out = Vec::new();
|
||||
// Two bytes — shorter than a 3-byte BOM.
|
||||
BomStripReader::new([0x41u8, 0x42u8].as_slice())
|
||||
.read_to_end(&mut out)
|
||||
.unwrap();
|
||||
assert_eq!(out, vec![0x41, 0x42]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
//! Error types specific to the data sources.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// Errors produced by the data layer.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("CSV error: {0}")]
|
||||
Csv(#[from] csv::Error),
|
||||
|
||||
#[error("invalid timeframe: {0}")]
|
||||
InvalidTimeframe(String),
|
||||
|
||||
#[error("indicator-core error: {0}")]
|
||||
Core(#[from] wickra_core::Error),
|
||||
|
||||
#[error("malformed payload: {0}")]
|
||||
Malformed(String),
|
||||
|
||||
/// A live-feed read exceeded its deadline.
|
||||
#[error("read timed out")]
|
||||
Timeout,
|
||||
|
||||
#[cfg(feature = "live-binance")]
|
||||
#[error("websocket error: {0}")]
|
||||
WebSocket(#[from] tokio_tungstenite::tungstenite::Error),
|
||||
|
||||
#[cfg(feature = "live-binance")]
|
||||
#[error("JSON decode error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
/// Boxed because ureq 2.x's `Error::Status` carries a full `Response`, which
|
||||
/// would otherwise make this the dominant enum variant (clippy
|
||||
/// `large_enum_variant`).
|
||||
#[cfg(feature = "live-binance")]
|
||||
#[error("HTTP error: {0}")]
|
||||
Http(#[from] Box<ureq::Error>),
|
||||
}
|
||||
|
||||
/// Convenience alias for `Result<T, wickra_data::Error>`.
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
@@ -0,0 +1,25 @@
|
||||
//! `wickra-data`: offline and online data sources for the Wickra indicator engine.
|
||||
//!
|
||||
//! - [`csv`]: stream OHLCV bars out of CSV files without buffering the whole
|
||||
//! history in memory.
|
||||
//! - [`aggregator`]: roll trade ticks up into candles of arbitrary timeframes.
|
||||
//! - [`resample`]: convert a stream of candles from one timeframe to a coarser one.
|
||||
//! - [`live`] (feature `live-binance`): connect to exchange websockets for live
|
||||
//! klines, or pull historical klines over REST — both yield typed candles
|
||||
//! compatible with the rest of the crate.
|
||||
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
// `tokio_tungstenite::Error` is large by itself (~200 B). Boxing every Err
|
||||
// variant per clippy::result_large_err just shifts allocation pressure into
|
||||
// the hot path. We accept the size because errors are rare in this crate.
|
||||
#![allow(clippy::result_large_err)]
|
||||
|
||||
pub mod aggregator;
|
||||
pub mod csv;
|
||||
pub mod error;
|
||||
pub mod resample;
|
||||
|
||||
#[cfg(feature = "live-binance")]
|
||||
pub mod live;
|
||||
|
||||
pub use error::{Error, Result};
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Live exchange feeds. Each adapter is feature-gated; the Binance adapter
|
||||
//! lives behind the `live-binance` feature.
|
||||
|
||||
pub mod binance;
|
||||
pub mod binance_rest;
|
||||
@@ -0,0 +1,915 @@
|
||||
//! Binance spot WebSocket kline feed.
|
||||
//!
|
||||
//! Subscribes to Binance's `<symbol>@kline_<interval>` stream and emits a
|
||||
//! [`KlineEvent`] every time the server pushes a new tick. The event tells you
|
||||
//! whether the current candle is still open or has just closed.
|
||||
//!
|
||||
//! Example (requires the `live-binance` feature):
|
||||
//!
|
||||
//! ```no_run
|
||||
//! use wickra_data::live::binance::{BinanceKlineStream, Interval};
|
||||
//! # async fn run() -> wickra_data::Result<()> {
|
||||
//! let mut stream = BinanceKlineStream::connect(&["BTCUSDT".to_string()], Interval::OneMinute).await?;
|
||||
//! while let Some(event) = stream.next_event().await? {
|
||||
//! if event.is_closed {
|
||||
//! println!("closed {} @ {}", event.symbol, event.candle.close);
|
||||
//! }
|
||||
//! }
|
||||
//! # Ok(()) }
|
||||
//! ```
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::SinkExt;
|
||||
use futures_util::StreamExt;
|
||||
use serde::Deserialize;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tokio_tungstenite::MaybeTlsStream;
|
||||
use tokio_tungstenite::WebSocketStream;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use wickra_core::Candle;
|
||||
|
||||
/// Tunable knobs for a [`BinanceKlineStream`]. The defaults match Binance's
|
||||
/// public production endpoint and are right for almost every caller; the
|
||||
/// fields exist so an integration test or a Binance Testnet user can point
|
||||
/// the stream at a different base URL and shrink the reconnect timing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BinanceConfig {
|
||||
/// WebSocket endpoint **without** path (e.g. `"wss://stream.binance.com:9443"`).
|
||||
/// The combined-stream path `/stream?streams=…` is appended internally.
|
||||
pub base_url: String,
|
||||
/// Maximum time to wait for the next inbound frame before treating the
|
||||
/// connection as stalled. Binance pings roughly every 3 minutes, so a
|
||||
/// healthy but quiet stream stays comfortably inside the 300 s default.
|
||||
pub read_timeout: Duration,
|
||||
/// Delay before the first reconnect attempt; doubles on each failure up
|
||||
/// to [`Self::max_reconnect_backoff`].
|
||||
pub initial_reconnect_delay: Duration,
|
||||
/// Upper bound on the exponential reconnect backoff.
|
||||
pub max_reconnect_backoff: Duration,
|
||||
/// How many times [`BinanceKlineStream::next_event`] retries a dropped
|
||||
/// connection before surfacing the last error. Must be `>= 1`.
|
||||
pub max_reconnect_attempts: u32,
|
||||
/// Upper bound on an inbound WebSocket message. Kline frames are tiny;
|
||||
/// this only caps a pathological or hostile server from forcing an
|
||||
/// unbounded allocation.
|
||||
pub max_message_size: usize,
|
||||
/// Upper bound on a single inbound WebSocket frame.
|
||||
pub max_frame_size: usize,
|
||||
}
|
||||
|
||||
impl Default for BinanceConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
base_url: "wss://stream.binance.com:9443".to_string(),
|
||||
read_timeout: Duration::from_secs(300),
|
||||
initial_reconnect_delay: Duration::from_secs(1),
|
||||
max_reconnect_backoff: Duration::from_secs(30),
|
||||
max_reconnect_attempts: 6,
|
||||
max_message_size: 8 << 20,
|
||||
max_frame_size: 2 << 20,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Supported Binance kline intervals. The `as_str` value matches Binance's
|
||||
/// wire-format strings (`"1m"`, `"5m"`, `"1h"`, etc.).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Interval {
|
||||
OneSecond,
|
||||
OneMinute,
|
||||
ThreeMinutes,
|
||||
FiveMinutes,
|
||||
FifteenMinutes,
|
||||
ThirtyMinutes,
|
||||
OneHour,
|
||||
TwoHours,
|
||||
FourHours,
|
||||
SixHours,
|
||||
EightHours,
|
||||
TwelveHours,
|
||||
OneDay,
|
||||
ThreeDays,
|
||||
OneWeek,
|
||||
OneMonth,
|
||||
}
|
||||
|
||||
impl Interval {
|
||||
/// Wire-format string used in the stream name.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::OneSecond => "1s",
|
||||
Self::OneMinute => "1m",
|
||||
Self::ThreeMinutes => "3m",
|
||||
Self::FiveMinutes => "5m",
|
||||
Self::FifteenMinutes => "15m",
|
||||
Self::ThirtyMinutes => "30m",
|
||||
Self::OneHour => "1h",
|
||||
Self::TwoHours => "2h",
|
||||
Self::FourHours => "4h",
|
||||
Self::SixHours => "6h",
|
||||
Self::EightHours => "8h",
|
||||
Self::TwelveHours => "12h",
|
||||
Self::OneDay => "1d",
|
||||
Self::ThreeDays => "3d",
|
||||
Self::OneWeek => "1w",
|
||||
Self::OneMonth => "1M",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One push from the Binance kline stream.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct KlineEvent {
|
||||
/// Symbol in lowercase form as sent by Binance (e.g. `"btcusdt"`).
|
||||
pub symbol: String,
|
||||
/// Interval the candle belongs to.
|
||||
pub interval: Interval,
|
||||
/// Candle in its current state (may still be open).
|
||||
pub candle: Candle,
|
||||
/// Whether the candle has been closed by the server. Closed events are the
|
||||
/// only ones safe to use for bar-completion logic.
|
||||
pub is_closed: bool,
|
||||
}
|
||||
|
||||
/// A live Binance kline stream.
|
||||
#[derive(Debug)]
|
||||
pub struct BinanceKlineStream {
|
||||
socket: WebSocketStream<MaybeTlsStream<TcpStream>>,
|
||||
/// Lowercased symbols the stream is subscribed to. Retained so the
|
||||
/// connection can be rebuilt on a reconnect.
|
||||
symbols: Vec<String>,
|
||||
/// Interval requested at connect time. Used to tag every event.
|
||||
interval: Interval,
|
||||
/// `true` once the caller invoked [`close`](Self::close). A closed stream
|
||||
/// is never polled or reconnected again.
|
||||
closed: bool,
|
||||
/// Timing / sizing knobs. Retained so reconnects honour the same config.
|
||||
config: BinanceConfig,
|
||||
}
|
||||
|
||||
/// Wire-format representation of an incoming Binance kline tick. Public so callers
|
||||
/// can deserialize it themselves if they prefer.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct RawWsEnvelope {
|
||||
/// Stream name, e.g. `"btcusdt@kline_1m"`.
|
||||
pub stream: String,
|
||||
pub data: RawKlinePayload,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct RawKlinePayload {
|
||||
#[serde(rename = "e")]
|
||||
pub event_type: String,
|
||||
#[serde(rename = "E")]
|
||||
pub event_time: i64,
|
||||
#[serde(rename = "s")]
|
||||
pub symbol: String,
|
||||
#[serde(rename = "k")]
|
||||
pub kline: RawKline,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct RawKline {
|
||||
#[serde(rename = "t")]
|
||||
pub open_time: i64,
|
||||
#[serde(rename = "T")]
|
||||
pub close_time: i64,
|
||||
#[serde(rename = "s")]
|
||||
pub symbol: String,
|
||||
#[serde(rename = "i")]
|
||||
pub interval: String,
|
||||
#[serde(rename = "o")]
|
||||
pub open: String,
|
||||
#[serde(rename = "c")]
|
||||
pub close: String,
|
||||
#[serde(rename = "h")]
|
||||
pub high: String,
|
||||
#[serde(rename = "l")]
|
||||
pub low: String,
|
||||
#[serde(rename = "v")]
|
||||
pub volume: String,
|
||||
#[serde(rename = "x")]
|
||||
pub is_closed: bool,
|
||||
}
|
||||
|
||||
impl BinanceKlineStream {
|
||||
/// Open a raw combined-stream WebSocket for the given (already-lowercased)
|
||||
/// symbols.
|
||||
async fn open(
|
||||
symbols: &[String],
|
||||
interval: Interval,
|
||||
config: &BinanceConfig,
|
||||
) -> Result<WebSocketStream<MaybeTlsStream<TcpStream>>> {
|
||||
let streams: Vec<String> = symbols
|
||||
.iter()
|
||||
.map(|s| format!("{}@kline_{}", s, interval.as_str()))
|
||||
.collect();
|
||||
let url = format!("{}/stream?streams={}", config.base_url, streams.join("/"));
|
||||
let url = url::Url::parse(&url).map_err(|e| Error::Malformed(e.to_string()))?;
|
||||
// tokio-tungstenite 0.29's WebSocketConfig is #[non_exhaustive],
|
||||
// so the only way to set fields is via the builder-style methods on
|
||||
// a fresh `default()` value.
|
||||
let ws_config = WebSocketConfig::default()
|
||||
.max_message_size(Some(config.max_message_size))
|
||||
.max_frame_size(Some(config.max_frame_size));
|
||||
let (socket, _) =
|
||||
tokio_tungstenite::connect_async_with_config(url.as_str(), Some(ws_config), false)
|
||||
.await?;
|
||||
Ok(socket)
|
||||
}
|
||||
|
||||
/// Connect to Binance's combined-stream endpoint for one or more symbols.
|
||||
///
|
||||
/// Symbols may be passed in either case; they are lowercased to match
|
||||
/// Binance's stream-name conventions. A dropped or stalled connection is
|
||||
/// re-established transparently by [`next_event`](Self::next_event).
|
||||
pub async fn connect(symbols: &[String], interval: Interval) -> Result<Self> {
|
||||
Self::connect_with_config(symbols, interval, BinanceConfig::default()).await
|
||||
}
|
||||
|
||||
/// Connect with a custom [`BinanceConfig`]. Useful for Binance Testnet
|
||||
/// (`"wss://testnet.binance.vision"`) or for shrinking the reconnect
|
||||
/// timing in integration tests.
|
||||
pub async fn connect_with_config(
|
||||
symbols: &[String],
|
||||
interval: Interval,
|
||||
config: BinanceConfig,
|
||||
) -> Result<Self> {
|
||||
if symbols.is_empty() {
|
||||
return Err(Error::Malformed(
|
||||
"BinanceKlineStream requires at least one symbol".into(),
|
||||
));
|
||||
}
|
||||
let symbols: Vec<String> = symbols.iter().map(|s| s.to_lowercase()).collect();
|
||||
let socket = Self::open(&symbols, interval, &config).await?;
|
||||
Ok(Self {
|
||||
socket,
|
||||
symbols,
|
||||
interval,
|
||||
closed: false,
|
||||
config,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the caller has closed the stream. Once closed, every further
|
||||
/// [`next_event`](Self::next_event) call yields `Ok(None)` immediately.
|
||||
pub fn is_closed(&self) -> bool {
|
||||
self.closed
|
||||
}
|
||||
|
||||
/// Re-establish a dropped connection with exponential backoff. Returns the
|
||||
/// last error if every attempt fails.
|
||||
async fn reconnect(&mut self) -> Result<()> {
|
||||
let mut delay = self.config.initial_reconnect_delay;
|
||||
let mut last_err = None;
|
||||
for _ in 0..self.config.max_reconnect_attempts {
|
||||
tokio::time::sleep(delay).await;
|
||||
match Self::open(&self.symbols, self.interval, &self.config).await {
|
||||
Ok(socket) => {
|
||||
self.socket = socket;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
delay = delay
|
||||
.saturating_mul(2)
|
||||
.min(self.config.max_reconnect_backoff);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(last_err.expect("max_reconnect_attempts is non-zero"))
|
||||
}
|
||||
|
||||
/// Receive the next kline event. A dropped, errored or stalled connection
|
||||
/// is re-established transparently (exponential backoff, up to
|
||||
/// [`BinanceConfig::max_reconnect_attempts`]); an exhausted reconnect
|
||||
/// surfaces as `Err`. `Ok(None)` is returned only after the caller has
|
||||
/// [`close`](Self::close)d the stream.
|
||||
pub async fn next_event(&mut self) -> Result<Option<KlineEvent>> {
|
||||
if self.closed {
|
||||
return Ok(None);
|
||||
}
|
||||
loop {
|
||||
// A protocol error, a clean server close, or a read stall are all
|
||||
// transient: reconnect with backoff and resume reading.
|
||||
let Ok(Some(Ok(msg))) =
|
||||
tokio::time::timeout(self.config.read_timeout, self.socket.next()).await
|
||||
else {
|
||||
self.reconnect().await?;
|
||||
continue;
|
||||
};
|
||||
match msg {
|
||||
Message::Text(text) => {
|
||||
if let Some(event) = Self::parse_frame(&text, self.interval)? {
|
||||
return Ok(Some(event));
|
||||
}
|
||||
// Non-kline frame (subscription ack / heartbeat / error):
|
||||
// skip it and keep reading.
|
||||
}
|
||||
Message::Binary(bytes) => {
|
||||
let text = String::from_utf8_lossy(&bytes);
|
||||
if let Some(event) = Self::parse_frame(&text, self.interval)? {
|
||||
return Ok(Some(event));
|
||||
}
|
||||
}
|
||||
Message::Ping(payload) => {
|
||||
// Best-effort Pong reply. If the write fails the
|
||||
// connection is already dead — the next read will
|
||||
// surface the error and trigger reconnect through
|
||||
// the timeout/err arm above.
|
||||
let _ = self.socket.send(Message::Pong(payload)).await;
|
||||
}
|
||||
Message::Pong(_) | Message::Frame(_) => {}
|
||||
Message::Close(_) => {
|
||||
self.reconnect().await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse one raw WebSocket text frame.
|
||||
///
|
||||
/// Returns `Ok(Some(event))` for a kline frame, `Ok(None)` for any other
|
||||
/// frame (subscription acknowledgements, error objects, heartbeats), and
|
||||
/// `Err` only when a frame that *is* a kline fails to decode.
|
||||
fn parse_frame(text: &str, interval: Interval) -> Result<Option<KlineEvent>> {
|
||||
let value: serde_json::Value = serde_json::from_str(text)?;
|
||||
// Combined-stream kline frames carry `data.e == "kline"`. Everything
|
||||
// else on the socket is control traffic that must not abort the feed.
|
||||
let is_kline = value
|
||||
.get("data")
|
||||
.and_then(|d| d.get("e"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some("kline");
|
||||
if !is_kline {
|
||||
return Ok(None);
|
||||
}
|
||||
let envelope: RawWsEnvelope = serde_json::from_value(value)?;
|
||||
Ok(Some(envelope.into_event(interval)?))
|
||||
}
|
||||
|
||||
/// Close the underlying socket cleanly and mark the stream closed. After
|
||||
/// this, [`next_event`](Self::next_event) yields `Ok(None)` and never
|
||||
/// reconnects.
|
||||
pub async fn close(&mut self) -> Result<()> {
|
||||
self.closed = true;
|
||||
self.socket.close(None).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RawWsEnvelope {
|
||||
fn into_event(self, interval: Interval) -> Result<KlineEvent> {
|
||||
let k = self.data.kline;
|
||||
let open: f64 = k
|
||||
.open
|
||||
.parse()
|
||||
.map_err(|_| Error::Malformed(format!("bad open '{}'", k.open)))?;
|
||||
let high: f64 = k
|
||||
.high
|
||||
.parse()
|
||||
.map_err(|_| Error::Malformed(format!("bad high '{}'", k.high)))?;
|
||||
let low: f64 = k
|
||||
.low
|
||||
.parse()
|
||||
.map_err(|_| Error::Malformed(format!("bad low '{}'", k.low)))?;
|
||||
let close: f64 = k
|
||||
.close
|
||||
.parse()
|
||||
.map_err(|_| Error::Malformed(format!("bad close '{}'", k.close)))?;
|
||||
let volume: f64 = k
|
||||
.volume
|
||||
.parse()
|
||||
.map_err(|_| Error::Malformed(format!("bad volume '{}'", k.volume)))?;
|
||||
let candle = Candle::new(open, high, low, close, volume, k.open_time)?;
|
||||
Ok(KlineEvent {
|
||||
symbol: self.data.symbol.to_lowercase(),
|
||||
interval,
|
||||
candle,
|
||||
is_closed: k.is_closed,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn interval_as_str_covers_every_variant() {
|
||||
// Wire-format strings are part of Binance's public protocol — pin
|
||||
// every mapping so a typo here is caught immediately.
|
||||
let pairs: &[(Interval, &str)] = &[
|
||||
(Interval::OneSecond, "1s"),
|
||||
(Interval::OneMinute, "1m"),
|
||||
(Interval::ThreeMinutes, "3m"),
|
||||
(Interval::FiveMinutes, "5m"),
|
||||
(Interval::FifteenMinutes, "15m"),
|
||||
(Interval::ThirtyMinutes, "30m"),
|
||||
(Interval::OneHour, "1h"),
|
||||
(Interval::TwoHours, "2h"),
|
||||
(Interval::FourHours, "4h"),
|
||||
(Interval::SixHours, "6h"),
|
||||
(Interval::EightHours, "8h"),
|
||||
(Interval::TwelveHours, "12h"),
|
||||
(Interval::OneDay, "1d"),
|
||||
(Interval::ThreeDays, "3d"),
|
||||
(Interval::OneWeek, "1w"),
|
||||
(Interval::OneMonth, "1M"),
|
||||
];
|
||||
for (iv, expected) in pairs {
|
||||
assert_eq!(iv.as_str(), *expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binance_config_default_matches_production_endpoint() {
|
||||
let cfg = BinanceConfig::default();
|
||||
assert_eq!(cfg.base_url, "wss://stream.binance.com:9443");
|
||||
assert_eq!(cfg.read_timeout, Duration::from_secs(300));
|
||||
assert_eq!(cfg.initial_reconnect_delay, Duration::from_secs(1));
|
||||
assert_eq!(cfg.max_reconnect_backoff, Duration::from_secs(30));
|
||||
assert_eq!(cfg.max_reconnect_attempts, 6);
|
||||
assert_eq!(cfg.max_message_size, 8 << 20);
|
||||
assert_eq!(cfg.max_frame_size, 2 << 20);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_rejects_an_empty_symbol_list() {
|
||||
let err = BinanceKlineStream::connect(&[], Interval::OneMinute)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_real_binance_payload() {
|
||||
// Sample event format from Binance's public docs (truncated).
|
||||
let json = r#"{
|
||||
"stream": "btcusdt@kline_1m",
|
||||
"data": {
|
||||
"e": "kline",
|
||||
"E": 1700000000000,
|
||||
"s": "BTCUSDT",
|
||||
"k": {
|
||||
"t": 1700000000000,
|
||||
"T": 1700000059999,
|
||||
"s": "BTCUSDT",
|
||||
"i": "1m",
|
||||
"f": 1,
|
||||
"L": 100,
|
||||
"o": "30000.0",
|
||||
"c": "30050.0",
|
||||
"h": "30100.0",
|
||||
"l": "29950.0",
|
||||
"v": "12.5",
|
||||
"n": 50,
|
||||
"x": false,
|
||||
"q": "375000.0",
|
||||
"V": "6.25",
|
||||
"Q": "187500.0",
|
||||
"B": "0"
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
let env: RawWsEnvelope = serde_json::from_str(json).unwrap();
|
||||
let evt = env.into_event(Interval::OneMinute).unwrap();
|
||||
assert_eq!(evt.symbol, "btcusdt");
|
||||
assert_eq!(evt.candle.open, 30_000.0);
|
||||
assert_eq!(evt.candle.close, 30_050.0);
|
||||
assert!(!evt.is_closed);
|
||||
assert_eq!(evt.interval, Interval::OneMinute);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_parsable_numbers() {
|
||||
let json = r#"{
|
||||
"stream": "btcusdt@kline_1m",
|
||||
"data": {
|
||||
"e": "kline", "E": 0, "s": "BTCUSDT",
|
||||
"k": {
|
||||
"t": 0, "T": 0, "s": "BTCUSDT", "i": "1m",
|
||||
"f": 0, "L": 0,
|
||||
"o": "not-a-number", "c": "0", "h": "0", "l": "0",
|
||||
"v": "0", "n": 0, "x": false, "q": "0", "V": "0", "Q": "0", "B": "0"
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
let env: RawWsEnvelope = serde_json::from_str(json).unwrap();
|
||||
let err = env.into_event(Interval::OneMinute).unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_non_kline_frames() {
|
||||
// Subscription acknowledgement: skipped, never an error.
|
||||
let ack = r#"{"result":null,"id":1}"#;
|
||||
assert!(BinanceKlineStream::parse_frame(ack, Interval::OneMinute)
|
||||
.unwrap()
|
||||
.is_none());
|
||||
// Error object: also skipped.
|
||||
let err = r#"{"error":{"code":2,"msg":"Invalid request"}}"#;
|
||||
assert!(BinanceKlineStream::parse_frame(err, Interval::OneMinute)
|
||||
.unwrap()
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_frame_decodes_a_kline() {
|
||||
let json = r#"{
|
||||
"stream": "btcusdt@kline_1m",
|
||||
"data": {
|
||||
"e": "kline", "E": 1700000000000, "s": "BTCUSDT",
|
||||
"k": {
|
||||
"t": 1700000000000, "T": 1700000059999, "s": "BTCUSDT", "i": "1m",
|
||||
"f": 1, "L": 100, "o": "30000.0", "c": "30050.0", "h": "30100.0",
|
||||
"l": "29950.0", "v": "12.5", "n": 50, "x": true,
|
||||
"q": "375000.0", "V": "6.25", "Q": "187500.0", "B": "0"
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
let event = BinanceKlineStream::parse_frame(json, Interval::OneMinute)
|
||||
.unwrap()
|
||||
.expect("a kline frame yields an event");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
assert!(event.is_closed);
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// Mock WebSocket server: drives the async / reconnect / control-frame
|
||||
// paths against a `127.0.0.1` listener instead of the real Binance
|
||||
// endpoint. Each test gets its own port (`bind("127.0.0.1:0")`).
|
||||
// ====================================================================
|
||||
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// A kline JSON frame matching Binance's combined-stream envelope. Always
|
||||
/// reports `is_closed = true` so the test can assert on the flag.
|
||||
fn sample_kline_text() -> String {
|
||||
r#"{"stream":"btcusdt@kline_1m","data":{"e":"kline","E":1700000000000,"s":"BTCUSDT","k":{"t":1700000000000,"T":1700000059999,"s":"BTCUSDT","i":"1m","f":1,"L":100,"o":"30000.0","c":"30050.0","h":"30100.0","l":"29950.0","v":"12.5","n":50,"x":true,"q":"375000.0","V":"6.25","Q":"187500.0","B":"0"}}}"#.to_string()
|
||||
}
|
||||
|
||||
/// Test-tuned [`BinanceConfig`]: aim at the given mock and shrink every
|
||||
/// timer so a failing reconnect loop completes in milliseconds.
|
||||
fn test_config(base_url: String) -> BinanceConfig {
|
||||
BinanceConfig {
|
||||
base_url,
|
||||
read_timeout: Duration::from_millis(200),
|
||||
initial_reconnect_delay: Duration::from_millis(5),
|
||||
max_reconnect_backoff: Duration::from_millis(10),
|
||||
max_reconnect_attempts: 3,
|
||||
..BinanceConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn a mock WS server that accepts one connection, drops the
|
||||
/// listener (so any reconnect lands on a refused port), and then hands
|
||||
/// the upgraded socket to `handler`. Every step `.unwrap()`s — a failure
|
||||
/// here is a bug in the test scaffolding, not in the production code.
|
||||
async fn one_shot_server<F, Fut>(handler: F) -> String
|
||||
where
|
||||
F: FnOnce(WebSocketStream<TcpStream>) -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("ws://{}", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
drop(listener);
|
||||
let ws = tokio_tungstenite::accept_async(stream).await.unwrap();
|
||||
handler(ws).await;
|
||||
});
|
||||
base_url
|
||||
}
|
||||
|
||||
/// Spawn a mock WS server that accepts exactly `n_accepts` connections
|
||||
/// and invokes `handler` for each (with a zero-based index). Returns a
|
||||
/// [`JoinHandle`](tokio::task::JoinHandle) the test can await so every
|
||||
/// spawned handler is guaranteed to reach its closing brace before the
|
||||
/// runtime is torn down.
|
||||
async fn multi_shot_server<F, Fut>(
|
||||
n_accepts: u32,
|
||||
handler: F,
|
||||
) -> (String, tokio::task::JoinHandle<()>)
|
||||
where
|
||||
F: Fn(u32, WebSocketStream<TcpStream>) -> Fut + Send + Sync + 'static,
|
||||
Fut: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("ws://{}", listener.local_addr().unwrap());
|
||||
let handler = Arc::new(handler);
|
||||
let h = tokio::spawn(async move {
|
||||
let mut joins = Vec::with_capacity(n_accepts as usize);
|
||||
for index in 0..n_accepts {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let handler = handler.clone();
|
||||
joins.push(tokio::spawn(async move {
|
||||
let ws = tokio_tungstenite::accept_async(stream).await.unwrap();
|
||||
handler(index, ws).await;
|
||||
}));
|
||||
}
|
||||
for j in joins {
|
||||
j.await.unwrap();
|
||||
}
|
||||
});
|
||||
(base_url, h)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_decodes_a_text_kline_frame() {
|
||||
let kline = sample_kline_text();
|
||||
let base = one_shot_server(move |mut ws| async move {
|
||||
let _ = ws.send(Message::Text(kline.into())).await;
|
||||
while let Some(Ok(_)) = ws.next().await {}
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!stream.is_closed());
|
||||
let event = stream
|
||||
.next_event()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("server pushes a kline");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
assert!(event.is_closed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_decodes_a_binary_kline_frame() {
|
||||
let kline = sample_kline_text();
|
||||
let base = one_shot_server(move |mut ws| async move {
|
||||
let bytes: Vec<u8> = kline.into_bytes();
|
||||
let _ = ws.send(Message::Binary(bytes.into())).await;
|
||||
while let Some(Ok(_)) = ws.next().await {}
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = stream
|
||||
.next_event()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("server pushes a kline as Binary");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_replies_to_a_ping_with_a_pong() {
|
||||
let kline = sample_kline_text();
|
||||
let base = one_shot_server(move |mut ws| async move {
|
||||
let _ = ws
|
||||
.send(Message::Ping(b"binance-ping".as_slice().into()))
|
||||
.await;
|
||||
let _ = ws.send(Message::Text(kline.into())).await;
|
||||
while let Some(Ok(_)) = ws.next().await {}
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// If the client never replied to the Ping the server's drain would
|
||||
// observe nothing — but for line coverage it's enough that the
|
||||
// client received the Ping, sent a Pong, then read the next frame.
|
||||
let event = stream
|
||||
.next_event()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("kline arrives right after the ping");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_skips_inbound_pong_frames() {
|
||||
let kline = sample_kline_text();
|
||||
let base = one_shot_server(move |mut ws| async move {
|
||||
let _ = ws
|
||||
.send(Message::Pong(b"unsolicited".as_slice().into()))
|
||||
.await;
|
||||
let _ = ws.send(Message::Text(kline.into())).await;
|
||||
while let Some(Ok(_)) = ws.next().await {}
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = stream
|
||||
.next_event()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("kline follows the ignored Pong");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_reconnects_after_a_server_close_frame() {
|
||||
let kline = sample_kline_text();
|
||||
let (base, server_done) = multi_shot_server(2, move |index, mut ws| {
|
||||
let kline = kline.clone();
|
||||
async move {
|
||||
let msg = if index == 0 {
|
||||
// First connection: send a clean Close so the client
|
||||
// exercises the Message::Close reconnect path.
|
||||
Message::Close(None)
|
||||
} else {
|
||||
Message::Text(kline.into())
|
||||
};
|
||||
let _ = ws.send(msg).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = stream
|
||||
.next_event()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("reconnect succeeds and the second connection serves a kline");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
// Wait for every spawned handler to reach its final state.
|
||||
tokio::time::timeout(Duration::from_secs(1), server_done)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_reconnects_after_a_read_timeout() {
|
||||
let kline = sample_kline_text();
|
||||
let stall_token = Arc::new(AtomicU32::new(0));
|
||||
let stall_token_h = stall_token.clone();
|
||||
let (base, server_done) = multi_shot_server(2, move |index, mut ws| {
|
||||
let kline = kline.clone();
|
||||
let stall_token = stall_token_h.clone();
|
||||
async move {
|
||||
if index == 0 {
|
||||
// First connection: never write anything. Outlast the
|
||||
// client's 80 ms read_timeout but bounded so the
|
||||
// handler still completes for the coverage check.
|
||||
stall_token.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
} else {
|
||||
let _ = ws.send(Message::Text(kline.into())).await;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let cfg = BinanceConfig {
|
||||
read_timeout: Duration::from_millis(80),
|
||||
..test_config(base)
|
||||
};
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
cfg,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = stream
|
||||
.next_event()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("client times out, reconnects, and reads the kline");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
assert!(stall_token.load(Ordering::SeqCst) >= 1);
|
||||
tokio::time::timeout(Duration::from_secs(1), server_done)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_yields_none_after_close() {
|
||||
let base = one_shot_server(|mut ws| async move {
|
||||
// Stay open until the client closes; this lets close() complete
|
||||
// its handshake cleanly.
|
||||
while let Some(Ok(_)) = ws.next().await {}
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
stream.close().await.unwrap();
|
||||
assert!(stream.is_closed());
|
||||
assert!(stream.next_event().await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_surfaces_an_error_when_reconnect_attempts_are_exhausted() {
|
||||
// After the first accept the listener is dropped (one_shot_server
|
||||
// does this), so every reconnect attempt lands on a closed port.
|
||||
let base = one_shot_server(|mut ws| async move {
|
||||
let _ = ws.send(Message::Close(None)).await;
|
||||
// Returning here also drops the socket, but the listener has
|
||||
// already been released — the client's subsequent connects
|
||||
// will be refused.
|
||||
})
|
||||
.await;
|
||||
let cfg = BinanceConfig {
|
||||
max_reconnect_attempts: 2,
|
||||
initial_reconnect_delay: Duration::from_millis(1),
|
||||
max_reconnect_backoff: Duration::from_millis(2),
|
||||
..test_config(base)
|
||||
};
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
cfg,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let err = stream
|
||||
.next_event()
|
||||
.await
|
||||
.expect_err("reconnect attempts are exhausted");
|
||||
// Either a WS-layer error or a Malformed error from URL parsing —
|
||||
// we only care that the call surfaced as Err rather than panicked.
|
||||
let _ = err;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_skips_non_kline_frames_and_returns_the_next_kline() {
|
||||
// Drives the loop through the Text- *and* Binary-arm "frame was
|
||||
// not a kline, keep reading" fall-throughs before serving the
|
||||
// actual kline.
|
||||
let kline = sample_kline_text();
|
||||
let base = one_shot_server(move |mut ws| async move {
|
||||
let _ = ws
|
||||
.send(Message::Text(r#"{"result":null,"id":1}"#.into()))
|
||||
.await;
|
||||
let _ = ws
|
||||
.send(Message::Binary(b"{\"id\":2}".to_vec().into()))
|
||||
.await;
|
||||
let _ = ws.send(Message::Text(kline.into())).await;
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = stream
|
||||
.next_event()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("kline arrives after the two skipped control frames");
|
||||
assert_eq!(event.symbol, "btcusdt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_event_propagates_a_parse_error_from_a_malformed_kline() {
|
||||
// A "kline" envelope whose open field is not a number — parse_frame
|
||||
// identifies it as a kline, into_event then fails, and next_event
|
||||
// bubbles the error rather than skipping the frame.
|
||||
let bad = r#"{"stream":"btcusdt@kline_1m","data":{"e":"kline","E":0,"s":"BTCUSDT","k":{"t":0,"T":0,"s":"BTCUSDT","i":"1m","o":"not-a-number","c":"0","h":"0","l":"0","v":"0","x":false}}}"#.to_string();
|
||||
let base = one_shot_server(move |mut ws| async move {
|
||||
let _ = ws.send(Message::Text(bad.into())).await;
|
||||
while let Some(Ok(_)) = ws.next().await {}
|
||||
})
|
||||
.await;
|
||||
let mut stream = BinanceKlineStream::connect_with_config(
|
||||
&["BTCUSDT".to_string()],
|
||||
Interval::OneMinute,
|
||||
test_config(base),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let err = stream.next_event().await.unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
//! Binance spot REST historical kline fetcher.
|
||||
//!
|
||||
//! Where [`super::binance`] streams *live* klines over a WebSocket, this module
|
||||
//! pulls *historical* candles from Binance's public REST endpoint
|
||||
//! (`GET /api/v3/klines`) with a single blocking request. It is the native,
|
||||
//! dependency-free replacement for hand-rolled `urllib` / `jackson` / `jsonlite`
|
||||
//! download helpers in every binding.
|
||||
//!
|
||||
//! Example (requires the `live-binance` feature):
|
||||
//!
|
||||
//! ```no_run
|
||||
//! use wickra_data::live::binance::Interval;
|
||||
//! use wickra_data::live::binance_rest::fetch_klines;
|
||||
//! # fn run() -> wickra_data::Result<()> {
|
||||
//! let candles = fetch_klines("BTCUSDT", Interval::OneHour, 500, None, None)?;
|
||||
//! println!("got {} candles", candles.len());
|
||||
//! # Ok(()) }
|
||||
//! ```
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::binance::Interval;
|
||||
use crate::error::{Error, Result};
|
||||
use wickra_core::Candle;
|
||||
|
||||
/// Binance allows at most 1000 klines per REST request.
|
||||
const MAX_LIMIT: u16 = 1000;
|
||||
|
||||
/// Endpoint configuration for [`fetch_klines_with_config`]. The default points
|
||||
/// at Binance's public production REST host; tests and Testnet users override
|
||||
/// `base_url` to aim the request elsewhere.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BinanceRestConfig {
|
||||
/// REST host **without** path, e.g. `"https://api.binance.com"`. The
|
||||
/// `/api/v3/klines` path and query string are appended internally.
|
||||
pub base_url: String,
|
||||
}
|
||||
|
||||
impl Default for BinanceRestConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
base_url: "https://api.binance.com".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One row of Binance's `/api/v3/klines` response. The wire format is a fixed
|
||||
/// 12-element JSON array of mixed types; only the first seven fields carry the
|
||||
/// OHLCV candle, the rest are ignored. Deserializing positionally lets serde
|
||||
/// reject a malformed shape before we touch the numbers.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RawRestKline(
|
||||
i64, // open time (ms)
|
||||
String, // open
|
||||
String, // high
|
||||
String, // low
|
||||
String, // close
|
||||
String, // volume
|
||||
serde::de::IgnoredAny, // close time
|
||||
serde::de::IgnoredAny, // quote asset volume
|
||||
serde::de::IgnoredAny, // number of trades
|
||||
serde::de::IgnoredAny, // taker buy base volume
|
||||
serde::de::IgnoredAny, // taker buy quote volume
|
||||
serde::de::IgnoredAny, // unused
|
||||
);
|
||||
|
||||
/// Parse one of the five OHLCV string fields into an `f64`, tagging the field
|
||||
/// name on failure so a bad payload is diagnosable.
|
||||
fn parse_field(raw: &str, field: &str) -> Result<f64> {
|
||||
raw.parse()
|
||||
.map_err(|_| Error::Malformed(format!("bad {field} '{raw}'")))
|
||||
}
|
||||
|
||||
impl RawRestKline {
|
||||
fn into_candle(self) -> Result<Candle> {
|
||||
let open = parse_field(&self.1, "open")?;
|
||||
let high = parse_field(&self.2, "high")?;
|
||||
let low = parse_field(&self.3, "low")?;
|
||||
let close = parse_field(&self.4, "close")?;
|
||||
let volume = parse_field(&self.5, "volume")?;
|
||||
Ok(Candle::new(open, high, low, close, volume, self.0)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch historical klines from Binance's production endpoint.
|
||||
///
|
||||
/// `symbol` is upper-cased to match Binance's convention; `limit` must be in
|
||||
/// `1..=1000`. `start_ms` / `end_ms` are optional inclusive Unix-millisecond
|
||||
/// bounds (`None` lets Binance pick the most recent `limit` candles). Returns
|
||||
/// the candles in ascending open-time order.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Malformed`] for an out-of-range `limit` or an unparseable
|
||||
/// price field, [`Error::Http`] for a transport or non-2xx response,
|
||||
/// [`Error::Json`] for a malformed body, and [`Error::Core`] when a row's OHLC
|
||||
/// values violate candle invariants.
|
||||
pub fn fetch_klines(
|
||||
symbol: &str,
|
||||
interval: Interval,
|
||||
limit: u16,
|
||||
start_ms: Option<i64>,
|
||||
end_ms: Option<i64>,
|
||||
) -> Result<Vec<Candle>> {
|
||||
fetch_klines_with_config(
|
||||
symbol,
|
||||
interval,
|
||||
limit,
|
||||
start_ms,
|
||||
end_ms,
|
||||
&BinanceRestConfig::default(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Like [`fetch_klines`] but against a custom [`BinanceRestConfig`] (Testnet or
|
||||
/// a local mock server).
|
||||
///
|
||||
/// # Errors
|
||||
/// See [`fetch_klines`].
|
||||
pub fn fetch_klines_with_config(
|
||||
symbol: &str,
|
||||
interval: Interval,
|
||||
limit: u16,
|
||||
start_ms: Option<i64>,
|
||||
end_ms: Option<i64>,
|
||||
config: &BinanceRestConfig,
|
||||
) -> Result<Vec<Candle>> {
|
||||
if limit == 0 || limit > MAX_LIMIT {
|
||||
return Err(Error::Malformed(format!(
|
||||
"limit must be in 1..={MAX_LIMIT}, got {limit}"
|
||||
)));
|
||||
}
|
||||
let mut url = format!(
|
||||
"{}/api/v3/klines?symbol={}&interval={}&limit={}",
|
||||
config.base_url,
|
||||
symbol.to_uppercase(),
|
||||
interval.as_str(),
|
||||
limit
|
||||
);
|
||||
if let Some(start) = start_ms {
|
||||
let _ = write!(url, "&startTime={start}");
|
||||
}
|
||||
if let Some(end) = end_ms {
|
||||
let _ = write!(url, "&endTime={end}");
|
||||
}
|
||||
|
||||
// ureq 2.x does not auto-configure native-tls, so build an agent with an
|
||||
// explicit native-tls connector (verifies against the OS trust store; no
|
||||
// bundled CA roots). The connector is cheap and a one-shot fetch needs no
|
||||
// pooling, so we build it per call.
|
||||
let connector = native_tls::TlsConnector::new()
|
||||
.map_err(|e| Error::Malformed(format!("native-tls init failed: {e}")))?;
|
||||
let agent = ureq::AgentBuilder::new()
|
||||
.tls_connector(std::sync::Arc::new(connector))
|
||||
.build();
|
||||
let body = agent.get(&url).call().map_err(Box::new)?.into_string()?;
|
||||
let rows: Vec<RawRestKline> = serde_json::from_str(&body)?;
|
||||
rows.into_iter().map(RawRestKline::into_candle).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// A canonical single-row klines response (12-element array, OHLCV in the
|
||||
/// first seven slots) — mirrors Binance's documented wire format.
|
||||
fn sample_response() -> String {
|
||||
r#"[[1700000000000,"30000.0","30100.0","29950.0","30050.0","12.5",1700000059999,"375000.0",50,"6.25","187500.0","0"]]"#.to_string()
|
||||
}
|
||||
|
||||
/// Spawn a one-shot mock HTTP server: accept a single connection, read the
|
||||
/// request, and reply `200 OK` with `body`. Returns the base URL plus a
|
||||
/// [`JoinHandle`] the test joins so the handler thread always reaches its
|
||||
/// closing brace (every step `.unwrap()`s — a failure is a test bug).
|
||||
fn mock_http(status_line: &'static str, body: String) -> (String, JoinHandle<()>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let handle = std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = stream.read(&mut buf).unwrap();
|
||||
let response = format!(
|
||||
"{status_line}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(response.as_bytes()).unwrap();
|
||||
});
|
||||
(format!("http://{addr}"), handle)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_parses_a_real_klines_response() {
|
||||
let (base, handle) = mock_http("HTTP/1.1 200 OK", sample_response());
|
||||
let candles = fetch_klines_with_config(
|
||||
"btcusdt",
|
||||
Interval::OneHour,
|
||||
1,
|
||||
Some(1_700_000_000_000),
|
||||
Some(1_700_000_059_999),
|
||||
&BinanceRestConfig { base_url: base },
|
||||
)
|
||||
.unwrap();
|
||||
handle.join().unwrap();
|
||||
assert_eq!(candles.len(), 1);
|
||||
assert_eq!(candles[0].open, 30_000.0);
|
||||
assert_eq!(candles[0].high, 30_100.0);
|
||||
assert_eq!(candles[0].low, 29_950.0);
|
||||
assert_eq!(candles[0].close, 30_050.0);
|
||||
assert_eq!(candles[0].volume, 12.5);
|
||||
assert_eq!(candles[0].timestamp, 1_700_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_handles_an_empty_result() {
|
||||
let (base, handle) = mock_http("HTTP/1.1 200 OK", "[]".to_string());
|
||||
let candles = fetch_klines_with_config(
|
||||
"BTCUSDT",
|
||||
Interval::OneMinute,
|
||||
10,
|
||||
None,
|
||||
None,
|
||||
&BinanceRestConfig { base_url: base },
|
||||
)
|
||||
.unwrap();
|
||||
handle.join().unwrap();
|
||||
assert!(candles.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_rejects_a_zero_limit() {
|
||||
let err = fetch_klines("BTCUSDT", Interval::OneHour, 0, None, None).unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_rejects_a_limit_above_the_cap() {
|
||||
let err =
|
||||
fetch_klines("BTCUSDT", Interval::OneHour, MAX_LIMIT + 1, None, None).unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_surfaces_a_transport_error_for_an_unreachable_host() {
|
||||
// Port 1 is privileged and unbound — the connection is refused.
|
||||
let err = fetch_klines_with_config(
|
||||
"BTCUSDT",
|
||||
Interval::OneHour,
|
||||
1,
|
||||
None,
|
||||
None,
|
||||
&BinanceRestConfig {
|
||||
base_url: "http://127.0.0.1:1".to_string(),
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, Error::Http(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_surfaces_a_json_error_for_a_malformed_body() {
|
||||
let (base, handle) = mock_http("HTTP/1.1 200 OK", "not json".to_string());
|
||||
let err = fetch_klines_with_config(
|
||||
"BTCUSDT",
|
||||
Interval::OneHour,
|
||||
1,
|
||||
None,
|
||||
None,
|
||||
&BinanceRestConfig { base_url: base },
|
||||
)
|
||||
.unwrap_err();
|
||||
handle.join().unwrap();
|
||||
assert!(matches!(err, Error::Json(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_rejects_an_unparseable_price_field() {
|
||||
let body =
|
||||
r#"[[1700000000000,"not-a-number","0","0","0","0",0,"0",0,"0","0","0"]]"#.to_string();
|
||||
let (base, handle) = mock_http("HTTP/1.1 200 OK", body);
|
||||
let err = fetch_klines_with_config(
|
||||
"BTCUSDT",
|
||||
Interval::OneHour,
|
||||
1,
|
||||
None,
|
||||
None,
|
||||
&BinanceRestConfig { base_url: base },
|
||||
)
|
||||
.unwrap_err();
|
||||
handle.join().unwrap();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_rejects_a_row_that_violates_candle_invariants() {
|
||||
// high (1) below low (100) — Candle::new rejects it.
|
||||
let body = r#"[[1700000000000,"50","1","100","50","1",0,"0",0,"0","0","0"]]"#.to_string();
|
||||
let (base, handle) = mock_http("HTTP/1.1 200 OK", body);
|
||||
let err = fetch_klines_with_config(
|
||||
"BTCUSDT",
|
||||
Interval::OneHour,
|
||||
1,
|
||||
None,
|
||||
None,
|
||||
&BinanceRestConfig { base_url: base },
|
||||
)
|
||||
.unwrap_err();
|
||||
handle.join().unwrap();
|
||||
assert!(matches!(err, Error::Core(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rest_config_default_targets_production() {
|
||||
assert_eq!(
|
||||
BinanceRestConfig::default().base_url,
|
||||
"https://api.binance.com"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
//! Resample an existing candle stream from a finer timeframe to a coarser one.
|
||||
|
||||
use crate::aggregator::Timeframe;
|
||||
use crate::error::{Error, Result};
|
||||
use wickra_core::Candle;
|
||||
|
||||
/// Roll a stream of candles up to a coarser timeframe.
|
||||
///
|
||||
/// Used to derive 5m bars from a 1m feed, or 1h bars from 5m bars, without
|
||||
/// touching the original tick stream. The output timeframe's bucket must be a
|
||||
/// strict multiple of the input timeframe's bucket, but this is not enforced
|
||||
/// — callers are responsible for picking sensible aggregations.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Resampler {
|
||||
timeframe: Timeframe,
|
||||
open: Option<RolledBar>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RolledBar {
|
||||
bucket_start: i64,
|
||||
open: f64,
|
||||
high: f64,
|
||||
low: f64,
|
||||
close: f64,
|
||||
volume: f64,
|
||||
}
|
||||
|
||||
impl RolledBar {
|
||||
fn from_candle(c: Candle, bucket_start: i64) -> Self {
|
||||
Self {
|
||||
bucket_start,
|
||||
open: c.open,
|
||||
high: c.high,
|
||||
low: c.low,
|
||||
close: c.close,
|
||||
volume: c.volume,
|
||||
}
|
||||
}
|
||||
|
||||
fn absorb(&mut self, c: Candle) {
|
||||
if c.high > self.high {
|
||||
self.high = c.high;
|
||||
}
|
||||
if c.low < self.low {
|
||||
self.low = c.low;
|
||||
}
|
||||
self.close = c.close;
|
||||
self.volume += c.volume;
|
||||
}
|
||||
|
||||
/// Finalise the rolled bar into a validated [`Candle`].
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Core`] if the accumulated `volume` is no longer finite.
|
||||
/// `volume` is summed across every absorbed candle, so a long or large run
|
||||
/// can drift it to `inf`; emitting such a candle would silently poison
|
||||
/// every downstream indicator, so it is surfaced instead. The OHLC fields
|
||||
/// are finite and correctly ordered by construction.
|
||||
fn into_candle(self) -> Result<Candle> {
|
||||
Candle::new(
|
||||
self.open,
|
||||
self.high,
|
||||
self.low,
|
||||
self.close,
|
||||
self.volume,
|
||||
self.bucket_start,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl Resampler {
|
||||
/// Build a resampler targeting the given output timeframe.
|
||||
pub fn new(timeframe: Timeframe) -> Self {
|
||||
Self {
|
||||
timeframe,
|
||||
open: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a finer-grained candle. Returns the coarser candle that just closed,
|
||||
/// if any.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Malformed`] if `candle.timestamp` falls into a bucket
|
||||
/// strictly before the currently open bar — out-of-order candles are not
|
||||
/// supported, matching [`crate::aggregator::TickAggregator::push`].
|
||||
pub fn push(&mut self, candle: Candle) -> Result<Option<Candle>> {
|
||||
let bucket = self.timeframe.floor(candle.timestamp);
|
||||
match self.open {
|
||||
Some(mut bar) if bucket == bar.bucket_start => {
|
||||
bar.absorb(candle);
|
||||
self.open = Some(bar);
|
||||
Ok(None)
|
||||
}
|
||||
Some(bar) if bucket > bar.bucket_start => {
|
||||
let closed = bar.into_candle()?;
|
||||
self.open = Some(RolledBar::from_candle(candle, bucket));
|
||||
Ok(Some(closed))
|
||||
}
|
||||
Some(bar) => Err(Error::Malformed(format!(
|
||||
"candle timestamp {} is older than the open bar start {}",
|
||||
candle.timestamp, bar.bucket_start
|
||||
))),
|
||||
None => {
|
||||
self.open = Some(RolledBar::from_candle(candle, bucket));
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Flush the currently open coarser bar, if any.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the open bar's accumulated volume is non-finite
|
||||
/// (see the internal `RolledBar::into_candle`).
|
||||
pub fn flush(&mut self) -> Result<Option<Candle>> {
|
||||
self.open.take().map(RolledBar::into_candle).transpose()
|
||||
}
|
||||
}
|
||||
|
||||
/// Roll an entire iterator of candles into a `Vec` of coarser candles. The final
|
||||
/// open bar (if any) is appended via [`Resampler::flush`].
|
||||
pub fn resample_all<I>(timeframe: Timeframe, iter: I) -> Result<Vec<Candle>>
|
||||
where
|
||||
I: IntoIterator<Item = Result<Candle>>,
|
||||
{
|
||||
let mut r = Resampler::new(timeframe);
|
||||
let mut out = Vec::new();
|
||||
for c in iter {
|
||||
let c = c?;
|
||||
if let Some(closed) = r.push(c)? {
|
||||
out.push(closed);
|
||||
}
|
||||
}
|
||||
if let Some(last) = r.flush()? {
|
||||
out.push(last);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn c(ts: i64, o: f64, h: f64, l: f64, cl: f64, v: f64) -> Candle {
|
||||
Candle::new(o, h, l, cl, v, ts).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resamples_1m_to_5m() {
|
||||
let tf = Timeframe::new(5).unwrap();
|
||||
let one_m = vec![
|
||||
c(0, 10.0, 11.0, 9.0, 10.5, 10.0),
|
||||
c(1, 10.5, 12.0, 10.0, 11.5, 12.0),
|
||||
c(2, 11.5, 13.0, 11.0, 12.5, 15.0),
|
||||
c(3, 12.5, 12.8, 11.5, 12.0, 8.0),
|
||||
c(4, 12.0, 12.2, 11.0, 11.5, 6.0),
|
||||
c(5, 11.5, 11.9, 11.0, 11.5, 4.0),
|
||||
];
|
||||
let rolled = resample_all(tf, one_m.into_iter().map(Ok)).unwrap();
|
||||
// First 5 candles share bucket 0 -> aggregate. Last candle opens bucket 5.
|
||||
assert_eq!(rolled.len(), 2);
|
||||
let a = rolled[0];
|
||||
assert_eq!(a.open, 10.0);
|
||||
assert_eq!(a.close, 11.5);
|
||||
assert_eq!(a.high, 13.0);
|
||||
assert_eq!(a.low, 9.0);
|
||||
assert!((a.volume - 51.0).abs() < 1e-12);
|
||||
let b = rolled[1];
|
||||
assert_eq!(b.open, 11.5);
|
||||
assert_eq!(b.timestamp, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_out_of_order_candle() {
|
||||
let mut r = Resampler::new(Timeframe::new(5).unwrap());
|
||||
assert!(r.push(c(10, 10.0, 11.0, 9.0, 10.5, 1.0)).unwrap().is_none());
|
||||
// A candle in an earlier bucket than the open bar is rejected.
|
||||
let err = r.push(c(2, 10.0, 11.0, 9.0, 10.5, 1.0)).unwrap_err();
|
||||
assert!(matches!(err, Error::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_bucket_candles_aggregate() {
|
||||
let mut r = Resampler::new(Timeframe::new(5).unwrap());
|
||||
assert!(r.push(c(0, 10.0, 11.0, 9.0, 10.5, 1.0)).unwrap().is_none());
|
||||
assert!(r.push(c(3, 10.5, 12.0, 10.0, 11.0, 1.0)).unwrap().is_none());
|
||||
let bar = r.flush().unwrap().unwrap();
|
||||
assert_eq!(bar.high, 12.0);
|
||||
assert_eq!(bar.low, 9.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absorb_lowers_low_on_dipping_candle() {
|
||||
// The first candle in the bucket sets low = 10.0; the second dips to
|
||||
// 8.0 and must overwrite. Exercises the `c.low < self.low` branch in
|
||||
// RolledBar::absorb that the other resampler tests never trigger
|
||||
// because their follow-up candles always have a higher low.
|
||||
let mut r = Resampler::new(Timeframe::new(5).unwrap());
|
||||
r.push(c(0, 10.0, 11.0, 10.0, 10.5, 1.0)).unwrap();
|
||||
r.push(c(1, 10.5, 11.5, 8.0, 9.0, 1.0)).unwrap();
|
||||
let bar = r.flush().unwrap().unwrap();
|
||||
assert_eq!(bar.low, 8.0);
|
||||
assert_eq!(bar.high, 11.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flushes_a_non_finite_volume_as_an_error() {
|
||||
let mut r = Resampler::new(Timeframe::new(5).unwrap());
|
||||
// Two near-max volumes in the same bucket sum to +inf.
|
||||
assert!(r
|
||||
.push(c(0, 10.0, 11.0, 9.0, 10.5, f64::MAX))
|
||||
.unwrap()
|
||||
.is_none());
|
||||
assert!(r
|
||||
.push(c(1, 10.0, 11.0, 9.0, 10.5, f64::MAX))
|
||||
.unwrap()
|
||||
.is_none());
|
||||
let err = r.flush().unwrap_err();
|
||||
assert!(matches!(err, Error::Core(_)));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user