扩展指标

This commit is contained in:
2026-07-09 05:08:16 +08:00
commit 308c46ab9a
537 changed files with 152299 additions and 0 deletions
@@ -0,0 +1,17 @@
"""
ferro_ta.data — Data ingestion, streaming, batch, and resampling utilities.
Sub-modules
-----------
* :mod:`ferro_ta.data.streaming` — Streaming / incremental indicator state machines
* :mod:`ferro_ta.data.batch` — Batch execution across multiple series (2-D arrays)
* :mod:`ferro_ta.data.chunked` — Chunked / windowed processing for large datasets
* :mod:`ferro_ta.data.resampling` — OHLCV resampling and multi-timeframe support
* :mod:`ferro_ta.data.aggregation` — Tick / trade aggregation pipelines
* :mod:`ferro_ta.data.adapters` — DataFrame adapters (pandas, polars, numpy)
Example usage::
from ferro_ta.data.streaming import StreamingSMA
from ferro_ta.data.batch import batch_sma
"""
@@ -0,0 +1,271 @@
"""
ferro_ta.adapters — Market data adapters (pluggable).
Defines an abstract ``DataAdapter`` interface and a concrete
``CsvAdapter`` that loads OHLCV data from a CSV file. Users can
subclass ``DataAdapter`` to add their own data sources (e.g. Alpaca,
Yahoo Finance, a database, etc.) while keeping the rest of the pipeline
unchanged.
Classes
-------
DataAdapter
Abstract base class. Subclasses must implement :meth:`fetch`.
CsvAdapter
Load OHLCV data from a CSV file. Requires pandas.
InMemoryAdapter
Wrap an already-loaded pandas DataFrame or dict of arrays.
Functions
---------
register_adapter(name, adapter_class)
Register an adapter class under a name for lookup by string.
get_adapter(name)
Return an adapter class previously registered under *name*.
Examples
--------
>>> from ferro_ta.data.adapters import InMemoryAdapter
>>> import numpy as np
>>> n = 50
>>> rng = np.random.default_rng(0)
>>> close = np.cumprod(1 + rng.normal(0, 0.01, n)) * 100
>>> adapter = InMemoryAdapter({
... "open": close, "high": close * 1.001,
... "low": close * 0.999, "close": close,
... "volume": np.ones(n) * 1000,
... })
>>> ohlcv = adapter.fetch()
>>> "close" in ohlcv
True
"""
from __future__ import annotations
import abc
from typing import Any, Optional
__all__ = [
"DataAdapter",
"CsvAdapter",
"InMemoryAdapter",
"register_adapter",
"get_adapter",
]
# ---------------------------------------------------------------------------
# Registry
# ---------------------------------------------------------------------------
_ADAPTER_REGISTRY: dict[str, type[DataAdapter]] = {}
def register_adapter(name: str, adapter_class: type[DataAdapter]) -> None:
"""Register *adapter_class* under *name*.
Parameters
----------
name : str
adapter_class : type — must subclass :class:`DataAdapter`
Examples
--------
>>> from ferro_ta.data.adapters import register_adapter, DataAdapter
>>> class MyAdapter(DataAdapter):
... def fetch(self, **kwargs): return {}
>>> register_adapter("my_source", MyAdapter)
"""
if not issubclass(adapter_class, DataAdapter):
raise TypeError(f"{adapter_class!r} must subclass DataAdapter")
_ADAPTER_REGISTRY[name] = adapter_class
def get_adapter(name: str) -> type[DataAdapter]:
"""Return the adapter class registered under *name*.
Parameters
----------
name : str
Raises
------
KeyError
If *name* is not registered.
"""
if name not in _ADAPTER_REGISTRY:
available = sorted(_ADAPTER_REGISTRY.keys())
raise KeyError(f"No adapter registered under {name!r}. Available: {available}")
return _ADAPTER_REGISTRY[name]
# ---------------------------------------------------------------------------
# Abstract base
# ---------------------------------------------------------------------------
class DataAdapter(abc.ABC):
"""Abstract base class for market data adapters.
Subclasses must implement :meth:`fetch`, which returns OHLCV data as
a ``pandas.DataFrame`` (preferred) or a ``dict`` of numpy arrays.
The contract for the returned data:
- Keys/columns: ``open``, ``high``, ``low``, ``close``, ``volume``
(additional columns are allowed but not required).
- Values: numeric (float64-compatible).
- Index (for DataFrames): ideally a ``DatetimeIndex``; not required.
"""
@abc.abstractmethod
def fetch(self, **kwargs: Any) -> Any:
"""Return OHLCV data.
Returns
-------
pandas.DataFrame or dict
OHLCV data with keys/columns ``open``, ``high``, ``low``,
``close``, ``volume``.
"""
def __repr__(self) -> str:
return f"{type(self).__name__}()"
# ---------------------------------------------------------------------------
# CsvAdapter
# ---------------------------------------------------------------------------
class CsvAdapter(DataAdapter):
"""Load OHLCV data from a CSV file.
The CSV must have a header row. Column names are configurable.
Parameters
----------
path : str
Path to the CSV file.
open_col, high_col, low_col, close_col, volume_col : str
CSV column names for each OHLCV field.
index_col : str or None
Column to use as the DataFrame index (e.g. ``'timestamp'``).
parse_dates : bool
If ``True`` (default), attempt to parse the index as dates.
Requires
--------
pandas
Examples
--------
>>> from ferro_ta.data.adapters import CsvAdapter
>>> # adapter = CsvAdapter("data.csv", index_col="date")
>>> # ohlcv = adapter.fetch()
"""
def __init__(
self,
path: str,
*,
open_col: str = "open",
high_col: str = "high",
low_col: str = "low",
close_col: str = "close",
volume_col: str = "volume",
index_col: Optional[str] = None,
parse_dates: bool = True,
) -> None:
self.path = path
self.open_col = open_col
self.high_col = high_col
self.low_col = low_col
self.close_col = close_col
self.volume_col = volume_col
self.index_col = index_col
self.parse_dates = parse_dates
def fetch(self, **kwargs: Any) -> Any:
"""Load the CSV and return a pandas DataFrame.
Raises
------
ImportError
If pandas is not installed.
"""
try:
import pandas as pd
except ImportError as exc:
raise ImportError(
"pandas is required for CsvAdapter. Install with: pip install pandas"
) from exc
df = pd.read_csv(
self.path,
index_col=self.index_col,
parse_dates=self.parse_dates if self.index_col is not None else False,
)
# Rename columns if they differ from the standard names
rename = {}
for src, dst in [
(self.open_col, "open"),
(self.high_col, "high"),
(self.low_col, "low"),
(self.close_col, "close"),
(self.volume_col, "volume"),
]:
if src != dst and src in df.columns:
rename[src] = dst
if rename:
df = df.rename(columns=rename)
return df
def __repr__(self) -> str:
return f"CsvAdapter(path={self.path!r})"
# ---------------------------------------------------------------------------
# InMemoryAdapter
# ---------------------------------------------------------------------------
class InMemoryAdapter(DataAdapter):
"""Wrap already-loaded OHLCV data (dict or DataFrame).
Parameters
----------
data : dict or pandas.DataFrame
OHLCV data.
Examples
--------
>>> import numpy as np
>>> from ferro_ta.data.adapters import InMemoryAdapter
>>> n = 10
>>> close = np.ones(n) * 100.0
>>> adapter = InMemoryAdapter({"open": close, "high": close,
... "low": close, "close": close,
... "volume": close})
>>> ohlcv = adapter.fetch()
>>> "close" in ohlcv
True
"""
def __init__(self, data: Any) -> None:
self._data = data
def fetch(self, **kwargs: Any) -> Any:
"""Return the wrapped data as-is."""
return self._data
def __repr__(self) -> str:
return "InMemoryAdapter(...)"
# ---------------------------------------------------------------------------
# Register built-in adapters
# ---------------------------------------------------------------------------
register_adapter("csv", CsvAdapter)
register_adapter("memory", InMemoryAdapter)
@@ -0,0 +1,238 @@
"""
ferro_ta.aggregation — Tick and trade aggregation pipeline.
Aggregates raw tick or trade data (streams of (timestamp, price, size)) into
OHLCV bars using three bar types:
- **time bars** — fixed time intervals (e.g. every 1 minute)
- **volume bars** — fixed volume threshold per bar
- **tick bars** — fixed number of ticks per bar
The compute-intensive bar accumulation is implemented in Rust; this module
provides the Python-facing API with DataFrame support.
Functions
---------
aggregate_ticks(ticks, rule)
Aggregate a tick stream to OHLCV bars. The *rule* string specifies the
bar type and parameter:
- ``'time:<seconds>'`` — e.g. ``'time:60'`` for 1-minute bars
- ``'volume:<threshold>'`` — e.g. ``'volume:1000'`` for 1000-unit volume bars
- ``'tick:<n>'`` — e.g. ``'tick:100'`` for 100-tick bars
Rust backend
------------
All accumulation logic delegates to::
ferro_ta._ferro_ta.aggregate_tick_bars
ferro_ta._ferro_ta.aggregate_volume_bars_ticks
ferro_ta._ferro_ta.aggregate_time_bars
"""
from __future__ import annotations
from typing import Any, Optional
import numpy as np
from numpy.typing import NDArray
from ferro_ta._ferro_ta import aggregate_tick_bars as _rust_tick_bars
from ferro_ta._ferro_ta import aggregate_time_bars as _rust_time_bars
from ferro_ta._ferro_ta import aggregate_volume_bars_ticks as _rust_volume_bars_ticks
from ferro_ta._utils import _to_f64
from ferro_ta.core.exceptions import FerroTAValueError
__all__ = [
"aggregate_ticks",
"TickAggregator",
]
# ---------------------------------------------------------------------------
# _parse_rule
# ---------------------------------------------------------------------------
def _parse_rule(rule: str) -> tuple[str, float]:
"""Parse a rule string into (bar_type, parameter).
Supported formats::
'time:60' → ('time', 60.0)
'volume:1000' → ('volume', 1000.0)
'tick:100' → ('tick', 100.0)
"""
parts = rule.split(":", 1)
if len(parts) != 2:
raise FerroTAValueError(
f"Invalid rule format: {rule!r}. "
"Expected 'time:<seconds>', 'volume:<threshold>', or 'tick:<n>'."
)
bar_type = parts[0].lower().strip()
if bar_type not in ("time", "volume", "tick"):
raise FerroTAValueError(
f"Unknown bar type {bar_type!r}. Supported types: 'time', 'volume', 'tick'."
)
try:
param = float(parts[1].strip())
except ValueError as exc:
raise FerroTAValueError(
f"Cannot parse parameter {parts[1]!r} as a number in rule {rule!r}."
) from exc
if param <= 0:
raise FerroTAValueError(
f"Rule parameter must be > 0, got {param} in rule {rule!r}."
)
return bar_type, param
# ---------------------------------------------------------------------------
# aggregate_ticks
# ---------------------------------------------------------------------------
def aggregate_ticks(
ticks: Any,
rule: str = "tick:100",
*,
timestamp_col: str = "timestamp",
price_col: str = "price",
size_col: str = "size",
) -> Any:
"""Aggregate tick/trade data into OHLCV bars.
Parameters
----------
ticks : pandas.DataFrame, list of (timestamp, price, size), or dict of arrays
Tick data. Accepted formats:
1. **pandas DataFrame** with columns ``timestamp``, ``price``, ``size``
(column names configurable via *_col* parameters). The timestamp
column must contain numeric Unix timestamps (seconds) for time bars.
2. **list of tuples** ``[(ts, price, size), …]``.
3. **dict** ``{'timestamp': array, 'price': array, 'size': array}``.
rule : str
Bar specification:
- ``'tick:<n>'`` — every N ticks become one bar (default ``'tick:100'``)
- ``'volume:<threshold>'`` — every N units of volume become one bar
- ``'time:<seconds>'`` — every N seconds become one bar
timestamp_col, price_col, size_col : str
Column names when *ticks* is a DataFrame.
Returns
-------
pandas.DataFrame or tuple of numpy arrays
If a DataFrame was passed in (or pandas is available), returns a
DataFrame with columns ``open``, ``high``, ``low``, ``close``,
``volume``, and (for time bars) ``timestamp``.
Otherwise returns a tuple ``(open, high, low, close, volume)``.
Examples
--------
>>> import numpy as np
>>> from ferro_ta.data.aggregation import aggregate_ticks
>>> rng = np.random.default_rng(42)
>>> n = 500
>>> price = 100 + np.cumsum(rng.normal(0, 0.1, n))
>>> size = rng.uniform(10, 100, n)
>>> bars = aggregate_ticks({"price": price, "size": size}, rule="tick:50")
>>> len(bars["open"]) == n // 50 + (1 if n % 50 != 0 else 0)
True
"""
bar_type, param = _parse_rule(rule)
# --- Normalise input ---
ts_arr: Optional[NDArray[np.float64]] = None
if isinstance(ticks, list):
# list of (ts, price, size) tuples
arr = np.ascontiguousarray(ticks, dtype=np.float64)
ts_arr = np.ascontiguousarray(arr[:, 0])
price_arr = np.ascontiguousarray(arr[:, 1])
size_arr = np.ascontiguousarray(arr[:, 2])
elif isinstance(ticks, dict):
price_arr = _to_f64(ticks[price_col])
size_arr = _to_f64(ticks[size_col])
if timestamp_col in ticks:
ts_arr = _to_f64(ticks[timestamp_col])
else:
# pandas DataFrame
try:
import pandas as pd
except ImportError as exc:
raise ImportError("pandas is required for DataFrame input") from exc
price_arr = _to_f64(ticks[price_col].values)
size_arr = _to_f64(ticks[size_col].values)
if timestamp_col in ticks.columns:
ts_arr = _to_f64(ticks[timestamp_col].values)
# --- Aggregate ---
if bar_type == "tick":
ro, rh, rl, rc, rv = _rust_tick_bars(price_arr, size_arr, int(param))
extra: Optional[NDArray] = None
elif bar_type == "volume":
ro, rh, rl, rc, rv = _rust_volume_bars_ticks(price_arr, size_arr, param)
extra = None
else: # time
if ts_arr is None:
raise FerroTAValueError(
"Time bars require a timestamp column in the tick data."
)
period_secs = int(param)
labels = (ts_arr // period_secs).astype(np.int64)
ro, rh, rl, rc, rv, lbl = _rust_time_bars(price_arr, size_arr, labels)
extra = lbl
# --- Return ---
try:
import pandas as pd
df: dict[str, Any] = {
"open": ro,
"high": rh,
"low": rl,
"close": rc,
"volume": rv,
}
if extra is not None:
df["timestamp"] = (extra * int(param)).astype(np.int64)
return pd.DataFrame(df)
except ImportError:
return {"open": ro, "high": rh, "low": rl, "close": rc, "volume": rv}
# ---------------------------------------------------------------------------
# TickAggregator — class-based API
# ---------------------------------------------------------------------------
class TickAggregator:
"""Class-based API for tick aggregation.
Parameters
----------
rule : str
Bar specification (see :func:`aggregate_ticks`).
Examples
--------
>>> from ferro_ta.data.aggregation import TickAggregator
>>> agg = TickAggregator(rule="tick:50")
>>> import numpy as np
>>> rng = np.random.default_rng(0)
>>> ticks = {"price": rng.uniform(99, 101, 200), "size": rng.uniform(1, 10, 200)}
>>> bars = agg.aggregate(ticks)
>>> len(bars["open"]) >= 4
True
"""
def __init__(self, rule: str = "tick:100") -> None:
self.rule = rule
# Validate rule at construction time
_parse_rule(rule)
def aggregate(self, ticks: Any, **kwargs: Any) -> Any:
"""Aggregate *ticks* into bars. See :func:`aggregate_ticks`."""
return aggregate_ticks(ticks, rule=self.rule, **kwargs)
def __repr__(self) -> str:
return f"TickAggregator(rule={self.rule!r})"
+446
View File
@@ -0,0 +1,446 @@
"""
Batch Execution API — run indicators on multiple series in a single call.
This module provides a 2-D batch API that accepts a 2-D numpy array
(n_samples × n_series) and applies an indicator to every column, returning
a 2-D output array of the same shape.
For the most common indicators — SMA, EMA, RSI — the 2-D path is handled
entirely in Rust (a single GIL release for all columns). ``batch_apply``
also dispatches these indicators to Rust when possible; other indicators
use the generic Python fallback path.
Functions
---------
batch_sma — SMA on every column of a 2-D array (Rust fast path for 2-D)
batch_ema — EMA on every column of a 2-D array (Rust fast path for 2-D)
batch_rsi — RSI on every column of a 2-D array (Rust fast path for 2-D)
batch_apply — Generic batch wrapper with Rust fast-path for SMA/EMA/RSI
Usage
-----
>>> import numpy as np
>>> from ferro_ta.data.batch import batch_sma
>>> data = np.random.rand(100, 5) # 100 bars, 5 symbols
>>> result = batch_sma(data, timeperiod=14)
>>> result.shape
(100, 5)
"""
from __future__ import annotations
from collections.abc import Callable, Sequence
import numpy as np
from numpy.typing import ArrayLike
from ferro_ta._ferro_ta import (
batch_adx as _rust_batch_adx,
)
from ferro_ta._ferro_ta import (
batch_atr as _rust_batch_atr,
)
from ferro_ta._ferro_ta import (
batch_ema as _rust_batch_ema,
)
from ferro_ta._ferro_ta import (
batch_rsi as _rust_batch_rsi,
)
from ferro_ta._ferro_ta import (
batch_sma as _rust_batch_sma,
)
from ferro_ta._ferro_ta import (
batch_stoch as _rust_batch_stoch,
)
from ferro_ta._ferro_ta import (
run_close_indicators as _rust_run_close_indicators,
)
from ferro_ta._ferro_ta import (
run_hlc_indicators as _rust_run_hlc_indicators,
)
from ferro_ta.core.registry import run as _registry_run
from ferro_ta.indicators.momentum import RSI
from ferro_ta.indicators.overlap import EMA, SMA
__all__ = [
"batch_sma",
"batch_ema",
"batch_rsi",
"batch_apply",
"compute_many",
]
_CLOSE_FASTPATH_DEFAULTS: dict[str, int] = {
"SMA": 30,
"EMA": 30,
"RSI": 14,
"STDDEV": 5,
"VAR": 5,
"LINEARREG": 14,
"LINEARREG_SLOPE": 14,
"LINEARREG_INTERCEPT": 14,
"LINEARREG_ANGLE": 14,
"TSF": 14,
}
_HLC_FASTPATH_DEFAULTS: dict[str, int] = {
"ATR": 14,
"NATR": 14,
"ADX": 14,
"ADXR": 14,
"CCI": 14,
"WILLR": 14,
}
_BATCH_FASTPATH_DEFAULTS: dict[str, int] = {
"SMA": 30,
"EMA": 30,
"RSI": 14,
}
def _resolve_batch_fastpath(
fn: Callable[..., np.ndarray],
kwargs: dict[str, object],
) -> tuple[str, int] | None:
name = getattr(fn, "__name__", "").upper()
if name not in _BATCH_FASTPATH_DEFAULTS:
return None
if set(kwargs) - {"timeperiod"}:
return None
raw = kwargs.get("timeperiod", _BATCH_FASTPATH_DEFAULTS[name])
if not isinstance(raw, int):
return None
return name, int(raw)
def _normalize_indicator_spec(
spec: str | tuple[str, dict[str, object]] | tuple[str, dict[str, object], object],
) -> tuple[str, dict[str, object], object | None]:
if isinstance(spec, str):
return spec, {}, None
if len(spec) == 2:
name, kwargs = spec
return name, kwargs, None
name, kwargs, out_key = spec
return name, kwargs, out_key
def _extract_timeperiod(
name: str, kwargs: dict[str, object], defaults: dict[str, int]
) -> int | None:
if name not in defaults:
return None
extra_keys = set(kwargs) - {"timeperiod"}
if extra_keys:
return None
raw_value = kwargs.get("timeperiod", defaults[name])
if not isinstance(raw_value, int):
return None
return raw_value
def compute_many(
indicators: Sequence[
str | tuple[str, dict[str, object]] | tuple[str, dict[str, object], object]
],
*,
close: ArrayLike,
high: ArrayLike | None = None,
low: ArrayLike | None = None,
volume: ArrayLike | None = None,
parallel: bool = True,
) -> list[object]:
"""Compute multiple indicators over the same arrays with grouped Rust calls.
Supported single-output indicators are grouped into one Rust boundary crossing
per input-shape family (`close` only or `high/low/close`). Unsupported specs
fall back to the regular registry path, preserving behavior.
"""
close_arr = np.ascontiguousarray(close, dtype=np.float64)
high_arr = None if high is None else np.ascontiguousarray(high, dtype=np.float64)
low_arr = None if low is None else np.ascontiguousarray(low, dtype=np.float64)
volume_arr = (
None if volume is None else np.ascontiguousarray(volume, dtype=np.float64)
)
normalized = [_normalize_indicator_spec(spec) for spec in indicators]
results: list[object | None] = [None] * len(normalized)
close_indices: list[int] = []
close_names: list[str] = []
close_periods: list[int] = []
hlc_indices: list[int] = []
hlc_names: list[str] = []
hlc_periods: list[int] = []
for idx, (name, kwargs, out_key) in enumerate(normalized):
if out_key is None:
close_period = _extract_timeperiod(name, kwargs, _CLOSE_FASTPATH_DEFAULTS)
if close_period is not None:
close_indices.append(idx)
close_names.append(name)
close_periods.append(close_period)
continue
hlc_period = _extract_timeperiod(name, kwargs, _HLC_FASTPATH_DEFAULTS)
if hlc_period is not None and high_arr is not None and low_arr is not None:
hlc_indices.append(idx)
hlc_names.append(name)
hlc_periods.append(hlc_period)
continue
if close_names:
grouped = _rust_run_close_indicators(
close_arr, close_names, close_periods, parallel
)
for idx, value in zip(close_indices, grouped):
results[idx] = np.asarray(value, dtype=np.float64)
if hlc_names and high_arr is not None and low_arr is not None:
grouped = _rust_run_hlc_indicators(
high_arr, low_arr, close_arr, hlc_names, hlc_periods, parallel
)
for idx, value in zip(hlc_indices, grouped):
results[idx] = np.asarray(value, dtype=np.float64)
for idx, (name, kwargs, _) in enumerate(normalized):
if results[idx] is not None:
continue
try:
results[idx] = _registry_run(name, close_arr, **kwargs)
continue
except (TypeError, Exception):
pass
if high_arr is not None and low_arr is not None:
try:
results[idx] = _registry_run(
name, high_arr, low_arr, close_arr, **kwargs
)
continue
except Exception:
pass
if volume_arr is not None:
try:
results[idx] = _registry_run(
name, high_arr, low_arr, close_arr, volume_arr, **kwargs
)
continue
except Exception:
pass
raise ValueError(
f"Cannot call indicator '{name}': insufficient data columns or incompatible parameters."
)
return [result for result in results]
def batch_apply(
data: ArrayLike,
fn: Callable[..., np.ndarray],
**kwargs,
) -> np.ndarray:
"""Apply any single-series indicator *fn* to every column of *data*.
For recognized close-only indicators (SMA/EMA/RSI with default or
``timeperiod`` argument only), this function dispatches to the Rust
batch kernels. Otherwise it falls back to a Python per-column loop.
Parameters
----------
data : array-like, shape (n_samples,) or (n_samples, n_series)
Input data. If 1-D, the function is called directly on the array
and the result is returned without adding a column dimension.
fn : callable
Single-series indicator function (e.g. ``SMA``, ``EMA``, ``RSI``).
It must accept a 1-D array as first positional argument and return
a 1-D array of the same length.
**kwargs
Extra keyword arguments forwarded to *fn* (e.g. ``timeperiod=14``).
Returns
-------
numpy.ndarray
Same shape as *data*. Leading values are ``NaN`` for the warm-up
period, identical to calling *fn* on each column individually.
Examples
--------
>>> import numpy as np
>>> from ferro_ta import SMA
>>> from ferro_ta.data.batch import batch_apply
>>> data = np.random.rand(50, 3)
>>> out = batch_apply(data, SMA, timeperiod=5)
>>> out.shape
(50, 3)
"""
arr = np.asarray(data, dtype=np.float64)
if arr.ndim == 1:
return fn(arr, **kwargs)
if arr.ndim != 2:
raise ValueError(f"batch_apply expects 1-D or 2-D input; got {arr.ndim}-D")
fastpath = _resolve_batch_fastpath(fn, kwargs)
if fastpath is not None:
indicator, timeperiod = fastpath
contiguous = np.ascontiguousarray(arr)
if indicator == "SMA":
return np.asarray(_rust_batch_sma(contiguous, timeperiod, True))
if indicator == "EMA":
return np.asarray(_rust_batch_ema(contiguous, timeperiod, True))
return np.asarray(_rust_batch_rsi(contiguous, timeperiod, True))
n_samples, n_series = arr.shape
result = np.empty((n_samples, n_series), dtype=np.float64)
for j in range(n_series):
result[:, j] = fn(arr[:, j], **kwargs)
return result
def batch_sma(
data: ArrayLike,
timeperiod: int = 30,
parallel: bool = True,
) -> np.ndarray:
"""Simple Moving Average on every column of *data*.
For 2-D inputs uses a Rust-side column loop (single GIL release).
When *parallel* is ``True`` (default), columns are processed in parallel
via Rayon across all available CPU cores.
1-D input is passed directly to the single-series SMA.
Parameters
----------
data : array-like, shape (n_samples,) or (n_samples, n_series)
timeperiod : int, default 30
parallel : bool, default True
Enable multi-threaded parallel column processing via Rayon.
Set to ``False`` for small inputs where thread overhead dominates.
Returns
-------
numpy.ndarray — same shape as *data*.
Examples
--------
>>> import numpy as np
>>> from ferro_ta.data.batch import batch_sma
>>> data = np.arange(1.0, 101.0).reshape(100, 1).repeat(3, axis=1)
>>> out = batch_sma(data, timeperiod=10)
>>> out.shape
(100, 3)
"""
arr = np.ascontiguousarray(data, dtype=np.float64)
if arr.ndim == 1:
return SMA(arr, timeperiod=timeperiod)
if arr.ndim != 2:
raise ValueError(f"batch_sma expects 1-D or 2-D input; got {arr.ndim}-D")
return np.asarray(_rust_batch_sma(arr, timeperiod, parallel))
def batch_ema(
data: ArrayLike,
timeperiod: int = 30,
parallel: bool = True,
) -> np.ndarray:
"""Exponential Moving Average on every column of *data*.
For 2-D inputs uses a Rust-side column loop (single GIL release).
When *parallel* is ``True`` (default), columns are processed in parallel
via Rayon across all available CPU cores.
Parameters
----------
data : array-like, shape (n_samples,) or (n_samples, n_series)
timeperiod : int, default 30
parallel : bool, default True
Enable multi-threaded parallel column processing via Rayon.
Returns
-------
numpy.ndarray — same shape as *data*.
"""
arr = np.ascontiguousarray(data, dtype=np.float64)
if arr.ndim == 1:
return EMA(arr, timeperiod=timeperiod)
if arr.ndim != 2:
raise ValueError(f"batch_ema expects 1-D or 2-D input; got {arr.ndim}-D")
return np.asarray(_rust_batch_ema(arr, timeperiod, parallel))
def batch_rsi(
data: ArrayLike,
timeperiod: int = 14,
parallel: bool = True,
) -> np.ndarray:
"""Relative Strength Index on every column of *data*.
For 2-D inputs uses a Rust-side column loop (single GIL release).
When *parallel* is ``True`` (default), columns are processed in parallel
via Rayon across all available CPU cores.
Parameters
----------
data : array-like, shape (n_samples,) or (n_samples, n_series)
timeperiod : int, default 14
parallel : bool, default True
Enable multi-threaded parallel column processing via Rayon.
Returns
-------
numpy.ndarray — same shape as *data*. Values in [0, 100].
"""
arr = np.ascontiguousarray(data, dtype=np.float64)
if arr.ndim == 1:
return RSI(arr, timeperiod=timeperiod)
if arr.ndim != 2:
raise ValueError(f"batch_rsi expects 1-D or 2-D input; got {arr.ndim}-D")
return np.asarray(_rust_batch_rsi(arr, timeperiod, parallel))
def batch_atr(
high: ArrayLike,
low: ArrayLike,
close: ArrayLike,
timeperiod: int = 14,
parallel: bool = True,
) -> np.ndarray:
h = np.ascontiguousarray(high, dtype=np.float64)
low_arr = np.ascontiguousarray(low, dtype=np.float64)
c = np.ascontiguousarray(close, dtype=np.float64)
return np.asarray(_rust_batch_atr(h, low_arr, c, timeperiod, parallel))
def batch_stoch(
high: ArrayLike,
low: ArrayLike,
close: ArrayLike,
fastk_period: int = 5,
slowk_period: int = 3,
slowd_period: int = 3,
parallel: bool = True,
) -> tuple[np.ndarray, np.ndarray]:
h = np.ascontiguousarray(high, dtype=np.float64)
low_arr = np.ascontiguousarray(low, dtype=np.float64)
c = np.ascontiguousarray(close, dtype=np.float64)
k, d = _rust_batch_stoch(
h, low_arr, c, fastk_period, slowk_period, slowd_period, parallel
)
return np.asarray(k), np.asarray(d)
def batch_adx(
high: ArrayLike,
low: ArrayLike,
close: ArrayLike,
timeperiod: int = 14,
parallel: bool = True,
) -> np.ndarray:
h = np.ascontiguousarray(high, dtype=np.float64)
low_arr = np.ascontiguousarray(low, dtype=np.float64)
c = np.ascontiguousarray(close, dtype=np.float64)
return np.asarray(_rust_batch_adx(h, low_arr, c, timeperiod, parallel))
@@ -0,0 +1,251 @@
"""
ferro_ta.chunked — Chunked / out-of-core processing.
====================================================
Run ferro-ta indicators on data that is too large to fit in memory by
processing it in overlapping chunks. Each chunk contains a warm-up prefix
(``overlap`` bars) from the previous chunk so that indicator state is
correct. After computing the indicator, the warm-up prefix is discarded and
the resulting arrays are concatenated.
Functions
---------
chunk_apply(fn, series, chunk_size, overlap, **fn_kwargs)
Run a single-input indicator function on a large series in chunks.
make_chunk_ranges(n, chunk_size, overlap)
Return (start, end) index pairs for chunked processing.
trim_overlap(chunk_out, overlap)
Discard the first *overlap* elements from an array.
stitch_chunks(chunks)
Concatenate trimmed chunk outputs into one array.
Rust backend
------------
ferro_ta._ferro_ta.make_chunk_ranges
ferro_ta._ferro_ta.trim_overlap
ferro_ta._ferro_ta.stitch_chunks
ferro_ta._ferro_ta.chunk_apply_close_indicator
Notes
-----
Indicators that rely on the full history (e.g. HT_TRENDLINE) cannot
produce exact results in chunked mode; the approximation improves with
larger ``overlap`` values. Indicators with a finite look-back period
(SMA, EMA, RSI, etc.) are exact when ``overlap >= timeperiod - 1``.
For very large datasets or distributed execution, the optional Dask
integration (``dask.dataframe.map_partitions``) can be used directly
by passing any ferro-ta indicator function. See the example in the
docstring of ``chunk_apply``.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Any
import numpy as np
from numpy.typing import ArrayLike, NDArray
from ferro_ta._ferro_ta import (
chunk_apply_close_indicator as _rust_chunk_apply_close_indicator,
)
from ferro_ta._ferro_ta import (
make_chunk_ranges as _rust_make_chunk_ranges,
)
from ferro_ta._ferro_ta import (
stitch_chunks as _rust_stitch_chunks,
)
from ferro_ta._ferro_ta import (
trim_overlap as _rust_trim_overlap,
)
from ferro_ta._utils import _to_f64
__all__ = [
"chunk_apply",
"make_chunk_ranges",
"trim_overlap",
"stitch_chunks",
]
_FASTPATH_DEFAULT_PERIODS: dict[str, int] = {
"SMA": 30,
"EMA": 30,
"RSI": 14,
}
def _resolve_chunk_fastpath(
fn: Callable[..., Any], fn_kwargs: dict[str, Any]
) -> tuple[str, int] | None:
name = getattr(fn, "__name__", "").upper()
if name not in _FASTPATH_DEFAULT_PERIODS:
return None
if set(fn_kwargs) - {"timeperiod"}:
return None
raw = fn_kwargs.get("timeperiod", _FASTPATH_DEFAULT_PERIODS[name])
if not isinstance(raw, int):
return None
return name, int(raw)
def make_chunk_ranges(
n: int,
chunk_size: int,
overlap: int,
) -> NDArray[np.int64]:
"""Compute start/end index pairs for chunked processing.
Parameters
----------
n : int — total length of the series
chunk_size : int — desired output bars per chunk (>= 1)
overlap : int — warm-up bars prepended to each chunk (>= 0)
Returns
-------
numpy.ndarray of int64 with shape (n_chunks, 2) — each row is
``[start_index, end_index)`` of the slice to pass to the indicator.
Examples
--------
>>> from ferro_ta.data.chunked import make_chunk_ranges
>>> make_chunk_ranges(10, 4, 2)
array([[ 0, 6],
[ 4, 10]])
"""
raw = np.asarray(
_rust_make_chunk_ranges(int(n), int(chunk_size), int(overlap)),
dtype=np.int64,
)
if len(raw) == 0:
return raw.reshape(0, 2)
return raw.reshape(-1, 2)
def trim_overlap(
chunk_out: ArrayLike,
overlap: int,
) -> NDArray[np.float64]:
"""Discard the first *overlap* elements from a chunk's indicator output.
Parameters
----------
chunk_out : array-like — indicator output for a chunk
overlap : int — number of leading warm-up elements to discard
Returns
-------
numpy.ndarray of float64 — the remaining elements
"""
arr = np.ascontiguousarray(_to_f64(chunk_out))
return np.asarray(_rust_trim_overlap(arr, int(overlap)), dtype=np.float64)
def stitch_chunks(
chunks: list[ArrayLike],
) -> NDArray[np.float64]:
"""Concatenate trimmed chunk outputs into a single array.
Parameters
----------
chunks : list of array-like — trimmed indicator outputs
Returns
-------
numpy.ndarray of float64 — full concatenated result
"""
converted = [np.ascontiguousarray(_to_f64(c)) for c in chunks]
return np.asarray(_rust_stitch_chunks(converted), dtype=np.float64)
def chunk_apply(
fn: Callable[..., Any],
series: ArrayLike,
chunk_size: int = 10_000,
overlap: int = 100,
**fn_kwargs: Any,
) -> NDArray[np.float64]:
"""Run a 1-D indicator function on a large series in overlapping chunks.
Parameters
----------
fn : callable — indicator function with signature ``fn(series, **kwargs)``
that accepts a 1-D numpy array and returns a 1-D numpy array of the
same length. Examples: ``ferro_ta.SMA``, ``ferro_ta.RSI``.
series : array-like — the full (possibly large) input series
chunk_size : int — output bars per chunk (default 10 000). Tune this
for memory/performance.
overlap : int — warm-up bars prepended to each chunk (default 100).
Set to at least ``timeperiod - 1`` for the indicator to be accurate.
**fn_kwargs : extra keyword arguments forwarded to *fn* on every chunk.
Returns
-------
numpy.ndarray of float64 — full indicator output over the entire series.
Notes
-----
For Dask DataFrames, call ``dask.dataframe.map_partitions`` directly::
import dask.dataframe as dd
from ferro_ta import RSI
ddf = dd.from_pandas(pd.Series(close), npartitions=4)
result = ddf.map_partitions(lambda s: pd.Series(RSI(s.values)))
Examples
--------
>>> import numpy as np
>>> from ferro_ta import SMA
>>> from ferro_ta.data.chunked import chunk_apply
>>> rng = np.random.default_rng(0)
>>> big_series = rng.standard_normal(50_000).cumsum() + 100
>>> out = chunk_apply(SMA, big_series, chunk_size=5000, overlap=30,
... timeperiod=20)
>>> out.shape
(50000,)
"""
s = _to_f64(series)
n = len(s)
if n == 0:
return np.empty(0, dtype=np.float64)
fastpath = _resolve_chunk_fastpath(fn, fn_kwargs)
if fastpath is not None:
indicator, timeperiod = fastpath
return np.asarray(
_rust_chunk_apply_close_indicator(
np.ascontiguousarray(s),
indicator,
int(timeperiod),
int(chunk_size),
int(overlap),
),
dtype=np.float64,
)
ranges = make_chunk_ranges(n, chunk_size, overlap)
if len(ranges) == 0:
result = fn(s, **fn_kwargs)
return np.asarray(result, dtype=np.float64)
trimmed_chunks: list[NDArray[np.float64]] = []
for i, (start, end) in enumerate(ranges):
chunk = s[int(start) : int(end)]
result = fn(chunk, **fn_kwargs)
result_arr = np.asarray(result, dtype=np.float64)
# Determine how many leading bars to discard:
# - first chunk: keep everything (no prior overlap)
# - subsequent chunks: discard the leading `overlap` bars
discard = 0 if i == 0 else int(overlap)
trimmed = trim_overlap(result_arr, discard)
trimmed_chunks.append(trimmed)
return stitch_chunks(trimmed_chunks) # type: ignore[arg-type]
@@ -0,0 +1,278 @@
"""
ferro_ta.resampling — OHLCV resampling and multi-timeframe API.
Provides functions to resample OHLCV data into coarser time bars or volume
bars, and a multi-timeframe helper that runs an indicator on two or more
resampled timeframes in one call.
The heavy OHLCV aggregation logic lives in the Rust backend
(``_ferro_ta.volume_bars`` and ``_ferro_ta.ohlcv_agg``); this module provides
the Python-facing API with:
- Time-based resampling via pandas (requires ``pandas``).
- Volume-bar resampling via Rust (no extra dependencies).
- Multi-timeframe helper that returns a dict of DataFrames.
Functions
---------
resample(ohlcv, rule, *, label='right', closed='right')
Resample a pandas OHLCV DataFrame by a time rule (e.g. ``'5min'``,
``'1h'``). Requires pandas.
volume_bars(ohlcv, volume_threshold)
Aggregate OHLCV data into volume bars using the Rust backend.
Accepts a pandas DataFrame or separate numpy arrays.
multi_timeframe(ohlcv, rules, *, indicator=None, indicator_kwargs=None)
Resample OHLCV to multiple timeframes and optionally run an indicator
on each. Returns a dict mapping each rule to a DataFrame (or to an
indicator result when *indicator* is given).
Rust backend
------------
All bar-accumulation logic delegates to::
ferro_ta._ferro_ta.volume_bars
ferro_ta._ferro_ta.ohlcv_agg
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Optional
from ferro_ta._ferro_ta import volume_bars as _rust_volume_bars
from ferro_ta._utils import _to_f64
__all__ = [
"resample",
"volume_bars",
"multi_timeframe",
]
# ---------------------------------------------------------------------------
# resample — time-based resampling (pandas required)
# ---------------------------------------------------------------------------
def resample(
ohlcv: Any,
rule: str,
*,
label: str = "right",
closed: str = "right",
) -> Any:
"""Resample an OHLCV DataFrame to a coarser time rule.
Uses ``pandas.DataFrame.resample`` under the hood; the index must be a
``DatetimeIndex`` (timezone-aware or naive).
Parameters
----------
ohlcv : pandas.DataFrame
Must have columns ``open``, ``high``, ``low``, ``close``, ``volume``
(case-sensitive; use the column-name helpers in :mod:`ferro_ta._utils`
if your column names differ). Index must be a ``DatetimeIndex``.
rule : str
Pandas offset alias (e.g. ``'5min'``, ``'1h'``, ``'1D'``).
label : str
Which bin edge to label the bucket with (``'left'`` or ``'right'``).
Default ``'right'``.
closed : str
Which side of the interval is closed (``'left'`` or ``'right'``).
Default ``'right'``.
Returns
-------
pandas.DataFrame
Resampled OHLCV DataFrame with the same column names.
Raises
------
ImportError
If pandas is not installed.
ValueError
If required columns are missing or the index is not a DatetimeIndex.
Examples
--------
>>> import pandas as pd, numpy as np
>>> from ferro_ta.data.resampling import resample
>>> idx = pd.date_range("2024-01-01", periods=60, freq="1min")
>>> df = pd.DataFrame({
... "open": np.random.rand(60) + 100,
... "high": np.random.rand(60) + 101,
... "low": np.random.rand(60) + 99,
... "close": np.random.rand(60) + 100,
... "volume": np.random.randint(100, 1000, 60).astype(float),
... }, index=idx)
>>> df5 = resample(df, "5min")
>>> df5.shape[0]
12
"""
try:
import pandas as pd
except ImportError as exc:
raise ImportError(
"pandas is required for time-based resampling. "
"Install it with: pip install pandas"
) from exc
required = {"open", "high", "low", "close", "volume"}
missing = required - set(ohlcv.columns)
if missing:
raise ValueError(f"OHLCV DataFrame missing columns: {missing}")
if not isinstance(ohlcv.index, pd.DatetimeIndex):
raise ValueError(
"ohlcv.index must be a pandas DatetimeIndex for time-based resampling."
)
agg = {
"open": "first",
"high": "max",
"low": "min",
"close": "last",
"volume": "sum",
}
return ohlcv.resample(rule, label=label, closed=closed).agg(agg).dropna(how="all")
# ---------------------------------------------------------------------------
# volume_bars — volume-based resampling (Rust backend)
# ---------------------------------------------------------------------------
def volume_bars(
ohlcv: Any,
volume_threshold: float,
*,
open_col: str = "open",
high_col: str = "high",
low_col: str = "low",
close_col: str = "close",
volume_col: str = "volume",
) -> Any:
"""Aggregate OHLCV data into volume bars using the Rust backend.
Each output bar accumulates input bars until ``volume_threshold`` units of
volume have been consumed.
Parameters
----------
ohlcv : pandas.DataFrame or tuple of arrays
Either a pandas DataFrame with OHLCV columns, or a tuple
``(open, high, low, close, volume)`` of array-like objects.
volume_threshold : float
Target volume per output bar (must be > 0).
open_col, high_col, low_col, close_col, volume_col : str
Column names when ``ohlcv`` is a DataFrame.
Returns
-------
pandas.DataFrame or tuple of numpy arrays
If a DataFrame was passed in, returns a DataFrame with the same column
names. Otherwise returns a tuple
``(open, high, low, close, volume)`` of numpy arrays.
Examples
--------
>>> import numpy as np
>>> from ferro_ta.data.resampling import volume_bars
>>> n = 100
>>> o = np.random.rand(n) + 100
>>> h = o + np.random.rand(n)
>>> l = o - np.random.rand(n)
>>> c = np.random.rand(n) + 100
>>> v = np.random.randint(50, 150, n).astype(float)
>>> bars = volume_bars((o, h, l, c, v), volume_threshold=500)
>>> len(bars[0]) > 0
True
"""
if isinstance(ohlcv, tuple):
o, h, low, c, v = (_to_f64(x) for x in ohlcv)
return _rust_volume_bars(o, h, low, c, v, float(volume_threshold))
# pandas DataFrame path
try:
import pandas as pd
except ImportError as exc:
raise ImportError("pandas is required when passing a DataFrame") from exc
o = _to_f64(ohlcv[open_col].values)
h = _to_f64(ohlcv[high_col].values)
low = _to_f64(ohlcv[low_col].values)
c = _to_f64(ohlcv[close_col].values)
v = _to_f64(ohlcv[volume_col].values)
ro, rh, rl, rc, rv = _rust_volume_bars(o, h, low, c, v, float(volume_threshold))
return pd.DataFrame(
{
open_col: ro,
high_col: rh,
low_col: rl,
close_col: rc,
volume_col: rv,
}
)
# ---------------------------------------------------------------------------
# multi_timeframe — run indicator on multiple resampled timeframes
# ---------------------------------------------------------------------------
def multi_timeframe(
ohlcv: Any,
rules: list[str],
*,
indicator: Optional[Callable[..., Any]] = None,
indicator_kwargs: Optional[dict[str, Any]] = None,
) -> dict[str, Any]:
"""Resample OHLCV to multiple timeframes and optionally run an indicator.
Parameters
----------
ohlcv : pandas.DataFrame
OHLCV data with a ``DatetimeIndex``.
rules : list of str
Pandas offset aliases, e.g. ``['5min', '1h']``.
indicator : callable, optional
A function ``indicator(close, **kwargs) -> array`` (or multi-output).
When provided it is called on the resampled ``close`` column for each
rule, and the result is stored in the returned dict instead of the
full DataFrame.
indicator_kwargs : dict, optional
Keyword arguments forwarded to *indicator*.
Returns
-------
dict
Mapping from each rule string to:
- a resampled pandas DataFrame when *indicator* is ``None``, or
- the indicator output (numpy array or tuple) when *indicator* is given.
Examples
--------
>>> import pandas as pd, numpy as np
>>> from ferro_ta import RSI
>>> from ferro_ta.data.resampling import multi_timeframe
>>> idx = pd.date_range("2024-01-01", periods=200, freq="1min")
>>> close = np.cumprod(1 + np.random.randn(200) * 0.001) * 100
>>> df = pd.DataFrame({
... "open": close, "high": close * 1.001, "low": close * 0.999,
... "close": close, "volume": np.ones(200) * 1000,
... }, index=idx)
>>> result = multi_timeframe(df, ["5min", "15min"], indicator=RSI,
... indicator_kwargs={"timeperiod": 14})
>>> sorted(result.keys())
['15min', '5min']
"""
kw = indicator_kwargs or {}
out: dict[str, Any] = {}
for rule in rules:
df_r = resample(ohlcv, rule)
if indicator is not None:
out[rule] = indicator(_to_f64(df_r["close"].values), **kw)
else:
out[rule] = df_r
return out
@@ -0,0 +1,69 @@
"""
Streaming / Incremental Indicators — bar-by-bar stateful classes.
All streaming classes are implemented in Rust (PyO3) for maximum performance.
The Python module re-exports the Rust classes from the ``_ferro_ta`` extension.
The extension must be built; there is no Python fallback.
Usage
-----
>>> from ferro_ta.data.streaming import StreamingSMA, StreamingEMA, StreamingRSI
>>> import numpy as np
>>> sma = StreamingSMA(period=3)
>>> for close in [10.0, 11.0, 12.0, 13.0, 14.0]:
... val = sma.update(close)
... print(f"{close}{val:.4f}" if not np.isnan(val) else f"{close} → NaN")
10.0 → NaN
11.0 → NaN
12.0 → 11.0000
13.0 → 12.0000
14.0 → 13.0000
Available classes
-----------------
StreamingSMA — Simple Moving Average
StreamingEMA — Exponential Moving Average
StreamingRSI — Relative Strength Index (Wilder seeding)
StreamingATR — Average True Range (Wilder seeding)
StreamingBBands — Bollinger Bands (upper, middle, lower)
StreamingMACD — MACD line, signal, histogram
StreamingStoch — Slow Stochastic (slowk, slowd)
StreamingVWAP — Volume Weighted Average Price (cumulative)
StreamingSupertrend — ATR-based Supertrend
Rust backend
------------
All classes are PyO3 classes compiled into the ``_ferro_ta`` extension module.
Import them directly from the extension for zero-overhead access::
from ferro_ta._ferro_ta import StreamingSMA
"""
from __future__ import annotations
# ---------------------------------------------------------------------------
# Import Rust-backed streaming classes from the compiled extension.
# ---------------------------------------------------------------------------
from ferro_ta._ferro_ta import ( # noqa: F401
StreamingATR,
StreamingBBands,
StreamingEMA,
StreamingMACD,
StreamingRSI,
StreamingSMA,
StreamingStoch,
StreamingSupertrend,
StreamingVWAP,
)
__all__ = [
"StreamingSMA",
"StreamingEMA",
"StreamingRSI",
"StreamingATR",
"StreamingBBands",
"StreamingMACD",
"StreamingStoch",
"StreamingVWAP",
"StreamingSupertrend",
]