paracas downloader 🚧

This commit is contained in:
Andreas Bigger
2025-12-29 13:21:48 -05:00
parent ab0afbc1d2
commit 7be4eded90
46 changed files with 4570 additions and 22 deletions
+39
View File
@@ -0,0 +1,39 @@
[package]
name = "paracas-format"
description = "Output formatters for paracas tick data downloader"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
documentation.workspace = true
authors.workspace = true
keywords.workspace = true
categories.workspace = true
[lints]
workspace = true
[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--cfg", "docsrs"]
[features]
default = ["csv", "json", "parquet"]
csv = []
json = []
parquet = ["dep:arrow", "dep:parquet"]
[dependencies]
paracas-types = { workspace = true }
paracas-aggregate = { workspace = true }
chrono = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
arrow = { workspace = true, optional = true }
parquet = { workspace = true, optional = true }
[dev-dependencies]
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+36
View File
@@ -0,0 +1,36 @@
# paracas-format
Output formatters for the paracas tick data downloader.
## Supported Formats
- **CSV** - Comma-separated values
- **JSON** - JSON array or newline-delimited JSON (NDJSON)
- **Parquet** - Apache Parquet columnar format (requires `parquet` feature)
## Usage
```rust,no_run
use paracas_format::{CsvFormatter, Formatter, OutputFormat};
use paracas_types::Tick;
use std::io::Cursor;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let ticks: Vec<Tick> = vec![];
let mut output = Cursor::new(Vec::new());
let formatter = CsvFormatter::new();
formatter.write_ticks(&ticks, &mut output)?;
Ok(())
}
```
## Features
- `csv` - CSV format support (default)
- `json` - JSON format support (default)
- `parquet` - Parquet format support (default)
## License
MIT License - see [LICENSE](../../LICENSE) for details.
+163
View File
@@ -0,0 +1,163 @@
//! CSV output format.
use paracas_aggregate::Ohlcv;
use paracas_types::Tick;
use std::io::Write;
use crate::{FormatError, Formatter};
/// CSV formatter.
#[derive(Debug, Clone, Default)]
pub struct CsvFormatter {
/// Field delimiter (default: comma).
delimiter: char,
/// Whether to include header row.
include_header: bool,
}
impl CsvFormatter {
/// Creates a new CSV formatter with default settings.
#[must_use]
pub const fn new() -> Self {
Self {
delimiter: ',',
include_header: true,
}
}
/// Sets the field delimiter.
#[must_use]
pub const fn with_delimiter(mut self, delimiter: char) -> Self {
self.delimiter = delimiter;
self
}
/// Sets whether to include a header row.
#[must_use]
pub const fn with_header(mut self, include: bool) -> Self {
self.include_header = include;
self
}
/// Creates a tab-separated values (TSV) formatter.
#[must_use]
pub const fn tsv() -> Self {
Self {
delimiter: '\t',
include_header: true,
}
}
}
impl Formatter for CsvFormatter {
fn write_ticks<W: Write + Send>(
&self,
ticks: &[Tick],
mut writer: W,
) -> Result<(), FormatError> {
let d = self.delimiter;
if self.include_header {
writeln!(writer, "timestamp{d}ask{d}bid{d}ask_volume{d}bid_volume")?;
}
for tick in ticks {
writeln!(
writer,
"{}{d}{}{d}{}{d}{}{d}{}",
tick.timestamp.format("%Y-%m-%dT%H:%M:%S%.3fZ"),
tick.ask,
tick.bid,
tick.ask_volume,
tick.bid_volume
)?;
}
Ok(())
}
fn write_ohlcv<W: Write + Send>(
&self,
bars: &[Ohlcv],
mut writer: W,
) -> Result<(), FormatError> {
let d = self.delimiter;
if self.include_header {
writeln!(
writer,
"timestamp{d}open{d}high{d}low{d}close{d}volume{d}tick_count"
)?;
}
for bar in bars {
writeln!(
writer,
"{}{d}{}{d}{}{d}{}{d}{}{d}{}{d}{}",
bar.timestamp.format("%Y-%m-%dT%H:%M:%SZ"),
bar.open,
bar.high,
bar.low,
bar.close,
bar.volume,
bar.tick_count
)?;
}
Ok(())
}
fn extension(&self) -> &str {
"csv"
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{TimeZone, Utc};
use std::io::Cursor;
fn create_test_tick() -> Tick {
let timestamp = Utc.with_ymd_and_hms(2024, 1, 15, 12, 30, 45).unwrap();
Tick::new(timestamp, 1.1001, 1.1000, 100.0, 200.0)
}
#[test]
fn test_csv_ticks() {
let formatter = CsvFormatter::new();
let ticks = vec![create_test_tick()];
let mut output = Cursor::new(Vec::new());
formatter.write_ticks(&ticks, &mut output).unwrap();
let result = String::from_utf8(output.into_inner()).unwrap();
assert!(result.contains("timestamp,ask,bid,ask_volume,bid_volume"));
assert!(result.contains("2024-01-15T12:30:45.000Z"));
assert!(result.contains("1.1001"));
}
#[test]
fn test_csv_no_header() {
let formatter = CsvFormatter::new().with_header(false);
let ticks = vec![create_test_tick()];
let mut output = Cursor::new(Vec::new());
formatter.write_ticks(&ticks, &mut output).unwrap();
let result = String::from_utf8(output.into_inner()).unwrap();
assert!(!result.contains("timestamp,ask"));
}
#[test]
fn test_tsv() {
let formatter = CsvFormatter::tsv();
let ticks = vec![create_test_tick()];
let mut output = Cursor::new(Vec::new());
formatter.write_ticks(&ticks, &mut output).unwrap();
let result = String::from_utf8(output.into_inner()).unwrap();
assert!(result.contains("timestamp\task\tbid"));
}
}
+99
View File
@@ -0,0 +1,99 @@
//! Output format abstraction.
use paracas_aggregate::Ohlcv;
use paracas_types::Tick;
use std::io::Write;
use thiserror::Error;
/// Output format identifier.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub enum OutputFormat {
/// CSV format.
#[default]
Csv,
/// JSON array format.
Json,
/// Newline-delimited JSON format.
Ndjson,
/// Apache Parquet format.
Parquet,
}
impl OutputFormat {
/// Returns the file extension for this format.
#[must_use]
pub const fn extension(&self) -> &'static str {
match self {
Self::Csv => "csv",
Self::Json => "json",
Self::Ndjson => "ndjson",
Self::Parquet => "parquet",
}
}
/// Returns all available formats.
#[must_use]
pub const fn all() -> &'static [Self] {
&[Self::Csv, Self::Json, Self::Ndjson, Self::Parquet]
}
}
impl std::fmt::Display for OutputFormat {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.extension())
}
}
impl std::str::FromStr for OutputFormat {
type Err = FormatError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_lowercase().as_str() {
"csv" => Ok(Self::Csv),
"json" => Ok(Self::Json),
"ndjson" | "jsonl" => Ok(Self::Ndjson),
"parquet" | "pq" => Ok(Self::Parquet),
_ => Err(FormatError::UnknownFormat(s.to_string())),
}
}
}
/// Errors that can occur during formatting.
#[derive(Error, Debug)]
pub enum FormatError {
/// Unknown output format.
#[error("Unknown format: {0}")]
UnknownFormat(String),
/// I/O error.
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
/// JSON serialization error.
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
/// Arrow/Parquet error.
#[error("Parquet error: {0}")]
Parquet(String),
}
/// Trait for output formatters.
pub trait Formatter: Send + Sync {
/// Writes tick data to the output.
///
/// # Errors
///
/// Returns an error if writing fails.
fn write_ticks<W: Write + Send>(&self, ticks: &[Tick], writer: W) -> Result<(), FormatError>;
/// Writes OHLCV data to the output.
///
/// # Errors
///
/// Returns an error if writing fails.
fn write_ohlcv<W: Write + Send>(&self, bars: &[Ohlcv], writer: W) -> Result<(), FormatError>;
/// Returns the file extension for this format.
fn extension(&self) -> &str;
}
+169
View File
@@ -0,0 +1,169 @@
//! JSON output format.
use paracas_aggregate::Ohlcv;
use paracas_types::Tick;
use std::io::Write;
use crate::{FormatError, Formatter};
/// JSON output style.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum JsonStyle {
/// JSON array (standard JSON).
#[default]
Array,
/// Newline-delimited JSON (NDJSON/JSONL).
Ndjson,
}
/// JSON formatter.
#[derive(Debug, Clone, Default)]
pub struct JsonFormatter {
/// Output style.
style: JsonStyle,
/// Whether to pretty-print (only for array style).
pretty: bool,
}
impl JsonFormatter {
/// Creates a new JSON formatter with default settings (array style).
#[must_use]
pub const fn new() -> Self {
Self {
style: JsonStyle::Array,
pretty: false,
}
}
/// Creates a new NDJSON formatter.
#[must_use]
pub const fn ndjson() -> Self {
Self {
style: JsonStyle::Ndjson,
pretty: false,
}
}
/// Sets whether to pretty-print output (array style only).
#[must_use]
pub const fn with_pretty(mut self, pretty: bool) -> Self {
self.pretty = pretty;
self
}
/// Sets the output style.
#[must_use]
pub const fn with_style(mut self, style: JsonStyle) -> Self {
self.style = style;
self
}
}
impl Formatter for JsonFormatter {
fn write_ticks<W: Write + Send>(
&self,
ticks: &[Tick],
mut writer: W,
) -> Result<(), FormatError> {
match self.style {
JsonStyle::Array => {
if self.pretty {
serde_json::to_writer_pretty(&mut writer, ticks)?;
} else {
serde_json::to_writer(&mut writer, ticks)?;
}
writeln!(writer)?;
}
JsonStyle::Ndjson => {
for tick in ticks {
serde_json::to_writer(&mut writer, tick)?;
writeln!(writer)?;
}
}
}
Ok(())
}
fn write_ohlcv<W: Write + Send>(
&self,
bars: &[Ohlcv],
mut writer: W,
) -> Result<(), FormatError> {
match self.style {
JsonStyle::Array => {
if self.pretty {
serde_json::to_writer_pretty(&mut writer, bars)?;
} else {
serde_json::to_writer(&mut writer, bars)?;
}
writeln!(writer)?;
}
JsonStyle::Ndjson => {
for bar in bars {
serde_json::to_writer(&mut writer, bar)?;
writeln!(writer)?;
}
}
}
Ok(())
}
fn extension(&self) -> &str {
match self.style {
JsonStyle::Array => "json",
JsonStyle::Ndjson => "ndjson",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{TimeZone, Utc};
use std::io::Cursor;
fn create_test_tick() -> Tick {
let timestamp = Utc.with_ymd_and_hms(2024, 1, 15, 12, 30, 45).unwrap();
Tick::new(timestamp, 1.1001, 1.1000, 100.0, 200.0)
}
#[test]
fn test_json_array() {
let formatter = JsonFormatter::new();
let ticks = vec![create_test_tick()];
let mut output = Cursor::new(Vec::new());
formatter.write_ticks(&ticks, &mut output).unwrap();
let result = String::from_utf8(output.into_inner()).unwrap();
assert!(result.starts_with('['));
assert!(result.contains("\"ask\":1.1001"));
}
#[test]
fn test_ndjson() {
let formatter = JsonFormatter::ndjson();
let ticks = vec![create_test_tick(), create_test_tick()];
let mut output = Cursor::new(Vec::new());
formatter.write_ticks(&ticks, &mut output).unwrap();
let result = String::from_utf8(output.into_inner()).unwrap();
let lines: Vec<_> = result.lines().collect();
assert_eq!(lines.len(), 2);
assert!(lines[0].starts_with('{'));
}
#[test]
fn test_pretty_json() {
let formatter = JsonFormatter::new().with_pretty(true);
let ticks = vec![create_test_tick()];
let mut output = Cursor::new(Vec::new());
formatter.write_ticks(&ticks, &mut output).unwrap();
let result = String::from_utf8(output.into_inner()).unwrap();
assert!(result.contains('\n'));
assert!(result.contains(" ")); // Indentation
}
}
+28
View File
@@ -0,0 +1,28 @@
//! Output formatters for paracas tick data downloader.
//!
//! This crate provides formatters for writing tick and OHLCV data
//! to various output formats:
//!
//! - [`CsvFormatter`] - CSV format
//! - [`JsonFormatter`] - JSON array or NDJSON format
//! - [`ParquetFormatter`] - Apache Parquet columnar format
#![doc = include_str!("../README.md")]
#![doc(issue_tracker_base_url = "https://github.com/factordynamics/paracas/issues/")]
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
#![warn(missing_docs)]
#![forbid(unsafe_code)]
mod csv;
mod formatter;
mod json;
#[cfg(feature = "parquet")]
mod parquet;
pub use crate::csv::CsvFormatter;
pub use formatter::{FormatError, Formatter, OutputFormat};
pub use json::{JsonFormatter, JsonStyle};
#[cfg(feature = "parquet")]
pub use crate::parquet::ParquetFormatter;
+236
View File
@@ -0,0 +1,236 @@
//! Apache Parquet output format.
use arrow::array::{Float32Array, Float64Array, TimestampMicrosecondArray, UInt32Array};
use arrow::datatypes::{DataType, Field, Schema, TimeUnit};
use arrow::record_batch::RecordBatch;
use paracas_aggregate::Ohlcv;
use paracas_types::Tick;
use parquet::arrow::ArrowWriter;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use std::io::Write;
use std::sync::Arc;
use crate::{FormatError, Formatter};
/// Parquet formatter.
#[derive(Debug, Clone)]
pub struct ParquetFormatter {
/// Row group size (number of rows per group).
row_group_size: usize,
/// Compression codec.
compression: Compression,
}
impl Default for ParquetFormatter {
fn default() -> Self {
Self {
row_group_size: 100_000,
compression: Compression::SNAPPY,
}
}
}
impl ParquetFormatter {
/// Creates a new Parquet formatter with default settings.
#[must_use]
pub fn new() -> Self {
Self::default()
}
/// Sets the row group size.
#[must_use]
pub const fn with_row_group_size(mut self, size: usize) -> Self {
self.row_group_size = size;
self
}
/// Sets the compression codec.
#[must_use]
pub const fn with_compression(mut self, compression: Compression) -> Self {
self.compression = compression;
self
}
/// Creates the Arrow schema for tick data.
fn tick_schema() -> Schema {
Schema::new(vec![
Field::new(
"timestamp",
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
false,
),
Field::new("ask", DataType::Float64, false),
Field::new("bid", DataType::Float64, false),
Field::new("ask_volume", DataType::Float32, false),
Field::new("bid_volume", DataType::Float32, false),
])
}
/// Creates the Arrow schema for OHLCV data.
fn ohlcv_schema() -> Schema {
Schema::new(vec![
Field::new(
"timestamp",
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
false,
),
Field::new("open", DataType::Float64, false),
Field::new("high", DataType::Float64, false),
Field::new("low", DataType::Float64, false),
Field::new("close", DataType::Float64, false),
Field::new("volume", DataType::Float64, false),
Field::new("tick_count", DataType::UInt32, false),
])
}
/// Converts ticks to Arrow RecordBatch.
fn ticks_to_batch(ticks: &[Tick]) -> Result<RecordBatch, FormatError> {
let timestamps: Vec<_> = ticks
.iter()
.map(|t| t.timestamp.timestamp_micros())
.collect();
let asks: Vec<_> = ticks.iter().map(|t| t.ask).collect();
let bids: Vec<_> = ticks.iter().map(|t| t.bid).collect();
let ask_vols: Vec<_> = ticks.iter().map(|t| t.ask_volume).collect();
let bid_vols: Vec<_> = ticks.iter().map(|t| t.bid_volume).collect();
RecordBatch::try_new(
Arc::new(Self::tick_schema()),
vec![
Arc::new(TimestampMicrosecondArray::from(timestamps).with_timezone("UTC")),
Arc::new(Float64Array::from(asks)),
Arc::new(Float64Array::from(bids)),
Arc::new(Float32Array::from(ask_vols)),
Arc::new(Float32Array::from(bid_vols)),
],
)
.map_err(|e| FormatError::Parquet(e.to_string()))
}
/// Converts OHLCV bars to Arrow RecordBatch.
fn ohlcv_to_batch(bars: &[Ohlcv]) -> Result<RecordBatch, FormatError> {
let timestamps: Vec<_> = bars
.iter()
.map(|b| b.timestamp.timestamp_micros())
.collect();
let opens: Vec<_> = bars.iter().map(|b| b.open).collect();
let highs: Vec<_> = bars.iter().map(|b| b.high).collect();
let lows: Vec<_> = bars.iter().map(|b| b.low).collect();
let closes: Vec<_> = bars.iter().map(|b| b.close).collect();
let volumes: Vec<_> = bars.iter().map(|b| b.volume).collect();
let tick_counts: Vec<_> = bars.iter().map(|b| b.tick_count).collect();
RecordBatch::try_new(
Arc::new(Self::ohlcv_schema()),
vec![
Arc::new(TimestampMicrosecondArray::from(timestamps).with_timezone("UTC")),
Arc::new(Float64Array::from(opens)),
Arc::new(Float64Array::from(highs)),
Arc::new(Float64Array::from(lows)),
Arc::new(Float64Array::from(closes)),
Arc::new(Float64Array::from(volumes)),
Arc::new(UInt32Array::from(tick_counts)),
],
)
.map_err(|e| FormatError::Parquet(e.to_string()))
}
}
impl Formatter for ParquetFormatter {
fn write_ticks<W: Write + Send>(&self, ticks: &[Tick], writer: W) -> Result<(), FormatError> {
let schema = Arc::new(Self::tick_schema());
let props = WriterProperties::builder()
.set_compression(self.compression)
.set_max_row_group_size(self.row_group_size)
.build();
let mut arrow_writer = ArrowWriter::try_new(writer, schema, Some(props))
.map_err(|e| FormatError::Parquet(e.to_string()))?;
// Write in batches
for chunk in ticks.chunks(self.row_group_size) {
let batch = Self::ticks_to_batch(chunk)?;
arrow_writer
.write(&batch)
.map_err(|e| FormatError::Parquet(e.to_string()))?;
}
arrow_writer
.close()
.map_err(|e| FormatError::Parquet(e.to_string()))?;
Ok(())
}
fn write_ohlcv<W: Write + Send>(&self, bars: &[Ohlcv], writer: W) -> Result<(), FormatError> {
let schema = Arc::new(Self::ohlcv_schema());
let props = WriterProperties::builder()
.set_compression(self.compression)
.set_max_row_group_size(self.row_group_size)
.build();
let mut arrow_writer = ArrowWriter::try_new(writer, schema, Some(props))
.map_err(|e| FormatError::Parquet(e.to_string()))?;
// Write in batches
for chunk in bars.chunks(self.row_group_size) {
let batch = Self::ohlcv_to_batch(chunk)?;
arrow_writer
.write(&batch)
.map_err(|e| FormatError::Parquet(e.to_string()))?;
}
arrow_writer
.close()
.map_err(|e| FormatError::Parquet(e.to_string()))?;
Ok(())
}
fn extension(&self) -> &str {
"parquet"
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{TimeZone, Utc};
use std::io::Cursor;
fn create_test_tick() -> Tick {
let timestamp = Utc.with_ymd_and_hms(2024, 1, 15, 12, 30, 45).unwrap();
Tick::new(timestamp, 1.1001, 1.1000, 100.0, 200.0)
}
#[test]
fn test_parquet_ticks() {
let formatter = ParquetFormatter::new();
let ticks = vec![create_test_tick()];
let mut output = Cursor::new(Vec::new());
formatter.write_ticks(&ticks, &mut output).unwrap();
// Parquet files start with "PAR1" magic bytes
let data = output.into_inner();
assert!(data.len() > 4);
assert_eq!(&data[0..4], b"PAR1");
}
#[test]
fn test_tick_schema() {
let schema = ParquetFormatter::tick_schema();
assert_eq!(schema.fields().len(), 5);
assert!(schema.field_with_name("timestamp").is_ok());
assert!(schema.field_with_name("ask").is_ok());
}
#[test]
fn test_ohlcv_schema() {
let schema = ParquetFormatter::ohlcv_schema();
assert_eq!(schema.fields().len(), 7);
assert!(schema.field_with_name("open").is_ok());
assert!(schema.field_with_name("close").is_ok());
}
}