paracas downloader 🚧
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
[package]
|
||||
name = "paracas-format"
|
||||
description = "Output formatters for paracas tick data downloader"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
documentation.workspace = true
|
||||
authors.workspace = true
|
||||
keywords.workspace = true
|
||||
categories.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
all-features = true
|
||||
rustdoc-args = ["--cfg", "docsrs"]
|
||||
|
||||
[features]
|
||||
default = ["csv", "json", "parquet"]
|
||||
csv = []
|
||||
json = []
|
||||
parquet = ["dep:arrow", "dep:parquet"]
|
||||
|
||||
[dependencies]
|
||||
paracas-types = { workspace = true }
|
||||
paracas-aggregate = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
arrow = { workspace = true, optional = true }
|
||||
parquet = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
@@ -0,0 +1,36 @@
|
||||
# paracas-format
|
||||
|
||||
Output formatters for the paracas tick data downloader.
|
||||
|
||||
## Supported Formats
|
||||
|
||||
- **CSV** - Comma-separated values
|
||||
- **JSON** - JSON array or newline-delimited JSON (NDJSON)
|
||||
- **Parquet** - Apache Parquet columnar format (requires `parquet` feature)
|
||||
|
||||
## Usage
|
||||
|
||||
```rust,no_run
|
||||
use paracas_format::{CsvFormatter, Formatter, OutputFormat};
|
||||
use paracas_types::Tick;
|
||||
use std::io::Cursor;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let ticks: Vec<Tick> = vec![];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
let formatter = CsvFormatter::new();
|
||||
formatter.write_ticks(&ticks, &mut output)?;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- `csv` - CSV format support (default)
|
||||
- `json` - JSON format support (default)
|
||||
- `parquet` - Parquet format support (default)
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see [LICENSE](../../LICENSE) for details.
|
||||
@@ -0,0 +1,163 @@
|
||||
//! CSV output format.
|
||||
|
||||
use paracas_aggregate::Ohlcv;
|
||||
use paracas_types::Tick;
|
||||
use std::io::Write;
|
||||
|
||||
use crate::{FormatError, Formatter};
|
||||
|
||||
/// CSV formatter.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CsvFormatter {
|
||||
/// Field delimiter (default: comma).
|
||||
delimiter: char,
|
||||
/// Whether to include header row.
|
||||
include_header: bool,
|
||||
}
|
||||
|
||||
impl CsvFormatter {
|
||||
/// Creates a new CSV formatter with default settings.
|
||||
#[must_use]
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
delimiter: ',',
|
||||
include_header: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the field delimiter.
|
||||
#[must_use]
|
||||
pub const fn with_delimiter(mut self, delimiter: char) -> Self {
|
||||
self.delimiter = delimiter;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets whether to include a header row.
|
||||
#[must_use]
|
||||
pub const fn with_header(mut self, include: bool) -> Self {
|
||||
self.include_header = include;
|
||||
self
|
||||
}
|
||||
|
||||
/// Creates a tab-separated values (TSV) formatter.
|
||||
#[must_use]
|
||||
pub const fn tsv() -> Self {
|
||||
Self {
|
||||
delimiter: '\t',
|
||||
include_header: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Formatter for CsvFormatter {
|
||||
fn write_ticks<W: Write + Send>(
|
||||
&self,
|
||||
ticks: &[Tick],
|
||||
mut writer: W,
|
||||
) -> Result<(), FormatError> {
|
||||
let d = self.delimiter;
|
||||
|
||||
if self.include_header {
|
||||
writeln!(writer, "timestamp{d}ask{d}bid{d}ask_volume{d}bid_volume")?;
|
||||
}
|
||||
|
||||
for tick in ticks {
|
||||
writeln!(
|
||||
writer,
|
||||
"{}{d}{}{d}{}{d}{}{d}{}",
|
||||
tick.timestamp.format("%Y-%m-%dT%H:%M:%S%.3fZ"),
|
||||
tick.ask,
|
||||
tick.bid,
|
||||
tick.ask_volume,
|
||||
tick.bid_volume
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_ohlcv<W: Write + Send>(
|
||||
&self,
|
||||
bars: &[Ohlcv],
|
||||
mut writer: W,
|
||||
) -> Result<(), FormatError> {
|
||||
let d = self.delimiter;
|
||||
|
||||
if self.include_header {
|
||||
writeln!(
|
||||
writer,
|
||||
"timestamp{d}open{d}high{d}low{d}close{d}volume{d}tick_count"
|
||||
)?;
|
||||
}
|
||||
|
||||
for bar in bars {
|
||||
writeln!(
|
||||
writer,
|
||||
"{}{d}{}{d}{}{d}{}{d}{}{d}{}{d}{}",
|
||||
bar.timestamp.format("%Y-%m-%dT%H:%M:%SZ"),
|
||||
bar.open,
|
||||
bar.high,
|
||||
bar.low,
|
||||
bar.close,
|
||||
bar.volume,
|
||||
bar.tick_count
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extension(&self) -> &str {
|
||||
"csv"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::{TimeZone, Utc};
|
||||
use std::io::Cursor;
|
||||
|
||||
fn create_test_tick() -> Tick {
|
||||
let timestamp = Utc.with_ymd_and_hms(2024, 1, 15, 12, 30, 45).unwrap();
|
||||
Tick::new(timestamp, 1.1001, 1.1000, 100.0, 200.0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_csv_ticks() {
|
||||
let formatter = CsvFormatter::new();
|
||||
let ticks = vec![create_test_tick()];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
formatter.write_ticks(&ticks, &mut output).unwrap();
|
||||
|
||||
let result = String::from_utf8(output.into_inner()).unwrap();
|
||||
assert!(result.contains("timestamp,ask,bid,ask_volume,bid_volume"));
|
||||
assert!(result.contains("2024-01-15T12:30:45.000Z"));
|
||||
assert!(result.contains("1.1001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_csv_no_header() {
|
||||
let formatter = CsvFormatter::new().with_header(false);
|
||||
let ticks = vec![create_test_tick()];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
formatter.write_ticks(&ticks, &mut output).unwrap();
|
||||
|
||||
let result = String::from_utf8(output.into_inner()).unwrap();
|
||||
assert!(!result.contains("timestamp,ask"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tsv() {
|
||||
let formatter = CsvFormatter::tsv();
|
||||
let ticks = vec![create_test_tick()];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
formatter.write_ticks(&ticks, &mut output).unwrap();
|
||||
|
||||
let result = String::from_utf8(output.into_inner()).unwrap();
|
||||
assert!(result.contains("timestamp\task\tbid"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
//! Output format abstraction.
|
||||
|
||||
use paracas_aggregate::Ohlcv;
|
||||
use paracas_types::Tick;
|
||||
use std::io::Write;
|
||||
use thiserror::Error;
|
||||
|
||||
/// Output format identifier.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
|
||||
pub enum OutputFormat {
|
||||
/// CSV format.
|
||||
#[default]
|
||||
Csv,
|
||||
/// JSON array format.
|
||||
Json,
|
||||
/// Newline-delimited JSON format.
|
||||
Ndjson,
|
||||
/// Apache Parquet format.
|
||||
Parquet,
|
||||
}
|
||||
|
||||
impl OutputFormat {
|
||||
/// Returns the file extension for this format.
|
||||
#[must_use]
|
||||
pub const fn extension(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Csv => "csv",
|
||||
Self::Json => "json",
|
||||
Self::Ndjson => "ndjson",
|
||||
Self::Parquet => "parquet",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns all available formats.
|
||||
#[must_use]
|
||||
pub const fn all() -> &'static [Self] {
|
||||
&[Self::Csv, Self::Json, Self::Ndjson, Self::Parquet]
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for OutputFormat {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.extension())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for OutputFormat {
|
||||
type Err = FormatError;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"csv" => Ok(Self::Csv),
|
||||
"json" => Ok(Self::Json),
|
||||
"ndjson" | "jsonl" => Ok(Self::Ndjson),
|
||||
"parquet" | "pq" => Ok(Self::Parquet),
|
||||
_ => Err(FormatError::UnknownFormat(s.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors that can occur during formatting.
|
||||
#[derive(Error, Debug)]
|
||||
pub enum FormatError {
|
||||
/// Unknown output format.
|
||||
#[error("Unknown format: {0}")]
|
||||
UnknownFormat(String),
|
||||
|
||||
/// I/O error.
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// JSON serialization error.
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
/// Arrow/Parquet error.
|
||||
#[error("Parquet error: {0}")]
|
||||
Parquet(String),
|
||||
}
|
||||
|
||||
/// Trait for output formatters.
|
||||
pub trait Formatter: Send + Sync {
|
||||
/// Writes tick data to the output.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if writing fails.
|
||||
fn write_ticks<W: Write + Send>(&self, ticks: &[Tick], writer: W) -> Result<(), FormatError>;
|
||||
|
||||
/// Writes OHLCV data to the output.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if writing fails.
|
||||
fn write_ohlcv<W: Write + Send>(&self, bars: &[Ohlcv], writer: W) -> Result<(), FormatError>;
|
||||
|
||||
/// Returns the file extension for this format.
|
||||
fn extension(&self) -> &str;
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
//! JSON output format.
|
||||
|
||||
use paracas_aggregate::Ohlcv;
|
||||
use paracas_types::Tick;
|
||||
use std::io::Write;
|
||||
|
||||
use crate::{FormatError, Formatter};
|
||||
|
||||
/// JSON output style.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum JsonStyle {
|
||||
/// JSON array (standard JSON).
|
||||
#[default]
|
||||
Array,
|
||||
/// Newline-delimited JSON (NDJSON/JSONL).
|
||||
Ndjson,
|
||||
}
|
||||
|
||||
/// JSON formatter.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct JsonFormatter {
|
||||
/// Output style.
|
||||
style: JsonStyle,
|
||||
/// Whether to pretty-print (only for array style).
|
||||
pretty: bool,
|
||||
}
|
||||
|
||||
impl JsonFormatter {
|
||||
/// Creates a new JSON formatter with default settings (array style).
|
||||
#[must_use]
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
style: JsonStyle::Array,
|
||||
pretty: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new NDJSON formatter.
|
||||
#[must_use]
|
||||
pub const fn ndjson() -> Self {
|
||||
Self {
|
||||
style: JsonStyle::Ndjson,
|
||||
pretty: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets whether to pretty-print output (array style only).
|
||||
#[must_use]
|
||||
pub const fn with_pretty(mut self, pretty: bool) -> Self {
|
||||
self.pretty = pretty;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the output style.
|
||||
#[must_use]
|
||||
pub const fn with_style(mut self, style: JsonStyle) -> Self {
|
||||
self.style = style;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Formatter for JsonFormatter {
|
||||
fn write_ticks<W: Write + Send>(
|
||||
&self,
|
||||
ticks: &[Tick],
|
||||
mut writer: W,
|
||||
) -> Result<(), FormatError> {
|
||||
match self.style {
|
||||
JsonStyle::Array => {
|
||||
if self.pretty {
|
||||
serde_json::to_writer_pretty(&mut writer, ticks)?;
|
||||
} else {
|
||||
serde_json::to_writer(&mut writer, ticks)?;
|
||||
}
|
||||
writeln!(writer)?;
|
||||
}
|
||||
JsonStyle::Ndjson => {
|
||||
for tick in ticks {
|
||||
serde_json::to_writer(&mut writer, tick)?;
|
||||
writeln!(writer)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_ohlcv<W: Write + Send>(
|
||||
&self,
|
||||
bars: &[Ohlcv],
|
||||
mut writer: W,
|
||||
) -> Result<(), FormatError> {
|
||||
match self.style {
|
||||
JsonStyle::Array => {
|
||||
if self.pretty {
|
||||
serde_json::to_writer_pretty(&mut writer, bars)?;
|
||||
} else {
|
||||
serde_json::to_writer(&mut writer, bars)?;
|
||||
}
|
||||
writeln!(writer)?;
|
||||
}
|
||||
JsonStyle::Ndjson => {
|
||||
for bar in bars {
|
||||
serde_json::to_writer(&mut writer, bar)?;
|
||||
writeln!(writer)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extension(&self) -> &str {
|
||||
match self.style {
|
||||
JsonStyle::Array => "json",
|
||||
JsonStyle::Ndjson => "ndjson",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::{TimeZone, Utc};
|
||||
use std::io::Cursor;
|
||||
|
||||
fn create_test_tick() -> Tick {
|
||||
let timestamp = Utc.with_ymd_and_hms(2024, 1, 15, 12, 30, 45).unwrap();
|
||||
Tick::new(timestamp, 1.1001, 1.1000, 100.0, 200.0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_array() {
|
||||
let formatter = JsonFormatter::new();
|
||||
let ticks = vec![create_test_tick()];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
formatter.write_ticks(&ticks, &mut output).unwrap();
|
||||
|
||||
let result = String::from_utf8(output.into_inner()).unwrap();
|
||||
assert!(result.starts_with('['));
|
||||
assert!(result.contains("\"ask\":1.1001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ndjson() {
|
||||
let formatter = JsonFormatter::ndjson();
|
||||
let ticks = vec![create_test_tick(), create_test_tick()];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
formatter.write_ticks(&ticks, &mut output).unwrap();
|
||||
|
||||
let result = String::from_utf8(output.into_inner()).unwrap();
|
||||
let lines: Vec<_> = result.lines().collect();
|
||||
assert_eq!(lines.len(), 2);
|
||||
assert!(lines[0].starts_with('{'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pretty_json() {
|
||||
let formatter = JsonFormatter::new().with_pretty(true);
|
||||
let ticks = vec![create_test_tick()];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
formatter.write_ticks(&ticks, &mut output).unwrap();
|
||||
|
||||
let result = String::from_utf8(output.into_inner()).unwrap();
|
||||
assert!(result.contains('\n'));
|
||||
assert!(result.contains(" ")); // Indentation
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
//! Output formatters for paracas tick data downloader.
|
||||
//!
|
||||
//! This crate provides formatters for writing tick and OHLCV data
|
||||
//! to various output formats:
|
||||
//!
|
||||
//! - [`CsvFormatter`] - CSV format
|
||||
//! - [`JsonFormatter`] - JSON array or NDJSON format
|
||||
//! - [`ParquetFormatter`] - Apache Parquet columnar format
|
||||
|
||||
#![doc = include_str!("../README.md")]
|
||||
#![doc(issue_tracker_base_url = "https://github.com/factordynamics/paracas/issues/")]
|
||||
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
|
||||
#![warn(missing_docs)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
mod csv;
|
||||
mod formatter;
|
||||
mod json;
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
mod parquet;
|
||||
|
||||
pub use crate::csv::CsvFormatter;
|
||||
pub use formatter::{FormatError, Formatter, OutputFormat};
|
||||
pub use json::{JsonFormatter, JsonStyle};
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
pub use crate::parquet::ParquetFormatter;
|
||||
@@ -0,0 +1,236 @@
|
||||
//! Apache Parquet output format.
|
||||
|
||||
use arrow::array::{Float32Array, Float64Array, TimestampMicrosecondArray, UInt32Array};
|
||||
use arrow::datatypes::{DataType, Field, Schema, TimeUnit};
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use paracas_aggregate::Ohlcv;
|
||||
use paracas_types::Tick;
|
||||
use parquet::arrow::ArrowWriter;
|
||||
use parquet::basic::Compression;
|
||||
use parquet::file::properties::WriterProperties;
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::{FormatError, Formatter};
|
||||
|
||||
/// Parquet formatter.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ParquetFormatter {
|
||||
/// Row group size (number of rows per group).
|
||||
row_group_size: usize,
|
||||
/// Compression codec.
|
||||
compression: Compression,
|
||||
}
|
||||
|
||||
impl Default for ParquetFormatter {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
row_group_size: 100_000,
|
||||
compression: Compression::SNAPPY,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ParquetFormatter {
|
||||
/// Creates a new Parquet formatter with default settings.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Sets the row group size.
|
||||
#[must_use]
|
||||
pub const fn with_row_group_size(mut self, size: usize) -> Self {
|
||||
self.row_group_size = size;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the compression codec.
|
||||
#[must_use]
|
||||
pub const fn with_compression(mut self, compression: Compression) -> Self {
|
||||
self.compression = compression;
|
||||
self
|
||||
}
|
||||
|
||||
/// Creates the Arrow schema for tick data.
|
||||
fn tick_schema() -> Schema {
|
||||
Schema::new(vec![
|
||||
Field::new(
|
||||
"timestamp",
|
||||
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
|
||||
false,
|
||||
),
|
||||
Field::new("ask", DataType::Float64, false),
|
||||
Field::new("bid", DataType::Float64, false),
|
||||
Field::new("ask_volume", DataType::Float32, false),
|
||||
Field::new("bid_volume", DataType::Float32, false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Creates the Arrow schema for OHLCV data.
|
||||
fn ohlcv_schema() -> Schema {
|
||||
Schema::new(vec![
|
||||
Field::new(
|
||||
"timestamp",
|
||||
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
|
||||
false,
|
||||
),
|
||||
Field::new("open", DataType::Float64, false),
|
||||
Field::new("high", DataType::Float64, false),
|
||||
Field::new("low", DataType::Float64, false),
|
||||
Field::new("close", DataType::Float64, false),
|
||||
Field::new("volume", DataType::Float64, false),
|
||||
Field::new("tick_count", DataType::UInt32, false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Converts ticks to Arrow RecordBatch.
|
||||
fn ticks_to_batch(ticks: &[Tick]) -> Result<RecordBatch, FormatError> {
|
||||
let timestamps: Vec<_> = ticks
|
||||
.iter()
|
||||
.map(|t| t.timestamp.timestamp_micros())
|
||||
.collect();
|
||||
let asks: Vec<_> = ticks.iter().map(|t| t.ask).collect();
|
||||
let bids: Vec<_> = ticks.iter().map(|t| t.bid).collect();
|
||||
let ask_vols: Vec<_> = ticks.iter().map(|t| t.ask_volume).collect();
|
||||
let bid_vols: Vec<_> = ticks.iter().map(|t| t.bid_volume).collect();
|
||||
|
||||
RecordBatch::try_new(
|
||||
Arc::new(Self::tick_schema()),
|
||||
vec![
|
||||
Arc::new(TimestampMicrosecondArray::from(timestamps).with_timezone("UTC")),
|
||||
Arc::new(Float64Array::from(asks)),
|
||||
Arc::new(Float64Array::from(bids)),
|
||||
Arc::new(Float32Array::from(ask_vols)),
|
||||
Arc::new(Float32Array::from(bid_vols)),
|
||||
],
|
||||
)
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))
|
||||
}
|
||||
|
||||
/// Converts OHLCV bars to Arrow RecordBatch.
|
||||
fn ohlcv_to_batch(bars: &[Ohlcv]) -> Result<RecordBatch, FormatError> {
|
||||
let timestamps: Vec<_> = bars
|
||||
.iter()
|
||||
.map(|b| b.timestamp.timestamp_micros())
|
||||
.collect();
|
||||
let opens: Vec<_> = bars.iter().map(|b| b.open).collect();
|
||||
let highs: Vec<_> = bars.iter().map(|b| b.high).collect();
|
||||
let lows: Vec<_> = bars.iter().map(|b| b.low).collect();
|
||||
let closes: Vec<_> = bars.iter().map(|b| b.close).collect();
|
||||
let volumes: Vec<_> = bars.iter().map(|b| b.volume).collect();
|
||||
let tick_counts: Vec<_> = bars.iter().map(|b| b.tick_count).collect();
|
||||
|
||||
RecordBatch::try_new(
|
||||
Arc::new(Self::ohlcv_schema()),
|
||||
vec![
|
||||
Arc::new(TimestampMicrosecondArray::from(timestamps).with_timezone("UTC")),
|
||||
Arc::new(Float64Array::from(opens)),
|
||||
Arc::new(Float64Array::from(highs)),
|
||||
Arc::new(Float64Array::from(lows)),
|
||||
Arc::new(Float64Array::from(closes)),
|
||||
Arc::new(Float64Array::from(volumes)),
|
||||
Arc::new(UInt32Array::from(tick_counts)),
|
||||
],
|
||||
)
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Formatter for ParquetFormatter {
|
||||
fn write_ticks<W: Write + Send>(&self, ticks: &[Tick], writer: W) -> Result<(), FormatError> {
|
||||
let schema = Arc::new(Self::tick_schema());
|
||||
let props = WriterProperties::builder()
|
||||
.set_compression(self.compression)
|
||||
.set_max_row_group_size(self.row_group_size)
|
||||
.build();
|
||||
|
||||
let mut arrow_writer = ArrowWriter::try_new(writer, schema, Some(props))
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))?;
|
||||
|
||||
// Write in batches
|
||||
for chunk in ticks.chunks(self.row_group_size) {
|
||||
let batch = Self::ticks_to_batch(chunk)?;
|
||||
arrow_writer
|
||||
.write(&batch)
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))?;
|
||||
}
|
||||
|
||||
arrow_writer
|
||||
.close()
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_ohlcv<W: Write + Send>(&self, bars: &[Ohlcv], writer: W) -> Result<(), FormatError> {
|
||||
let schema = Arc::new(Self::ohlcv_schema());
|
||||
let props = WriterProperties::builder()
|
||||
.set_compression(self.compression)
|
||||
.set_max_row_group_size(self.row_group_size)
|
||||
.build();
|
||||
|
||||
let mut arrow_writer = ArrowWriter::try_new(writer, schema, Some(props))
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))?;
|
||||
|
||||
// Write in batches
|
||||
for chunk in bars.chunks(self.row_group_size) {
|
||||
let batch = Self::ohlcv_to_batch(chunk)?;
|
||||
arrow_writer
|
||||
.write(&batch)
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))?;
|
||||
}
|
||||
|
||||
arrow_writer
|
||||
.close()
|
||||
.map_err(|e| FormatError::Parquet(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extension(&self) -> &str {
|
||||
"parquet"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::{TimeZone, Utc};
|
||||
use std::io::Cursor;
|
||||
|
||||
fn create_test_tick() -> Tick {
|
||||
let timestamp = Utc.with_ymd_and_hms(2024, 1, 15, 12, 30, 45).unwrap();
|
||||
Tick::new(timestamp, 1.1001, 1.1000, 100.0, 200.0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parquet_ticks() {
|
||||
let formatter = ParquetFormatter::new();
|
||||
let ticks = vec![create_test_tick()];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
|
||||
formatter.write_ticks(&ticks, &mut output).unwrap();
|
||||
|
||||
// Parquet files start with "PAR1" magic bytes
|
||||
let data = output.into_inner();
|
||||
assert!(data.len() > 4);
|
||||
assert_eq!(&data[0..4], b"PAR1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tick_schema() {
|
||||
let schema = ParquetFormatter::tick_schema();
|
||||
assert_eq!(schema.fields().len(), 5);
|
||||
assert!(schema.field_with_name("timestamp").is_ok());
|
||||
assert!(schema.field_with_name("ask").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ohlcv_schema() {
|
||||
let schema = ParquetFormatter::ohlcv_schema();
|
||||
assert_eq!(schema.fields().len(), 7);
|
||||
assert!(schema.field_with_name("open").is_ok());
|
||||
assert!(schema.field_with_name("close").is_ok());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user