diff --git a/Cargo.toml b/Cargo.toml index b53c60e..c4572db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,20 +55,20 @@ paracas-estimate = { path = "crates/paracas-estimate", version = "0.3.0" } paracas-daemon = { path = "crates/paracas-daemon", version = "0.3.0" } # Async runtime -tokio = { version = "1.42", features = ["full"] } futures = "0.3" async-trait = "0.1" +tokio = { version = "1.42", features = ["full"] } # HTTP client -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip", "stream"] } bytes = "1.9" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip", "stream"] } # Compression lzma-rs = "0.3" # Serialization -serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } csv-async = { version = "1.3", features = ["tokio"] } # Arrow/Parquet @@ -79,13 +79,13 @@ parquet = { version = "54", features = ["async", "arrow"] } chrono = { version = "0.4", features = ["serde"] } # CLI -clap = { version = "4.5", features = ["derive"] } -indicatif = "0.17" inquire = "0.7" +indicatif = "0.17" +clap = { version = "4.5", features = ["derive"] } # Error handling -thiserror = "2.0" anyhow = "1.0" +thiserror = "2.0" # Binary parsing byteorder = "1.5" @@ -97,9 +97,9 @@ derive_more = { version = "1.0", default-features = false, features = ["display" approx = "0.5" # Benchmarking -criterion = { version = "0.5", features = ["async_tokio", "html_reports"] } -tempfile = "3.14" which = "7.0" +tempfile = "3.14" +criterion = { version = "0.5", features = ["async_tokio", "html_reports"] } # UUID uuid = { version = "1.0", features = ["v4", "serde"] } diff --git a/README.md b/README.md index 6cfe0c3..128300b 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ High-performance Rust CLI for downloading historical tick data from Dukascopy. - **Flexible**: CSV, JSON, and Parquet output formats - **Complete**: All 1000+ Dukascopy instruments supported - **Aggregation**: Built-in OHLCV aggregation (1m, 5m, 15m, 30m, 1h, 4h, 1d) +- **Background Jobs**: Run long downloads as detached daemon processes ## Installation @@ -38,6 +39,19 @@ paracas download eurusd -s 2024-01-01 -e 2024-01-31 -t h1 # Specify custom output file paracas download eurusd -o my_data.csv + +# Run download in background +paracas download eurusd -s 2024-01-01 -e 2024-12-31 --background +``` + +### Download All Instruments + +```bash +# Download all forex instruments +paracas download-all --category forex -o ./data/ + +# Download all crypto as Parquet in background +paracas download-all --category crypto -f parquet --background ``` ### List Instruments @@ -59,6 +73,22 @@ paracas list --search btc paracas info eurusd ``` +### Background Jobs + +```bash +# Check job status +paracas status --all + +# Watch running jobs +paracas status --follow 5 + +# Manage jobs +paracas job pause +paracas job resume +paracas job kill +paracas job clean +``` + ## Output Formats | Format | Extension | Description | diff --git a/bin/README.md b/bin/README.md index e951840..0e3558d 100644 --- a/bin/README.md +++ b/bin/README.md @@ -20,6 +20,21 @@ paracas download eurusd -s 2024-01-01 -e 2024-01-31 -o data.csv # Download as Parquet with 1-hour aggregation paracas download btcusd -s 2024-01-01 -e 2024-12-31 -o data.parquet -f parquet -t h1 + +# Download in background +paracas download eurusd -s 2024-01-01 -e 2024-12-31 --background +``` + +### Download All + +Download all instruments (or filter by category): + +```bash +# Download all forex instruments +paracas download-all --category forex -o ./data/ + +# Download all crypto as Parquet in background +paracas download-all --category crypto -f parquet --background ``` ### List @@ -45,6 +60,48 @@ Show instrument details: paracas info eurusd ``` +### Status + +Check background job status: + +```bash +# Show all jobs +paracas status --all + +# Show only running jobs +paracas status --running + +# Check specific job +paracas status + +# Watch mode (refresh every 5 seconds) +paracas status --follow 5 + +# Cancel a running job +paracas status --cancel +``` + +### Job + +Manage background jobs: + +```bash +# Pause a running job +paracas job pause + +# Resume a paused job +paracas job resume + +# Kill a running or paused job +paracas job kill + +# Clean up finished jobs +paracas job clean + +# Clean all finished jobs +paracas job clean --all +``` + ## License MIT License - see [LICENSE](../LICENSE) for details. diff --git a/crates/paracas-daemon/Cargo.toml b/crates/paracas-daemon/Cargo.toml index a1a9576..34b36a5 100644 --- a/crates/paracas-daemon/Cargo.toml +++ b/crates/paracas-daemon/Cargo.toml @@ -1,16 +1,24 @@ [package] name = "paracas-daemon" +description = "Background job management for paracas tick data downloader" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true -description = "Background job management for paracas tick data downloader" +documentation.workspace = true +authors.workspace = true +keywords.workspace = true +categories.workspace = true [lints] workspace = true +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + [dependencies] serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/paracas-daemon/README.md b/crates/paracas-daemon/README.md new file mode 100644 index 0000000..8957672 --- /dev/null +++ b/crates/paracas-daemon/README.md @@ -0,0 +1,42 @@ +# paracas-daemon + +Background job management for the paracas tick data downloader. + +## Features + +- Job tracking with unique identifiers +- Persistent state storage +- Detached daemon process spawning +- Thread-safe progress tracking + +## Types + +- `JobId` - Unique identifier for download jobs +- `JobStatus` - Current status of a job (pending, running, completed, failed) +- `InstrumentTask` - Download task for a single instrument +- `DownloadJob` - Complete download job with multiple tasks +- `StateManager` - Persistent state storage and retrieval +- `DaemonSpawner` - Spawns detached daemon processes +- `DaemonProgress` - Thread-safe progress tracking + +## Usage + +```rust,ignore +use paracas_daemon::{StateManager, DownloadJob, JobId, JobStatus}; + +// Create a state manager +let manager = StateManager::new()?; + +// Create a new job +let job = DownloadJob::new(tasks); +manager.save_job(&job)?; + +// Retrieve job status +if let Some(job) = manager.get_job(&job.id)? { + println!("Job status: {:?}", job.status()); +} +``` + +## License + +MIT License - see [LICENSE](../../LICENSE) for details. diff --git a/crates/paracas-daemon/src/lib.rs b/crates/paracas-daemon/src/lib.rs index e239fa6..150d728 100644 --- a/crates/paracas-daemon/src/lib.rs +++ b/crates/paracas-daemon/src/lib.rs @@ -11,6 +11,7 @@ //! - [`DaemonSpawner`] - Spawns detached daemon processes for background downloads //! - [`DaemonProgress`] - Thread-safe progress tracking for daemon jobs +#![doc = include_str!("../README.md")] #![doc(issue_tracker_base_url = "https://github.com/factordynamics/paracas/issues/")] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![warn(missing_docs)] diff --git a/crates/paracas-estimate/Cargo.toml b/crates/paracas-estimate/Cargo.toml index fbd5c79..9690880 100644 --- a/crates/paracas-estimate/Cargo.toml +++ b/crates/paracas-estimate/Cargo.toml @@ -1,16 +1,24 @@ [package] name = "paracas-estimate" +description = "Download size and time estimation for paracas tick data downloader" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true -description = "Download size and time estimation for paracas tick data downloader" +documentation.workspace = true +authors.workspace = true +keywords.workspace = true +categories.workspace = true [lints] workspace = true +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + [dependencies] paracas-types = { workspace = true } serde = { workspace = true } diff --git a/crates/paracas-estimate/README.md b/crates/paracas-estimate/README.md new file mode 100644 index 0000000..c967041 --- /dev/null +++ b/crates/paracas-estimate/README.md @@ -0,0 +1,39 @@ +# paracas-estimate + +Download size and time estimation for the paracas tick data downloader. + +## Features + +- Historical size estimates per instrument category +- Download time estimation based on data volume +- Confidence levels for estimates + +## Types + +- `EstimateDatabase` - Database of historical size estimates per category +- `CategoryEstimate` - Size estimates for a single category +- `Estimator` - Computes download estimates for instruments and date ranges +- `DownloadEstimate` - Estimated download metrics +- `EstimateConfidence` - Confidence level of the estimate + +## Usage + +```rust,ignore +use paracas_estimate::{Estimator, EstimateDatabase}; +use paracas_types::{Category, DateRange}; + +// Create an estimator with the default database +let db = EstimateDatabase::default(); +let estimator = Estimator::new(&db); + +// Estimate download for a date range +let range = DateRange::new(start, end)?; +let estimate = estimator.estimate(Category::Forex, &range); + +println!("Estimated size: {} bytes", estimate.size_bytes); +println!("Confidence: {:?}", estimate.confidence); +``` + +## License + +MIT License - see [LICENSE](../../LICENSE) for details. diff --git a/crates/paracas-estimate/src/lib.rs b/crates/paracas-estimate/src/lib.rs index db6801d..ddc5edc 100644 --- a/crates/paracas-estimate/src/lib.rs +++ b/crates/paracas-estimate/src/lib.rs @@ -9,6 +9,7 @@ //! - [`DownloadEstimate`] - Estimated download metrics //! - [`EstimateConfidence`] - Confidence level of the estimate +#![doc = include_str!("../README.md")] #![doc(issue_tracker_base_url = "https://github.com/factordynamics/paracas/issues/")] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![warn(missing_docs)] diff --git a/crates/paracas-lib/README.md b/crates/paracas-lib/README.md index d0e495f..afed79b 100644 --- a/crates/paracas-lib/README.md +++ b/crates/paracas-lib/README.md @@ -51,6 +51,11 @@ This is a facade crate that re-exports functionality from: - `paracas-aggregate` - OHLCV aggregation - `paracas-format` - Output formatters +Related workspace crates (not re-exported): + +- `paracas-daemon` - Background job management +- `paracas-estimate` - Download size estimation + ## License MIT License - see [LICENSE](../../LICENSE) for details.