diff --git a/.gitignore b/.gitignore index c77fa9e..4433903 100755 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ Cargo.lock # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -.cargo/ \ No newline at end of file +.cargo/ +.claude/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 3147f66..ce72227 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,6 @@ serde-big-array = "0.5.1" futures = "0.3.31" futures-util = "0.3.31" base64 = "0.22.1" -bs58 = "0.5.1" rand = "0.9.0" bincode = "1.3.3" anyhow = "1.0.90" @@ -52,6 +51,7 @@ thiserror = "2.0.11" async-trait = "0.1.86" lazy_static = "1.5.0" once_cell = "1.20.3" +dashmap = "6.0.1" prost = "0.13.5" prost-types = "0.13.5" num_enum = "0.7.3" @@ -64,3 +64,10 @@ borsh-derive = "1.5.5" indicatif = "0.18.0" maplit = "1.0.2" env_logger = "0.11.8" +crossbeam = "0.8.4" +crossbeam-queue = "0.3.12" +parking_lot = "0.12.1" +wide = "0.7" + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } \ No newline at end of file diff --git a/README.md b/README.md index cc73cf0..011c2a2 100755 --- a/README.md +++ b/README.md @@ -24,11 +24,12 @@ A lightweight Rust library for real-time event streaming from Solana DEX trading 11. **Performance Monitoring**: Built-in performance metrics monitoring, including event processing speed, etc. 12. **Memory Optimization**: Object pooling and caching mechanisms to reduce memory allocations 13. **Flexible Configuration System**: Support for custom batch sizes, backpressure strategies, channel sizes, and other parameters -14. **Preset Configurations**: Provides high-performance, low-latency, ordered processing, and other preset configurations -15. **Backpressure Handling**: Supports blocking, dropping, retrying, ordered, and other backpressure strategies +14. **Preset Configurations**: Provides high-throughput and low-latency preset configurations optimized for different use cases +15. **Backpressure Handling**: Supports blocking and dropping backpressure strategies 16. **Runtime Configuration Updates**: Supports dynamic configuration parameter updates at runtime 17. **Full Function Performance Monitoring**: All subscribe_events functions support performance monitoring, automatically collecting and reporting performance metrics 18. **Graceful Shutdown**: Support for programmatic stop() method for clean shutdown +19. **Dynamic Subscription Management**: Runtime filter updates without reconnection, enabling adaptive monitoring strategies ## Installation @@ -55,6 +56,65 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" } solana-streamer-sdk = "0.3.10" ``` +## Configuration System + +### Preset Configurations + +The library provides three preset configurations optimized for different use cases: + +#### 1. High Throughput Configuration (`high_throughput()`) + +Optimized for high-concurrency scenarios, prioritizing throughput over latency: + +```rust +let config = StreamClientConfig::high_throughput(); +// Or use convenience methods +let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?; +let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?; +``` + +**Features:** +- **Backpressure Strategy**: Drop - drops messages during high load to avoid blocking +- **Buffer Size**: 5,000 permits to handle burst traffic +- **Use Case**: Scenarios where you need to process large volumes of data and can tolerate occasional message drops during peak loads + +#### 2. Low Latency Configuration (`low_latency()`) + +Optimized for real-time scenarios, prioritizing latency over throughput: + +```rust +let config = StreamClientConfig::low_latency(); +// Or use convenience methods +let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?; +let shred = ShredStreamGrpc::new_low_latency(endpoint).await?; +``` + +**Features:** +- **Backpressure Strategy**: Block - ensures no data loss +- **Buffer Size**: 4000 permits for balanced throughput and latency +- **Immediate Processing**: No buffering, processes events immediately +- **Use Case**: Scenarios where every millisecond counts and you cannot afford to lose any events, such as trading applications or real-time monitoring + + +### Custom Configuration + +You can also create custom configurations: + +```rust +let config = StreamClientConfig { + connection: ConnectionConfig { + connect_timeout: 30, + request_timeout: 120, + max_decoding_message_size: 20 * 1024 * 1024, // 20MB + }, + backpressure: BackpressureConfig { + permits: 2000, + strategy: BackpressureStrategy::Block, + }, + enable_metrics: true, +}; +``` + ## Usage Examples ### Quick Start - Parse Transaction Events @@ -72,6 +132,20 @@ This example demonstrates: The example uses a predefined transaction signature and shows how to extract protocol-specific events from the transaction data. +### Dynamic Subscription Management Example + +Test runtime filter updates without reconnection: + +```bash +cargo run --example dynamic_subscription +``` + +This example demonstrates: +- Creating initial subscriptions with specific protocol filters +- Updating subscription filters at runtime without reconnection +- Single subscription enforcement and proper error handling +- Clean shutdown and resource management + ### Advanced Usage - Complete Example ```rust @@ -470,6 +544,32 @@ let event_type_filter = Some(EventTypeFilter { }); ``` +## Dynamic Subscription Management + +Update subscription filters at runtime without reconnecting to the stream. + +```rust +// Update filters on existing subscription +grpc.update_subscription( + TransactionFilter { + account_include: vec!["new_program_id".to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, +).await?; +``` + +- **No Reconnection**: Filter changes apply immediately without closing the stream +- **Atomic Updates**: Both transaction and account filters updated together +- **Single Subscription**: One active subscription per client instance +- **Compatible**: Works with both immediate and advanced subscription methods + +Note: Multiple subscription attempts on the same client return an error. + ## Supported Protocols - **PumpFun**: Primary meme coin trading platform diff --git a/README_CN.md b/README_CN.md index 0a63871..e0c94dc 100644 --- a/README_CN.md +++ b/README_CN.md @@ -24,8 +24,8 @@ 11. **性能监控**: 内置性能指标监控,包括事件处理速度等 12. **内存优化**: 对象池和缓存机制减少内存分配 13. **灵活配置系统**: 支持自定义批处理大小、背压策略、通道大小等参数 -14. **预设配置**: 提供高性能、低延迟、有序处理等预设配置 -15. **背压处理**: 支持阻塞、丢弃、重试、有序等多种背压策略 +14. **预设配置**: 提供高吞吐量、低延迟等预设配置,针对不同使用场景优化 +15. **背压处理**: 支持阻塞、丢弃等背压策略 16. **运行时配置更新**: 支持在运行时动态更新配置参数 17. **全函数性能监控**: 所有subscribe_events函数都支持性能监控,自动收集和报告性能指标 18. **优雅关闭**: 支持编程式 stop() 方法进行干净的关闭 @@ -55,6 +55,65 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" } solana-streamer-sdk = "0.3.10" ``` +## 配置系统 + +### 预设配置 + +库提供了三种预设配置,针对不同的使用场景进行了优化: + +#### 1. 高吞吐量配置 (`high_throughput()`) + +专为高并发场景优化,优先考虑吞吐量而非延迟: + +```rust +let config = StreamClientConfig::high_throughput(); +// 或者使用便捷方法 +let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?; +let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?; +``` + +**特性:** +- **背压策略**: Drop(丢弃策略)- 在高负载时丢弃消息以避免阻塞 +- **缓冲区大小**: 5,000 个许可证,处理突发流量 +- **适用场景**: 需要处理大量数据且可以容忍在峰值负载时偶尔丢失消息的场景 + +#### 2. 低延迟配置 (`low_latency()`) + +专为实时场景优化,优先考虑延迟而非吞吐量: + +```rust +let config = StreamClientConfig::low_latency(); +// 或者使用便捷方法 +let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?; +let shred = ShredStreamGrpc::new_low_latency(endpoint).await?; +``` + +**特性:** +- **背压策略**: Block(阻塞策略)- 确保不丢失任何数据 +- **缓冲区大小**: 4000 个许可证,平衡吞吐量和延迟 +- **立即处理**: 不进行缓冲,立即处理事件 +- **适用场景**: 每毫秒都很重要且不能丢失任何事件的场景,如交易应用或实时监控 + + +### 自定义配置 + +您也可以创建自定义配置: + +```rust +let config = StreamClientConfig { + connection: ConnectionConfig { + connect_timeout: 30, + request_timeout: 120, + max_decoding_message_size: 20 * 1024 * 1024, // 20MB + }, + backpressure: BackpressureConfig { + permits: 2000, + strategy: BackpressureStrategy::Block, + }, + enable_metrics: true, +}; +``` + ## 使用示例 ### 快速开始 - 解析交易事件 @@ -502,7 +561,7 @@ let event_type_filter = Some(EventTypeFilter { - **Yellowstone gRPC 客户端**: 针对 Solana 事件流优化 - **ShredStream 客户端**: 替代流实现 -- **异步处理**: 非阻塞事件处理 +- **高性能处理**: 优化的事件处理机制 ## 项目结构 diff --git a/examples/dynamic_subscription.rs b/examples/dynamic_subscription.rs new file mode 100644 index 0000000..f5ffe5c --- /dev/null +++ b/examples/dynamic_subscription.rs @@ -0,0 +1,468 @@ +use anyhow::Result; +use solana_streamer_sdk::streaming::yellowstone_grpc::{AccountFilter, TransactionFilter, YellowstoneGrpc}; +use solana_streamer_sdk::streaming::event_parser::Protocol; +use solana_streamer_sdk::streaming::event_parser::common::filter::EventTypeFilter; +use solana_streamer_sdk::streaming::event_parser::common::types::EventType; +use solana_sdk::signature::{Keypair, Signer}; +use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; +use tokio::time::sleep; + +const PUMPFUN_PROGRAM_ID: &str = "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P"; +const RAYDIUM_CPMM_PROGRAM_ID: &str = "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C"; + +const GRPC_ENDPOINT: &str = "https://solana-yellowstone-grpc.publicnode.com:443"; +const API_KEY: Option<&str> = None; +const MONITORING_DURATION_SECS: u64 = 10; + +/// Demonstrates dynamic subscription updates and filter changes in real-time +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + + println!("Connecting to Yellowstone gRPC at {}", GRPC_ENDPOINT); + let client = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let event_counter = Arc::new(AtomicU64::new(0)); + let counter = event_counter.clone(); + + let callback = move |event: Box| { + let count = counter.fetch_add(1, Ordering::Relaxed); + + let protocol = match event.event_type() { + EventType::PumpFunBuy | EventType::PumpFunSell => "PumpFun", + EventType::RaydiumCpmmSwapBaseInput | EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmm", + _ => "Unknown" + }; + + println!("Event #{}: {:11} - {:.8}...", count + 1, protocol, event.signature()); + }; + + println!("\n=== Phase 1: PumpFun only ==="); + let pumpfun_filter = TransactionFilter { + account_include: vec![PUMPFUN_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }; + + let account_filter = AccountFilter { + account: vec![], + owner: vec![], + }; + let trade_event_filter = EventTypeFilter { + include: vec![ + EventType::PumpFunBuy, + EventType::PumpFunSell, + EventType::RaydiumCpmmSwapBaseInput, + EventType::RaydiumCpmmSwapBaseOutput, + ], + }; + + if let Err(e) = client.subscribe_events_immediate( + vec![Protocol::PumpFun, Protocol::RaydiumCpmm], + None, + pumpfun_filter, + account_filter, + Some(trade_event_filter), + None, + callback, + ).await { + println!("Failed to create subscription: {}", e); + return Ok(()); + } + + println!("Subscribed to PumpFun transactions with trade event filters, monitoring for {}s...", MONITORING_DURATION_SECS); + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + let phase1_count = event_counter.load(Ordering::Relaxed); + println!("Phase 1: {} events", phase1_count); + + println!("\n=== Phase 2: PumpFun + RaydiumCpmm ==="); + let multi_protocol_filter = TransactionFilter { + account_include: vec![ + PUMPFUN_PROGRAM_ID.to_string(), + RAYDIUM_CPMM_PROGRAM_ID.to_string(), + ], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + multi_protocol_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + println!("Updated to PumpFun + RaydiumCpmm transactions, monitoring for {}s...", MONITORING_DURATION_SECS); + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + let phase2_count = event_counter.load(Ordering::Relaxed); + println!("Phase 2: {} events", phase2_count - phase1_count); + + println!("\n=== Phase 3: RaydiumCpmm only ==="); + let raydium_cpmm_filter = TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + raydium_cpmm_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + println!("Updated to RaydiumCpmm transactions only, monitoring for {}s...", MONITORING_DURATION_SECS); + let phase3_count = event_counter.load(Ordering::Relaxed); + println!("Phase 3: {} events", phase3_count - phase2_count); + + println!("\n=== Phase 4: Back to PumpFun only ==="); + let pumpfun_only_filter = TransactionFilter { + account_include: vec![PUMPFUN_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + pumpfun_only_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + println!("Updated to PumpFun transactions only, monitoring for {}s...", MONITORING_DURATION_SECS); + let phase4_count = event_counter.load(Ordering::Relaxed); + println!("Phase 4: {} events", phase4_count - phase3_count); + + println!("\n=== Phase 5: All events ==="); + let empty_filter = TransactionFilter { + account_include: vec![], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + empty_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + println!("Updated to all transactions (no filters), monitoring for {}s...", MONITORING_DURATION_SECS); + let phase5_count = event_counter.load(Ordering::Relaxed); + println!("Phase 5: {} events", phase5_count - phase4_count); + + println!("\n=== Phase 6: Silence ==="); + + let random_keypair_1 = Keypair::new(); + let random_keypair_2 = Keypair::new(); + let random_pubkey_1 = random_keypair_1.pubkey(); + let random_pubkey_2 = random_keypair_2.pubkey(); + + let silence_filter = TransactionFilter { + account_include: vec![], + account_exclude: vec![], + account_required: vec![ + random_pubkey_1.to_string(), + random_pubkey_2.to_string(), + ], + }; + + if let Err(e) = client.update_subscription( + silence_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + println!("Updated to random addresses (expecting silence), monitoring for 3s..."); + let before_silence = event_counter.load(Ordering::Relaxed); + let start_time = Instant::now(); + let last_event_time = Arc::new(Mutex::new(start_time)); + let last_event_time_clone = last_event_time.clone(); + + let mut last_count = before_silence; + for _ in 0..6 { + sleep(Duration::from_millis(500)).await; + let current_count = event_counter.load(Ordering::Relaxed); + if current_count > last_count { + if let Ok(mut time) = last_event_time_clone.lock() { + *time = Instant::now(); + } + last_count = current_count; + } + } + + let final_count = event_counter.load(Ordering::Relaxed); + let events_during_silence = final_count - before_silence; + + if events_during_silence == 0 { + println!("Phase 6: 0 events (immediate filter application)"); + } else if let Ok(last_time) = last_event_time.lock() { + let propagation_time = last_time.duration_since(start_time); + println!("Phase 6: {} events during propagation, filter took {}ms", + events_during_silence, propagation_time.as_millis()); + } + + println!("\n=== Phase 7: Shutdown ==="); + + let shutdown_client = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let shutdown_event_counter = Arc::new(AtomicU64::new(0)); + let shutdown_counter = shutdown_event_counter.clone(); + let shutdown_callback = move |_event: Box| { + shutdown_counter.fetch_add(1, Ordering::Relaxed); + }; + + if let Err(e) = shutdown_client.subscribe_events_immediate( + vec![Protocol::PumpFun, Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + shutdown_callback, + ).await { + println!("Failed to subscribe shutdown client: {}", e); + return Ok(()); + } + + sleep(Duration::from_millis(1000)).await; + let pre_stop_count = shutdown_event_counter.load(Ordering::Relaxed); + println!("Received {} events before stop", pre_stop_count); + + let stop_time = Instant::now(); + shutdown_client.stop().await; + let shutdown_duration = stop_time.elapsed(); + println!("stop() completed in {:.1}ms", shutdown_duration.as_millis()); + + let post_stop_count = shutdown_event_counter.load(Ordering::Relaxed); + let during_stop = post_stop_count - pre_stop_count; + if during_stop > 0 { + println!(" {} events received during stop()", during_stop); + } + + let last_event_time = Arc::new(Mutex::new(stop_time)); + let last_event_time_clone = last_event_time.clone(); + let mut last_count = post_stop_count; + + for _ in 0..20 { + sleep(Duration::from_millis(100)).await; + let current_count = shutdown_event_counter.load(Ordering::Relaxed); + if current_count > last_count { + if let Ok(mut time) = last_event_time_clone.lock() { + *time = Instant::now(); + } + last_count = current_count; + } + } + + let final_count = shutdown_event_counter.load(Ordering::Relaxed); + let after_stop = final_count - post_stop_count; + + if after_stop == 0 { + println!("Phase 7: Clean shutdown - no events after stop()"); + } else if let Ok(last_time) = last_event_time.lock() { + let post_stop_duration = last_time.duration_since(stop_time); + let silence_duration = Instant::now().duration_since(*last_time); + println!("Phase 7: {} events arrived up to {}ms after stop(), then silent for {}ms", + after_stop, post_stop_duration.as_millis(), silence_duration.as_millis()); + } + + println!("\n=== Subscription enforcement ==="); + + let test_callback = |_event: Box| {}; + + match client.subscribe_events_immediate( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + test_callback, + ).await { + Ok(_) => println!("ERROR: Same client created second subscription"), + Err(e) if e.to_string().contains("Already subscribed") => { + println!("✓ Single subscription enforcement working"); + }, + Err(e) => println!("Unexpected error: {}", e), + } + + let client2 = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let client2_counter = Arc::new(AtomicU64::new(0)); + let counter2 = client2_counter.clone(); + let client2_callback = move |_event: Box| { + counter2.fetch_add(1, Ordering::Relaxed); + }; + + match client2.subscribe_events_immediate( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + client2_callback, + ).await { + Ok(_) => { + sleep(Duration::from_millis(500)).await; + let count = client2_counter.load(Ordering::Relaxed); + println!("✓ Second client: {} events", count); + client2.stop().await; + }, + Err(e) => println!("ERROR: Second client failed: {}", e), + } + + println!("\n=== Advanced subscription enforcement ==="); + + let test_callback_advanced = |_event: Box| {}; + + let client3 = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + // First subscription should succeed + match client3.subscribe_events_immediate( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + test_callback_advanced, + ).await { + Ok(_) => { + // Second subscription attempt on same client should fail + match client3.subscribe_events_immediate( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + |_| {}, + ).await { + Ok(_) => println!("ERROR: Same client created second advanced subscription"), + Err(e) if e.to_string().contains("Already subscribed") => { + println!("✓ Advanced single subscription enforcement working"); + }, + Err(e) => println!("Unexpected error: {}", e), + } + }, + Err(e) => println!("ERROR: First advanced subscription failed: {}", e), + } + + // Test that a second client can subscribe using advanced method + let client4 = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let client4_counter = Arc::new(AtomicU64::new(0)); + let counter4 = client4_counter.clone(); + let client4_callback = move |_event: Box| { + counter4.fetch_add(1, Ordering::Relaxed); + }; + + match client4.subscribe_events_immediate( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + client4_callback, + ).await { + Ok(_) => { + sleep(Duration::from_millis(500)).await; + let count = client4_counter.load(Ordering::Relaxed); + println!("✓ Second client (advanced): {} events", count); + client4.stop().await; + }, + Err(e) => println!("ERROR: Second client (advanced) failed: {}", e), + } + + client3.stop().await; + + client.stop().await; + + Ok(()) +} diff --git a/examples/parse_tx_events.rs b/examples/parse_tx_events.rs index cbb5bb6..5f54b8d 100644 --- a/examples/parse_tx_events.rs +++ b/examples/parse_tx_events.rs @@ -1,5 +1,7 @@ use anyhow::Result; use solana_sdk::commitment_config::CommitmentConfig; + +use solana_streamer_sdk::streaming::event_parser::UnifiedEvent; use solana_streamer_sdk::streaming::event_parser::{ protocols::MutilEventParser, EventParser, Protocol, }; @@ -10,7 +12,7 @@ use std::sync::Arc; #[tokio::main] async fn main() -> Result<()> { let signatures = vec![ - "42agNk1heHabNAVRzEKqQEt5adGkQzRYf9M1Q81uBJPCCHyP4cyCA1RNkgxXrtEAWeeGcytyh2TsnkBDgqnHeq4z", + "5sDWrTTkE69CNc6nrAX7SqPS7FiajJTg8TMog3Gve7KjVfrqYn8YZcX1kAoyKok976S4RTnK1EdCV8hRiDWg68Aj", ]; // Validate signature format let mut valid_signatures = Vec::new(); @@ -52,7 +54,7 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> { .get_transaction_with_config( &signature, solana_client::rpc_config::RpcTransactionConfig { - encoding: Some(UiTransactionEncoding::Binary), + encoding: Some(UiTransactionEncoding::Base64), commitment: Some(CommitmentConfig::confirmed()), max_supported_transaction_version: Some(0), }, @@ -98,30 +100,23 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> { Protocol::RaydiumAmmV4, ]; let parser: Arc = Arc::new(MutilEventParser::new(protocols, None)); - let start_time = std::time::Instant::now(); - let events = parser - .parse_transaction( - transaction.transaction.clone(), - &signature.to_string(), - Some(transaction.slot), - None, - 0, - None, + parser + .parse_encoded_confirmed_transaction_with_status_meta( + signature, + transaction, + Arc::new(move |event: &Box| { + println!("{:?}\n", event); + }), ) - .await - .unwrap_or_else(|_e| vec![]); - - let end_time = std::time::Instant::now(); - let duration = end_time.duration_since(start_time); - println!("Parsing time: {:?}", duration); - for event in events { - println!("{:?}\n", event); - } + .await?; } Err(e) => { println!("Failed to get transaction: {}", e); } } + println!("Press Ctrl+C to exit example..."); + tokio::signal::ctrl_c().await?; + Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 8416f5b..2e79b9d 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,3 @@ -pub mod streaming; +pub mod common; pub mod protos; -pub mod common; \ No newline at end of file +pub mod streaming; diff --git a/src/main.rs b/src/main.rs index dd110cd..d6ad5c3 100755 --- a/src/main.rs +++ b/src/main.rs @@ -61,7 +61,7 @@ async fn test_grpc() -> Result<(), Box> { println!("Subscribing to Yellowstone gRPC events..."); // Create low-latency configuration - let mut config = ClientConfig::low_latency(); + let mut config: ClientConfig = ClientConfig::low_latency(); // Enable performance monitoring, has performance overhead, disabled by default config.enable_metrics = true; let grpc = YellowstoneGrpc::new_with_config( @@ -112,7 +112,7 @@ async fn test_grpc() -> Result<(), Box> { // No event filtering, includes all events let event_type_filter = None; // Only include PumpSwapBuy events and PumpSwapSell events - // let event_type_filter = EventTypeFilter { include: vec![EventType::PumpSwapBuy, EventType::PumpSwapSell] }; + // let event_type_filter = Some(EventTypeFilter { include: vec![EventType::PumpFunBuy] }); println!("Starting to listen for events, press Ctrl+C to stop..."); println!("Monitoring programs: {:?}", account_include); @@ -188,11 +188,15 @@ async fn test_shreds() -> Result<(), Box> { fn create_event_callback() -> impl Fn(Box) { |event: Box| { - println!("🎉 Event received! Type: {:?}, ID: {}", event.event_type(), event.id()); + println!( + "🎉 Event received! Type: {:?}, transaction_index: {:?}", + event.event_type(), + event.transaction_index() + ); match_event!(event, { // -------------------------- block meta ----------------------- BlockMetaEvent => |e: BlockMetaEvent| { - println!("BlockMetaEvent: {e:?}"); + println!("BlockMetaEvent: {:?}", e.metadata.program_handle_time_consuming_us); }, // -------------------------- bonk ----------------------- BonkPoolCreateEvent => |e: BonkPoolCreateEvent| { diff --git a/src/streaming/common/batch.rs b/src/streaming/common/batch.rs deleted file mode 100644 index f9c092c..0000000 --- a/src/streaming/common/batch.rs +++ /dev/null @@ -1,131 +0,0 @@ -use crate::streaming::event_parser::UnifiedEvent; - -/// 通用批处理事件收集器 -pub struct EventBatchProcessor -where - F: FnMut(Vec>) + Send + Sync + 'static, -{ - pub(crate) callback: F, - batch: Vec>, - batch_size: usize, - timeout_ms: u64, - last_flush_time: std::time::Instant, -} - -impl EventBatchProcessor -where - F: FnMut(Vec>) + Send + Sync + 'static, -{ - /// 创建新的批处理器 - pub fn new(callback: F, batch_size: usize, timeout_ms: u64) -> Self { - Self { - callback, - batch: Vec::with_capacity(batch_size), - batch_size, - timeout_ms, - last_flush_time: std::time::Instant::now(), - } - } - - /// 添加事件到批次 - pub fn add_event(&mut self, event: Box) { - log::debug!("Adding event to batch: {} (type: {:?})", event.id(), event.event_type()); - self.batch.push(event); - - // 检查是否需要刷新批次 - if self.batch.len() >= self.batch_size || self.should_flush_by_timeout() { - log::debug!("Flushing batch: size={}, timeout={}", self.batch.len(), self.should_flush_by_timeout()); - self.flush(); - } - } - - /// 强制刷新当前批次 - pub fn flush(&mut self) { - if !self.batch.is_empty() { - let events = std::mem::replace(&mut self.batch, Vec::with_capacity(self.batch_size)); - log::debug!("Flushing {} events from batch processor", events.len()); - - // 添加调试信息(仅在debug模式下) - if log::log_enabled!(log::Level::Debug) { - for (i, event) in events.iter().enumerate() { - log::debug!("Event {}: Type={:?}, ID={}", i, event.event_type(), event.id()); - } - } - - // 执行回调并捕获可能的错误 - log::debug!("Executing batch callback with {} events", events.len()); - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - (self.callback)(events); - })) { - Ok(_) => { - log::debug!("Batch callback executed successfully"); - } - Err(e) => { - log::error!("Batch callback panicked: {:?}", e); - } - } - - self.last_flush_time = std::time::Instant::now(); - } else { - log::debug!("No events to flush"); - } - } - - /// 获取当前批次大小 - pub fn current_batch_size(&self) -> usize { - self.batch.len() - } - - /// 检查是否应该基于超时刷新 - fn should_flush_by_timeout(&self) -> bool { - self.last_flush_time.elapsed().as_millis() >= self.timeout_ms as u128 - } - - /// 检查批次是否已满 - pub fn is_batch_full(&self) -> bool { - self.batch.len() >= self.batch_size - } - - /// 检查是否需要刷新(大小或超时) - pub fn should_flush(&self) -> bool { - self.is_batch_full() || self.should_flush_by_timeout() - } -} - -/// 简单的事件批处理器,用于将单个事件回调转换为批量回调 -pub struct SimpleEventBatchProcessor -where - F: Fn(Box) + Send + Sync + 'static, -{ - callback: F, -} - -impl SimpleEventBatchProcessor -where - F: Fn(Box) + Send + Sync + 'static, -{ - pub fn new(callback: F) -> Self { - Self { callback } - } - - /// 将批量事件拆分为单个事件处理 - pub fn process_batch(&self, events: Vec>) { - for event in events { - (self.callback)(event); - } - } -} - -/// 批处理器包装器,用于将单个事件回调适配为批量处理 -pub fn create_batch_callback_adapter( - single_event_callback: F, -) -> impl FnMut(Vec>) + Send + Sync + 'static -where - F: Fn(Box) + Send + Sync + 'static, -{ - move |events: Vec>| { - for event in events { - single_event_callback(event); - } - } -} diff --git a/src/streaming/common/config.rs b/src/streaming/common/config.rs index c050016..ce21ec8 100644 --- a/src/streaming/common/config.rs +++ b/src/streaming/common/config.rs @@ -1,14 +1,12 @@ use super::constants::*; -/// 背压处理策略 +/// Backpressure handling strategy #[derive(Debug, Clone, Copy)] pub enum BackpressureStrategy { - /// 阻塞等待(默认) + /// Block and wait (default) Block, - /// 丢弃消息 + /// Drop messages Drop, - /// 重试有限次数后丢弃 - Retry { max_attempts: usize, wait_ms: u64 }, } impl Default for BackpressureStrategy { @@ -17,50 +15,29 @@ impl Default for BackpressureStrategy { } } -/// 批处理配置 -#[derive(Debug, Clone)] -pub struct BatchConfig { - /// 批处理大小(默认:100) - pub batch_size: usize, - /// 批处理超时时间(毫秒,默认:5ms) - pub batch_timeout_ms: u64, - /// 是否启用批处理(默认:true) - pub enabled: bool, -} - -impl Default for BatchConfig { - fn default() -> Self { - Self { - batch_size: DEFAULT_BATCH_SIZE, - batch_timeout_ms: DEFAULT_BATCH_TIMEOUT_MS, - enabled: true, - } - } -} - -/// 背压配置 +/// Backpressure configuration #[derive(Debug, Clone)] pub struct BackpressureConfig { - /// 通道大小(默认:1000) - pub channel_size: usize, - /// 背压处理策略(默认:Block) + /// Channel size (default: 1000) + pub permits: usize, + /// Backpressure handling strategy (default: Block) pub strategy: BackpressureStrategy, } impl Default for BackpressureConfig { fn default() -> Self { - Self { channel_size: DEFAULT_CHANNEL_SIZE, strategy: BackpressureStrategy::default() } + Self { permits: 1, strategy: BackpressureStrategy::default() } } } -/// 连接配置 +/// Connection configuration #[derive(Debug, Clone)] pub struct ConnectionConfig { - /// 连接超时时间(秒,默认:10) + /// Connection timeout in seconds (default: 10) pub connect_timeout: u64, - /// 请求超时时间(秒,默认:60) + /// Request timeout in seconds (default: 60) pub request_timeout: u64, - /// 最大解码消息大小(字节,默认:10MB) + /// Maximum decoding message size in bytes (default: 10MB) pub max_decoding_message_size: usize, } @@ -74,16 +51,14 @@ impl Default for ConnectionConfig { } } -/// 通用客户端配置 +/// Common client configuration #[derive(Debug, Clone)] pub struct StreamClientConfig { - /// 连接配置 + /// Connection configuration pub connection: ConnectionConfig, - /// 批处理配置 - pub batch: BatchConfig, - /// 背压配置 + /// Backpressure configuration pub backpressure: BackpressureConfig, - /// 是否启用性能监控(默认:false) + /// Whether performance monitoring is enabled (default: false) pub enable_metrics: bool, } @@ -91,7 +66,6 @@ impl Default for StreamClientConfig { fn default() -> Self { Self { connection: ConnectionConfig::default(), - batch: BatchConfig::default(), backpressure: BackpressureConfig::default(), enable_metrics: false, } @@ -99,33 +73,40 @@ impl Default for StreamClientConfig { } impl StreamClientConfig { - /// 创建高性能配置(适合高并发场景) - pub fn high_performance() -> Self { + /// Creates a high-throughput configuration optimized for high-concurrency scenarios. + /// + /// This configuration prioritizes throughput over latency by: + /// - Implementing a drop strategy for backpressure to avoid blocking + /// - Setting a large permit buffer (5,000) to handle burst traffic + /// + /// Ideal for scenarios where you need to process large volumes of data + /// and can tolerate occasional message drops during peak loads. + pub fn high_throughput() -> Self { Self { connection: ConnectionConfig::default(), - batch: BatchConfig { batch_size: 200, batch_timeout_ms: 5, enabled: true }, backpressure: BackpressureConfig { - channel_size: 20000, + permits: 20000, strategy: BackpressureStrategy::Drop, }, enable_metrics: false, } } - /// 创建低延迟配置(适合实时场景) + /// Creates a low-latency configuration optimized for real-time scenarios. + /// + /// This configuration prioritizes latency over throughput by: + /// - Processing events immediately without buffering + /// - Implementing a blocking backpressure strategy to ensure no data loss + /// - Setting optimal permits (4000) for balanced throughput and latency + /// + /// Ideal for scenarios where every millisecond counts and you cannot + /// afford to lose any events, such as trading applications or real-time monitoring. pub fn low_latency() -> Self { Self { connection: ConnectionConfig::default(), - batch: BatchConfig { - batch_size: 10, - batch_timeout_ms: 1, - enabled: false, // 禁用批处理,即时处理 - }, - backpressure: BackpressureConfig { - channel_size: 1000, - strategy: BackpressureStrategy::Block, - }, + backpressure: BackpressureConfig { permits: 4000, strategy: BackpressureStrategy::Block }, enable_metrics: false, } } + } diff --git a/src/streaming/common/constants.rs b/src/streaming/common/constants.rs index d4b3d18..e54309c 100644 --- a/src/streaming/common/constants.rs +++ b/src/streaming/common/constants.rs @@ -5,10 +5,8 @@ pub const DEFAULT_CONNECT_TIMEOUT: u64 = 10; pub const DEFAULT_REQUEST_TIMEOUT: u64 = 60; pub const DEFAULT_CHANNEL_SIZE: usize = 1000; pub const DEFAULT_MAX_DECODING_MESSAGE_SIZE: usize = 1024 * 1024 * 10; -pub const DEFAULT_BATCH_SIZE: usize = 100; -pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5; // 性能监控相关常量 pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5; pub const DEFAULT_METRICS_PRINT_INTERVAL_SECONDS: u64 = 10; -pub const SLOW_PROCESSING_THRESHOLD_MS: f64 = 10.0; +pub const SLOW_PROCESSING_THRESHOLD_US: f64 = 3000.0; diff --git a/src/streaming/common/event_processor.rs b/src/streaming/common/event_processor.rs new file mode 100644 index 0000000..7ae82f7 --- /dev/null +++ b/src/streaming/common/event_processor.rs @@ -0,0 +1,434 @@ +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use crossbeam_queue::SegQueue; +use solana_sdk::pubkey::Pubkey; + +use crate::common::AnyResult; +use crate::streaming::common::BackpressureStrategy; +use crate::streaming::common::{ + MetricsEventType, MetricsManager, StreamClientConfig as ClientConfig, +}; +use crate::streaming::event_parser::common::filter::EventTypeFilter; +use crate::streaming::event_parser::core::account_event_parser::AccountEventParser; +use crate::streaming::event_parser::core::common_event_parser::CommonEventParser; +use crate::streaming::event_parser::core::traits::get_high_perf_clock; +use crate::streaming::event_parser::EventParser; +use crate::streaming::event_parser::{ + core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol, +}; +use crate::streaming::grpc::{BackpressureConfig, EventPretty}; +use crate::streaming::shred::TransactionWithSlot; +use once_cell::sync::OnceCell; + +/// High-performance Event processor using SegQueue for all strategies +pub struct EventProcessor { + pub(crate) metrics_manager: MetricsManager, + pub(crate) config: ClientConfig, + pub(crate) parser_cache: OnceCell>, + pub(crate) protocols: Vec, + pub(crate) event_type_filter: Option, + pub(crate) callback: Option) + Send + Sync>>, + pub(crate) backpressure_config: BackpressureConfig, + /// High-performance lockfree queue for gRPC events + pub(crate) grpc_queue: Arc)>>, + /// High-performance lockfree queue for shred events + pub(crate) shred_queue: Arc)>>, + /// Fast O(1) counter for Drop strategy (avoids expensive SegQueue::len()) + pub(crate) grpc_pending_count: Arc, + pub(crate) shred_pending_count: Arc, + /// Processing thread control + pub(crate) processing_shutdown: Arc, +} + +impl EventProcessor { + /// Create a new high-performance event processor + pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self { + let backpressure_config = config.backpressure.clone(); + let grpc_queue = Arc::new(SegQueue::new()); + let shred_queue = Arc::new(SegQueue::new()); + let grpc_pending_count = Arc::new(AtomicUsize::new(0)); + let shred_pending_count = Arc::new(AtomicUsize::new(0)); + let processing_shutdown = Arc::new(AtomicBool::new(false)); + + Self { + metrics_manager, + config, + parser_cache: OnceCell::new(), + protocols: vec![], + event_type_filter: None, + backpressure_config, + callback: None, + grpc_queue, + shred_queue, + grpc_pending_count, + shred_pending_count, + processing_shutdown, + } + } + + pub fn set_protocols_and_event_type_filter( + &mut self, + protocols: Vec, + event_type_filter: Option, + backpressure_config: BackpressureConfig, + callback: Option) + Send + Sync>>, + ) { + self.protocols = protocols; + self.event_type_filter = event_type_filter; + + // Check if Block processing thread should be started (before moving backpressure_config) + let should_start_block_processing = true; + // matches!(backpressure_config.strategy, BackpressureStrategy::Block); + + self.backpressure_config = backpressure_config; + self.callback = callback; + // Use stored values to initialize parser_cache + let protocols_ref = &self.protocols; + let event_type_filter_ref = self.event_type_filter.as_ref(); + self.parser_cache.get_or_init(|| { + Arc::new(MutilEventParser::new(protocols_ref.clone(), event_type_filter_ref.cloned())) + }); + + // Start Block processing thread if using Block strategy + if should_start_block_processing { + self.start_block_processing_thread(); + } + } + + pub fn get_parser(&self) -> Arc { + self.parser_cache.get().unwrap().clone() + } + + /// Create adapter callback + fn create_adapter_callback(&self) -> Arc) + Send + Sync> { + let callback = self.callback.clone().unwrap(); + let metrics_manager = self.metrics_manager.clone(); + + Arc::new(move |event: Box| { + let processing_time_us = event.program_handle_time_consuming_us() as f64; + callback(event); + metrics_manager.update_metrics(MetricsEventType::Transaction, 1, processing_time_us); + }) + } + + pub async fn process_grpc_event_transaction_with_metrics( + &self, + event_pretty: EventPretty, + bot_wallet: Option, + ) -> AnyResult<()> { + self.apply_backpressure_control(event_pretty, bot_wallet).await + } + + /// Apply backpressure control strategy + async fn apply_backpressure_control( + &self, + event_pretty: EventPretty, + bot_wallet: Option, + ) -> AnyResult<()> { + match self.backpressure_config.strategy { + BackpressureStrategy::Block => { + // Block strategy: async wait if queue is full (backpressure control) + loop { + let current_pending = self.grpc_pending_count.load(Ordering::Relaxed); + if current_pending < self.backpressure_config.permits { + self.grpc_queue.push((event_pretty, bot_wallet)); + self.grpc_pending_count.fetch_add(1, Ordering::Relaxed); + break; + } + // Async yield to avoid blocking gRPC data source + tokio::task::yield_now().await; + } + Ok(()) + } + BackpressureStrategy::Drop => { + // Drop strategy: Use O(1) atomic counter instead of expensive O(n) len() + // If pending count >= permits, DROP the event immediately + let current_pending = self.grpc_pending_count.load(Ordering::Relaxed); + if current_pending >= self.backpressure_config.permits { + self.metrics_manager.increment_dropped_events(); + Ok(()) + } else { + self.grpc_pending_count.fetch_add(1, Ordering::Relaxed); + let processor = self.clone(); + tokio::spawn(async move { + match processor + .process_grpc_event_transaction(event_pretty, bot_wallet) + .await + { + Ok(_) => { + processor.grpc_pending_count.fetch_sub(1, Ordering::Relaxed); + } + Err(e) => { + log::error!("Error in async gRPC processing: {}", e); + } + } + }); + Ok(()) + } + } + } + } + + async fn process_grpc_event_transaction( + &self, + event_pretty: EventPretty, + bot_wallet: Option, + ) -> AnyResult<()> { + if self.callback.is_none() { + return Ok(()); + } + match event_pretty { + EventPretty::Account(account_pretty) => { + self.metrics_manager.add_account_process_count(); + let account_event = AccountEventParser::parse_account_event( + &self.protocols, + account_pretty, + self.event_type_filter.as_ref(), + ); + if let Some(event) = account_event { + let processing_time_us = event.program_handle_time_consuming_us() as f64; + self.invoke_callback(event); + self.update_metrics(MetricsEventType::Account, 1, processing_time_us); + } + } + EventPretty::Transaction(transaction_pretty) => { + self.metrics_manager.add_tx_process_count(); + let slot = transaction_pretty.slot; + let signature = transaction_pretty.signature; + let tx = transaction_pretty.tx; + let block_time = transaction_pretty.block_time; + let program_received_time_us = transaction_pretty.program_received_time_us; + let transaction_index = transaction_pretty.transaction_index; + // Use cache to get parser + let parser = self.get_parser(); + let adapter_callback = self.create_adapter_callback(); + parser + .parse_transaction_owned( + tx, + signature, + Some(slot), + block_time, + program_received_time_us, + bot_wallet, + transaction_index, + adapter_callback, + ) + .await?; + } + EventPretty::BlockMeta(block_meta_pretty) => { + self.metrics_manager.add_block_meta_process_count(); + let block_time_ms = block_meta_pretty + .block_time + .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000) + .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()); + let block_meta_event = CommonEventParser::generate_block_meta_event( + block_meta_pretty.slot, + &block_meta_pretty.block_hash, + block_time_ms, + block_meta_pretty.program_received_time_us, + ); + let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64; + self.invoke_callback(block_meta_event); + self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us); + } + } + + Ok(()) + } + + pub fn invoke_callback(&self, event: Box) { + if let Some(callback) = self.callback.as_ref() { + callback(event); + } + } + + /// Process a single transaction immediately + pub async fn process_shred_transaction_immediate( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) -> AnyResult<()> { + self.process_shred_transaction(transaction_with_slot, bot_wallet).await + } + + /// Process shred transaction with backpressure control and performance monitoring + pub async fn process_shred_transaction_with_metrics( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) -> AnyResult<()> { + // Backpressure control logic + self.apply_shred_backpressure_control(transaction_with_slot, bot_wallet).await + } + + /// Apply shred backpressure control strategy + async fn apply_shred_backpressure_control( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) -> AnyResult<()> { + match self.backpressure_config.strategy { + BackpressureStrategy::Block => { + // Block strategy: async wait if queue is full (backpressure control) + loop { + let current_pending = self.shred_pending_count.load(Ordering::Relaxed); + if current_pending < self.backpressure_config.permits { + self.shred_queue.push((transaction_with_slot, bot_wallet)); + self.shred_pending_count.fetch_add(1, Ordering::Relaxed); + break; + } + // Async yield to avoid blocking shred data source + tokio::task::yield_now().await; + } + Ok(()) + } + BackpressureStrategy::Drop => { + // Drop strategy: Use O(1) atomic counter instead of expensive O(n) len() + let current_pending = self.shred_pending_count.load(Ordering::Relaxed); + if current_pending >= self.backpressure_config.permits { + self.metrics_manager.increment_dropped_events(); + Ok(()) + } else { + self.shred_pending_count.fetch_add(1, Ordering::Relaxed); + let processor = self.clone(); + tokio::spawn(async move { + match processor + .process_shred_transaction(transaction_with_slot, bot_wallet) + .await + { + Ok(_) => { + processor.shred_pending_count.fetch_sub(1, Ordering::Relaxed); + } + Err(e) => { + log::error!("Error in async shred processing: {}", e); + } + } + }); + Ok(()) + } + } + } + } + + pub async fn process_shred_transaction( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) -> AnyResult<()> { + if self.callback.is_none() { + return Ok(()); + } + self.metrics_manager.add_tx_process_count(); + let tx = transaction_with_slot.transaction; + + let slot = transaction_with_slot.slot; + let signature = tx.signatures[0]; + let program_received_time_us = transaction_with_slot.program_received_time_us; + // Use cache to get parser + let parser = self.get_parser(); + let adapter_callback = self.create_adapter_callback(); + parser + .parse_versioned_transaction_owned( + tx, + signature, + Some(slot), + None, + program_received_time_us, + bot_wallet, + None, + &[], + adapter_callback, + ) + .await?; + + Ok(()) + } + + fn update_metrics(&self, ty: MetricsEventType, count: u64, time_us: f64) { + self.metrics_manager.update_metrics(ty, count, time_us); + } + + /// Start dedicated processing threads for all strategies + fn start_block_processing_thread(&self) { + // Reset shutdown flag + self.processing_shutdown.store(false, Ordering::Relaxed); + + let grpc_queue = Arc::clone(&self.grpc_queue); + let shred_queue = Arc::clone(&self.shred_queue); + let grpc_pending_count = Arc::clone(&self.grpc_pending_count); + let shred_pending_count = Arc::clone(&self.shred_pending_count); + let shutdown_flag = Arc::clone(&self.processing_shutdown); + let shutdown_flag_clone = Arc::clone(&self.processing_shutdown); + let processor = self.clone(); + let processor_clone = self.clone(); + // 1. 专用线程 + 2. Busy-wait + 4. 无锁处理 + std::thread::spawn(move || { + // 创建blocking runtime for async processing + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + while !shutdown_flag.load(Ordering::Relaxed) { + if let Some((event_pretty, bot_wallet)) = grpc_queue.pop() { + // Decrement pending counter when consuming from queue + grpc_pending_count.fetch_sub(1, Ordering::Relaxed); + // Process event in blocking runtime + if let Err(e) = rt.block_on( + processor.process_grpc_event_transaction(event_pretty, bot_wallet), + ) { + println!("Error processing gRPC event: {}", e); + } + } else { + // 2. 优化忙等待: 使用轻量级休眠减少CPU占用 + std::thread::yield_now(); + } + } + }); + + // Shred处理也使用相同的低延迟优化 + std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + + while !shutdown_flag_clone.load(Ordering::Relaxed) { + if let Some((transaction_with_slot, bot_wallet)) = shred_queue.pop() { + // Decrement pending counter when consuming from queue + shred_pending_count.fetch_sub(1, Ordering::Relaxed); + // Process transaction in blocking runtime + if let Err(e) = rt.block_on( + processor_clone + .process_shred_transaction(transaction_with_slot, bot_wallet), + ) { + log::error!("Error processing shred transaction: {}", e); + } + } else { + // 优化忙等待: 使用轻量级休眠减少CPU占用 + std::thread::yield_now(); + } + } + }); + } + + /// Stop processing threads + pub fn stop_processing(&self) { + self.processing_shutdown.store(true, Ordering::Relaxed); + } +} + +// Implement Clone trait to support sharing between modules +impl Clone for EventProcessor { + fn clone(&self) -> Self { + Self { + metrics_manager: self.metrics_manager.clone(), + config: self.config.clone(), + parser_cache: self.parser_cache.clone(), + protocols: self.protocols.clone(), + event_type_filter: self.event_type_filter.clone(), + backpressure_config: self.backpressure_config.clone(), + callback: self.callback.clone(), + grpc_queue: self.grpc_queue.clone(), + shred_queue: self.shred_queue.clone(), + grpc_pending_count: self.grpc_pending_count.clone(), + shred_pending_count: self.shred_pending_count.clone(), + processing_shutdown: self.processing_shutdown.clone(), + } + } +} diff --git a/src/streaming/common/metrics.rs b/src/streaming/common/metrics.rs index 2ccdebe..402f909 100644 --- a/src/streaming/common/metrics.rs +++ b/src/streaming/common/metrics.rs @@ -1,305 +1,656 @@ +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; -use tokio::sync::Mutex; -use super::config::StreamClientConfig; use super::constants::*; -/// 单个事件类型的指标 +/// Event type enumeration +#[derive(Debug, Clone, Copy)] +pub enum EventType { + Transaction = 0, + Account = 1, + BlockMeta = 2, +} + +/// Compatibility alias +pub type MetricsEventType = EventType; + +impl EventType { + #[inline] + const fn as_index(self) -> usize { + self as usize + } + + const fn name(self) -> &'static str { + match self { + EventType::Transaction => "TX", + EventType::Account => "Account", + EventType::BlockMeta => "Block Meta", + } + } + + // Compatibility constants + pub const TX: EventType = EventType::Transaction; +} + +/// High-performance atomic event metrics +#[derive(Debug)] +struct AtomicEventMetrics { + process_count: AtomicU64, + events_processed: AtomicU64, + events_in_window: AtomicU64, + window_start_nanos: AtomicU64, + events_per_second_bits: AtomicU64, // Bit representation of f64 +} + +impl AtomicEventMetrics { + fn new(now_nanos: u64) -> Self { + Self { + process_count: AtomicU64::new(0), + events_processed: AtomicU64::new(0), + events_in_window: AtomicU64::new(0), + window_start_nanos: AtomicU64::new(now_nanos), + events_per_second_bits: AtomicU64::new(0), + } + } + + /// Atomically increment process count + #[inline] + fn add_process_count(&self) { + self.process_count.fetch_add(1, Ordering::Relaxed); + } + + /// Atomically increment event processing count + #[inline] + fn add_events_processed(&self, count: u64) { + self.events_processed.fetch_add(count, Ordering::Relaxed); + self.events_in_window.fetch_add(count, Ordering::Relaxed); + } + + /// Get current count (non-blocking) + #[inline] + fn get_counts(&self) -> (u64, u64, u64) { + ( + self.process_count.load(Ordering::Relaxed), + self.events_processed.load(Ordering::Relaxed), + self.events_in_window.load(Ordering::Relaxed), + ) + } + + /// Atomically update events per second + #[inline] + fn update_events_per_second(&self, eps: f64) { + self.events_per_second_bits.store(eps.to_bits(), Ordering::Relaxed); + } + + /// Get events per second + #[inline] + fn get_events_per_second(&self) -> f64 { + f64::from_bits(self.events_per_second_bits.load(Ordering::Relaxed)) + } + + /// Reset window count + #[inline] + fn reset_window(&self, new_start_nanos: u64) { + self.events_in_window.store(0, Ordering::Relaxed); + self.window_start_nanos.store(new_start_nanos, Ordering::Relaxed); + } + + #[inline] + fn get_window_start(&self) -> u64 { + self.window_start_nanos.load(Ordering::Relaxed) + } +} + +/// High-performance atomic processing time statistics +#[derive(Debug)] +struct AtomicProcessingTimeStats { + min_time_bits: AtomicU64, + max_time_bits: AtomicU64, + min_time_timestamp_nanos: AtomicU64, // Timestamp of min value update (nanoseconds) + max_time_timestamp_nanos: AtomicU64, // Timestamp of max value update (nanoseconds) + total_time_us: AtomicU64, // Store integer part of microseconds + total_events: AtomicU64, +} + +impl AtomicProcessingTimeStats { + fn new() -> Self { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + + Self { + min_time_bits: AtomicU64::new(f64::INFINITY.to_bits()), + max_time_bits: AtomicU64::new(0), + min_time_timestamp_nanos: AtomicU64::new(now_nanos), + max_time_timestamp_nanos: AtomicU64::new(now_nanos), + total_time_us: AtomicU64::new(0), + total_events: AtomicU64::new(0), + } + } + + /// Atomically update processing time statistics + #[inline] + fn update(&self, time_us: f64, event_count: u64) { + let time_bits = time_us.to_bits(); + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + + // Update minimum value, check time difference and reset if over 10 seconds + let mut current_min = self.min_time_bits.load(Ordering::Relaxed); + let min_timestamp = self.min_time_timestamp_nanos.load(Ordering::Relaxed); + + // Check if min value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds) + let min_time_diff_nanos = now_nanos.saturating_sub(min_timestamp); + if min_time_diff_nanos > 10_000_000_000 { + // Over 10 seconds, reset min value + self.min_time_bits.store(f64::INFINITY.to_bits(), Ordering::Relaxed); + self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + current_min = f64::INFINITY.to_bits(); + } + + // If current time is less than min value, update min value and timestamp + while time_bits < current_min { + match self.min_time_bits.compare_exchange_weak( + current_min, + time_bits, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + // Successfully updated min value, also update timestamp + self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + break; + } + Err(x) => current_min = x, + } + } + + // Update maximum value, check time difference and reset if over 10 seconds + let mut current_max = self.max_time_bits.load(Ordering::Relaxed); + let max_timestamp = self.max_time_timestamp_nanos.load(Ordering::Relaxed); + + // Check if max value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds) + let time_diff_nanos = now_nanos.saturating_sub(max_timestamp); + if time_diff_nanos > 10_000_000_000 { + // Over 10 seconds, reset max value + self.max_time_bits.store(0, Ordering::Relaxed); + self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + current_max = 0; + } + + // If current time is greater than max value, update max value and timestamp + while time_bits > current_max { + match self.max_time_bits.compare_exchange_weak( + current_max, + time_bits, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + // Successfully updated max value, also update timestamp + self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + break; + } + Err(x) => current_max = x, + } + } + + // Update cumulative values (convert microseconds to integers to avoid floating point accumulation issues) + let total_time_us_int = (time_us * event_count as f64) as u64; + self.total_time_us.fetch_add(total_time_us_int, Ordering::Relaxed); + self.total_events.fetch_add(event_count, Ordering::Relaxed); + } + + /// Get statistics (non-blocking) + #[inline] + fn get_stats(&self) -> ProcessingTimeStats { + let min_bits = self.min_time_bits.load(Ordering::Relaxed); + let max_bits = self.max_time_bits.load(Ordering::Relaxed); + let total_time_us_int = self.total_time_us.load(Ordering::Relaxed); + let total_events = self.total_events.load(Ordering::Relaxed); + + let min_time = f64::from_bits(min_bits); + let max_time = f64::from_bits(max_bits); + let avg_time = + if total_events > 0 { total_time_us_int as f64 / total_events as f64 } else { 0.0 }; + + ProcessingTimeStats { + min_us: if min_time == f64::INFINITY { 0.0 } else { min_time }, + max_us: max_time, + avg_us: avg_time, + } + } +} + +/// Processing time statistics result #[derive(Debug, Clone)] -pub struct EventMetrics { +pub struct ProcessingTimeStats { + pub min_us: f64, + pub max_us: f64, + pub avg_us: f64, +} + +/// Event metrics snapshot +#[derive(Debug, Clone)] +pub struct EventMetricsSnapshot { pub process_count: u64, pub events_processed: u64, pub events_per_second: f64, - pub events_in_window: u64, - pub window_start_time: std::time::Instant, } -impl EventMetrics { - fn new(now: std::time::Instant) -> Self { - Self { - process_count: 0, - events_processed: 0, - events_per_second: 0.0, - events_in_window: 0, - window_start_time: now, - } - } -} - -/// 通用性能监控指标 +/// Compatibility structure - complete performance metrics #[derive(Debug, Clone)] pub struct PerformanceMetrics { - pub start_time: std::time::Instant, - pub event_metrics: [EventMetrics; 3], // [Tx, Account, BlockMeta] - pub average_processing_time_ms: f64, - pub min_processing_time_ms: f64, - pub max_processing_time_ms: f64, - pub last_update_time: std::time::Instant, -} - -impl Default for PerformanceMetrics { - fn default() -> Self { - Self::new() - } -} - -pub enum MetricsEventType { - Tx, - Account, - BlockMeta, -} - -impl MetricsEventType { - fn as_index(&self) -> usize { - match self { - MetricsEventType::Tx => 0, - MetricsEventType::Account => 1, - MetricsEventType::BlockMeta => 2, - } - } + pub uptime: std::time::Duration, + pub tx_metrics: EventMetricsSnapshot, + pub account_metrics: EventMetricsSnapshot, + pub block_meta_metrics: EventMetricsSnapshot, + pub processing_stats: ProcessingTimeStats, + pub dropped_events_count: u64, } impl PerformanceMetrics { + /// Create default performance metrics (compatibility method) pub fn new() -> Self { - let now = std::time::Instant::now(); + let default_metrics = + EventMetricsSnapshot { process_count: 0, events_processed: 0, events_per_second: 0.0 }; + let default_stats = ProcessingTimeStats { min_us: 0.0, max_us: 0.0, avg_us: 0.0 }; + Self { - start_time: now, - event_metrics: [EventMetrics::new(now), EventMetrics::new(now), EventMetrics::new(now)], - average_processing_time_ms: 0.0, - min_processing_time_ms: 0.0, - max_processing_time_ms: 0.0, - last_update_time: now, + uptime: std::time::Duration::ZERO, + tx_metrics: default_metrics.clone(), + account_metrics: default_metrics.clone(), + block_meta_metrics: default_metrics, + processing_stats: default_stats, + dropped_events_count: 0, + } + } +} + +/// High-performance metrics system +#[derive(Debug)] +pub struct HighPerformanceMetrics { + start_nanos: u64, + event_metrics: [AtomicEventMetrics; 3], + processing_stats: AtomicProcessingTimeStats, + // 丢弃事件指标 + dropped_events_count: AtomicU64, +} + +impl HighPerformanceMetrics { + fn new() -> Self { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + + Self { + start_nanos: now_nanos, + event_metrics: [ + AtomicEventMetrics::new(now_nanos), + AtomicEventMetrics::new(now_nanos), + AtomicEventMetrics::new(now_nanos), + ], + processing_stats: AtomicProcessingTimeStats::new(), + // 初始化丢弃事件指标 + dropped_events_count: AtomicU64::new(0), } } - /// 更新时间窗口指标 - fn update_window_metrics( - &mut self, - event_type: &MetricsEventType, - now: std::time::Instant, - window_duration: std::time::Duration, - ) { + /// 获取运行时长(秒) + #[inline] + pub fn get_uptime_seconds(&self) -> f64 { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + (now_nanos - self.start_nanos) as f64 / 1_000_000_000.0 + } + + /// 获取事件指标快照 + #[inline] + pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot { let index = event_type.as_index(); - let event_metric = &mut self.event_metrics[index]; + let (process_count, events_processed, _) = self.event_metrics[index].get_counts(); + let events_per_second = self.calculate_real_time_eps(event_type); - if now.duration_since(event_metric.window_start_time) >= window_duration { - let window_seconds = now.duration_since(event_metric.window_start_time).as_secs_f64(); - // 修复:正确计算每秒事件数,避免除零错误 - event_metric.events_per_second = if window_seconds > 0.001 { - // 避免极小的时间差 - event_metric.events_in_window as f64 / window_seconds - } else { - 0.0 // 时间太短时设为0,而不是事件总数 - }; - - // 重置窗口 - event_metric.events_in_window = 0; - event_metric.window_start_time = now; - } + EventMetricsSnapshot { process_count, events_processed, events_per_second } } - /// 计算实时每秒事件数(用于显示) - fn calculate_real_time_events_per_second( - &self, - event_type: &MetricsEventType, - now: std::time::Instant, - ) -> f64 { + /// 获取处理时间统计 + #[inline] + pub fn get_processing_stats(&self) -> ProcessingTimeStats { + self.processing_stats.get_stats() + } + + /// 获取丢弃事件计数 + #[inline] + pub fn get_dropped_events_count(&self) -> u64 { + self.dropped_events_count.load(Ordering::Relaxed) + } + + /// 计算实时每秒事件数(非阻塞) + fn calculate_real_time_eps(&self, event_type: EventType) -> f64 { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + let index = event_type.as_index(); let event_metric = &self.event_metrics[index]; - let current_window_duration = - now.duration_since(event_metric.window_start_time).as_secs_f64(); + let window_start = event_metric.get_window_start(); + let current_window_duration_secs = + (now_nanos.saturating_sub(window_start)) as f64 / 1_000_000_000.0; + let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed); - // 如果当前窗口有足够的时间和事件,使用当前窗口的数据 - if current_window_duration > 1.0 && event_metric.events_in_window > 0 { - event_metric.events_in_window as f64 / current_window_duration + // 优先级1: 当前窗口实时数据(≥2秒且有事件) + if current_window_duration_secs >= 2.0 && events_in_window > 0 { + return events_in_window as f64 / current_window_duration_secs; } - // 如果当前窗口时间太短或没有事件,使用上一个完整窗口的值 - else if event_metric.events_per_second > 0.0 { - event_metric.events_per_second + + // 优先级2: 上一个窗口的结果 + let stored_eps = event_metric.get_events_per_second(); + if stored_eps > 0.0 { + return stored_eps; } - // 如果都没有,计算总体平均值 - else { - let total_duration = now.duration_since(self.start_time).as_secs_f64(); - if total_duration > 1.0 && event_metric.events_processed > 0 { - event_metric.events_processed as f64 / total_duration - } else { - 0.0 + + // 优先级3: 总体平均值(≥3秒运行时间) + let total_duration_secs = self.get_uptime_seconds(); + let total_events = event_metric.events_processed.load(Ordering::Relaxed); + if total_duration_secs >= 3.0 && total_events > 0 { + return total_events as f64 / total_duration_secs; + } + + 0.0 + } + + /// 更新窗口指标(后台任务调用) + fn update_window_metrics(&self, event_type: EventType, window_duration_nanos: u64) { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + + let index = event_type.as_index(); + let event_metric = &self.event_metrics[index]; + + let window_start = event_metric.get_window_start(); + if now_nanos.saturating_sub(window_start) >= window_duration_nanos { + let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed); + let window_duration_secs = window_duration_nanos as f64 / 1_000_000_000.0; + + if window_duration_secs > 0.001 && events_in_window > 0 { + let eps = events_in_window as f64 / window_duration_secs; + event_metric.update_events_per_second(eps); } + + event_metric.reset_window(now_nanos); } } } -/// 通用性能监控管理器 +/// 高性能指标管理器 pub struct MetricsManager { - metrics: Arc>, - config: Arc, + metrics: Arc, + enable_metrics: bool, stream_name: String, + background_task_running: AtomicBool, } impl MetricsManager { - /// 创建新的性能监控管理器 - pub fn new( - metrics: Arc>, - config: Arc, - stream_name: String, - ) -> Self { - Self { metrics, config, stream_name } + /// 创建新的指标管理器 + pub fn new(enable_metrics: bool, stream_name: String) -> Self { + let manager = Self { + metrics: Arc::new(HighPerformanceMetrics::new()), + enable_metrics, + stream_name, + background_task_running: AtomicBool::new(false), + }; + + // 启动后台任务 + manager.start_background_tasks(); + manager } - /// 获取性能指标 - pub async fn get_metrics(&self) -> PerformanceMetrics { - let metrics = self.metrics.lock().await; - metrics.clone() + /// 启动后台任务 + fn start_background_tasks(&self) { + if self + .background_task_running + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + if !self.enable_metrics { + return; + } + + let metrics = self.metrics.clone(); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_millis(500)); + + loop { + interval.tick().await; + + let window_duration_nanos = DEFAULT_METRICS_WINDOW_SECONDS * 1_000_000_000; + + // 更新所有事件类型的窗口指标 + metrics.update_window_metrics(EventType::Transaction, window_duration_nanos); + metrics.update_window_metrics(EventType::Account, window_duration_nanos); + metrics.update_window_metrics(EventType::BlockMeta, window_duration_nanos); + } + }); + } } - /// 打印性能指标 - pub async fn print_metrics(&self) { - let metrics = self.get_metrics().await; - let event_names = ["TX", "Account", "Block Meta"]; - let event_types = - [MetricsEventType::Tx, MetricsEventType::Account, MetricsEventType::BlockMeta]; - let now = std::time::Instant::now(); + /// 记录处理次数(非阻塞) + #[inline] + pub fn record_process(&self, event_type: EventType) { + if self.enable_metrics { + self.metrics.event_metrics[event_type.as_index()].add_process_count(); + } + } + /// 记录事件处理(非阻塞) + #[inline] + pub fn record_events(&self, event_type: EventType, count: u64, processing_time_us: f64) { + if !self.enable_metrics { + return; + } + + // 原子更新事件计数 + self.metrics.event_metrics[event_type.as_index()].add_events_processed(count); + + // 原子更新处理时间统计 + self.metrics.processing_stats.update(processing_time_us, count); + } + + /// 记录慢处理操作 + #[inline] + pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) { + if processing_time_us > SLOW_PROCESSING_THRESHOLD_US { + log::debug!( + "{} slow processing: {:.2}us for {} events", + self.stream_name, + processing_time_us, + event_count, + ); + } + } + + /// 获取运行时长 + pub fn get_uptime(&self) -> std::time::Duration { + std::time::Duration::from_secs_f64(self.metrics.get_uptime_seconds()) + } + + /// 获取事件指标 + pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot { + self.metrics.get_event_metrics(event_type) + } + + /// 获取处理时间统计 + pub fn get_processing_stats(&self) -> ProcessingTimeStats { + self.metrics.get_processing_stats() + } + + /// 获取丢弃事件计数 + pub fn get_dropped_events_count(&self) -> u64 { + self.metrics.get_dropped_events_count() + } + + /// 打印性能指标(非阻塞) + pub fn print_metrics(&self) { println!("\n📊 {} Performance Metrics", self.stream_name); - println!(" Run Time: {:?}", metrics.start_time.elapsed()); - - // 打印表格头部 + println!(" Run Time: {:?}", self.get_uptime()); + + // 打印丢弃事件指标 + let dropped_count = self.get_dropped_events_count(); + if dropped_count > 0 { + println!("\n⚠️ Dropped Events: {}", dropped_count); + } + + // 打印事件指标表格 println!("┌─────────────┬──────────────┬──────────────────┬─────────────────┐"); println!("│ Event Type │ Process Count│ Events Processed │ Events/Second │"); println!("├─────────────┼──────────────┼──────────────────┼─────────────────┤"); - // 打印每种事件类型的数据 - for (i, name) in event_names.iter().enumerate() { - let event_metric = &metrics.event_metrics[i]; - // 使用实时计算的每秒事件数,而不是窗口更新的值 - let real_time_eps = metrics.calculate_real_time_events_per_second(&event_types[i], now); - + for event_type in [EventType::Transaction, EventType::Account, EventType::BlockMeta] { + let metrics = self.get_event_metrics(event_type); println!( "│ {:11} │ {:12} │ {:16} │ {:13.2} │", - name, - event_metric.process_count, - event_metric.events_processed, - real_time_eps + event_type.name(), + metrics.process_count, + metrics.events_processed, + metrics.events_per_second ); } println!("└─────────────┴──────────────┴──────────────────┴─────────────────┘"); // 打印处理时间统计表格 + let stats = self.get_processing_stats(); println!("\n⏱️ Processing Time Statistics"); - println!("┌─────────────────────┬─────────────┐"); - println!("│ Metric │ Value (ms) │"); - println!("├─────────────────────┼─────────────┤"); - println!("│ Average │ {:9.2} │", metrics.average_processing_time_ms); - println!("│ Minimum │ {:9.2} │", metrics.min_processing_time_ms); - println!("│ Maximum │ {:9.2} │", metrics.max_processing_time_ms); - println!("└─────────────────────┴─────────────┘"); + println!("┌───────────────────────┬─────────────┐"); + println!("│ Metric │ Value (us) │"); + println!("├───────────────────────┼─────────────┤"); + println!("│ Average │ {:9.2} │", stats.avg_us); + println!("│ Minimum within 10s │ {:9.2} │", stats.min_us); + println!("│ Maximum within 10s │ {:9.2} │", stats.max_us); + println!("└───────────────────────┴─────────────┘"); + println!(); } /// 启动自动性能监控任务 pub async fn start_auto_monitoring(&self) -> Option> { - // 检查是否启用性能监控 - if !self.config.enable_metrics { - return None; // 如果未启用性能监控,不启动监控任务 + if !self.enable_metrics { + return None; } - let metrics_manager = self.clone(); + let manager = self.clone(); let handle = tokio::spawn(async move { - let mut interval = tokio::time::interval(tokio::time::Duration::from_secs( + let mut interval = tokio::time::interval(std::time::Duration::from_secs( DEFAULT_METRICS_PRINT_INTERVAL_SECONDS, )); loop { interval.tick().await; - metrics_manager.print_metrics().await; + manager.print_metrics(); } }); Some(handle) } - /// 更新处理次数 - pub async fn add_process_count(&self, event_type: MetricsEventType) { - if !self.config.enable_metrics { - return; + // === 兼容性方法 === + + /// 兼容性构造函数 + pub fn new_with_metrics( + _metrics: Arc>, + enable_metrics: bool, + stream_name: String, + ) -> Self { + Self::new(enable_metrics, stream_name) + } + + /// 获取完整的性能指标(兼容性方法) + pub fn get_metrics(&self) -> PerformanceMetrics { + PerformanceMetrics { + uptime: self.get_uptime(), + tx_metrics: self.get_event_metrics(EventType::Transaction), + account_metrics: self.get_event_metrics(EventType::Account), + block_meta_metrics: self.get_event_metrics(EventType::BlockMeta), + processing_stats: self.get_processing_stats(), + dropped_events_count: self.metrics.get_dropped_events_count(), } - let mut metrics = self.metrics.lock().await; - metrics.event_metrics[event_type.as_index()].process_count += 1; } - // 保持向后兼容的方法 - pub async fn add_tx_process_count(&self) { - self.add_process_count(MetricsEventType::Tx).await; + /// 兼容性方法 - 添加交易处理计数 + #[inline] + pub fn add_tx_process_count(&self) { + self.record_process(EventType::Transaction); } - pub async fn add_account_process_count(&self) { - self.add_process_count(MetricsEventType::Account).await; + /// 兼容性方法 - 添加账户处理计数 + #[inline] + pub fn add_account_process_count(&self) { + self.record_process(EventType::Account); } - pub async fn add_block_meta_process_count(&self) { - self.add_process_count(MetricsEventType::BlockMeta).await; + /// 兼容性方法 - 添加区块元数据处理计数 + #[inline] + pub fn add_block_meta_process_count(&self) { + self.record_process(EventType::BlockMeta); } - /// 更新性能指标 - pub async fn update_metrics( + /// 兼容性方法 - 更新指标 + #[inline] + pub fn update_metrics( &self, event_type: MetricsEventType, events_processed: u64, - processing_time_ms: f64, + processing_time_us: f64, ) { - // 检查是否启用性能监控 - if !self.config.enable_metrics { + self.record_events(event_type, events_processed, processing_time_us); + self.log_slow_processing(processing_time_us, events_processed as usize); + } + + /// 增加丢弃事件计数 + #[inline] + pub fn increment_dropped_events(&self) { + if !self.enable_metrics { return; } - let mut metrics = self.metrics.lock().await; - let now = std::time::Instant::now(); - let index = event_type.as_index(); + // 原子地增加丢弃事件计数 + let new_count = self.metrics.dropped_events_count.fetch_add(1, Ordering::Relaxed) + 1; - // 更新事件计数 - metrics.event_metrics[index].events_processed += events_processed; - metrics.event_metrics[index].events_in_window += events_processed; - - metrics.last_update_time = now; - - // 更新处理时间统计 - if processing_time_ms < metrics.min_processing_time_ms - || metrics.min_processing_time_ms == 0.0 - { - metrics.min_processing_time_ms = processing_time_ms; + // 每丢弃1000个事件记录一次警告日志 + if new_count % 1000 == 0 { + log::debug!("{} dropped events count reached: {}", self.stream_name, new_count); } - if processing_time_ms > metrics.max_processing_time_ms { - metrics.max_processing_time_ms = processing_time_ms; - } - - // 计算平均处理时间 - 使用增量更新避免重复计算 - let total_events = metrics.event_metrics[index].events_processed; - if total_events > 0 { - let total_events_f64 = total_events as f64; - let old_total = (total_events_f64 - events_processed as f64).max(0.0); - - metrics.average_processing_time_ms = if old_total > 0.0 { - (metrics.average_processing_time_ms * old_total - + processing_time_ms * events_processed as f64) - / total_events_f64 - } else { - processing_time_ms - }; - } - - // 更新时间窗口指标 - let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS); - metrics.update_window_metrics(&event_type, now, window_duration); } - /// 记录慢处理操作 - pub fn log_slow_processing(&self, processing_time_ms: f64, event_count: usize) { - if processing_time_ms > SLOW_PROCESSING_THRESHOLD_MS { - log::warn!( - "{} slow processing: {processing_time_ms}ms for {event_count} events", - self.stream_name + /// 批量增加丢弃事件计数 + #[inline] + pub fn increment_dropped_events_by(&self, count: u64) { + if !self.enable_metrics || count == 0 { + return; + } + + // 原子地增加丢弃事件计数 + let new_count = + self.metrics.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count; + + // 记录批量丢弃事件的日志 + if count > 1 { + log::debug!( + "{} dropped batch of {} events, total dropped: {}", + self.stream_name, + count, + new_count ); } + + // 每丢弃1000个事件记录一次警告日志 + if new_count % 1000 == 0 || (new_count / 1000) != ((new_count - count) / 1000) { + log::debug!("{} dropped events count reached: {}", self.stream_name, new_count); + } } } @@ -307,8 +658,9 @@ impl Clone for MetricsManager { fn clone(&self) -> Self { Self { metrics: self.metrics.clone(), - config: self.config.clone(), + enable_metrics: self.enable_metrics, stream_name: self.stream_name.clone(), + background_task_running: AtomicBool::new(false), // 新实例不自动启动后台任务 } } } diff --git a/src/streaming/common/mod.rs b/src/streaming/common/mod.rs index bbbc5e1..55a76aa 100644 --- a/src/streaming/common/mod.rs +++ b/src/streaming/common/mod.rs @@ -1,13 +1,15 @@ // 公用模块 - 包含流处理相关的通用功能 pub mod config; pub mod metrics; -pub mod batch; pub mod constants; pub mod subscription; +pub mod event_processor; +pub mod simd_utils; // 重新导出主要类型 pub use config::*; pub use metrics::*; -pub use batch::*; pub use constants::*; pub use subscription::*; +pub use event_processor::*; +pub use simd_utils::*; \ No newline at end of file diff --git a/src/streaming/common/simd_utils.rs b/src/streaming/common/simd_utils.rs new file mode 100644 index 0000000..8aaf1f0 --- /dev/null +++ b/src/streaming/common/simd_utils.rs @@ -0,0 +1,295 @@ +use wide::*; + +/// SIMD-accelerated data parsing utilities +pub struct SimdUtils; + +impl SimdUtils { + /// SIMD-accelerated byte array comparison + /// For arrays with length >= 16, uses SIMD instructions for fast comparison + #[inline(always)] + pub fn fast_bytes_equal(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + + let len = a.len(); + + // For small arrays, use standard comparison directly + if len < 16 { + return a == b; + } + + // Use SIMD to process 16-byte chunks + let chunks = len / 16; + let remainder = len % 16; + + // Process complete 16-byte chunks + for i in 0..chunks { + let offset = i * 16; + let chunk_a = u8x16::from(&a[offset..offset + 16]); + let chunk_b = u8x16::from(&b[offset..offset + 16]); + + if !chunk_a.cmp_eq(chunk_b).all() { + return false; + } + } + + // Process remaining bytes + if remainder > 0 { + let start = chunks * 16; + return &a[start..] == &b[start..]; + } + + true + } + + /// Fast discriminator matching, specifically for instruction discriminator comparison + #[inline(always)] + pub fn fast_discriminator_match(data: &[u8], discriminator: &[u8]) -> bool { + if data.len() < discriminator.len() { + return false; + } + + let disc_len = discriminator.len(); + + // Optimize for common discriminator lengths + match disc_len { + 1 => data[0] == discriminator[0], + 2 => { + let data_u16 = u16::from_le_bytes([data[0], data[1]]); + let disc_u16 = u16::from_le_bytes([discriminator[0], discriminator[1]]); + data_u16 == disc_u16 + } + 4 => { + let data_u32 = u32::from_le_bytes([data[0], data[1], data[2], data[3]]); + let disc_u32 = u32::from_le_bytes([ + discriminator[0], + discriminator[1], + discriminator[2], + discriminator[3], + ]); + data_u32 == disc_u32 + } + 8 => { + let data_u64 = u64::from_le_bytes([ + data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], + ]); + let disc_u64 = u64::from_le_bytes([ + discriminator[0], + discriminator[1], + discriminator[2], + discriminator[3], + discriminator[4], + discriminator[5], + discriminator[6], + discriminator[7], + ]); + data_u64 == disc_u64 + } + 16 => { + // Use SIMD to process 16-byte discriminators + let data_chunk = u8x16::from(&data[..16]); + let disc_chunk = u8x16::from(discriminator); + data_chunk.cmp_eq(disc_chunk).all() + } + _ => { + // For other lengths, use generic SIMD comparison + Self::fast_bytes_equal(&data[..disc_len], discriminator) + } + } + } + + /// SIMD-accelerated memory search to find specific patterns in data + #[inline(always)] + pub fn find_pattern_simd(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || haystack.len() < needle.len() { + return None; + } + + let needle_len = needle.len(); + let haystack_len = haystack.len(); + + // For single-byte search, use optimized method + if needle_len == 1 { + let target = needle[0]; + return haystack.iter().position(|&b| b == target); + } + + // For multi-byte search, use SIMD acceleration + if needle_len <= 16 && haystack_len >= 16 { + let first_byte = needle[0]; + let chunks = (haystack_len - needle_len + 1) / 16; + + for chunk_idx in 0..chunks { + let start = chunk_idx * 16; + let end = std::cmp::min(start + 16, haystack_len - needle_len + 1); + + // Use SIMD to find first byte matches + let chunk = &haystack[start..start + 16]; + let target_vec = u8x16::splat(first_byte); + let chunk_vec = u8x16::from(chunk); + let matches = chunk_vec.cmp_eq(target_vec); + + // Check each match position + let matches_array: [u8; 16] = matches.into(); + for i in 0..16 { + if start + i >= end { + break; + } + + if matches_array[i] != 0 && start + i + needle_len <= haystack_len { + if Self::fast_bytes_equal( + &haystack[start + i..start + i + needle_len], + needle, + ) { + return Some(start + i); + } + } + } + } + + // Process remaining part + let remaining_start = chunks * 16; + for i in remaining_start..=(haystack_len - needle_len) { + if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) { + return Some(i); + } + } + } else { + // Fallback to standard search + for i in 0..=(haystack_len - needle_len) { + if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) { + return Some(i); + } + } + } + + None + } + + /// SIMD-accelerated data validation to check if data conforms to specific format + #[inline(always)] + pub fn validate_data_format(data: &[u8], min_length: usize) -> bool { + if data.len() < min_length { + return false; + } + + true + } + + /// Fast checksum calculation (maintains API consistency) + #[inline(always)] + pub fn fast_checksum(data: &[u8]) -> u32 { + // Simplified implementation, directly sum all bytes + data.iter().map(|&b| b as u32).sum() + } + + /// SIMD-accelerated data copy (for large data blocks) + #[inline(always)] + pub fn fast_copy(src: &[u8], dst: &mut [u8]) { + if src.len() != dst.len() { + panic!("Source and destination must have the same length"); + } + + let len = src.len(); + + if len >= 32 { + // Use 32-byte SIMD copy + let chunks = len / 32; + + for i in 0..chunks { + let start = i * 32; + let src_chunk1 = u8x16::from(&src[start..start + 16]); + let src_chunk2 = u8x16::from(&src[start + 16..start + 32]); + + let chunk1_array: [u8; 16] = src_chunk1.into(); + let chunk2_array: [u8; 16] = src_chunk2.into(); + + dst[start..start + 16].copy_from_slice(&chunk1_array); + dst[start + 16..start + 32].copy_from_slice(&chunk2_array); + } + + // Process remaining bytes + let remaining_start = chunks * 32; + dst[remaining_start..].copy_from_slice(&src[remaining_start..]); + } else { + // For small data, use standard copy + dst.copy_from_slice(src); + } + } + + /// SIMD-accelerated account indices validation + /// Validates that all indices in the account index array are less than the total account count + #[inline(always)] + pub fn validate_account_indices_simd(indices: &[u8], account_count: usize) -> bool { + if indices.is_empty() { + return true; + } + + let max_valid_index = account_count as u8; + + // For small arrays, use standard comparison directly + if indices.len() < 16 { + return indices.iter().all(|&idx| idx < max_valid_index); + } + + // Use SIMD for batch loading and comparison + let chunks = indices.len() / 16; + let remainder = indices.len() % 16; + + // Process complete 16-byte chunks + for i in 0..chunks { + let start = i * 16; + let indices_chunk = u8x16::from(&indices[start..start + 16]); + + // Convert SIMD vector to array for fast batch checking + let indices_array: [u8; 16] = indices_chunk.into(); + + // Use unrolled loop for fast comparison, compiler will optimize this + if indices_array[0] >= max_valid_index + || indices_array[1] >= max_valid_index + || indices_array[2] >= max_valid_index + || indices_array[3] >= max_valid_index + || indices_array[4] >= max_valid_index + || indices_array[5] >= max_valid_index + || indices_array[6] >= max_valid_index + || indices_array[7] >= max_valid_index + || indices_array[8] >= max_valid_index + || indices_array[9] >= max_valid_index + || indices_array[10] >= max_valid_index + || indices_array[11] >= max_valid_index + || indices_array[12] >= max_valid_index + || indices_array[13] >= max_valid_index + || indices_array[14] >= max_valid_index + || indices_array[15] >= max_valid_index + { + return false; + } + } + + // Process remaining bytes + if remainder > 0 { + let remaining_start = chunks * 16; + return indices[remaining_start..].iter().all(|&idx| idx < max_valid_index); + } + + true + } + + /// SIMD-accelerated instruction data validation + /// Validates basic format and length requirements of instruction data + #[inline(always)] + pub fn validate_instruction_data_simd( + data: &[u8], + min_length: usize, + discriminator_length: usize, + ) -> bool { + // Basic length check + if data.len() < min_length || data.len() < discriminator_length { + return false; + } + + // Use existing data format validation + Self::validate_data_format(data, min_length) + } +} diff --git a/src/streaming/common/subscription.rs b/src/streaming/common/subscription.rs index ca29cbb..c0abf16 100644 --- a/src/streaming/common/subscription.rs +++ b/src/streaming/common/subscription.rs @@ -3,7 +3,7 @@ use tokio::task::JoinHandle; /// Subscription handle for managing and stopping subscriptions pub struct SubscriptionHandle { stream_handle: JoinHandle<()>, - event_handle: JoinHandle<()>, + event_handle: Option>, metrics_handle: Option>, } @@ -11,7 +11,7 @@ impl SubscriptionHandle { /// Create a new subscription handle pub fn new( stream_handle: JoinHandle<()>, - event_handle: JoinHandle<()>, + event_handle: Option>, metrics_handle: Option>, ) -> Self { Self { stream_handle, event_handle, metrics_handle } @@ -20,7 +20,9 @@ impl SubscriptionHandle { /// Stop subscription and abort all related tasks pub fn stop(self) { self.stream_handle.abort(); - self.event_handle.abort(); + if let Some(handle) = self.event_handle { + handle.abort(); + } if let Some(handle) = self.metrics_handle { handle.abort(); } @@ -29,7 +31,9 @@ impl SubscriptionHandle { /// Asynchronously wait for all tasks to complete pub async fn join(self) -> Result<(), tokio::task::JoinError> { let _ = self.stream_handle.await; - let _ = self.event_handle.await; + if let Some(handle) = self.event_handle { + let _ = handle.await; + } if let Some(handle) = self.metrics_handle { let _ = handle.await; } diff --git a/src/streaming/event_parser/common/filter.rs b/src/streaming/event_parser/common/filter.rs index 8fa3296..2ac0743 100644 --- a/src/streaming/event_parser/common/filter.rs +++ b/src/streaming/event_parser/common/filter.rs @@ -2,7 +2,7 @@ use crate::streaming::event_parser::common::{ types::EventType, ACCOUNT_EVENT_TYPES, BLOCK_EVENT_TYPES, }; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct EventTypeFilter { pub include: Vec, } diff --git a/src/streaming/event_parser/common/mod.rs b/src/streaming/event_parser/common/mod.rs index 3ecc161..3e9d088 100755 --- a/src/streaming/event_parser/common/mod.rs +++ b/src/streaming/event_parser/common/mod.rs @@ -8,10 +8,6 @@ macro_rules! impl_unified_event { // 带有自定义ID表达式的版本 ($struct_name:ident, $($field:ident),*) => { impl $crate::streaming::event_parser::core::traits::UnifiedEvent for $struct_name { - fn id(&self) -> &str { - &self.metadata.id - } - fn event_type(&self) -> $crate::streaming::event_parser::common::types::EventType { self.metadata.event_type.clone() } @@ -24,16 +20,16 @@ macro_rules! impl_unified_event { self.metadata.slot } - fn program_received_time_ms(&self) -> i64 { - self.metadata.program_received_time_ms + fn program_received_time_us(&self) -> i64 { + self.metadata.program_received_time_us } - fn program_handle_time_consuming_ms(&self) -> i64 { - self.metadata.program_handle_time_consuming_ms + fn program_handle_time_consuming_us(&self) -> i64 { + self.metadata.program_handle_time_consuming_us } - fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64) { - self.metadata.program_handle_time_consuming_ms = program_handle_time_consuming_ms; + fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64) { + self.metadata.program_handle_time_consuming_us = program_handle_time_consuming_us; } fn as_any(&self) -> &dyn std::any::Any { @@ -48,7 +44,7 @@ macro_rules! impl_unified_event { Box::new(self.clone()) } - fn merge(&mut self, other: Box) { + fn merge(&mut self, other: &dyn $crate::streaming::event_parser::core::traits::UnifiedEvent) { if let Some(_e) = other.as_any().downcast_ref::<$struct_name>() { $( self.$field = _e.$field.clone(); @@ -56,12 +52,23 @@ macro_rules! impl_unified_event { } } - fn set_transfer_datas(&mut self, transfer_datas: Vec<$crate::streaming::event_parser::common::types::TransferData>, swap_data: Option<$crate::streaming::event_parser::common::types::SwapData>) { - self.metadata.set_transfer_datas(transfer_datas, swap_data); + fn set_swap_data(&mut self, swap_data: $crate::streaming::event_parser::common::types::SwapData) { + self.metadata.set_swap_data(swap_data); } - fn index(&self) -> String { - self.metadata.index.clone() + fn swap_data_is_parsed(&self) -> bool { + self.metadata.swap_data.is_some() + } + + fn instruction_outer_index(&self) -> i64 { + self.metadata.instruction_outer_index + } + + fn instruction_inner_index(&self) -> Option { + self.metadata.instruction_inner_index + } + fn transaction_index(&self) -> Option { + self.metadata.transaction_index } } }; diff --git a/src/streaming/event_parser/common/types.rs b/src/streaming/event_parser/common/types.rs index 9794cc8..b086d8f 100755 --- a/src/streaming/event_parser/common/types.rs +++ b/src/streaming/event_parser/common/types.rs @@ -1,26 +1,24 @@ use borsh::{BorshDeserialize, BorshSerialize}; +use crossbeam_queue::ArrayQueue; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; -use solana_transaction_status::UiInstruction; -use std::{ - hash::{DefaultHasher, Hash, Hasher}, - str::FromStr, - sync::Arc, -}; -use tokio::sync::Mutex; +use std::{borrow::Cow, fmt, str::FromStr, sync::Arc}; use crate::{ match_event, - streaming::event_parser::{ - protocols::{ - bonk::BonkTradeEvent, - pumpfun::PumpFunTradeEvent, - pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}, - raydium_amm_v4::RaydiumAmmV4SwapEvent, - raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event}, - raydium_cpmm::RaydiumCpmmSwapEvent, + streaming::{ + common::SimdUtils, + event_parser::{ + protocols::{ + bonk::BonkTradeEvent, + pumpfun::PumpFunTradeEvent, + pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}, + raydium_amm_v4::RaydiumAmmV4SwapEvent, + raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event}, + raydium_cpmm::RaydiumCpmmSwapEvent, + }, + UnifiedEvent, }, - UnifiedEvent, }, }; @@ -30,7 +28,7 @@ const TRANSFER_DATA_POOL_SIZE: usize = 2000; /// Event metadata object pool pub struct EventMetadataPool { - pool: Arc>>, + pool: Arc>, } impl Default for EventMetadataPool { @@ -41,55 +39,22 @@ impl Default for EventMetadataPool { impl EventMetadataPool { pub fn new() -> Self { - Self { pool: Arc::new(Mutex::new(Vec::with_capacity(EVENT_METADATA_POOL_SIZE))) } + Self { pool: Arc::new(ArrayQueue::new(EVENT_METADATA_POOL_SIZE)) } } - pub async fn acquire(&self) -> Option { - let mut pool = self.pool.lock().await; - pool.pop() + pub fn acquire(&self) -> Option { + self.pool.pop() } - pub async fn release(&self, metadata: EventMetadata) { - let mut pool = self.pool.lock().await; - if pool.len() < EVENT_METADATA_POOL_SIZE { - pool.push(metadata); - } - } -} - -/// Transfer data object pool -pub struct TransferDataPool { - pool: Arc>>, -} - -impl Default for TransferDataPool { - fn default() -> Self { - Self::new() - } -} - -impl TransferDataPool { - pub fn new() -> Self { - Self { pool: Arc::new(Mutex::new(Vec::with_capacity(TRANSFER_DATA_POOL_SIZE))) } - } - - pub async fn acquire(&self) -> Option { - let mut pool = self.pool.lock().await; - pool.pop() - } - - pub async fn release(&self, transfer_data: TransferData) { - let mut pool = self.pool.lock().await; - if pool.len() < TRANSFER_DATA_POOL_SIZE { - pool.push(transfer_data); - } + pub fn release(&self, metadata: EventMetadata) { + // 如果队列已满,push 会失败,但不会阻塞 + let _ = self.pool.push(metadata); } } // Global object pool instances lazy_static::lazy_static! { pub static ref EVENT_METADATA_POOL: EventMetadataPool = EventMetadataPool::new(); - pub static ref TRANSFER_DATA_POOL: TransferDataPool = TransferDataPool::new(); } #[derive( @@ -199,70 +164,69 @@ pub const ACCOUNT_EVENT_TYPES: &[EventType] = &[ ]; pub const BLOCK_EVENT_TYPES: &[EventType] = &[EventType::BlockMeta]; -impl EventType { - #[allow(clippy::inherent_to_string)] - pub fn to_string(&self) -> String { +impl fmt::Display for EventType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - EventType::PumpSwapBuy => "PumpSwapBuy".to_string(), - EventType::PumpSwapSell => "PumpSwapSell".to_string(), - EventType::PumpSwapCreatePool => "PumpSwapCreatePool".to_string(), - EventType::PumpSwapDeposit => "PumpSwapDeposit".to_string(), - EventType::PumpSwapWithdraw => "PumpSwapWithdraw".to_string(), - EventType::PumpFunCreateToken => "PumpFunCreateToken".to_string(), - EventType::PumpFunBuy => "PumpFunBuy".to_string(), - EventType::PumpFunSell => "PumpFunSell".to_string(), - EventType::PumpFunMigrate => "PumpFunMigrate".to_string(), - EventType::BonkBuyExactIn => "BonkBuyExactIn".to_string(), - EventType::BonkBuyExactOut => "BonkBuyExactOut".to_string(), - EventType::BonkSellExactIn => "BonkSellExactIn".to_string(), - EventType::BonkSellExactOut => "BonkSellExactOut".to_string(), - EventType::BonkInitialize => "BonkInitialize".to_string(), - EventType::BonkInitializeV2 => "BonkInitializeV2".to_string(), - EventType::BonkMigrateToAmm => "BonkMigrateToAmm".to_string(), - EventType::BonkMigrateToCpswap => "BonkMigrateToCpswap".to_string(), - EventType::AccountPumpFunBondingCurve => "AccountPumpFunBondingCurve".to_string(), - EventType::AccountPumpFunGlobal => "AccountPumpFunGlobal".to_string(), - EventType::AccountPumpSwapGlobalConfig => "AccountPumpSwapGlobalConfig".to_string(), - EventType::AccountPumpSwapPool => "AccountPumpSwapPool".to_string(), - EventType::AccountBonkPoolState => "AccountBonkPoolState".to_string(), - EventType::AccountBonkGlobalConfig => "AccountBonkGlobalConfig".to_string(), - EventType::AccountBonkPlatformConfig => "AccountBonkPlatformConfig".to_string(), - EventType::AccountBonkVestingRecord => "AccountBonkVestingRecord".to_string(), - EventType::RaydiumCpmmSwapBaseInput => "RaydiumCpmmSwapBaseInput".to_string(), - EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmmSwapBaseOutput".to_string(), - EventType::RaydiumCpmmDeposit => "RaydiumCpmmDeposit".to_string(), - EventType::RaydiumCpmmInitialize => "RaydiumCpmmInitialize".to_string(), - EventType::RaydiumCpmmWithdraw => "RaydiumCpmmWithdraw".to_string(), - EventType::RaydiumClmmSwap => "RaydiumClmmSwap".to_string(), - EventType::RaydiumClmmSwapV2 => "RaydiumClmmSwapV2".to_string(), - EventType::RaydiumClmmClosePosition => "RaydiumClmmClosePosition".to_string(), + EventType::PumpSwapBuy => write!(f, "PumpSwapBuy"), + EventType::PumpSwapSell => write!(f, "PumpSwapSell"), + EventType::PumpSwapCreatePool => write!(f, "PumpSwapCreatePool"), + EventType::PumpSwapDeposit => write!(f, "PumpSwapDeposit"), + EventType::PumpSwapWithdraw => write!(f, "PumpSwapWithdraw"), + EventType::PumpFunCreateToken => write!(f, "PumpFunCreateToken"), + EventType::PumpFunBuy => write!(f, "PumpFunBuy"), + EventType::PumpFunSell => write!(f, "PumpFunSell"), + EventType::PumpFunMigrate => write!(f, "PumpFunMigrate"), + EventType::BonkBuyExactIn => write!(f, "BonkBuyExactIn"), + EventType::BonkBuyExactOut => write!(f, "BonkBuyExactOut"), + EventType::BonkSellExactIn => write!(f, "BonkSellExactIn"), + EventType::BonkSellExactOut => write!(f, "BonkSellExactOut"), + EventType::BonkInitialize => write!(f, "BonkInitialize"), + EventType::BonkInitializeV2 => write!(f, "BonkInitializeV2"), + EventType::BonkMigrateToAmm => write!(f, "BonkMigrateToAmm"), + EventType::BonkMigrateToCpswap => write!(f, "BonkMigrateToCpswap"), + EventType::RaydiumCpmmSwapBaseInput => write!(f, "RaydiumCpmmSwapBaseInput"), + EventType::RaydiumCpmmSwapBaseOutput => write!(f, "RaydiumCpmmSwapBaseOutput"), + EventType::RaydiumCpmmDeposit => write!(f, "RaydiumCpmmDeposit"), + EventType::RaydiumCpmmInitialize => write!(f, "RaydiumCpmmInitialize"), + EventType::RaydiumCpmmWithdraw => write!(f, "RaydiumCpmmWithdraw"), + EventType::RaydiumClmmSwap => write!(f, "RaydiumClmmSwap"), + EventType::RaydiumClmmSwapV2 => write!(f, "RaydiumClmmSwapV2"), + EventType::RaydiumClmmClosePosition => write!(f, "RaydiumClmmClosePosition"), EventType::RaydiumClmmDecreaseLiquidityV2 => { - "RaydiumClmmDecreaseLiquidityV2".to_string() + write!(f, "RaydiumClmmDecreaseLiquidityV2") } - EventType::RaydiumClmmCreatePool => "RaydiumClmmCreatePool".to_string(), + EventType::RaydiumClmmCreatePool => write!(f, "RaydiumClmmCreatePool"), EventType::RaydiumClmmIncreaseLiquidityV2 => { - "RaydiumClmmIncreaseLiquidityV2".to_string() + write!(f, "RaydiumClmmIncreaseLiquidityV2") } EventType::RaydiumClmmOpenPositionWithToken22Nft => { - "RaydiumClmmOpenPositionWithToken22Nft".to_string() + write!(f, "RaydiumClmmOpenPositionWithToken22Nft") } - EventType::RaydiumClmmOpenPositionV2 => "RaydiumClmmOpenPositionV2".to_string(), - EventType::RaydiumAmmV4SwapBaseIn => "RaydiumAmmV4SwapBaseIn".to_string(), - EventType::RaydiumAmmV4SwapBaseOut => "RaydiumAmmV4SwapBaseOut".to_string(), - EventType::RaydiumAmmV4Deposit => "RaydiumAmmV4Deposit".to_string(), - EventType::RaydiumAmmV4Initialize2 => "RaydiumAmmV4Initialize2".to_string(), - EventType::RaydiumAmmV4Withdraw => "RaydiumAmmV4Withdraw".to_string(), - EventType::RaydiumAmmV4WithdrawPnl => "RaydiumAmmV4WithdrawPnl".to_string(), - EventType::AccountRaydiumAmmV4AmmInfo => "AccountRaydiumAmmV4AmmInfo".to_string(), - EventType::AccountRaydiumClmmAmmConfig => "AccountRaydiumClmmAmmConfig".to_string(), - EventType::AccountRaydiumClmmPoolState => "AccountRaydiumClmmPoolState".to_string(), + EventType::RaydiumClmmOpenPositionV2 => write!(f, "RaydiumClmmOpenPositionV2"), + EventType::RaydiumAmmV4SwapBaseIn => write!(f, "RaydiumAmmV4SwapBaseIn"), + EventType::RaydiumAmmV4SwapBaseOut => write!(f, "RaydiumAmmV4SwapBaseOut"), + EventType::RaydiumAmmV4Deposit => write!(f, "RaydiumAmmV4Deposit"), + EventType::RaydiumAmmV4Initialize2 => write!(f, "RaydiumAmmV4Initialize2"), + EventType::RaydiumAmmV4Withdraw => write!(f, "RaydiumAmmV4Withdraw"), + EventType::RaydiumAmmV4WithdrawPnl => write!(f, "RaydiumAmmV4WithdrawPnl"), + EventType::AccountRaydiumAmmV4AmmInfo => write!(f, "AccountRaydiumAmmV4AmmInfo"), + EventType::AccountPumpSwapGlobalConfig => write!(f, "AccountPumpSwapGlobalConfig"), + EventType::AccountPumpSwapPool => write!(f, "AccountPumpSwapPool"), + EventType::AccountBonkPoolState => write!(f, "AccountBonkPoolState"), + EventType::AccountBonkGlobalConfig => write!(f, "AccountBonkGlobalConfig"), + EventType::AccountBonkPlatformConfig => write!(f, "AccountBonkPlatformConfig"), + EventType::AccountBonkVestingRecord => write!(f, "AccountBonkVestingRecord"), + EventType::AccountPumpFunBondingCurve => write!(f, "AccountPumpFunBondingCurve"), + EventType::AccountPumpFunGlobal => write!(f, "AccountPumpFunGlobal"), + EventType::AccountRaydiumClmmAmmConfig => write!(f, "AccountRaydiumClmmAmmConfig"), + EventType::AccountRaydiumClmmPoolState => write!(f, "AccountRaydiumClmmPoolState"), EventType::AccountRaydiumClmmTickArrayState => { - "AccountRaydiumClmmTickArrayState".to_string() + write!(f, "AccountRaydiumClmmTickArrayState") } - EventType::AccountRaydiumCpmmAmmConfig => "AccountRaydiumCpmmAmmConfig".to_string(), - EventType::AccountRaydiumCpmmPoolState => "AccountRaydiumCpmmPoolState".to_string(), - EventType::BlockMeta => "BlockMeta".to_string(), - EventType::Unknown => "Unknown".to_string(), + EventType::AccountRaydiumCpmmAmmConfig => write!(f, "AccountRaydiumCpmmAmmConfig"), + EventType::AccountRaydiumCpmmPoolState => write!(f, "AccountRaydiumCpmmPoolState"), + EventType::BlockMeta => write!(f, "BlockMeta"), + EventType::Unknown => write!(f, "Unknown"), } } } @@ -310,20 +274,6 @@ impl ProtocolInfo { } } -/// Transfer data -#[derive( - Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, -)] -pub struct TransferData { - pub token_program: Pubkey, - pub source: Pubkey, - pub destination: Pubkey, - pub authority: Option, - pub amount: u64, - pub decimals: Option, - pub mint: Option, -} - #[derive( Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, )] @@ -340,285 +290,234 @@ pub struct SwapData { Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, )] pub struct EventMetadata { - pub id: String, - pub signature: String, + pub signature: Cow<'static, str>, pub slot: u64, + pub transaction_index: Option, // 新增:交易在slot中的索引 pub block_time: i64, pub block_time_ms: i64, - pub program_received_time_ms: i64, - pub program_handle_time_consuming_ms: i64, + pub program_received_time_us: i64, + pub program_handle_time_consuming_us: i64, pub protocol: ProtocolType, pub event_type: EventType, pub program_id: Pubkey, - pub transfer_datas: Vec, pub swap_data: Option, - pub index: String, + pub instruction_outer_index: i64, + pub instruction_inner_index: Option, } impl EventMetadata { #[allow(clippy::too_many_arguments)] pub fn new( - id: String, - signature: String, + signature: Cow<'static, str>, slot: u64, block_time: i64, block_time_ms: i64, protocol: ProtocolType, event_type: EventType, program_id: Pubkey, - index: String, - program_received_time_ms: i64, + instruction_outer_index: i64, + instruction_inner_index: Option, + program_received_time_us: i64, + transaction_index: Option, ) -> Self { Self { - id, signature, slot, block_time, block_time_ms, - program_received_time_ms, - program_handle_time_consuming_ms: 0, + program_received_time_us, + program_handle_time_consuming_us: 0, protocol, event_type, program_id, - transfer_datas: Vec::with_capacity(4), // Pre-allocate capacity swap_data: None, - index, + instruction_outer_index, + instruction_inner_index, + transaction_index, } } - pub fn set_id(&mut self, id: String) { - let _id = format!("{}-{}-{}", self.signature, self.event_type.to_string(), id); - let mut hasher = DefaultHasher::new(); - _id.hash(&mut hasher); - let hash_value = hasher.finish(); - self.id = format!("{:x}", hash_value); - } - - pub fn set_transfer_datas( - &mut self, - transfer_datas: Vec, - swap_data: Option, - ) { - self.transfer_datas = transfer_datas; - self.swap_data = swap_data; + pub fn set_swap_data(&mut self, swap_data: SwapData) { + self.swap_data = Some(swap_data); } /// Recycle EventMetadata to object pool - pub async fn recycle(self) { - EVENT_METADATA_POOL.release(self).await; + pub fn recycle(self) { + EVENT_METADATA_POOL.release(self); } } -/// Parse token transfer data from next instructions -pub fn parse_transfer_datas_from_next_instructions( - event: Box, - inner_instruction: &solana_transaction_status::UiInnerInstructions, - current_index: i8, - accounts: &[Pubkey], -) -> (Vec, Option) { - let mut transfer_datas = vec![]; - // Get the next two instructions after the current instruction - let next_instructions: Vec<&UiInstruction> = - inner_instruction.instructions.iter().skip((current_index + 1) as usize).collect(); - - let system_programs = vec![ - // Token Program +lazy_static::lazy_static! { + static ref SOL_MINT: Pubkey = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap(); + static ref SYSTEM_PROGRAMS: [Pubkey; 3] = [ Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), - // Token 2022 Program Pubkey::from_str("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb").unwrap(), - // System Program Pubkey::from_str("11111111111111111111111111111111").unwrap(), ]; - for instruction in next_instructions { - if let UiInstruction::Compiled(compiled) = instruction { - if !system_programs.contains(&accounts[compiled.program_id_index as usize]) { - break; - } - if let Ok(data) = bs58::decode(compiled.data.clone()).into_vec() { - // Token Program: transferChecked - // Token 2022 Program: transferChecked - if data[0] == 12 { - let account_pubkeys: Vec = - compiled.accounts.iter().map(|a| accounts[*a as usize]).collect(); - if account_pubkeys.len() < 4 { - continue; - } - let (source, mint, destination, authority) = ( - account_pubkeys[0], - account_pubkeys[1], - account_pubkeys[2], - account_pubkeys[3], - ); - let amount = u64::from_le_bytes(data[1..9].try_into().unwrap()); - let decimals = data[9]; - let token_program = accounts[compiled.program_id_index as usize]; - transfer_datas.push(TransferData { - amount, - decimals: Some(decimals), - mint: Some(mint), - source, - destination, - authority: Some(authority), - token_program, - }); - } - // Token Program: transfer - else if data[0] == 3 { - let account_pubkeys: Vec = - compiled.accounts.iter().map(|a| accounts[*a as usize]).collect(); - if account_pubkeys.len() < 3 { - continue; - } - let (source, destination, authority) = - (account_pubkeys[0], account_pubkeys[1], account_pubkeys[2]); - let amount = u64::from_le_bytes(data[1..9].try_into().unwrap()); - let token_program = accounts[compiled.program_id_index as usize]; - transfer_datas.push(TransferData { - amount, - decimals: None, - mint: None, - source, - destination, - authority: Some(authority), - token_program, - }); - } - //System Program: transfer - else if data[0] == 2 { - let account_pubkeys: Vec = - compiled.accounts.iter().map(|a| accounts[*a as usize]).collect(); - if account_pubkeys.len() < 2 { - continue; - } - let (source, destination) = (account_pubkeys[0], account_pubkeys[1]); - let amount = u64::from_le_bytes(data[4..12].try_into().unwrap()); - let token_program = accounts[compiled.program_id_index as usize]; - transfer_datas.push(TransferData { - amount, - decimals: None, - mint: None, - source, - destination, - authority: None, - token_program, - }); - } - } - } - } - let mut swap_data: SwapData = SwapData { +} + +/// Parse token transfer data from next instructions +pub fn parse_swap_data_from_next_instructions( + event: &dyn UnifiedEvent, + inner_instruction: &solana_transaction_status::InnerInstructions, + current_index: i8, + accounts: &[Pubkey], +) -> Option { + let mut swap_data = SwapData { from_mint: Pubkey::default(), to_mint: Pubkey::default(), from_amount: 0, to_amount: 0, description: None, }; - let sol_mint = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap(); - if transfer_datas.len() > 0 { - let mut user: Option = None; - let mut from_mint: Option = None; - let mut to_mint: Option = None; - let mut user_from_token: Option = None; - let mut user_to_token: Option = None; - let mut from_vault: Option = None; - let mut to_vault: Option = None; - match_event!(event, { - BonkTradeEvent => |e: BonkTradeEvent| { - user = Some(e.payer); - from_mint = Some(e.base_token_mint); - to_mint = Some(e.quote_token_mint); - user_from_token = Some(e.user_base_token); - user_to_token = Some(e.user_quote_token); - from_vault = Some(e.base_vault); - to_vault = Some(e.quote_vault); - }, - PumpFunTradeEvent => |e: PumpFunTradeEvent| { - swap_data.from_mint = if e.is_buy { - sol_mint - } else { - e.mint - }; - swap_data.to_mint = if e.is_buy { - e.mint - } else { - sol_mint - }; - }, - PumpSwapBuyEvent => |e: PumpSwapBuyEvent| { - swap_data.from_mint = e.quote_mint; - swap_data.to_mint = e.base_mint; - }, - PumpSwapSellEvent => |e: PumpSwapSellEvent| { - swap_data.from_mint = e.base_mint; - swap_data.to_mint = e.quote_mint; - }, - RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| { - user = Some(e.payer); - from_mint = Some(e.input_token_mint); - to_mint = Some(e.output_token_mint); - user_from_token = Some(e.input_token_account); - user_to_token = Some(e.output_token_account); - from_vault = Some(e.input_vault); - to_vault = Some(e.output_vault); - }, - RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| { - user = Some(e.payer); - swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string()); - user_from_token = Some(e.input_token_account); - user_to_token = Some(e.output_token_account); - from_vault = Some(e.input_vault); - to_vault = Some(e.output_vault); - }, - RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| { - user = Some(e.payer); - from_mint = Some(e.input_vault_mint); - to_mint = Some(e.output_vault_mint); - user_from_token = Some(e.input_token_account); - user_to_token = Some(e.output_token_account); - from_vault = Some(e.input_vault); - to_vault = Some(e.output_vault); - }, - RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| { - user = Some(e.user_source_owner); - swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string()); - user_from_token = Some(e.user_source_token_account); - user_to_token = Some(e.user_destination_token_account); - from_vault = Some(e.pool_pc_token_account); - to_vault = Some(e.pool_coin_token_account); - }, - }); - for transfer_data in transfer_datas.clone() { - if transfer_data.source == user_to_token.unwrap_or_default() - && transfer_data.destination == to_vault.unwrap_or_default() - { - swap_data.from_mint = to_mint.unwrap_or_default(); - swap_data.from_amount = transfer_data.amount; - } else if transfer_data.source == from_vault.unwrap_or_default() - && transfer_data.destination == user_from_token.unwrap_or_default() - { - swap_data.to_mint = from_mint.unwrap_or_default(); - swap_data.to_amount = transfer_data.amount; - } else if transfer_data.source == user_from_token.unwrap_or_default() - && transfer_data.destination == from_vault.unwrap_or_default() - { - swap_data.from_mint = from_mint.unwrap_or_default(); - swap_data.from_amount = transfer_data.amount; - } else if transfer_data.source == to_vault.unwrap_or_default() - && transfer_data.destination == user_to_token.unwrap_or_default() - { - swap_data.to_mint = to_mint.unwrap_or_default(); - swap_data.to_amount = transfer_data.amount; + // 先根据 event 取出关键信息 + let mut user: Option = None; + let mut from_mint: Option = None; + let mut to_mint: Option = None; + let mut user_from_token: Option = None; + let mut user_to_token: Option = None; + let mut from_vault: Option = None; + let mut to_vault: Option = None; + + match_event!(&*event, { + BonkTradeEvent => |e: BonkTradeEvent| { + user = Some(e.payer); + from_mint = Some(e.base_token_mint); + to_mint = Some(e.quote_token_mint); + user_from_token = Some(e.user_base_token); + user_to_token = Some(e.user_quote_token); + from_vault = Some(e.base_vault); + to_vault = Some(e.quote_vault); + }, + PumpFunTradeEvent => |e: PumpFunTradeEvent| { + swap_data.from_mint = if e.is_buy { *SOL_MINT } else { e.mint }; + swap_data.to_mint = if e.is_buy { e.mint } else { *SOL_MINT }; + }, + PumpSwapBuyEvent => |e: PumpSwapBuyEvent| { + swap_data.from_mint = e.quote_mint; + swap_data.to_mint = e.base_mint; + }, + PumpSwapSellEvent => |e: PumpSwapSellEvent| { + swap_data.from_mint = e.base_mint; + swap_data.to_mint = e.quote_mint; + }, + RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| { + user = Some(e.payer); + from_mint = Some(e.input_token_mint); + to_mint = Some(e.output_token_mint); + user_from_token = Some(e.input_token_account); + user_to_token = Some(e.output_token_account); + from_vault = Some(e.input_vault); + to_vault = Some(e.output_vault); + }, + RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| { + user = Some(e.payer); + swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string()); + user_from_token = Some(e.input_token_account); + user_to_token = Some(e.output_token_account); + from_vault = Some(e.input_vault); + to_vault = Some(e.output_vault); + }, + RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| { + user = Some(e.payer); + from_mint = Some(e.input_vault_mint); + to_mint = Some(e.output_vault_mint); + user_from_token = Some(e.input_token_account); + user_to_token = Some(e.output_token_account); + from_vault = Some(e.input_vault); + to_vault = Some(e.output_vault); + }, + RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| { + user = Some(e.user_source_owner); + swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string()); + user_from_token = Some(e.user_source_token_account); + user_to_token = Some(e.user_destination_token_account); + from_vault = Some(e.pool_pc_token_account); + to_vault = Some(e.pool_coin_token_account); + }, + }); + + let user_to_token = user_to_token.unwrap_or_default(); + let user_from_token = user_from_token.unwrap_or_default(); + let to_vault = to_vault.unwrap_or_default(); + let from_vault = from_vault.unwrap_or_default(); + let to_mint = to_mint.unwrap_or_default(); + let from_mint = from_mint.unwrap_or_default(); + + // 单次循环完成提取和判断 + for instruction in inner_instruction.instructions.iter().skip((current_index + 1) as usize) { + let compiled = &instruction.instruction; + let program_id = accounts[compiled.program_id_index as usize]; + if !SYSTEM_PROGRAMS.contains(&program_id) { + break; + } + let data = &compiled.data; + + // 使用 SIMD 验证数据格式 + if !SimdUtils::validate_data_format(data, 8) { + continue; + } + + let get_pubkey = |i: usize| accounts[compiled.accounts[i] as usize]; + let (source, destination, amount) = match data[0] { + 12 if compiled.accounts.len() >= 4 => { + let amt = u64::from_le_bytes(data[1..9].try_into().unwrap()); + (get_pubkey(0), get_pubkey(2), amt) } + 3 if compiled.accounts.len() >= 3 => { + let amt = u64::from_le_bytes(data[1..9].try_into().unwrap()); + (get_pubkey(0), get_pubkey(1), amt) + } + 2 if compiled.accounts.len() >= 2 => { + let amt = u64::from_le_bytes(data[4..12].try_into().unwrap()); + (get_pubkey(0), get_pubkey(1), amt) + } + _ => continue, + }; + + match (source, destination) { + (s, d) if s == user_to_token && d == to_vault => { + swap_data.from_mint = to_mint; + swap_data.from_amount = amount; + } + (s, d) if s == from_vault && d == user_from_token => { + swap_data.to_mint = from_mint; + swap_data.to_amount = amount; + } + (s, d) if s == user_from_token && d == from_vault => { + swap_data.from_mint = from_mint; + swap_data.from_amount = amount; + } + (s, d) if s == to_vault && d == user_to_token => { + swap_data.to_mint = to_mint; + swap_data.to_amount = amount; + } + (s, d) if s == user_from_token && d == to_vault => { + swap_data.from_mint = from_mint; + swap_data.from_amount = amount; + } + (s, d) if s == from_vault && d == user_to_token => { + swap_data.to_mint = to_mint; + swap_data.to_amount = amount; + } + _ => {} + } + if swap_data.from_mint != Pubkey::default() && swap_data.to_mint != Pubkey::default() { + break; + } + if swap_data.from_amount != 0 && swap_data.to_amount != 0 { + break; } } + if swap_data.from_mint != Pubkey::default() || swap_data.to_mint != Pubkey::default() || swap_data.from_amount != 0 || swap_data.to_amount != 0 { - (transfer_datas, Some(swap_data)) + Some(swap_data) } else { - (transfer_datas, None) + None } } diff --git a/src/streaming/event_parser/common/utils.rs b/src/streaming/event_parser/common/utils.rs index 2d372b2..56ec1a5 100755 --- a/src/streaming/event_parser/common/utils.rs +++ b/src/streaming/event_parser/common/utils.rs @@ -1,5 +1,3 @@ -use base64::engine::general_purpose; -use base64::Engine; use std::time::{SystemTime, UNIX_EPOCH}; /// 获取当前时间戳 @@ -7,16 +5,6 @@ pub fn current_timestamp() -> i64 { SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards").as_secs() as i64 } -/// 从base64字符串解码数据 -pub fn decode_base64(data: &str) -> Result, base64::DecodeError> { - general_purpose::STANDARD.decode(data) -} - -/// 将数据编码为base64字符串 -pub fn encode_base64(data: &[u8]) -> String { - general_purpose::STANDARD.encode(data) -} - /// 从字节数组中提取鉴别器和剩余数据 pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8])> { if data.len() < length { @@ -25,15 +13,6 @@ pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8] Some((&data[..length], &data[length..])) } -/// 检查鉴别器是否匹配 - 优化版本 -pub fn discriminator_matches(data: &str, expected: &str) -> bool { - if data.len() < expected.len() { - return false; - } - // 使用字节比较而不是字符串比较,更高效 - data.as_bytes().starts_with(expected.as_bytes()) -} - /// 从日志中提取程序数据 pub fn extract_program_data(log: &str) -> Option<&str> { const PROGRAM_DATA_PREFIX: &str = "Program data: "; diff --git a/src/streaming/event_parser/core/account_event_parser.rs b/src/streaming/event_parser/core/account_event_parser.rs index 7e95698..0efa7be 100644 --- a/src/streaming/event_parser/core/account_event_parser.rs +++ b/src/streaming/event_parser/core/account_event_parser.rs @@ -1,11 +1,13 @@ +use std::borrow::Cow; use std::collections::HashMap; use std::sync::OnceLock; use solana_sdk::pubkey::Pubkey; +use crate::streaming::common::SimdUtils; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::common::{EventMetadata, EventType, ProtocolType}; -use crate::streaming::event_parser::core::traits::UnifiedEvent; +use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock}; use crate::streaming::event_parser::protocols::bonk::parser::BONK_PROGRAM_ID; use crate::streaming::event_parser::protocols::pumpfun::parser::PUMPFUN_PROGRAM_ID; use crate::streaming::event_parser::protocols::pumpswap::parser::PUMPSWAP_PROGRAM_ID; @@ -35,7 +37,10 @@ static PROTOCOL_CONFIGS_CACHE: OnceLock, event_type_filter: Option) -> Vec { + pub fn configs( + protocols: &[Protocol], + event_type_filter: Option<&EventTypeFilter>, + ) -> Vec { let protocols_map = PROTOCOL_CONFIGS_CACHE.get_or_init(|| { let mut map: HashMap> = HashMap::new(); map.insert(Protocol::PumpSwap, vec![ @@ -145,43 +150,49 @@ impl AccountEventParser { }); let mut configs = vec![]; + let empty_vec = vec![]; for protocol in protocols { - let protocol_configs = protocols_map.get(&protocol).unwrap_or(&vec![]).clone(); - let filtered_configs: Vec = protocol_configs.into_iter().filter(|config| { - event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true) - }).collect(); + let protocol_configs = protocols_map.get(protocol).unwrap_or(&empty_vec); + let filtered_configs: Vec = protocol_configs + .iter() + .filter(|config| { + event_type_filter + .map(|filter| filter.include.contains(&config.event_type)) + .unwrap_or(true) + }) + .cloned() + .collect(); configs.extend(filtered_configs); } configs } pub fn parse_account_event( - protocols: Vec, + protocols: &[Protocol], account: AccountPretty, - program_received_time_ms: i64, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> Option> { let configs = Self::configs(protocols, event_type_filter); for config in configs { - if account.owner == config.program_id.to_string() - && account.data[..config.account_discriminator.len()] - == *config.account_discriminator + if account.owner == config.program_id + && SimdUtils::fast_discriminator_match(&account.data, config.account_discriminator) { + let signature_str = Cow::Owned(account.signature.to_string()); let event = (config.account_parser)( &account, EventMetadata { slot: account.slot, - signature: account.signature.clone(), + signature: signature_str, protocol: config.protocol_type, event_type: config.event_type, program_id: config.program_id, - program_received_time_ms, + program_received_time_us: account.program_received_time_us, ..Default::default() }, ); if let Some(mut event) = event { - event.set_program_handle_time_consuming_ms( - chrono::Utc::now().timestamp_millis() - program_received_time_ms, + event.set_program_handle_time_consuming_us( + get_high_perf_clock().elapsed_micros_since(account.program_received_time_us), ); return Some(event); } diff --git a/src/streaming/event_parser/core/common_event_parser.rs b/src/streaming/event_parser/core/common_event_parser.rs index 8442446..ef44d96 100644 --- a/src/streaming/event_parser/core/common_event_parser.rs +++ b/src/streaming/event_parser/core/common_event_parser.rs @@ -1,4 +1,4 @@ -use crate::streaming::event_parser::core::traits::UnifiedEvent; +use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock}; use crate::streaming::event_parser::protocols::block::block_meta_event::BlockMetaEvent; pub struct CommonEventParser {} @@ -8,8 +8,17 @@ impl CommonEventParser { slot: u64, block_hash: &str, block_time_ms: i64, + program_received_time_us: i64, ) -> Box { - let block_meta_event = BlockMetaEvent::new(slot, block_hash.to_string(), block_time_ms); + let mut block_meta_event = BlockMetaEvent::new( + slot, + block_hash.to_string(), + block_time_ms, + program_received_time_us, + ); + block_meta_event.set_program_handle_time_consuming_us( + get_high_perf_clock().elapsed_micros_since(program_received_time_us), + ); Box::new(block_meta_event) } } diff --git a/src/streaming/event_parser/core/global_state.rs b/src/streaming/event_parser/core/global_state.rs new file mode 100644 index 0000000..c14142e --- /dev/null +++ b/src/streaming/event_parser/core/global_state.rs @@ -0,0 +1,243 @@ +use solana_sdk::pubkey::Pubkey; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use dashmap::DashMap; +use std::collections::BTreeSet; + +const MAX_SLOTS: usize = 1000; +const CLEANUP_BATCH_SIZE: usize = 100; + +/// Slot-based trader addresses, completely lock-free +#[derive(Default)] +struct SlotAddresses { + /// Developer addresses for this slot + dev_addresses: BTreeSet, + /// Bonk developer addresses for this slot + bonk_dev_addresses: BTreeSet, +} + +/// High-performance global state with lock-free slot-based storage +pub struct GlobalState { + /// Slot -> trader addresses mapping (lock-free concurrent hashmap) + slot_data: DashMap, + /// Current slot count for capacity management + slot_count: AtomicUsize, + /// Generation counter to handle cleanup races + generation: AtomicU64, +} + +impl GlobalState { + /// Create a new high-performance global state instance + pub fn new() -> Self { + Self { + slot_data: DashMap::new(), + slot_count: AtomicUsize::new(0), + generation: AtomicU64::new(0), + } + } + + /// Lock-free capacity management - cleanup old slots when limit exceeded + fn maybe_cleanup(&self) { + let current_count = self.slot_count.load(Ordering::Relaxed); + if current_count <= MAX_SLOTS { + return; + } + + // Use CAS to ensure only one thread performs cleanup + let gen = self.generation.load(Ordering::Relaxed); + if self.generation.compare_exchange_weak(gen, gen + 1, Ordering::Acquire, Ordering::Relaxed).is_err() { + return; // Another thread is cleaning up + } + + // Collect oldest slots (BTreeMap naturally orders by key) + let mut slots_to_remove: Vec = self.slot_data.iter() + .map(|entry| *entry.key()) + .collect(); + + if slots_to_remove.len() <= MAX_SLOTS { + return; // Race condition, already cleaned up + } + + slots_to_remove.sort_unstable(); + slots_to_remove.truncate(CLEANUP_BATCH_SIZE); + + // Remove old slots atomically + for slot in slots_to_remove { + self.slot_data.remove(&slot); + self.slot_count.fetch_sub(1, Ordering::Relaxed); + } + } + + /// Add developer address for a specific slot (lock-free) + pub fn add_dev_address(&self, slot: u64, address: Pubkey) { + self.maybe_cleanup(); + + self.slot_data.entry(slot) + .and_modify(|addresses| { + addresses.dev_addresses.insert(address); + }) + .or_insert_with(|| { + self.slot_count.fetch_add(1, Ordering::Relaxed); + let mut slot_addr = SlotAddresses::default(); + slot_addr.dev_addresses.insert(address); + slot_addr + }); + } + + /// Add Bonk developer address for a specific slot (lock-free) + pub fn add_bonk_dev_address(&self, slot: u64, address: Pubkey) { + self.maybe_cleanup(); + + self.slot_data.entry(slot) + .and_modify(|addresses| { + addresses.bonk_dev_addresses.insert(address); + }) + .or_insert_with(|| { + self.slot_count.fetch_add(1, Ordering::Relaxed); + let mut slot_addr = SlotAddresses::default(); + slot_addr.bonk_dev_addresses.insert(address); + slot_addr + }); + } + + /// High-performance: Check if address is a developer address in specific slot (O(log m)) + pub fn is_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool { + self.slot_data.get(&slot) + .map(|entry| entry.dev_addresses.contains(address)) + .unwrap_or(false) + } + + /// High-performance: Check if address is a Bonk developer address in specific slot (O(log m)) + pub fn is_bonk_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool { + self.slot_data.get(&slot) + .map(|entry| entry.bonk_dev_addresses.contains(address)) + .unwrap_or(false) + } + + /// Check if address is a developer address in any slot (lock-free scan, slower) + pub fn is_dev_address(&self, address: &Pubkey) -> bool { + self.slot_data.iter().any(|entry| entry.dev_addresses.contains(address)) + } + + /// Check if address is a Bonk developer address in any slot (lock-free scan, slower) + pub fn is_bonk_dev_address(&self, address: &Pubkey) -> bool { + self.slot_data.iter().any(|entry| entry.bonk_dev_addresses.contains(address)) + } + + /// Get all developer addresses from all slots (lock-free aggregation) + pub fn get_dev_addresses(&self) -> Vec { + let mut all_addresses = BTreeSet::new(); + for entry in self.slot_data.iter() { + for addr in &entry.dev_addresses { + all_addresses.insert(*addr); + } + } + all_addresses.into_iter().collect() + } + + /// Get all Bonk developer addresses from all slots (lock-free aggregation) + pub fn get_bonk_dev_addresses(&self) -> Vec { + let mut all_addresses = BTreeSet::new(); + for entry in self.slot_data.iter() { + for addr in &entry.bonk_dev_addresses { + all_addresses.insert(*addr); + } + } + all_addresses.into_iter().collect() + } + + /// Get developer addresses for a specific slot + pub fn get_dev_addresses_for_slot(&self, slot: u64) -> Vec { + self.slot_data.get(&slot) + .map(|entry| entry.dev_addresses.iter().copied().collect()) + .unwrap_or_default() + } + + /// Get Bonk developer addresses for a specific slot + pub fn get_bonk_dev_addresses_for_slot(&self, slot: u64) -> Vec { + self.slot_data.get(&slot) + .map(|entry| entry.bonk_dev_addresses.iter().copied().collect()) + .unwrap_or_default() + } + + /// Get current slot count + pub fn get_slot_count(&self) -> usize { + self.slot_count.load(Ordering::Relaxed) + } + + /// Clear all data (lock-free) + pub fn clear_all_data(&self) { + self.slot_data.clear(); + self.slot_count.store(0, Ordering::Relaxed); + self.generation.store(0, Ordering::Relaxed); + } +} + +impl Default for GlobalState { + fn default() -> Self { + Self::new() + } +} + +/// Global state instance +static GLOBAL_STATE: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(GlobalState::new); + +/// Get global state instance +pub fn get_global_state() -> &'static GlobalState { + &GLOBAL_STATE +} + +/// Convenience function: Add developer address for a specific slot +pub fn add_dev_address(slot: u64, address: Pubkey) { + get_global_state().add_dev_address(slot, address); +} + +/// Convenience function: Check if address is a developer address +pub fn is_dev_address(address: &Pubkey) -> bool { + get_global_state().is_dev_address(address) +} + +/// Convenience function: Add Bonk developer address for a specific slot +pub fn add_bonk_dev_address(slot: u64, address: Pubkey) { + get_global_state().add_bonk_dev_address(slot, address); +} + +/// Convenience function: Check if address is a Bonk developer address +pub fn is_bonk_dev_address(address: &Pubkey) -> bool { + get_global_state().is_bonk_dev_address(address) +} + +/// Convenience function: Get all developer addresses +pub fn get_dev_addresses() -> Vec { + get_global_state().get_dev_addresses() +} + +/// Convenience function: Get all Bonk developer addresses +pub fn get_bonk_dev_addresses() -> Vec { + get_global_state().get_bonk_dev_addresses() +} + +/// Convenience function: Get developer addresses for a specific slot +pub fn get_dev_addresses_for_slot(slot: u64) -> Vec { + get_global_state().get_dev_addresses_for_slot(slot) +} + +/// Convenience function: Get Bonk developer addresses for a specific slot +pub fn get_bonk_dev_addresses_for_slot(slot: u64) -> Vec { + get_global_state().get_bonk_dev_addresses_for_slot(slot) +} + +/// Convenience function: Get current slot count +pub fn get_slot_count() -> usize { + get_global_state().get_slot_count() +} + +/// High-performance: Check if address is a developer address in specific slot +pub fn is_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool { + get_global_state().is_dev_address_in_slot(slot, address) +} + +/// High-performance: Check if address is a Bonk developer address in specific slot +pub fn is_bonk_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool { + get_global_state().is_bonk_dev_address_in_slot(slot, address) +} diff --git a/src/streaming/event_parser/core/macros.rs b/src/streaming/event_parser/core/macros.rs new file mode 100644 index 0000000..724ccc0 --- /dev/null +++ b/src/streaming/event_parser/core/macros.rs @@ -0,0 +1,96 @@ +/// Macro to generate boilerplate EventParser implementation for protocol parsers +/// +/// This macro eliminates the repetitive code where each parser simply delegates +/// all EventParser trait methods to its inner GenericEventParser. +/// +/// Usage: +/// ```rust +/// impl_event_parser_delegate!(MyEventParser); +/// ``` +/// +/// This will generate the complete EventParser implementation that delegates +/// all methods to `self.inner`. +#[macro_export] +macro_rules! impl_event_parser_delegate { + ($parser_type:ty) => { + #[async_trait::async_trait] + impl $crate::streaming::event_parser::core::traits::EventParser for $parser_type { + fn instruction_configs( + &self, + ) -> std::collections::HashMap< + Vec, + Vec<$crate::streaming::event_parser::core::traits::GenericEventParseConfig>, + > { + self.inner.instruction_configs() + } + + fn parse_events_from_inner_instruction( + &self, + inner_instruction: &solana_sdk::instruction::CompiledInstruction, + signature: solana_sdk::signature::Signature, + slot: u64, + block_time: Option, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + transaction_index: Option, + config: &GenericEventParseConfig, + ) -> Vec> { + self.inner.parse_events_from_inner_instruction( + inner_instruction, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + config, + ) + } + + fn parse_events_from_instruction( + &self, + instruction: &solana_sdk::instruction::CompiledInstruction, + accounts: &[solana_sdk::pubkey::Pubkey], + signature: solana_sdk::signature::Signature, + slot: u64, + block_time: Option, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + bot_wallet: Option, + transaction_index: Option, + inner_instructions: Option<&solana_transaction_status::InnerInstructions>, + callback: std::sync::Arc< + dyn for<'a> Fn(&'a Box) + + Send + + Sync, + >, + ) -> anyhow::Result<()> { + self.inner.parse_events_from_instruction( + instruction, + accounts, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + bot_wallet, + transaction_index, + inner_instructions, + callback, + ) + } + + fn should_handle(&self, program_id: &solana_sdk::pubkey::Pubkey) -> bool { + self.inner.should_handle(program_id) + } + + fn supported_program_ids(&self) -> Vec { + self.inner.supported_program_ids() + } + } + }; +} diff --git a/src/streaming/event_parser/core/mod.rs b/src/streaming/event_parser/core/mod.rs index 27d61db..2e9f50e 100755 --- a/src/streaming/event_parser/core/mod.rs +++ b/src/streaming/event_parser/core/mod.rs @@ -1,4 +1,6 @@ pub mod common_event_parser; pub mod traits; pub mod account_event_parser; +pub mod macros; +pub mod global_state; pub use traits::{EventParser, UnifiedEvent}; diff --git a/src/streaming/event_parser/core/traits.rs b/src/streaming/event_parser/core/traits.rs index a6d8574..65c93de 100755 --- a/src/streaming/event_parser/core/traits.rs +++ b/src/streaming/event_parser/core/traits.rs @@ -1,31 +1,148 @@ -use anyhow::Result; use prost_types::Timestamp; +use solana_sdk::bs58; +use solana_sdk::signature::Signature; use solana_sdk::{ instruction::CompiledInstruction, pubkey::Pubkey, transaction::VersionedTransaction, }; use solana_transaction_status::{ - EncodedTransactionWithStatusMeta, UiCompiledInstruction, UiInnerInstructions, UiInstruction, + EncodedConfirmedTransactionWithStatusMeta, InnerInstruction, InnerInstructions, + TransactionWithStatusMeta, UiInstruction, }; +use std::borrow::Cow; +use std::collections::HashMap; use std::fmt::Debug; -use std::{collections::HashMap, str::FromStr}; +use std::sync::Arc; +use std::time::Instant; -use crate::streaming::event_parser::common::{ - parse_transfer_datas_from_next_instructions, SwapData, TransferData, +use super::global_state::{ + add_bonk_dev_address, add_dev_address, is_bonk_dev_address, is_dev_address, }; + +use crate::streaming::common::simd_utils::SimdUtils; +use crate::streaming::event_parser::common::{parse_swap_data_from_next_instructions, SwapData}; use crate::streaming::event_parser::protocols::pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}; use crate::streaming::event_parser::{ - common::{utils::*, EventMetadata, EventType, ProtocolType}, + common::{EventMetadata, EventType, ProtocolType}, protocols::{ bonk::{BonkPoolCreateEvent, BonkTradeEvent}, pumpfun::{PumpFunCreateTokenEvent, PumpFunTradeEvent}, }, }; +/// 高性能时钟管理器,减少系统调用开销 +#[derive(Debug)] +pub struct HighPerformanceClock { + /// 基准时间点(程序启动时的单调时钟时间) + base_instant: Instant, + /// 基准时间点对应的UTC时间戳(微秒) + base_timestamp_us: i64, +} + +impl HighPerformanceClock { + /// 创建新的高性能时钟 + pub fn new() -> Self { + let base_instant = Instant::now(); + let base_timestamp_us = chrono::Utc::now().timestamp_micros(); + + Self { base_instant, base_timestamp_us } + } + + /// 获取当前时间戳(微秒),使用单调时钟计算,避免系统调用 + #[inline(always)] + pub fn now_micros(&self) -> i64 { + let elapsed = self.base_instant.elapsed(); + self.base_timestamp_us + elapsed.as_micros() as i64 + } + + /// 计算从指定时间戳到现在的消耗时间(微秒) + #[inline(always)] + pub fn elapsed_micros_since(&self, start_timestamp_us: i64) -> i64 { + self.now_micros() - start_timestamp_us + } +} + +impl Default for HighPerformanceClock { + fn default() -> Self { + Self::new() + } +} + +/// 全局高性能时钟实例(使用OnceCell避免重复初始化) +static HIGH_PERF_CLOCK: once_cell::sync::OnceCell = + once_cell::sync::OnceCell::new(); + +/// 获取全局高性能时钟实例 +#[inline(always)] +pub fn get_high_perf_clock() -> &'static HighPerformanceClock { + HIGH_PERF_CLOCK.get_or_init(HighPerformanceClock::new) +} + +/// 轻量级事件包装器,避免频繁的Box分配 +#[derive(Debug)] +pub struct EventWrapper { + pub event: T, +} + +impl EventWrapper { + #[inline] + pub fn new(event: T) -> Self { + Self { event } + } + + #[inline] + pub fn into_boxed(self) -> Box { + Box::new(self.event) + } +} + +/// 高性能账户公钥缓存,避免重复Vec分配 +#[derive(Debug)] +pub struct AccountPubkeyCache { + /// 预分配的账户公钥向量,避免每次重新分配 + cache: Vec, +} + +impl AccountPubkeyCache { + /// 创建新的账户公钥缓存 + pub fn new() -> Self { + Self { + cache: Vec::with_capacity(32), // 预分配32个位置,覆盖大多数交易 + } + } + + /// 从指令账户索引构建账户公钥向量,重用缓存内存 + #[inline] + pub fn build_account_pubkeys( + &mut self, + instruction_accounts: &[u8], + all_accounts: &[Pubkey], + ) -> &[Pubkey] { + self.cache.clear(); + + // 确保容量足够,避免动态扩容 + if self.cache.capacity() < instruction_accounts.len() { + self.cache.reserve(instruction_accounts.len() - self.cache.capacity()); + } + + // 快速填充账户公钥 + for &idx in instruction_accounts.iter() { + if (idx as usize) < all_accounts.len() { + self.cache.push(all_accounts[idx as usize]); + } + } + + &self.cache + } +} + +impl Default for AccountPubkeyCache { + fn default() -> Self { + Self::new() + } +} + /// Unified Event Interface - All protocol events must implement this trait pub trait UnifiedEvent: Debug + Send + Sync { - /// Get event ID - fn id(&self) -> &str; - /// Get event type fn event_type(&self) -> EventType; @@ -36,13 +153,13 @@ pub trait UnifiedEvent: Debug + Send + Sync { fn slot(&self) -> u64; /// Get program received timestamp (milliseconds) - fn program_received_time_ms(&self) -> i64; + fn program_received_time_us(&self) -> i64; /// Processing time consumption (milliseconds) - fn program_handle_time_consuming_ms(&self) -> i64; + fn program_handle_time_consuming_us(&self) -> i64; /// Set processing time consumption (milliseconds) - fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64); + fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64); /// Convert event to Any for downcasting fn as_any(&self) -> &dyn std::any::Any; @@ -54,38 +171,42 @@ pub trait UnifiedEvent: Debug + Send + Sync { fn clone_boxed(&self) -> Box; /// Merge events (optional implementation) - fn merge(&mut self, _other: Box) { + fn merge(&mut self, _other: &dyn UnifiedEvent) { // Default implementation: no merging operation } - /// Set transfer datas - fn set_transfer_datas( - &mut self, - transfer_datas: Vec, - swap_data: Option, - ); + /// Set swap data + fn set_swap_data(&mut self, swap_data: SwapData); + + /// swap_data is parsed + fn swap_data_is_parsed(&self) -> bool; /// Get index - fn index(&self) -> String; + fn instruction_outer_index(&self) -> i64; + fn instruction_inner_index(&self) -> Option; + + /// Get transaction index in slot + fn transaction_index(&self) -> Option; } /// 事件解析器trait - 定义了事件解析的核心方法 #[async_trait::async_trait] pub trait EventParser: Send + Sync { - /// 获取内联指令解析配置 - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec>; /// 获取指令解析配置 fn instruction_configs(&self) -> HashMap, Vec>; /// 从内联指令中解析事件数据 #[allow(clippy::too_many_arguments)] fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, - index: String, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + transaction_index: Option, + config: &GenericEventParseConfig, ) -> Vec>; /// 从指令中解析事件数据 @@ -94,31 +215,36 @@ pub trait EventParser: Send + Sync { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec>; + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + bot_wallet: Option, + transaction_index: Option, + inner_instructions: Option<&InnerInstructions>, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()>; /// 从VersionedTransaction中解析指令事件的通用方法 #[allow(clippy::too_many_arguments)] async fn parse_instruction_events_from_versioned_transaction( &self, transaction: &VersionedTransaction, - signature: &str, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, accounts: &[Pubkey], - inner_instructions: &[UiInnerInstructions], - ) -> Result>> { - // 预分配容量,避免动态扩容 - let mut instruction_events = Vec::with_capacity(16); + inner_instructions: &[InnerInstructions], + bot_wallet: Option, + transaction_index: Option, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { // 获取交易的指令和账户 let compiled_instructions = transaction.message.instructions(); let mut accounts: Vec = accounts.to_vec(); - // 检查交易中是否包含程序 let has_program = accounts.iter().any(|account| self.should_handle(account)); if has_program { @@ -133,365 +259,347 @@ pub trait EventParser: Send + Sync { accounts.push(Pubkey::default()); } } - if let Ok(mut events) = self - .parse_instruction( - instruction, - &accounts, - signature, - slot, - block_time, - program_received_time_ms, - format!("{index}"), - ) - .await - { - if !events.is_empty() { - if let Some(inn) = - inner_instructions.iter().find(|inner_instruction| { - inner_instruction.index == index as u8 - }) - { - events.iter_mut().for_each(|event| { - let (transfer_datas, swap_data) = - parse_transfer_datas_from_next_instructions( - event.clone_boxed(), - inn, - -1_i8, - &accounts, - ); - event.set_transfer_datas(transfer_datas, swap_data); - }); - } - instruction_events.extend(events); - } - } + let inner_instructions = inner_instructions + .iter() + .find(|inner_instruction| inner_instruction.index == index as u8); + self.parse_instruction( + instruction, + &accounts, + signature, + slot, + block_time, + program_received_time_us, + index as i64, + None, + bot_wallet, + transaction_index, + inner_instructions, + Arc::clone(&callback), + ) + .await?; } } } } - Ok(instruction_events) + Ok(()) + } + + async fn parse_versioned_transaction_owned( + &self, + versioned_tx: VersionedTransaction, + signature: Signature, + slot: Option, + block_time: Option, + program_received_time_us: i64, + bot_wallet: Option, + transaction_index: Option, + inner_instructions: &[InnerInstructions], + callback: Arc) + Send + Sync>, + ) -> anyhow::Result<()> { + // 创建适配器回调,将所有权回调转换为引用回调 + let adapter_callback = Arc::new(move |event: &Box| { + callback(event.clone_boxed()); + }); + self.parse_versioned_transaction( + &versioned_tx, + signature, + slot, + block_time, + program_received_time_us, + bot_wallet, + transaction_index, + inner_instructions, + adapter_callback, + ) + .await?; + Ok(()) } async fn parse_versioned_transaction( &self, versioned_tx: &VersionedTransaction, - signature: &str, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, bot_wallet: Option, - ) -> Result>> { + transaction_index: Option, + inner_instructions: &[InnerInstructions], + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { let accounts: Vec = versioned_tx.message.static_account_keys().to_vec(); - let events = self - .parse_instruction_events_from_versioned_transaction( - versioned_tx, - signature, - slot, - block_time, - program_received_time_ms, - &accounts, - &[], - ) - .await - .unwrap_or_else(|_e| vec![]); - Ok(self.process_events(events, bot_wallet)) + self.parse_instruction_events_from_versioned_transaction( + versioned_tx, + signature, + slot, + block_time, + program_received_time_us, + &accounts, + inner_instructions, + bot_wallet, + transaction_index, + callback, + ) + .await?; + Ok(()) + } + + /// 解析交易,使用所有权语义的回调以避免不必要的克隆 + async fn parse_transaction_owned( + &self, + tx: TransactionWithStatusMeta, + signature: Signature, + slot: Option, + block_time: Option, + program_received_time_us: i64, + bot_wallet: Option, + transaction_index: Option, + callback: Arc) + Send + Sync>, + ) -> anyhow::Result<()> { + // 创建适配器回调,将所有权回调转换为引用回调 + let adapter_callback = Arc::new(move |event: &Box| { + callback(event.clone_boxed()); + }); + // 调用原始方法 + self.parse_transaction( + tx, + signature, + slot, + block_time, + program_received_time_us, + bot_wallet, + transaction_index, + adapter_callback, + ) + .await } async fn parse_transaction( &self, - tx: EncodedTransactionWithStatusMeta, - signature: &str, + tx: TransactionWithStatusMeta, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, bot_wallet: Option, - ) -> Result>> { - // TODO: bug - 待优化 - // // 生成缓存键 - // let cache_key = format!("{}_{}_{}", signature, slot.unwrap_or(0), program_received_time_ms); - - // // 尝试从缓存获取 - // if let Some(cached_events) = PARSE_CACHE.get(&cache_key).await { - // return Ok(cached_events); - // } - - let transaction = tx.transaction; - // 检查交易元数据 - let meta = - tx.meta.as_ref().ok_or_else(|| anyhow::anyhow!("Missing transaction metadata"))?; - + transaction_index: Option, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { + let versioned_tx = tx.get_transaction(); + let meta = tx.get_status_meta(); let mut address_table_lookups: Vec = vec![]; - let mut inner_instructions: Vec = vec![]; - if meta.err.is_none() { - // 正确处理OptionSerializer类型 + let mut inner_instructions: Vec = vec![]; + if let Some(meta) = meta { + inner_instructions = meta.inner_instructions.unwrap_or_default(); + address_table_lookups.reserve( + meta.loaded_addresses.writable.len() + meta.loaded_addresses.readonly.len(), + ); + address_table_lookups.extend( + meta.loaded_addresses.writable.into_iter().chain(meta.loaded_addresses.readonly), + ); + } + let mut accounts = Vec::with_capacity( + versioned_tx.message.static_account_keys().len() + address_table_lookups.len(), + ); + accounts.extend_from_slice(versioned_tx.message.static_account_keys()); + accounts.extend(address_table_lookups); + // 使用 Arc 包装共享数据,避免不必要的克隆 + let accounts_arc = Arc::new(accounts); + let inner_instructions_arc = Arc::new(inner_instructions); + // 解析指令事件 + self.parse_instruction_events_from_versioned_transaction( + &versioned_tx, + signature, + slot, + block_time, + program_received_time_us, + &accounts_arc, + &inner_instructions_arc, + bot_wallet, + transaction_index, + callback.clone(), + ) + .await?; + + // 解析嵌套指令事件 + for inner_instruction in inner_instructions_arc.iter() { + for (index, instruction) in inner_instruction.instructions.iter().enumerate() { + self.parse_instruction( + &instruction.instruction, + &accounts_arc, + signature, + slot, + block_time, + program_received_time_us, + inner_instruction.index as i64, + Some(index as i64), + bot_wallet, + transaction_index, + Some(&inner_instruction), + callback.clone(), + ) + .await?; + } + } + + Ok(()) + } + + async fn parse_encoded_confirmed_transaction_with_status_meta( + &self, + signature: Signature, + transaction: EncodedConfirmedTransactionWithStatusMeta, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { + let versioned_tx = match transaction.transaction.transaction.decode() { + Some(tx) => tx, + None => { + return Ok(()); + } + }; + let mut inner_instructions_vec: Vec = Vec::new(); + if let Some(meta) = &transaction.transaction.meta { + // 从meta中获取inner_instructions,处理OptionSerializer类型 if let solana_transaction_status::option_serializer::OptionSerializer::Some( - meta_inner_instructions, + ui_inner_insts, ) = &meta.inner_instructions { - inner_instructions = meta_inner_instructions.clone(); + // 将UiInnerInstructions转换为InnerInstructions + for ui_inner in ui_inner_insts { + let mut converted_instructions = Vec::new(); + + // 转换每个UiInstruction为InnerInstruction + for ui_instruction in &ui_inner.instructions { + if let UiInstruction::Compiled(ui_compiled) = ui_instruction { + // 解码base58编码的data + if let Ok(data) = bs58::decode(&ui_compiled.data).into_vec() { + // base64解码 + let compiled_instruction = CompiledInstruction { + program_id_index: ui_compiled.program_id_index, + accounts: ui_compiled.accounts.clone(), + data, + }; + + let inner_instruction = InnerInstruction { + instruction: compiled_instruction, + stack_height: ui_compiled.stack_height, + }; + + converted_instructions.push(inner_instruction); + } + } + } + + let inner_instructions = InnerInstructions { + index: ui_inner.index, + instructions: converted_instructions, + }; + + inner_instructions_vec.push(inner_instructions); + } } + } + let inner_instructions: &[InnerInstructions] = &inner_instructions_vec; + + let meta = transaction.transaction.meta; + let mut address_table_lookups: Vec = vec![]; + if let Some(meta) = meta { if let solana_transaction_status::option_serializer::OptionSerializer::Some( loaded_addresses, ) = &meta.loaded_addresses { - for lookup in &loaded_addresses.writable { - if let Ok(pubkey) = Pubkey::from_str(lookup) { - address_table_lookups.push(pubkey); - } - } - for lookup in &loaded_addresses.readonly { - if let Ok(pubkey) = Pubkey::from_str(lookup) { - address_table_lookups.push(pubkey); - } - } + address_table_lookups + .reserve(loaded_addresses.writable.len() + loaded_addresses.readonly.len()); + address_table_lookups.extend( + loaded_addresses + .writable + .iter() + .filter_map(|s| s.parse::().ok()) + .chain( + loaded_addresses + .readonly + .iter() + .filter_map(|s| s.parse::().ok()), + ), + ); } } - let mut accounts: Vec = vec![]; - - // 预分配容量,避免动态扩容 - let mut instruction_events = Vec::with_capacity(16); + let mut accounts = Vec::with_capacity( + versioned_tx.message.static_account_keys().len() + address_table_lookups.len(), + ); + accounts.extend_from_slice(versioned_tx.message.static_account_keys()); + accounts.extend(address_table_lookups); + // 使用 Arc 包装共享数据,避免不必要的克隆 + let accounts_arc = Arc::new(accounts); + let inner_instructions_arc = Arc::new(inner_instructions); + let slot = transaction.slot; + let block_time = transaction.block_time.map(|t| Timestamp { seconds: t as i64, nanos: 0 }); + let program_received_time_us = chrono::Utc::now().timestamp_micros(); + let bot_wallet = None; + let transaction_index = None; // 解析指令事件 - if let Some(versioned_tx) = transaction.decode() { - accounts = versioned_tx.message.static_account_keys().to_vec(); - accounts.extend(address_table_lookups.clone()); + self.parse_instruction_events_from_versioned_transaction( + &versioned_tx, + signature, + Some(slot), + block_time, + program_received_time_us, + &accounts_arc, + &inner_instructions_arc, + bot_wallet, + transaction_index, + callback.clone(), + ) + .await?; - instruction_events = self - .parse_instruction_events_from_versioned_transaction( - &versioned_tx, + // 解析嵌套指令事件 + for inner_instruction in inner_instructions_arc.iter() { + for (index, instruction) in inner_instruction.instructions.iter().enumerate() { + self.parse_instruction( + &instruction.instruction, + &accounts_arc, signature, - slot, + Some(slot), block_time, - program_received_time_ms, - &accounts, - &inner_instructions, + program_received_time_us, + inner_instruction.index as i64, + Some(index as i64), + bot_wallet, + transaction_index, + Some(&inner_instruction), + callback.clone(), ) - .await - .unwrap_or_else(|_e| vec![]); - } else { - accounts.extend(address_table_lookups.clone()); - } - - // 解析内联指令事件 - // 预分配容量,避免动态扩容 - let mut inner_instruction_events = Vec::with_capacity(8); - // 检查交易是否成功 - if meta.err.is_none() { - for inner_instruction in inner_instructions { - for (index, instruction) in inner_instruction.instructions.iter().enumerate() { - if let UiInstruction::Compiled(compiled) = instruction { - // 解析嵌套指令 - let compiled_instruction = CompiledInstruction { - program_id_index: compiled.program_id_index, - accounts: compiled.accounts.clone(), - data: bs58::decode(compiled.data.clone()) - .into_vec() - .unwrap_or_else(|_| vec![]), - }; - if let Ok(mut events) = self - .parse_instruction( - &compiled_instruction, - &accounts, - signature, - slot, - block_time, - program_received_time_ms, - format!("{}.{}", inner_instruction.index, index), - ) - .await - { - if !events.is_empty() { - events.iter_mut().for_each(|event| { - let (transfer_datas, swap_data) = - parse_transfer_datas_from_next_instructions( - event.clone_boxed(), - &inner_instruction, - index as i8, - &accounts, - ); - event.set_transfer_datas(transfer_datas, swap_data); - }); - instruction_events.extend(events); - } - } - if let Ok(mut events) = self - .parse_inner_instruction( - compiled, - signature, - slot, - block_time, - program_received_time_ms, - format!("{}.{}", inner_instruction.index, index), - ) - .await - { - if !events.is_empty() { - events.iter_mut().for_each(|event| { - let (transfer_datas, swap_data) = - parse_transfer_datas_from_next_instructions( - event.clone_boxed(), - &inner_instruction, - index as i8, - &accounts, - ); - event.set_transfer_datas(transfer_datas, swap_data); - }); - inner_instruction_events.extend(events); - } - } - } - } + .await?; } } - if !instruction_events.is_empty() && !inner_instruction_events.is_empty() { - for instruction_event in &mut instruction_events { - for inner_instruction_event in &inner_instruction_events { - if instruction_event.id() == inner_instruction_event.id() { - let i_index = instruction_event.index(); - let in_index = inner_instruction_event.index(); - if !i_index.contains(".") && in_index.contains(".") { - let in_index_parts: Vec<&str> = in_index.split(".").collect(); - if !in_index_parts.is_empty() && in_index_parts[0] == i_index { - instruction_event.merge(inner_instruction_event.clone_boxed()); - break; - } - } else if i_index.contains(".") && in_index.contains(".") { - // 嵌套指令 - let i_index_parts: Vec<&str> = i_index.split(".").collect(); - let in_index_parts: Vec<&str> = in_index.split(".").collect(); - - if !i_index_parts.is_empty() - && !in_index_parts.is_empty() - && i_index_parts[0] == in_index_parts[0] - { - let i_index_child_index = i_index_parts - .get(1) - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - let in_index_child_index = in_index_parts - .get(1) - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - if in_index_child_index > i_index_child_index { - instruction_event.merge(inner_instruction_event.clone_boxed()); - break; - } - } - } - } - } - } - } - - let result = self.process_events(instruction_events, bot_wallet); - - // 缓存结果 - // PARSE_CACHE.set(cache_key, result.clone()).await; - - Ok(result) - } - - fn process_events( - &self, - mut events: Vec>, - bot_wallet: Option, - ) -> Vec> { - let start_time = std::time::Instant::now(); - let mut dev_address = vec![]; - let mut bonk_dev_address = None; - for event in &mut events { - if let Some(token_info) = event.as_any().downcast_ref::() { - dev_address.push(token_info.user); - if token_info.creator != Pubkey::default() && token_info.creator != token_info.user - { - dev_address.push(token_info.creator); - } - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() - { - if dev_address.contains(&trade_info.user) - || dev_address.contains(&trade_info.creator) - { - trade_info.is_dev_create_token_trade = true; - } else if Some(trade_info.user) == bot_wallet { - trade_info.is_bot = true; - } else { - trade_info.is_dev_create_token_trade = false; - } - if trade_info.metadata.swap_data.is_some() { - trade_info.metadata.swap_data.as_mut().unwrap().from_amount = - if trade_info.is_buy { - trade_info.sol_amount - } else { - trade_info.token_amount - }; - trade_info.metadata.swap_data.as_mut().unwrap().to_amount = if trade_info.is_buy - { - trade_info.token_amount - } else { - trade_info.sol_amount - }; - } - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { - if trade_info.metadata.swap_data.is_some() { - trade_info.metadata.swap_data.as_mut().unwrap().from_amount = - trade_info.user_quote_amount_in; - trade_info.metadata.swap_data.as_mut().unwrap().to_amount = - trade_info.base_amount_out; - } - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() - { - if trade_info.metadata.swap_data.is_some() { - trade_info.metadata.swap_data.as_mut().unwrap().from_amount = - trade_info.base_amount_in; - trade_info.metadata.swap_data.as_mut().unwrap().to_amount = - trade_info.user_quote_amount_out; - } - } else if let Some(pool_info) = event.as_any().downcast_ref::() { - bonk_dev_address = Some(pool_info.creator); - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { - if Some(trade_info.payer) == bonk_dev_address { - trade_info.is_dev_create_token_trade = true; - } else if Some(trade_info.payer) == bot_wallet { - trade_info.is_bot = true; - } else { - trade_info.is_dev_create_token_trade = false; - } - } - let now = chrono::Utc::now().timestamp_millis(); - event.set_program_handle_time_consuming_ms(now - event.program_received_time_ms()); - } - - // 记录处理时间 - let processing_time = start_time.elapsed(); - if processing_time.as_millis() > 10 { - log::warn!( - "Event processing took {}ms for {} events", - processing_time.as_millis(), - events.len() - ); - } - - events + Ok(()) } async fn parse_inner_instruction( &self, - instruction: &UiCompiledInstruction, - signature: &str, + instruction: &CompiledInstruction, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Result>> { + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + transaction_index: Option, + config: &GenericEventParseConfig, + ) -> anyhow::Result>> { let slot = slot.unwrap_or(0); let events = self.parse_events_from_inner_instruction( instruction, signature, slot, block_time, - program_received_time_ms, - index, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + config, ); Ok(events) } @@ -501,23 +609,32 @@ pub trait EventParser: Send + Sync { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Result>> { + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + bot_wallet: Option, + transaction_index: Option, + inner_instructions: Option<&InnerInstructions>, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { let slot = slot.unwrap_or(0); - let events = self.parse_events_from_instruction( + self.parse_events_from_instruction( instruction, accounts, signature, slot, block_time, - program_received_time_ms, - index, - ); - Ok(events) + program_received_time_us, + outer_index, + inner_index, + bot_wallet, + transaction_index, + inner_instructions, + callback, + ) } /// 检查是否应该处理此程序ID @@ -539,7 +656,7 @@ impl Clone for Box { pub struct GenericEventParseConfig { pub program_id: Pubkey, pub protocol_type: ProtocolType, - pub inner_instruction_discriminator: &'static str, + pub inner_instruction_discriminator: &'static [u8], pub instruction_discriminator: &'static [u8], pub event_type: EventType, pub inner_instruction_parser: Option, @@ -557,29 +674,29 @@ pub type InstructionEventParser = /// 通用事件解析器基类 pub struct GenericEventParser { pub program_ids: Vec, - pub inner_instruction_configs: HashMap<&'static str, Vec>, + // pub inner_instruction_configs: HashMap, Vec>, pub instruction_configs: HashMap, Vec>, + /// 账户公钥缓存,避免重复分配 + pub account_cache: parking_lot::Mutex, } impl GenericEventParser { /// 创建新的通用事件解析器 pub fn new(program_ids: Vec, configs: Vec) -> Self { // 预分配容量,避免动态扩容 - let mut inner_instruction_configs = HashMap::with_capacity(configs.len()); let mut instruction_configs = HashMap::with_capacity(configs.len()); for config in configs { - inner_instruction_configs - .entry(config.inner_instruction_discriminator) - .or_insert_with(Vec::new) - .push(config.clone()); instruction_configs .entry(config.instruction_discriminator.to_vec()) .or_insert_with(Vec::new) .push(config.clone()); } - Self { program_ids, inner_instruction_configs, instruction_configs } + // 初始化账户缓存 + let account_cache = parking_lot::Mutex::new(AccountPubkeyCache::new()); + + Self { program_ids, instruction_configs, account_cache } } /// 通用的内联指令解析方法 @@ -588,26 +705,30 @@ impl GenericEventParser { &self, config: &GenericEventParseConfig, data: &[u8], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, - index: String, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + transaction_index: Option, ) -> Option> { if let Some(parser) = config.inner_instruction_parser { + let signature_str = Cow::Owned(signature.to_string()); let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); let block_time_ms = timestamp.seconds * 1000 + (timestamp.nanos as i64) / 1_000_000; let metadata = EventMetadata::new( - signature.to_string(), - signature.to_string(), + signature_str, slot, timestamp.seconds, block_time_ms, config.protocol_type.clone(), config.event_type.clone(), config.program_id, - index, - program_received_time_ms, + outer_index, + inner_index, + program_received_time_us, + transaction_index, ); parser(data, metadata) } else { @@ -622,26 +743,30 @@ impl GenericEventParser { config: &GenericEventParseConfig, data: &[u8], account_pubkeys: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, - index: String, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + transaction_index: Option, ) -> Option> { if let Some(parser) = config.instruction_parser { + let signature_str = Cow::Owned(signature.to_string()); let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); let block_time_ms = timestamp.seconds * 1000 + (timestamp.nanos as i64) / 1_000_000; let metadata = EventMetadata::new( - signature.to_string(), - signature.to_string(), + signature_str, slot, timestamp.seconds, block_time_ms, config.protocol_type.clone(), config.event_type.clone(), config.program_id, - index, - program_received_time_ms, + outer_index, + inner_index, + program_received_time_us, + transaction_index, ); parser(data, account_pubkeys, metadata) } else { @@ -652,9 +777,6 @@ impl GenericEventParser { #[async_trait::async_trait] impl EventParser for GenericEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner_instruction_configs.clone() - } fn instruction_configs(&self) -> HashMap, Vec> { self.instruction_configs.clone() } @@ -662,39 +784,34 @@ impl EventParser for GenericEventParser { #[allow(clippy::too_many_arguments)] fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, - index: String, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + transaction_index: Option, + config: &GenericEventParseConfig, ) -> Vec> { - let inner_instruction_data = inner_instruction.data.clone(); - let inner_instruction_data_decoded = - bs58::decode(inner_instruction_data).into_vec().unwrap_or_else(|_| vec![]); - if inner_instruction_data_decoded.len() < 16 { + // Use SIMD-optimized data validation + if !SimdUtils::validate_instruction_data_simd(&inner_instruction.data, 16, 0) { return Vec::new(); } - let inner_instruction_data_decoded_str = - format!("0x{}", hex::encode(&inner_instruction_data_decoded)); - let data = &inner_instruction_data_decoded[16..]; + let data = &inner_instruction.data[16..]; let mut events = Vec::new(); - for (disc, configs) in &self.inner_instruction_configs { - if discriminator_matches(&inner_instruction_data_decoded_str, disc) { - for config in configs { - if let Some(event) = self.parse_inner_instruction_event( - config, - data, - signature, - slot, - block_time, - program_received_time_ms, - index.clone(), - ) { - events.push(event); - } - } - } + if let Some(event) = self.parse_inner_instruction_event( + config, + data, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + ) { + events.push(event); } events } @@ -705,52 +822,132 @@ impl EventParser for GenericEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + bot_wallet: Option, + transaction_index: Option, + inner_instructions: Option<&InnerInstructions>, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { let program_id = accounts[instruction.program_id_index as usize]; if !self.should_handle(&program_id) { - return Vec::new(); + return Ok(()); } - let mut events = Vec::new(); - for (disc, configs) in &self.instruction_configs { - if instruction.data.len() < disc.len() { - continue; - } - let discriminator = &instruction.data[..disc.len()]; - let data = &instruction.data[disc.len()..]; - if discriminator == disc { - // 验证账户索引 - if !validate_account_indices(&instruction.accounts, accounts.len()) { - continue; - } + // 一维化并行处理:将所有 (discriminator, config) 组合展开并行处理 + let all_processing_params: Vec<_> = self + .instruction_configs + .iter() + .filter(|(disc, _)| { + // Use SIMD-optimized data validation and discriminator matching + SimdUtils::validate_instruction_data_simd(&instruction.data, disc.len(), disc.len()) + && SimdUtils::fast_discriminator_match(&instruction.data, disc) + }) + .flat_map(|(disc, configs)| { + configs + .iter() + .filter(|config| config.program_id == program_id) + .map(move |config| (disc, config)) + }) + .collect(); - let account_pubkeys: Vec = - instruction.accounts.iter().map(|&idx| accounts[idx as usize]).collect(); - for config in configs { - if config.program_id != program_id { - continue; - } - if let Some(event) = self.parse_instruction_event( - config, - data, - &account_pubkeys, - signature, - slot, - block_time, - program_received_time_ms, - index.clone(), - ) { - events.push(event); - } - } - } + // Use SIMD-optimized account indices validation (只需检查一次) + if !SimdUtils::validate_account_indices_simd(&instruction.accounts, accounts.len()) { + return Ok(()); } - events + // 使用缓存构建账户公钥列表,避免重复分配 (只需构建一次) + let account_pubkeys = { + let mut cache_guard = self.account_cache.lock(); + cache_guard.build_account_pubkeys(&instruction.accounts, accounts).to_vec() + }; + + // 并行处理所有 (discriminator, config) 组合 + let all_results: Vec<_> = all_processing_params + .iter() + .filter_map(|(disc, config)| { + let data = &instruction.data[disc.len()..]; + self.parse_instruction_event( + config, + data, + &account_pubkeys, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + ) + .map(|event| ((*disc).clone(), (*config).clone(), event)) + }) + .collect(); + + for (_disc, config, mut event) in all_results { + // 阻塞处理:原有的同步逻辑 + let mut inner_instruction_event: Option> = None; + if inner_instructions.is_some() { + let inner_instructions_ref = inner_instructions.unwrap(); + + // 并行执行两个任务 + let (inner_event_result, swap_data_result) = std::thread::scope(|s| { + let inner_event_handle = s.spawn(|| { + for inner_instruction in inner_instructions_ref.instructions.iter() { + let result = self.parse_events_from_inner_instruction( + &inner_instruction.instruction, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + &config, + ); + if result.len() > 0 { + return Some(result[0].clone()); + } + } + None + }); + + let swap_data_handle = s.spawn(|| { + if !event.swap_data_is_parsed() { + parse_swap_data_from_next_instructions( + &*event, + inner_instructions_ref, + inner_index.unwrap_or(-1_i64) as i8, + &accounts, + ) + } else { + None + } + }); + + // 等待两个任务完成 + (inner_event_handle.join().unwrap(), swap_data_handle.join().unwrap()) + }); + + inner_instruction_event = inner_event_result; + if let Some(swap_data) = swap_data_result { + event.set_swap_data(swap_data); + } + } + // 合并事件 + if let Some(inner_instruction_event) = inner_instruction_event { + event.merge(&*inner_instruction_event); + } + // 设置处理时间(使用高性能时钟) + event.set_program_handle_time_consuming_us( + get_high_perf_clock().elapsed_micros_since(program_received_time_us), + ); + event = process_event(event, bot_wallet); + callback(&event); + } + Ok(()) } fn should_handle(&self, program_id: &Pubkey) -> bool { @@ -761,3 +958,53 @@ impl EventParser for GenericEventParser { self.program_ids.clone() } } + +fn process_event( + mut event: Box, + bot_wallet: Option, +) -> Box { + let slot = event.slot(); + if let Some(token_info) = event.as_any().downcast_ref::() { + add_dev_address(slot, token_info.user); + if token_info.creator != Pubkey::default() && token_info.creator != token_info.user { + add_dev_address(slot, token_info.creator); + } + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if is_dev_address(&trade_info.user) || is_dev_address(&trade_info.creator) { + trade_info.is_dev_create_token_trade = true; + } else if Some(trade_info.user) == bot_wallet { + trade_info.is_bot = true; + } else { + trade_info.is_dev_create_token_trade = false; + } + if trade_info.metadata.swap_data.is_some() { + trade_info.metadata.swap_data.as_mut().unwrap().from_amount = + if trade_info.is_buy { trade_info.sol_amount } else { trade_info.token_amount }; + trade_info.metadata.swap_data.as_mut().unwrap().to_amount = + if trade_info.is_buy { trade_info.token_amount } else { trade_info.sol_amount }; + } + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if trade_info.metadata.swap_data.is_some() { + trade_info.metadata.swap_data.as_mut().unwrap().from_amount = + trade_info.user_quote_amount_in; + trade_info.metadata.swap_data.as_mut().unwrap().to_amount = trade_info.base_amount_out; + } + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if trade_info.metadata.swap_data.is_some() { + trade_info.metadata.swap_data.as_mut().unwrap().from_amount = trade_info.base_amount_in; + trade_info.metadata.swap_data.as_mut().unwrap().to_amount = + trade_info.user_quote_amount_out; + } + } else if let Some(pool_info) = event.as_any().downcast_ref::() { + add_bonk_dev_address(slot, pool_info.creator); + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if is_bonk_dev_address(&trade_info.payer) { + trade_info.is_dev_create_token_trade = true; + } else if Some(trade_info.payer) == bot_wallet { + trade_info.is_bot = true; + } else { + trade_info.is_dev_create_token_trade = false; + } + } + event +} diff --git a/src/streaming/event_parser/protocols/block/block_meta_event.rs b/src/streaming/event_parser/protocols/block/block_meta_event.rs index f4bee6d..8fe4051 100644 --- a/src/streaming/event_parser/protocols/block/block_meta_event.rs +++ b/src/streaming/event_parser/protocols/block/block_meta_event.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use crate::impl_unified_event; use crate::streaming::event_parser::common::{types::EventType, EventMetadata}; use borsh::BorshDeserialize; @@ -13,18 +15,24 @@ pub struct BlockMetaEvent { } impl BlockMetaEvent { - pub fn new(slot: u64, block_hash: String, block_time_ms: i64) -> Self { + pub fn new( + slot: u64, + block_hash: String, + block_time_ms: i64, + program_received_time_us: i64, + ) -> Self { let metadata = EventMetadata::new( - format!("block_{}_{}", slot, block_hash), - "".to_string(), + Cow::Borrowed(""), slot, block_time_ms / 1000, block_time_ms, crate::streaming::event_parser::common::types::ProtocolType::Common, EventType::BlockMeta, solana_sdk::pubkey::Pubkey::default(), - "".to_string(), - chrono::Utc::now().timestamp_millis(), + 0, + None, + program_received_time_us, + None, ); Self { metadata, slot, block_hash } } diff --git a/src/streaming/event_parser/protocols/bonk/events.rs b/src/streaming/event_parser/protocols/bonk/events.rs index e2a2d50..cb450ae 100755 --- a/src/streaming/event_parser/protocols/bonk/events.rs +++ b/src/streaming/event_parser/protocols/bonk/events.rs @@ -314,8 +314,12 @@ impl_unified_event!(BonkPlatformConfigAccountEvent,); /// Event discriminator constants pub mod discriminators { // Event discriminators - pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; - pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae"; + // pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; + pub const TRADE_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238]; + // pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae"; + pub const POOL_CREATE_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 151, 215, 226, 9, 118, 161, 115, 174]; // Instruction discriminators pub const BUY_EXACT_IN: &[u8] = &[250, 234, 13, 123, 213, 156, 19, 236]; diff --git a/src/streaming/event_parser/protocols/bonk/parser.rs b/src/streaming/event_parser/protocols/bonk/parser.rs index 056b39d..e16fc8f 100755 --- a/src/streaming/event_parser/protocols/bonk/parser.rs +++ b/src/streaming/event_parser/protocols/bonk/parser.rs @@ -1,14 +1,16 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; - -use crate::streaming::event_parser::{ - common::{utils::*, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::bonk::{ - bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection, VestingParams +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{utils::*, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::bonk::{ + bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, + AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, + BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, + TradeDirection, VestingParams, + }, }, }; @@ -88,7 +90,7 @@ impl BonkEventParser { GenericEventParseConfig { program_id: BONK_PROGRAM_ID, protocol_type: ProtocolType::Bonk, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::MIGRATE_TO_AMM, event_type: EventType::BonkMigrateToAmm, inner_instruction_parser: None, @@ -97,7 +99,7 @@ impl BonkEventParser { GenericEventParseConfig { program_id: BONK_PROGRAM_ID, protocol_type: ProtocolType::Bonk, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::MIGRATE_TO_CP_SWAP, event_type: EventType::BonkMigrateToCpswap, inner_instruction_parser: None, @@ -116,8 +118,6 @@ impl BonkEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = bonk_pool_create_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); Some(Box::new(BonkPoolCreateEvent { metadata, ..event })) } else { None @@ -130,8 +130,6 @@ impl BonkEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = bonk_trade_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, event.pool_state)); if metadata.event_type == EventType::BonkBuyExactIn || metadata.event_type == EventType::BonkBuyExactOut { @@ -164,9 +162,6 @@ impl BonkEventParser { let minimum_amount_out = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_in, @@ -205,9 +200,6 @@ impl BonkEventParser { let maximum_amount_in = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_out, @@ -246,9 +238,6 @@ impl BonkEventParser { let minimum_amount_out = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_in, @@ -287,9 +276,6 @@ impl BonkEventParser { let maximum_amount_in = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_out, @@ -330,9 +316,6 @@ impl BonkEventParser { let curve_param = Self::parse_curve_params(data, &mut offset)?; let vesting_param = Self::parse_vesting_params(data, &mut offset)?; - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkPoolCreateEvent { metadata, payer: accounts[0], @@ -367,9 +350,6 @@ impl BonkEventParser { let vesting_param = Self::parse_vesting_params(data, &mut offset)?; let amm_fee_on = data[offset]; - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkPoolCreateEvent { metadata, payer: accounts[0], @@ -512,9 +492,6 @@ impl BonkEventParser { let quote_lot_size = u64::from_le_bytes(data[8..16].try_into().unwrap()); let market_vault_signer_nonce = data[16]; - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkMigrateToAmmEvent { metadata, base_lot_size, @@ -562,9 +539,6 @@ impl BonkEventParser { accounts: &[Pubkey], metadata: EventMetadata, ) -> Option> { - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkMigrateToCpswapEvent { metadata, payer: accounts[0], @@ -601,59 +575,4 @@ impl BonkEventParser { } } -#[async_trait::async_trait] -impl EventParser for BonkEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(BonkEventParser); diff --git a/src/streaming/event_parser/protocols/mutil/parser.rs b/src/streaming/event_parser/protocols/mutil/parser.rs index af3902d..e64edfd 100755 --- a/src/streaming/event_parser/protocols/mutil/parser.rs +++ b/src/streaming/event_parser/protocols/mutil/parser.rs @@ -1,13 +1,10 @@ -use std::collections::HashMap; - -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; - -use crate::streaming::event_parser::common::filter::EventTypeFilter; -use crate::streaming::event_parser::{ - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - EventParserFactory, Protocol, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::filter::EventTypeFilter, + core::traits::{GenericEventParseConfig, GenericEventParser}, + EventParserFactory, Protocol, + }, }; pub struct MutilEventParser { @@ -21,20 +18,22 @@ impl MutilEventParser { for protocol in protocols { let parse = EventParserFactory::create_parser(protocol); - // Merge inner_instruction_configs, append configurations to existing Vec - for (key, configs) in parse.inner_instruction_configs() { - let filtered_configs: Vec = configs.into_iter().filter(|config| { - event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true) - }).collect(); - inner.inner_instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs); - } - // Merge instruction_configs, append configurations to existing Vec for (key, configs) in parse.instruction_configs() { - let filtered_configs: Vec = configs.into_iter().filter(|config| { - event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true) - }).collect(); - inner.instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs); + let filtered_configs: Vec = configs + .into_iter() + .filter(|config| { + event_type_filter + .as_ref() + .map(|filter| filter.include.contains(&config.event_type)) + .unwrap_or(true) + }) + .collect(); + inner + .instruction_configs + .entry(key) + .or_insert_with(Vec::new) + .extend(filtered_configs); } // Append program_ids (this is already appending) @@ -44,59 +43,4 @@ impl MutilEventParser { } } -#[async_trait::async_trait] -impl EventParser for MutilEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(MutilEventParser); diff --git a/src/streaming/event_parser/protocols/pumpfun/events.rs b/src/streaming/event_parser/protocols/pumpfun/events.rs index 89b02d7..5e06a84 100755 --- a/src/streaming/event_parser/protocols/pumpfun/events.rs +++ b/src/streaming/event_parser/protocols/pumpfun/events.rs @@ -255,9 +255,15 @@ impl_unified_event!(PumpFunGlobalAccountEvent,); /// 事件鉴别器常量 pub mod discriminators { // 事件鉴别器 - pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376"; - pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; - pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94"; + // pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376"; + pub const CREATE_TOKEN_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 27, 114, 169, 77, 222, 235, 99, 118]; + // pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; + pub const TRADE_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238]; + // pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94"; + pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 189, 233, 93, 185, 92, 148, 234, 148]; // 指令鉴别器 pub const CREATE_TOKEN_IX: &[u8] = &[24, 30, 200, 40, 5, 28, 7, 119]; diff --git a/src/streaming/event_parser/protocols/pumpfun/parser.rs b/src/streaming/event_parser/protocols/pumpfun/parser.rs index 4faaf01..0de1634 100755 --- a/src/streaming/event_parser/protocols/pumpfun/parser.rs +++ b/src/streaming/event_parser/protocols/pumpfun/parser.rs @@ -1,16 +1,15 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; - -use crate::streaming::event_parser::{ - common::{EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::pumpfun::{ - discriminators, pumpfun_create_token_event_log_decode, pumpfun_migrate_event_log_decode, - pumpfun_trade_event_log_decode, PumpFunCreateTokenEvent, PumpFunMigrateEvent, - PumpFunTradeEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::pumpfun::{ + discriminators, pumpfun_create_token_event_log_decode, + pumpfun_migrate_event_log_decode, pumpfun_trade_event_log_decode, + PumpFunCreateTokenEvent, PumpFunMigrateEvent, PumpFunTradeEvent, + }, }, }; @@ -82,8 +81,6 @@ impl PumpFunEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pumpfun_migrate_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, event.user, event.mint)); Some(Box::new(PumpFunMigrateEvent { metadata, ..event })) } else { None @@ -96,11 +93,6 @@ impl PumpFunEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pumpfun_create_token_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.name, event.symbol, event.mint - )); Some(Box::new(PumpFunCreateTokenEvent { metadata, ..event })) } else { None @@ -113,11 +105,6 @@ impl PumpFunEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pumpfun_trade_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.mint, event.user, event.is_buy - )); Some(Box::new(PumpFunTradeEvent { metadata, ..event })) } else { None @@ -152,9 +139,6 @@ impl PumpFunEventParser { Pubkey::default() }; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, name, symbol, accounts[0])); - Some(Box::new(PumpFunCreateTokenEvent { metadata, name: name.to_string(), @@ -181,8 +165,6 @@ impl PumpFunEventParser { } let amount = u64::from_le_bytes(data[0..8].try_into().unwrap()); let max_sol_cost = u64::from_le_bytes(data[8..16].try_into().unwrap()); - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], true)); Some(Box::new(PumpFunTradeEvent { metadata, global: accounts[0], @@ -217,9 +199,6 @@ impl PumpFunEventParser { } let amount = u64::from_le_bytes(data[0..8].try_into().unwrap()); let min_sol_output = u64::from_le_bytes(data[8..16].try_into().unwrap()); - let mut metadata = metadata; - metadata - .set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], false)); Some(Box::new(PumpFunTradeEvent { metadata, global: accounts[0], @@ -252,8 +231,6 @@ impl PumpFunEventParser { if accounts.len() < 24 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[5], accounts[2])); Some(Box::new(PumpFunMigrateEvent { metadata, global: accounts[0], @@ -285,59 +262,4 @@ impl PumpFunEventParser { } } -#[async_trait::async_trait] -impl EventParser for PumpFunEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(PumpFunEventParser); diff --git a/src/streaming/event_parser/protocols/pumpswap/events.rs b/src/streaming/event_parser/protocols/pumpswap/events.rs index 5b27449..63a85b7 100755 --- a/src/streaming/event_parser/protocols/pumpswap/events.rs +++ b/src/streaming/event_parser/protocols/pumpswap/events.rs @@ -392,11 +392,21 @@ impl_unified_event!(PumpSwapPoolAccountEvent,); /// 事件鉴别器常量 pub mod discriminators { // 事件鉴别器 - pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777"; - pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a"; - pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774"; - pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90"; - pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0"; + // pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777"; + pub const BUY_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 103, 244, 82, 31, 44, 245, 119, 119]; + // pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a"; + pub const SELL_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 62, 47, 55, 10, 165, 3, 220, 42]; + // pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774"; + pub const CREATE_POOL_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 177, 49, 12, 210, 160, 118, 167, 116]; + // pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90"; + pub const DEPOSIT_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 120, 248, 61, 83, 31, 142, 107, 144]; + // pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0"; + pub const WITHDRAW_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 22, 9, 133, 26, 160, 44, 71, 192]; // 指令鉴别器 pub const BUY_IX: &[u8] = &[102, 6, 61, 18, 1, 218, 235, 234]; diff --git a/src/streaming/event_parser/protocols/pumpswap/parser.rs b/src/streaming/event_parser/protocols/pumpswap/parser.rs index ec84ad1..d50372e 100755 --- a/src/streaming/event_parser/protocols/pumpswap/parser.rs +++ b/src/streaming/event_parser/protocols/pumpswap/parser.rs @@ -1,17 +1,16 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; - -use crate::streaming::event_parser::{ - common::{read_u64_le, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::pumpswap::{ - discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode, - pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode, - pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent, - PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{read_u64_le, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::pumpswap::{ + discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode, + pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode, + pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent, + PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent, + }, }, }; @@ -92,11 +91,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_buy_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.user, event.pool, event.base_amount_out - )); Some(Box::new(PumpSwapBuyEvent { metadata, ..event })) } else { None @@ -109,11 +103,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_sell_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.user, event.pool, event.base_amount_in - )); Some(Box::new(PumpSwapSellEvent { metadata, ..event })) } else { None @@ -126,11 +115,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_create_pool_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.pool, event.creator, event.base_amount_in - )); Some(Box::new(PumpSwapCreatePoolEvent { metadata, ..event })) } else { None @@ -143,11 +127,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_deposit_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.pool, event.user, event.lp_token_amount_out - )); Some(Box::new(PumpSwapDepositEvent { metadata, ..event })) } else { None @@ -160,11 +139,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_withdraw_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.pool, event.user, event.lp_token_amount_in - )); Some(Box::new(PumpSwapWithdrawEvent { metadata, ..event })) } else { None @@ -184,12 +158,6 @@ impl PumpSwapEventParser { let base_amount_out = read_u64_le(data, 0)?; let max_quote_amount_in = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[1], accounts[0], base_amount_out - )); - Some(Box::new(PumpSwapBuyEvent { metadata, base_amount_out, @@ -225,12 +193,6 @@ impl PumpSwapEventParser { let base_amount_in = read_u64_le(data, 0)?; let min_quote_amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[1], accounts[0], base_amount_in - )); - Some(Box::new(PumpSwapSellEvent { metadata, base_amount_in, @@ -272,12 +234,6 @@ impl PumpSwapEventParser { Pubkey::default() }; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[0], accounts[2], base_amount_in - )); - Some(Box::new(PumpSwapCreatePoolEvent { metadata, index, @@ -312,12 +268,6 @@ impl PumpSwapEventParser { let max_base_amount_in = u64::from_le_bytes(data[8..16].try_into().ok()?); let max_quote_amount_in = u64::from_le_bytes(data[16..24].try_into().ok()?); - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[0], accounts[2], lp_token_amount_out - )); - Some(Box::new(PumpSwapDepositEvent { metadata, lp_token_amount_out, @@ -350,12 +300,6 @@ impl PumpSwapEventParser { let min_base_amount_out = u64::from_le_bytes(data[8..16].try_into().ok()?); let min_quote_amount_out = u64::from_le_bytes(data[16..24].try_into().ok()?); - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[0], accounts[2], lp_token_amount_in - )); - Some(Box::new(PumpSwapWithdrawEvent { metadata, lp_token_amount_in, @@ -375,59 +319,4 @@ impl PumpSwapEventParser { } } -#[async_trait::async_trait] -impl EventParser for PumpSwapEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(PumpSwapEventParser); diff --git a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs index ba4a5cf..f9124e3 100755 --- a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs @@ -1,15 +1,14 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; - -use crate::streaming::event_parser::{ - common::{read_u64_le, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::raydium_amm_v4::{ - discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event, - RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{read_u64_le, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::raydium_amm_v4::{ + discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event, + RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent, + }, }, }; @@ -35,7 +34,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_IN, event_type: EventType::RaydiumAmmV4SwapBaseIn, inner_instruction_parser: None, @@ -44,7 +43,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_OUT, event_type: EventType::RaydiumAmmV4SwapBaseOut, inner_instruction_parser: None, @@ -53,7 +52,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::DEPOSIT, event_type: EventType::RaydiumAmmV4Deposit, inner_instruction_parser: None, @@ -62,7 +61,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::INITIALIZE2, event_type: EventType::RaydiumAmmV4Initialize2, inner_instruction_parser: None, @@ -71,7 +70,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::WITHDRAW, event_type: EventType::RaydiumAmmV4Withdraw, inner_instruction_parser: None, @@ -80,7 +79,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::WITHDRAW_PNL, event_type: EventType::RaydiumAmmV4WithdrawPnl, inner_instruction_parser: None, @@ -103,12 +102,6 @@ impl RaydiumAmmV4EventParser { return None; } - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4WithdrawPnlEvent { metadata, token_program: accounts[0], @@ -142,12 +135,6 @@ impl RaydiumAmmV4EventParser { } let amount = read_u64_le(data, 0)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4WithdrawEvent { metadata, amount, @@ -191,12 +178,6 @@ impl RaydiumAmmV4EventParser { let init_pc_amount = read_u64_le(data, 9)?; let init_coin_amount = read_u64_le(data, 17)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4Initialize2Event { metadata, nonce, @@ -241,12 +222,6 @@ impl RaydiumAmmV4EventParser { let max_pc_amount = read_u64_le(data, 8)?; let base_side = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4DepositEvent { metadata, max_coin_amount, @@ -282,12 +257,6 @@ impl RaydiumAmmV4EventParser { let max_amount_in = read_u64_le(data, 0)?; let amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - let mut accounts = accounts.to_vec(); if accounts.len() == 17 { // 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符 @@ -335,12 +304,6 @@ impl RaydiumAmmV4EventParser { let amount_in = read_u64_le(data, 0)?; let minimum_amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - let mut accounts = accounts.to_vec(); if accounts.len() == 17 { // 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符 @@ -377,59 +340,4 @@ impl RaydiumAmmV4EventParser { } } -#[async_trait::async_trait] -impl EventParser for RaydiumAmmV4EventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(RaydiumAmmV4EventParser); diff --git a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs index f06f238..ced7677 100755 --- a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs @@ -1,20 +1,19 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; - -use crate::streaming::event_parser::{ - common::{ - read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata, - EventType, ProtocolType, - }, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::raydium_clmm::{ - discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent, - RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event, - RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent, - RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{ + read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata, + EventType, ProtocolType, + }, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::raydium_clmm::{ + discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent, + RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event, + RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent, + RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event, + }, }, }; @@ -40,7 +39,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP, event_type: EventType::RaydiumClmmSwap, inner_instruction_parser: None, @@ -49,7 +48,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_V2, event_type: EventType::RaydiumClmmSwapV2, inner_instruction_parser: None, @@ -58,7 +57,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::CLOSE_POSITION, event_type: EventType::RaydiumClmmClosePosition, inner_instruction_parser: None, @@ -67,7 +66,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::DECREASE_LIQUIDITY_V2, event_type: EventType::RaydiumClmmDecreaseLiquidityV2, inner_instruction_parser: None, @@ -76,7 +75,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::CREATE_POOL, event_type: EventType::RaydiumClmmCreatePool, inner_instruction_parser: None, @@ -85,7 +84,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::INCREASE_LIQUIDITY_V2, event_type: EventType::RaydiumClmmIncreaseLiquidityV2, inner_instruction_parser: None, @@ -94,7 +93,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::OPEN_POSITION_WITH_TOKEN_22_NFT, event_type: EventType::RaydiumClmmOpenPositionWithToken22Nft, inner_instruction_parser: None, @@ -103,7 +102,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::OPEN_POSITION_V2, event_type: EventType::RaydiumClmmOpenPositionV2, inner_instruction_parser: None, @@ -125,8 +124,6 @@ impl RaydiumClmmEventParser { if data.len() < 51 || accounts.len() < 22 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmOpenPositionV2Event { metadata, tick_lower_index: read_i32_le(data, 0)?, @@ -173,8 +170,6 @@ impl RaydiumClmmEventParser { if data.len() < 51 || accounts.len() < 20 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmOpenPositionWithToken22NftEvent { metadata, tick_lower_index: read_i32_le(data, 0)?, @@ -218,8 +213,6 @@ impl RaydiumClmmEventParser { if data.len() < 34 || accounts.len() < 15 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmIncreaseLiquidityV2Event { metadata, liquidity: read_u128_le(data, 0)?, @@ -253,8 +246,6 @@ impl RaydiumClmmEventParser { if data.len() < 24 || accounts.len() < 13 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmCreatePoolEvent { metadata, sqrt_price_x64: read_u128_le(data, 0)?, @@ -284,8 +275,6 @@ impl RaydiumClmmEventParser { if data.len() < 32 || accounts.len() < 16 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmDecreaseLiquidityV2Event { metadata, liquidity: read_u128_le(data, 0)?, @@ -320,8 +309,6 @@ impl RaydiumClmmEventParser { if accounts.len() < 6 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmClosePositionEvent { metadata, nft_owner: accounts[0], @@ -348,12 +335,6 @@ impl RaydiumClmmEventParser { let sqrt_price_limit_x64 = read_u128_le(data, 16)?; let is_base_input = read_u8_le(data, 32)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[2], accounts[3], accounts[4] - )); - Some(Box::new(RaydiumClmmSwapEvent { metadata, amount, @@ -388,12 +369,6 @@ impl RaydiumClmmEventParser { let sqrt_price_limit_x64 = read_u128_le(data, 16)?; let is_base_input = read_u8_le(data, 32)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[2], accounts[3], accounts[4] - )); - Some(Box::new(RaydiumClmmSwapV2Event { metadata, amount, @@ -418,59 +393,4 @@ impl RaydiumClmmEventParser { } } -#[async_trait::async_trait] -impl EventParser for RaydiumClmmEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(RaydiumClmmEventParser); diff --git a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs index fdc453a..d89f779 100755 --- a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs @@ -1,15 +1,14 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; - -use crate::streaming::event_parser::{ - common::{read_u64_le, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::raydium_cpmm::{ - discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent, RaydiumCpmmSwapEvent, - RaydiumCpmmWithdrawEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{read_u64_le, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::raydium_cpmm::{ + discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent, + RaydiumCpmmSwapEvent, RaydiumCpmmWithdrawEvent, + }, }, }; @@ -35,7 +34,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_IN, event_type: EventType::RaydiumCpmmSwapBaseInput, inner_instruction_parser: None, @@ -44,7 +43,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_OUT, event_type: EventType::RaydiumCpmmSwapBaseOutput, inner_instruction_parser: None, @@ -53,7 +52,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::DEPOSIT, event_type: EventType::RaydiumCpmmDeposit, inner_instruction_parser: None, @@ -62,7 +61,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::INITIALIZE, event_type: EventType::RaydiumCpmmInitialize, inner_instruction_parser: None, @@ -71,7 +70,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::WITHDRAW, event_type: EventType::RaydiumCpmmWithdraw, inner_instruction_parser: None, @@ -93,8 +92,6 @@ impl RaydiumCpmmEventParser { if data.len() < 24 || accounts.len() < 14 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumCpmmWithdrawEvent { metadata, lp_token_amount: read_u64_le(data, 0)?, @@ -126,8 +123,6 @@ impl RaydiumCpmmEventParser { if data.len() < 24 || accounts.len() < 20 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumCpmmInitializeEvent { metadata, init_amount0: read_u64_le(data, 0)?, @@ -165,8 +160,6 @@ impl RaydiumCpmmEventParser { if data.len() < 24 || accounts.len() < 13 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumCpmmDepositEvent { metadata, lp_token_amount: read_u64_le(data, 0)?, @@ -201,12 +194,6 @@ impl RaydiumCpmmEventParser { let amount_in = read_u64_le(data, 0)?; let minimum_amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumCpmmSwapEvent { metadata, amount_in, @@ -240,12 +227,6 @@ impl RaydiumCpmmEventParser { let max_amount_in = read_u64_le(data, 0)?; let amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumCpmmSwapEvent { metadata, max_amount_in, @@ -268,59 +249,4 @@ impl RaydiumCpmmEventParser { } } -#[async_trait::async_trait] -impl EventParser for RaydiumCpmmEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: &str, - slot: u64, - block_time: Option, - program_received_time_ms: i64, - index: String, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_ms, - index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(RaydiumCpmmEventParser); diff --git a/src/streaming/grpc/event_processor.rs b/src/streaming/grpc/event_processor.rs deleted file mode 100644 index 10454fe..0000000 --- a/src/streaming/grpc/event_processor.rs +++ /dev/null @@ -1,290 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use solana_sdk::pubkey::Pubkey; - -use super::types::EventPretty; -use crate::common::AnyResult; -use crate::streaming::common::{ - EventBatchProcessor as EventBatchCollector, MetricsEventType, MetricsManager, - StreamClientConfig as ClientConfig, -}; -use crate::streaming::event_parser::common::filter::EventTypeFilter; -use crate::streaming::event_parser::core::account_event_parser::AccountEventParser; -use crate::streaming::event_parser::core::common_event_parser::CommonEventParser; -use crate::streaming::event_parser::EventParser; -use crate::streaming::event_parser::{ - core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol, -}; - -/// 事件处理器 -pub struct EventProcessor { - pub(crate) metrics_manager: MetricsManager, - pub(crate) config: ClientConfig, - pub(crate) parser_cache: Arc>>>, -} - -impl EventProcessor { - /// 创建新的事件处理器 - pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self { - Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) } - } - - /// 获取或创建解析器,使用缓存机制避免重复创建 - fn get_or_create_parser( - &self, - protocols: Vec, - event_type_filter: Option, - ) -> Arc { - let mut cache = self.parser_cache.lock().unwrap(); - if let Some(cached_parser) = cache.clone() { - return cached_parser.clone(); - } - let parser: Arc = - Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone())); - *cache = Some(parser.clone()); - parser - } - - /// 使用性能监控处理事件交易 - pub async fn process_event_transaction_with_metrics( - &self, - event_pretty: EventPretty, - callback: &F, - bot_wallet: Option, - protocols: Vec, - event_type_filter: Option, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { - match event_pretty { - EventPretty::Account(account_pretty) => { - self.metrics_manager.add_account_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let account_event = AccountEventParser::parse_account_event( - protocols.clone(), - account_pretty, - program_received_time_ms, - event_type_filter, - ); - if let Some(event) = account_event { - callback(event); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 更新性能指标(如果启用) - self.metrics_manager - .update_metrics(MetricsEventType::Account, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - EventPretty::Transaction(transaction_pretty) => { - self.metrics_manager.add_tx_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_pretty.slot; - let signature = transaction_pretty.signature.to_string(); - - // 使用缓存获取解析器 - let parser = self.get_or_create_parser(protocols.clone(), event_type_filter); - let all_events = parser - .parse_transaction( - transaction_pretty.tx.clone(), - &signature, - Some(slot), - transaction_pretty.block_time.map(|ts| prost_types::Timestamp { - seconds: ts.seconds, - nanos: ts.nanos, - }), - program_received_time_ms, - bot_wallet, - ) - .await - .unwrap_or_else(|_e| vec![]); - - // 保存事件数量用于日志记录 - let event_count = all_events.len(); - - // 批量处理事件 - if !all_events.is_empty() { - for event in all_events { - callback(event); - } - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 更新性能指标(如果启用) - self.metrics_manager - .update_metrics(MetricsEventType::Tx, event_count as u64, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, event_count); - } - EventPretty::BlockMeta(block_meta_pretty) => { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_block_meta_process_count().await; - let block_time_ms = block_meta_pretty - .block_time - .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000) - .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()); - let block_meta_event = CommonEventParser::generate_block_meta_event( - block_meta_pretty.slot, - &block_meta_pretty.block_hash, - block_time_ms, - ); - callback(block_meta_event); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 更新性能指标(如果启用) - self.metrics_manager - .update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - - Ok(()) - } - - /// 使用批处理处理事件交易 - pub async fn process_event_transaction_with_batch( - &self, - event_pretty: EventPretty, - batch_processor: &mut EventBatchCollector, - bot_wallet: Option, - protocols: Vec, - event_type_filter: Option, - ) -> AnyResult<()> - where - F: Fn(Vec>) + Send + Sync + 'static, - { - match event_pretty { - EventPretty::Account(account_pretty) => { - self.metrics_manager.add_account_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let account_event = AccountEventParser::parse_account_event( - protocols.clone(), - account_pretty, - program_received_time_ms, - event_type_filter, - ); - if let Some(event) = account_event { - (batch_processor.callback)(vec![event]); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 实际调用性能指标更新 - self.metrics_manager - .update_metrics(MetricsEventType::Account, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - EventPretty::Transaction(transaction_pretty) => { - self.metrics_manager.add_tx_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_pretty.slot; - let signature = transaction_pretty.signature.to_string(); - - // 使用缓存获取解析器 - let parser = self.get_or_create_parser(protocols.clone(), event_type_filter); - let result = parser - .parse_transaction( - transaction_pretty.tx.clone(), - &signature, - Some(slot), - transaction_pretty.block_time.map(|ts| prost_types::Timestamp { - seconds: ts.seconds, - nanos: ts.nanos, - }), - program_received_time_ms, - bot_wallet, - ) - .await; - - // 处理解析结果并使用批处理器 - let total_events = match result { - Ok(events) => { - let event_count = events.len(); - if !events.is_empty() { - log::debug!("Parsed {} events", event_count); - log::debug!("Adding {} events to batch processor", event_count); - for event in events { - if self.config.batch.enabled { - batch_processor.add_event(event); - } else { - // 如果批处理被禁用,直接调用回调 - // 这里需要将单个事件包装成Vec来调用批处理回调 - let single_event_batch = vec![event]; - (batch_processor.callback)(single_event_batch); - } - } - } - event_count - } - Err(e) => { - log::warn!("Failed to parse transaction: {:?}", e); - 0 - } - }; - - // 添加调试信息 - if total_events > 0 { - log::debug!( - "Total events parsed: {} for transaction {}", - total_events, - signature - ); - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 实际调用性能指标更新 - self.metrics_manager - .update_metrics(MetricsEventType::Tx, total_events as u64, processing_time_ms) - .await; - - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, total_events); - } - EventPretty::BlockMeta(block_meta_pretty) => { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_block_meta_process_count().await; - let block_time_ms = block_meta_pretty - .block_time - .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000) - .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()); - let block_meta_event = CommonEventParser::generate_block_meta_event( - block_meta_pretty.slot, - &block_meta_pretty.block_hash, - block_time_ms, - ); - (batch_processor.callback)(vec![block_meta_event]); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 更新性能指标(如果启用) - self.metrics_manager - .update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - - Ok(()) - } -} diff --git a/src/streaming/grpc/mod.rs b/src/streaming/grpc/mod.rs index 44713a9..0d91713 100644 --- a/src/streaming/grpc/mod.rs +++ b/src/streaming/grpc/mod.rs @@ -1,25 +1,15 @@ // gRPC 相关模块 pub mod connection; -pub mod types; pub mod subscription; -pub mod stream_handler; -pub mod event_processor; +pub mod types; // 重新导出主要类型 pub use connection::*; -pub use types::*; pub use subscription::*; -pub use stream_handler::*; -pub use event_processor::*; +pub use types::*; // 从公用模块重新导出 pub use crate::streaming::common::{ + BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsManager, PerformanceMetrics, StreamClientConfig as ClientConfig, - PerformanceMetrics, - MetricsManager, - EventBatchProcessor as EventBatchCollector, - BackpressureStrategy, - BatchConfig, - BackpressureConfig, - ConnectionConfig, -}; \ No newline at end of file +}; diff --git a/src/streaming/grpc/stream_handler.rs b/src/streaming/grpc/stream_handler.rs deleted file mode 100644 index 362d963..0000000 --- a/src/streaming/grpc/stream_handler.rs +++ /dev/null @@ -1,134 +0,0 @@ -use chrono::Local; -use futures::{channel::mpsc, sink::Sink, SinkExt}; -use yellowstone_grpc_proto::geyser::{ - subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, SubscribeUpdate, -}; - -use super::types::{BlockMetaPretty, EventPretty, TransactionPretty}; -use crate::common::AnyResult; -use crate::streaming::common::BackpressureStrategy; -use crate::streaming::grpc::AccountPretty; - -/// 流消息处理器 -pub struct StreamHandler; - -impl StreamHandler { - /// 处理单个流消息 - pub async fn handle_stream_message( - msg: SubscribeUpdate, - tx: &mut mpsc::Sender, - subscribe_tx: &mut (impl Sink + Unpin), - backpressure_strategy: BackpressureStrategy, - ) -> AnyResult<()> { - let created_at = msg.created_at; - match msg.update_oneof { - Some(UpdateOneof::Account(account)) => { - let account_pretty = AccountPretty::from(account); - log::debug!("Received account: {:?}", account_pretty); - Self::handle_backpressure( - tx, - EventPretty::Account(account_pretty), - backpressure_strategy, - ) - .await?; - } - Some(UpdateOneof::BlockMeta(sut)) => { - let block_meta_pretty = BlockMetaPretty::from((sut, created_at)); - log::debug!("Received block meta: {:?}", block_meta_pretty); - Self::handle_backpressure( - tx, - EventPretty::BlockMeta(block_meta_pretty), - backpressure_strategy, - ) - .await?; - } - Some(UpdateOneof::Transaction(sut)) => { - let transaction_pretty = TransactionPretty::from((sut, created_at)); - log::debug!( - "Received transaction: {} at slot {}", - transaction_pretty.signature, - transaction_pretty.slot - ); - - // 根据背压策略处理发送 - Self::handle_backpressure( - tx, - EventPretty::Transaction(transaction_pretty), - backpressure_strategy, - ) - .await?; - } - Some(UpdateOneof::Ping(_)) => { - subscribe_tx - .send(SubscribeRequest { - ping: Some(SubscribeRequestPing { id: 1 }), - ..Default::default() - }) - .await?; - log::debug!("service is ping: {}", Local::now()); - } - Some(UpdateOneof::Pong(_)) => { - log::debug!("service is pong: {}", Local::now()); - } - _ => { - log::debug!("Received other message type"); - } - } - Ok(()) - } - - /// 处理背压策略 - async fn handle_backpressure( - tx: &mut mpsc::Sender, - event_pretty: EventPretty, - backpressure_strategy: BackpressureStrategy, - ) -> AnyResult<()> { - match backpressure_strategy { - BackpressureStrategy::Block => { - // 阻塞等待,直到有空间 - if let Err(e) = tx.send(event_pretty).await { - log::error!("Failed to send transaction to channel: {:?}", e); - return Err(anyhow::anyhow!("Channel send failed: {:?}", e)); - } - } - BackpressureStrategy::Drop => { - // 尝试发送,如果失败则丢弃 - if let Err(e) = tx.try_send(event_pretty) { - if e.is_full() { - log::warn!("Channel is full, dropping transaction"); - } else { - log::error!("Channel is closed: {:?}", e); - return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); - } - } - } - BackpressureStrategy::Retry { max_attempts, wait_ms } => { - // 重试有限次数 - let mut retry_count = 0; - loop { - match tx.try_send(event_pretty.clone()) { - Ok(_) => break, - Err(e) => { - if e.is_full() { - retry_count += 1; - if retry_count >= max_attempts { - log::warn!( - "Channel is full after {} attempts, dropping transaction", - retry_count - ); - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms)) - .await; - } else { - log::error!("Channel is closed: {:?}", e); - return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); - } - } - } - } - } - } - Ok(()) - } -} diff --git a/src/streaming/grpc/subscription.rs b/src/streaming/grpc/subscription.rs index 2efdddd..06d4d59 100644 --- a/src/streaming/grpc/subscription.rs +++ b/src/streaming/grpc/subscription.rs @@ -14,7 +14,7 @@ use crate::common::AnyResult; use crate::streaming::common::StreamClientConfig as ClientConfig; use crate::streaming::event_parser::common::filter::EventTypeFilter; -/// 订阅管理器 +/// Subscription manager #[derive(Clone)] pub struct SubscriptionManager { endpoint: String, @@ -23,12 +23,12 @@ pub struct SubscriptionManager { } impl SubscriptionManager { - /// 创建新的订阅管理器 + /// Create a new subscription manager pub fn new(endpoint: String, x_token: Option, config: ClientConfig) -> Self { Self { endpoint, x_token, config } } - /// 创建 gRPC 连接 + /// Create gRPC connection pub async fn connect(&self) -> AnyResult> { let builder = GeyserGrpcClient::build_from_shared(self.endpoint.clone())? .x_token(self.x_token.clone())? @@ -39,19 +39,20 @@ impl SubscriptionManager { Ok(builder.connect().await?) } - /// 创建订阅请求并返回流 + /// Create subscription request and return stream pub async fn subscribe_with_request( &self, transactions: Option, accounts: Option, commitment: Option, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> AnyResult<( impl Sink, impl Stream>, + SubscribeRequest, )> { let blocks_meta = if event_type_filter.is_some() - && event_type_filter.as_ref().unwrap().include_block_event() + && event_type_filter.unwrap().include_block_event() { hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} } } else if event_type_filter.is_none() { @@ -71,22 +72,22 @@ impl SubscriptionManager { ..Default::default() }; let mut client = self.connect().await?; - let (sink, stream) = client.subscribe_with_request(Some(subscribe_request)).await?; - Ok((sink, stream)) + let (sink, stream) = client.subscribe_with_request(Some(subscribe_request.clone())).await?; + Ok((sink, stream, subscribe_request)) } - /// 创建账户订阅请求并返回流 + /// Create account subscription request and return stream pub fn subscribe_with_account_request( &self, account: Vec, owner: Vec, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> Option { if account.len() == 0 && owner.len() == 0 { return None; } if event_type_filter.is_some() - && !event_type_filter.as_ref().unwrap().include_account_event() + && !event_type_filter.unwrap().include_account_event() { return None; } @@ -103,16 +104,16 @@ impl SubscriptionManager { Some(accounts) } - /// 生成订阅请求过滤器 + /// Generate subscription request filter pub fn get_subscribe_request_filter( &self, account_include: Vec, account_exclude: Vec, account_required: Vec, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> Option { if event_type_filter.is_some() - && !event_type_filter.as_ref().unwrap().include_transaction_event() + && !event_type_filter.unwrap().include_transaction_event() { return None; } @@ -131,7 +132,7 @@ impl SubscriptionManager { Some(transactions) } - /// 获取配置 + /// Get configuration pub fn get_config(&self) -> &ClientConfig { &self.config } diff --git a/src/streaming/grpc/types.rs b/src/streaming/grpc/types.rs index 8d80a7c..7cfd946 100644 --- a/src/streaming/grpc/types.rs +++ b/src/streaming/grpc/types.rs @@ -1,5 +1,5 @@ -use solana_sdk::signature::Signature; -use solana_transaction_status::{EncodedTransactionWithStatusMeta, UiTransactionEncoding}; +use solana_sdk::{pubkey::Pubkey, signature::Signature}; +use solana_transaction_status::TransactionWithStatusMeta; use std::{collections::HashMap, fmt}; use yellowstone_grpc_proto::{ geyser::{ @@ -22,13 +22,14 @@ pub enum EventPretty { #[derive(Clone)] pub struct AccountPretty { pub slot: u64, - pub signature: String, - pub pubkey: String, + pub signature: Signature, + pub pubkey: Pubkey, pub executable: bool, pub lamports: u64, - pub owner: String, + pub owner: Pubkey, pub rent_epoch: u64, pub data: Vec, + pub program_received_time_us: i64, } impl fmt::Debug for AccountPretty { @@ -51,6 +52,7 @@ pub struct BlockMetaPretty { pub slot: u64, pub block_hash: String, pub block_time: Option, + pub program_received_time_us: i64, } impl fmt::Debug for BlockMetaPretty { @@ -59,6 +61,7 @@ impl fmt::Debug for BlockMetaPretty { .field("slot", &self.slot) .field("block_hash", &self.block_hash) .field("block_time", &self.block_time) + .field("program_received_time_us", &self.program_received_time_us) .finish() } } @@ -66,28 +69,23 @@ impl fmt::Debug for BlockMetaPretty { #[derive(Clone)] pub struct TransactionPretty { pub slot: u64, + pub transaction_index: Option, // 新增:交易在slot中的索引 pub block_hash: String, pub block_time: Option, pub signature: Signature, pub is_vote: bool, - pub tx: EncodedTransactionWithStatusMeta, + pub tx: TransactionWithStatusMeta, + pub program_received_time_us: i64, } impl fmt::Debug for TransactionPretty { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - struct TxWrap<'a>(&'a EncodedTransactionWithStatusMeta); - impl<'a> fmt::Debug for TxWrap<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let serialized = serde_json::to_string(self.0).expect("failed to serialize"); - fmt::Display::fmt(&serialized, f) - } - } - f.debug_struct("TransactionPretty") .field("slot", &self.slot) + .field("transaction_index", &self.transaction_index) .field("signature", &self.signature) .field("is_vote", &self.is_vote) - .field("tx", &TxWrap(&self.tx)) + .field("program_received_time_us", &self.program_received_time_us) .finish() } } @@ -97,13 +95,18 @@ impl From for AccountPretty { let account_info = account.account.unwrap(); Self { slot: account.slot, - signature: bs58::encode(&account_info.txn_signature.unwrap_or_default()).into_string(), - pubkey: bs58::encode(&account_info.pubkey).into_string(), + signature: if let Some(txn_signature) = account_info.txn_signature { + Signature::try_from(txn_signature.as_slice()).expect("valid signature") + } else { + Signature::default() + }, + pubkey: Pubkey::try_from(account_info.pubkey.as_slice()).expect("valid pubkey"), executable: account_info.executable, lamports: account_info.lamports, - owner: bs58::encode(&account_info.owner).into_string(), + owner: Pubkey::try_from(account_info.owner.as_slice()).expect("valid pubkey"), rent_epoch: account_info.rent_epoch, data: account_info.data, + program_received_time_us: chrono::Utc::now().timestamp_micros(), } } } @@ -115,7 +118,12 @@ impl From<(SubscribeUpdateBlockMeta, Option)> for BlockMetaPretty { Option, ), ) -> Self { - Self { block_hash: blockhash.to_string(), block_time, slot } + Self { + block_hash: blockhash.to_string(), + block_time, + slot, + program_received_time_us: chrono::Utc::now().timestamp_micros(), + } } } @@ -127,16 +135,18 @@ impl From<(SubscribeUpdateTransaction, Option)> for TransactionPretty ), ) -> Self { let tx = transaction.expect("should be defined"); + // 根据用户说明,交易索引在 transaction.index 中 + let transaction_index = tx.index; Self { slot, + transaction_index: Some(transaction_index), // 提取交易索引 block_time, block_hash: "".to_string(), signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"), is_vote: tx.is_vote, tx: yellowstone_grpc_proto::convert_from::create_tx_with_meta(tx) - .expect("valid tx with meta") - .encode(UiTransactionEncoding::Base64, Some(u8::MAX), true) - .expect("failed to encode"), + .expect("valid tx with meta"), + program_received_time_us: chrono::Utc::now().timestamp_micros(), } } } diff --git a/src/streaming/shred/connection.rs b/src/streaming/shred/connection.rs index 8a27a8a..951993c 100644 --- a/src/streaming/shred/connection.rs +++ b/src/streaming/shred/connection.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::RwLock; use tokio::sync::Mutex; use tonic::transport::Channel; @@ -13,7 +14,7 @@ use crate::streaming::common::{ pub struct ShredStreamGrpc { pub shredstream_client: Arc>, pub config: StreamClientConfig, - pub metrics: Arc>, + pub metrics: Arc>, pub metrics_manager: MetricsManager, pub subscription_handle: Arc>>, } @@ -27,31 +28,38 @@ impl ShredStreamGrpc { /// 创建客户端,使用自定义配置 pub async fn new_with_config(endpoint: String, config: StreamClientConfig) -> AnyResult { let shredstream_client = ShredstreamProxyClient::connect(endpoint.clone()).await?; - let metrics = Arc::new(Mutex::new(PerformanceMetrics::new())); - let config_arc = Arc::new(config.clone()); + let metrics = Arc::new(RwLock::new(PerformanceMetrics::new())); - let metrics_manager = - MetricsManager::new(metrics.clone(), config_arc, "ShredStream".to_string()); + let metrics_manager = MetricsManager::new(config.enable_metrics, "ShredStream".to_string()); Ok(Self { shredstream_client: Arc::new(shredstream_client), config, - metrics, + metrics: metrics.clone(), metrics_manager, subscription_handle: Arc::new(Mutex::new(None)), }) } - /// 创建高性能客户端(适合高并发场景) - pub async fn new_high_performance(endpoint: String) -> AnyResult { - Self::new_with_config(endpoint, StreamClientConfig::high_performance()).await + /// Creates a new ShredStreamClient with high-throughput configuration. + /// + /// This is a convenience method that creates a client optimized for high-concurrency scenarios + /// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()` + /// for detailed configuration information. + pub async fn new_high_throughput(endpoint: String) -> AnyResult { + Self::new_with_config(endpoint, StreamClientConfig::high_throughput()).await } - /// 创建低延迟客户端(适合实时场景) + /// Creates a new ShredStreamClient with low-latency configuration. + /// + /// This is a convenience method that creates a client optimized for real-time scenarios + /// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()` + /// for detailed configuration information. pub async fn new_low_latency(endpoint: String) -> AnyResult { Self::new_with_config(endpoint, StreamClientConfig::low_latency()).await } + /// 获取当前配置 pub fn get_config(&self) -> &StreamClientConfig { &self.config @@ -63,8 +71,8 @@ impl ShredStreamGrpc { } /// 获取性能指标 - pub async fn get_metrics(&self) -> PerformanceMetrics { - self.metrics_manager.get_metrics().await + pub fn get_metrics(&self) -> PerformanceMetrics { + self.metrics_manager.get_metrics() } /// 启用或禁用性能监控 @@ -73,8 +81,8 @@ impl ShredStreamGrpc { } /// 打印性能指标 - pub async fn print_metrics(&self) { - self.metrics_manager.print_metrics().await; + pub fn print_metrics(&self) { + self.metrics_manager.print_metrics(); } /// 启动自动性能监控任务 diff --git a/src/streaming/shred/event_processor.rs b/src/streaming/shred/event_processor.rs deleted file mode 100644 index c72895a..0000000 --- a/src/streaming/shred/event_processor.rs +++ /dev/null @@ -1,170 +0,0 @@ -use solana_sdk::pubkey::Pubkey; -use std::sync::{Arc, Mutex}; - -use crate::common::AnyResult; -use crate::streaming::common::{ - EventBatchProcessor, MetricsEventType, MetricsManager, StreamClientConfig, -}; -use crate::streaming::event_parser::common::filter::EventTypeFilter; -use crate::streaming::event_parser::protocols::MutilEventParser; -use crate::streaming::event_parser::{EventParser, Protocol, UnifiedEvent}; -use crate::streaming::shred::TransactionWithSlot; - -/// ShredStream 事件处理器 -pub struct ShredEventProcessor { - pub(crate) metrics_manager: MetricsManager, - pub(crate) config: StreamClientConfig, - pub(crate) parser_cache: Arc>>>, -} - -impl ShredEventProcessor { - /// 创建新的事件处理器 - pub fn new(metrics_manager: MetricsManager, config: StreamClientConfig) -> Self { - Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) } - } - - /// 获取或创建解析器,使用缓存机制避免重复创建 - fn get_or_create_parser( - &self, - protocols: Vec, - event_type_filter: Option, - ) -> Arc { - let mut cache = self.parser_cache.lock().unwrap(); - if let Some(cached_parser) = cache.clone() { - return cached_parser.clone(); - } - let parser: Arc = - Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone())); - *cache = Some(parser.clone()); - parser - } - - /// 即时处理单个交易 - pub async fn process_transaction_immediate( - &self, - transaction_with_slot: TransactionWithSlot, - protocols: Vec, - bot_wallet: Option, - event_type_filter: Option, - callback: &F, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_tx_process_count().await; - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_with_slot.slot; - let versioned_tx = transaction_with_slot.transaction; - let signature = versioned_tx.signatures[0]; - - // 获取缓存的解析器 - let parser = self.get_or_create_parser(protocols, event_type_filter); - - let all_events = parser - .parse_versioned_transaction( - &versioned_tx, - &signature.to_string(), - Some(slot), - None, - program_received_time_ms, - bot_wallet, - ) - .await - .unwrap_or_else(|_e| vec![]); - - // 保存事件数量用于日志记录 - let event_count = all_events.len(); - - // 即时处理事件 - for event in all_events { - callback(event); - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 实际调用性能指标更新 - self.update_metrics(event_count as u64, processing_time_ms).await; - - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, event_count); - - Ok(()) - } - - /// 批处理模式处理单个交易 - pub async fn process_transaction_with_batch( - &self, - transaction_with_slot: TransactionWithSlot, - protocols: Vec, - bot_wallet: Option, - batch_processor: &mut EventBatchProcessor, - event_type_filter: Option, - ) -> AnyResult<()> - where - F: FnMut(Vec>) + Send + Sync + 'static, - { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_tx_process_count().await; - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_with_slot.slot; - let versioned_tx = transaction_with_slot.transaction; - let signature = versioned_tx.signatures[0]; - - // 获取缓存的解析器 - let parser = self.get_or_create_parser(protocols, event_type_filter); - - let all_events = parser - .parse_versioned_transaction( - &versioned_tx, - &signature.to_string(), - Some(slot), - None, - program_received_time_ms, - bot_wallet, - ) - .await - .unwrap_or_else(|_e| vec![]); - - // 保存事件数量用于日志记录 - let event_count = all_events.len(); - - // 使用批处理器处理事件 - for event in all_events { - batch_processor.add_event(event); - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 实际调用性能指标更新 - self.update_metrics(event_count as u64, processing_time_ms).await; - - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, event_count); - - Ok(()) - } - - /// 更新性能指标 - async fn update_metrics(&self, events_processed: u64, processing_time_ms: f64) { - // 使用统一的指标管理器,这里假设 ShredStream 主要处理交易事件 - self.metrics_manager - .update_metrics(MetricsEventType::Tx, events_processed, processing_time_ms) - .await; - } -} - -// 实现 Clone trait 以支持模块间共享 -impl Clone for ShredEventProcessor { - fn clone(&self) -> Self { - Self { - metrics_manager: self.metrics_manager.clone(), - config: self.config.clone(), - parser_cache: self.parser_cache.clone(), - } - } -} diff --git a/src/streaming/shred/mod.rs b/src/streaming/shred/mod.rs index 5ba7daf..1358fe9 100644 --- a/src/streaming/shred/mod.rs +++ b/src/streaming/shred/mod.rs @@ -1,17 +1,13 @@ // ShredStream 相关模块 pub mod connection; pub mod types; -pub mod stream_handler; -pub mod event_processor; // 重新导出主要类型 pub use connection::*; pub use types::*; -pub use stream_handler::*; -pub use event_processor::*; // 从公用模块重新导出 pub use crate::streaming::common::{ - BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, EventBatchProcessor, - MetricsEventType, MetricsManager, PerformanceMetrics, StreamClientConfig, + BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsEventType, MetricsManager, + PerformanceMetrics, StreamClientConfig, }; diff --git a/src/streaming/shred/stream_handler.rs b/src/streaming/shred/stream_handler.rs deleted file mode 100644 index b5de4c2..0000000 --- a/src/streaming/shred/stream_handler.rs +++ /dev/null @@ -1,116 +0,0 @@ -use futures::{channel::mpsc, StreamExt}; -use log::error; -use solana_entry::entry::Entry; -use tokio::task::JoinHandle; - -use crate::common::AnyResult; -use crate::protos::shredstream::{ - shredstream_proxy_client::ShredstreamProxyClient, SubscribeEntriesRequest, -}; -use crate::streaming::shred::TransactionWithSlot; - -/// ShredStream 流处理器 -pub struct ShredStreamHandler; - -impl ShredStreamHandler { - /// 启动 ShredStream 流处理任务 - /// - /// # 参数 - /// * `client` - ShredStream 客户端 - /// * `tx` - 事务发送通道 - /// * `channel_size` - 通道缓冲区大小 - /// - /// # 返回值 - /// 返回 ShredStream 流处理任务句柄和事务接收通道 - pub async fn start_stream_processing( - mut client: ShredstreamProxyClient, - channel_size: usize, - ) -> AnyResult<(JoinHandle<()>, mpsc::Receiver)> { - let request = tonic::Request::new(SubscribeEntriesRequest {}); - let stream = client.subscribe_entries(request).await?.into_inner(); - let (tx, rx) = mpsc::channel::(channel_size); - - let stream_task = tokio::spawn(Self::process_stream_messages(stream, tx)); - - Ok((stream_task, rx)) - } - - /// 处理流消息 - /// - /// # 参数 - /// * `stream` - ShredStream 数据流 - /// * `tx` - 事务发送通道 - async fn process_stream_messages( - mut stream: tonic::codec::Streaming, - mut tx: mpsc::Sender, - ) { - while let Some(message) = stream.next().await { - match message { - Ok(msg) => { - if let Err(e) = Self::handle_stream_message(msg, &mut tx).await { - error!("Error handling stream message: {e:?}"); - continue; - } - } - Err(error) => { - error!("Stream error: {error:?}"); - break; - } - } - } - } - - /// 处理单个流消息 - /// - /// # 参数 - /// * `msg` - ShredStream 消息 - /// * `tx` - 事务发送通道 - async fn handle_stream_message( - msg: crate::protos::shredstream::Entry, - tx: &mut mpsc::Sender, - ) -> AnyResult<()> { - if let Ok(entries) = bincode::deserialize::>(&msg.entries) { - for entry in entries { - for transaction in entry.transactions { - let transaction_with_slot = - TransactionWithSlot::new(transaction.clone(), msg.slot); - - if let Err(e) = tx.try_send(transaction_with_slot) { - // 如果通道满了,记录警告但不中断处理 - if e.is_full() { - log::warn!("Transaction channel is full, dropping transaction"); - } else { - // 通道已关闭,返回错误 - return Err(e.into()); - } - } - } - } - } - Ok(()) - } - - /// 启动事务处理任务 - /// - /// # 参数 - /// * `rx` - 事务接收通道 - /// * `processor` - 事务处理器 - pub fn start_transaction_processing( - mut rx: mpsc::Receiver, - processor: F, - ) -> JoinHandle<()> - where - F: Fn(TransactionWithSlot) -> Result<(), Box> - + Send - + Sync - + 'static, - { - tokio::spawn(async move { - while let Some(transaction_with_slot) = rx.next().await { - if let Err(e) = processor(transaction_with_slot) { - error!("Error processing transaction: {e:?}"); - } - } - }) - } -} diff --git a/src/streaming/shred/types.rs b/src/streaming/shred/types.rs index 888622d..d10094e 100644 --- a/src/streaming/shred/types.rs +++ b/src/streaming/shred/types.rs @@ -5,16 +5,16 @@ use solana_sdk::transaction::VersionedTransaction; pub struct TransactionWithSlot { pub transaction: VersionedTransaction, pub slot: u64, + pub program_received_time_us: i64, } impl TransactionWithSlot { /// 创建新的带槽位的交易 - pub fn new(transaction: VersionedTransaction, slot: u64) -> Self { - Self { transaction, slot } - } - - /// 获取交易签名 - pub fn signature(&self) -> String { - self.transaction.signatures[0].to_string() + pub fn new( + transaction: VersionedTransaction, + slot: u64, + program_received_time_us: i64, + ) -> Self { + Self { transaction, slot, program_received_time_us } } } diff --git a/src/streaming/shred_stream.rs b/src/streaming/shred_stream.rs index 356e774..34101c0 100755 --- a/src/streaming/shred_stream.rs +++ b/src/streaming/shred_stream.rs @@ -1,10 +1,16 @@ +use std::sync::Arc; + +use futures::StreamExt; use solana_sdk::pubkey::Pubkey; use crate::common::AnyResult; -use crate::streaming::common::{EventBatchProcessor, SubscriptionHandle}; +use crate::protos::shredstream::SubscribeEntriesRequest; +use crate::streaming::common::{EventProcessor, SubscriptionHandle}; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::{Protocol, UnifiedEvent}; -use crate::streaming::shred::{ShredEventProcessor, ShredStreamHandler, TransactionWithSlot}; +use crate::streaming::shred::TransactionWithSlot; +use log::error; +use solana_entry::entry::Entry; use super::ShredStreamGrpc; @@ -29,120 +35,61 @@ impl ShredStreamGrpc { metrics_handle = self.metrics_manager.start_auto_monitoring().await; } - // 启动流处理 - let client = (*self.shredstream_client).clone(); - let (stream_task, rx) = ShredStreamHandler::start_stream_processing( - client, - self.config.backpressure.channel_size, - ) - .await?; + // 创建事件处理器 + let mut event_processor = + EventProcessor::new(self.metrics_manager.clone(), self.config.clone()); + event_processor.set_protocols_and_event_type_filter( + protocols, + event_type_filter, + self.config.backpressure.clone(), + Some(Arc::new(callback)), + ); - // 根据配置选择处理模式并获取事件处理任务句柄 - let event_handle = if self.config.batch.enabled { - // 批处理模式 - self.process_with_batch(rx, protocols, bot_wallet, event_type_filter, callback).await? - } else { - // 即时处理模式 - self.process_immediate(rx, protocols, bot_wallet, event_type_filter, callback).await? - }; + // 启动流处理 + let mut client = (*self.shredstream_client).clone(); + let request = tonic::Request::new(SubscribeEntriesRequest {}); + let mut stream = client.subscribe_entries(request).await?.into_inner(); + let event_processor_clone = event_processor.clone(); + let stream_task = tokio::spawn(async move { + while let Some(message) = stream.next().await { + match message { + Ok(msg) => { + if let Ok(entries) = bincode::deserialize::>(&msg.entries) { + for entry in entries { + for transaction in entry.transactions { + let transaction_with_slot = TransactionWithSlot::new( + transaction.clone(), + msg.slot, + chrono::Utc::now().timestamp_micros(), + ); + // 直接处理,背压控制在 EventProcessor 内部处理 + if let Err(e) = event_processor_clone + .process_shred_transaction_with_metrics( + transaction_with_slot, + bot_wallet, + ) + .await + { + error!("Error handling message: {e:?}"); + } + } + } + } + continue; + } + Err(error) => { + error!("Stream error: {error:?}"); + break; + } + } + } + }); // 保存订阅句柄 - let subscription_handle = SubscriptionHandle::new(stream_task, event_handle, metrics_handle); + let subscription_handle = SubscriptionHandle::new(stream_task, None, metrics_handle); let mut handle_guard = self.subscription_handle.lock().await; *handle_guard = Some(subscription_handle); Ok(()) } - - /// 批处理模式 - async fn process_with_batch( - &self, - mut rx: futures::channel::mpsc::Receiver, - protocols: Vec, - bot_wallet: Option, - event_type_filter: Option, - callback: F, - ) -> AnyResult> - where - F: Fn(Box) + Send + Sync + 'static, - { - use futures::StreamExt; - - // 创建批处理器,将单个事件回调转换为批量回调 - let batch_callback = move |events: Vec>| { - for event in events { - callback(event); - } - }; - - let mut batch_processor = EventBatchProcessor::new( - batch_callback, - self.config.batch.batch_size, - self.config.batch.batch_timeout_ms, - ); - - // 创建事件处理器 - let event_processor = - ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone()); - - let event_handle = tokio::spawn(async move { - while let Some(transaction_with_slot) = rx.next().await { - if let Err(e) = event_processor - .process_transaction_with_batch( - transaction_with_slot, - protocols.clone(), - bot_wallet, - &mut batch_processor, - event_type_filter.clone(), - ) - .await - { - log::error!("Error processing transaction: {e:?}"); - } - } - - // 处理剩余的事件 - batch_processor.flush(); - }); - - Ok(event_handle) - } - - /// 即时处理模式 - async fn process_immediate( - &self, - mut rx: futures::channel::mpsc::Receiver, - protocols: Vec, - bot_wallet: Option, - event_type_filter: Option, - callback: F, - ) -> AnyResult> - where - F: Fn(Box) + Send + Sync + 'static, - { - use futures::StreamExt; - - // 创建事件处理器 - let event_processor = - ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone()); - - let event_handle = tokio::spawn(async move { - while let Some(transaction_with_slot) = rx.next().await { - if let Err(e) = event_processor - .process_transaction_immediate( - transaction_with_slot, - protocols.clone(), - bot_wallet, - event_type_filter.clone(), - &callback, - ) - .await - { - log::error!("Error processing transaction: {e:?}"); - } - } - }); - - Ok(event_handle) - } } diff --git a/src/streaming/yellowstone_grpc.rs b/src/streaming/yellowstone_grpc.rs index f88e88b..f083b8b 100644 --- a/src/streaming/yellowstone_grpc.rs +++ b/src/streaming/yellowstone_grpc.rs @@ -1,19 +1,26 @@ -use futures::{channel::mpsc, StreamExt}; -use log::error; -use solana_sdk::pubkey::Pubkey; -use std::sync::Arc; -use tokio::sync::Mutex; -use yellowstone_grpc_proto::geyser::CommitmentLevel; - use crate::common::AnyResult; use crate::streaming::common::{ - EventBatchProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle, + EventProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle, }; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::{Protocol, UnifiedEvent}; -use crate::streaming::grpc::{EventPretty, EventProcessor, StreamHandler, SubscriptionManager}; +use crate::streaming::grpc::{ + AccountPretty, BlockMetaPretty, EventPretty, SubscriptionManager, TransactionPretty, +}; +use anyhow::anyhow; +use chrono::Local; +use futures::channel::mpsc; +use futures::{SinkExt, StreamExt}; +use log::error; +use solana_sdk::pubkey::Pubkey; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, RwLock}; +use tokio::sync::Mutex; +use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; +use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestPing}; /// 交易过滤器 +#[derive(Debug, Clone)] pub struct TransactionFilter { pub account_include: Vec, pub account_exclude: Vec, @@ -21,6 +28,7 @@ pub struct TransactionFilter { } /// 账户过滤器 +#[derive(Debug, Clone)] pub struct AccountFilter { pub account: Vec, pub owner: Vec, @@ -30,11 +38,15 @@ pub struct YellowstoneGrpc { pub endpoint: String, pub x_token: Option, pub config: StreamClientConfig, - pub metrics: Arc>, + pub metrics: Arc>, pub subscription_manager: SubscriptionManager, pub metrics_manager: MetricsManager, pub event_processor: EventProcessor, pub subscription_handle: Arc>>, + // Dynamic subscription management fields + pub active_subscription: Arc, + pub control_tx: Arc>>>, + pub current_request: Arc>>, } impl YellowstoneGrpc { @@ -50,43 +62,50 @@ impl YellowstoneGrpc { config: StreamClientConfig, ) -> AnyResult { let _ = rustls::crypto::ring::default_provider().install_default().ok(); - let metrics = Arc::new(Mutex::new(PerformanceMetrics::new())); - let config_arc = Arc::new(config.clone()); + let metrics = Arc::new(RwLock::new(PerformanceMetrics::new())); let subscription_manager = SubscriptionManager::new(endpoint.clone(), x_token.clone(), config.clone()); - let metrics_manager = - MetricsManager::new(metrics.clone(), config_arc.clone(), "YellowstoneGrpc".to_string()); + let metrics_manager = MetricsManager::new_with_metrics( + metrics.clone(), + config.enable_metrics, + "YellowstoneGrpc".to_string(), + ); let event_processor = EventProcessor::new(metrics_manager.clone(), config.clone()); Ok(Self { endpoint, x_token, config, - metrics, + metrics: metrics.clone(), subscription_manager, metrics_manager, event_processor, subscription_handle: Arc::new(Mutex::new(None)), + active_subscription: Arc::new(AtomicBool::new(false)), + control_tx: Arc::new(tokio::sync::Mutex::new(None)), + current_request: Arc::new(tokio::sync::RwLock::new(None)), }) } - /// 创建高性能客户端 - pub fn new_high_performance(endpoint: String, x_token: Option) -> AnyResult { - Self::new_with_config(endpoint, x_token, StreamClientConfig::high_performance()) + /// Creates a new YellowstoneGrpcClient with high-throughput configuration. + /// + /// This is a convenience method that creates a client optimized for high-concurrency scenarios + /// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()` + /// for detailed configuration information. + pub fn new_high_throughput(endpoint: String, x_token: Option) -> AnyResult { + Self::new_with_config(endpoint, x_token, StreamClientConfig::high_throughput()) } - /// 创建低延迟客户端 + /// Creates a new YellowstoneGrpcClient with low-latency configuration. + /// + /// This is a convenience method that creates a client optimized for real-time scenarios + /// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()` + /// for detailed configuration information. pub fn new_low_latency(endpoint: String, x_token: Option) -> AnyResult { Self::new_with_config(endpoint, x_token, StreamClientConfig::low_latency()) } - /// 创建即时处理客户端 - pub fn new_immediate(endpoint: String, x_token: Option) -> AnyResult { - let mut config = StreamClientConfig::low_latency(); - config.enable_metrics = false; - Self::new_with_config(endpoint, x_token, config) - } /// 获取配置 pub fn get_config(&self) -> &StreamClientConfig { @@ -99,13 +118,13 @@ impl YellowstoneGrpc { } /// 获取性能指标 - pub async fn get_metrics(&self) -> PerformanceMetrics { - self.metrics_manager.get_metrics().await + pub fn get_metrics(&self) -> PerformanceMetrics { + self.metrics_manager.get_metrics() } /// 打印性能指标 - pub async fn print_metrics(&self) { - self.metrics_manager.print_metrics().await; + pub fn print_metrics(&self) { + self.metrics_manager.print_metrics(); } /// 启用或禁用性能监控 @@ -119,6 +138,9 @@ impl YellowstoneGrpc { if let Some(handle) = handle_guard.take() { handle.stop(); } + *self.control_tx.lock().await = None; + *self.current_request.write().await = None; + self.active_subscription.store(false, Ordering::Release); } /// Simplified immediate event subscription (recommended for simple scenarios) @@ -147,8 +169,13 @@ impl YellowstoneGrpc { where F: Fn(Box) + Send + Sync + 'static, { - // 如果已有活跃订阅,先停止它 - self.stop().await; + if self + .active_subscription + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + return Err(anyhow!("Already subscribed. Use update_subscription() to modify filters")); + } let mut metrics_handle = None; // 启动自动性能监控(如果启用) @@ -160,205 +187,187 @@ impl YellowstoneGrpc { transaction_filter.account_include, transaction_filter.account_exclude, transaction_filter.account_required, - event_type_filter.clone(), + event_type_filter.as_ref(), ); let accounts = self.subscription_manager.subscribe_with_account_request( account_filter.account, account_filter.owner, - event_type_filter.clone(), + event_type_filter.as_ref(), ); // 订阅事件 - let (mut subscribe_tx, mut stream) = self + let (mut subscribe_tx, mut stream, subscribe_request) = self .subscription_manager - .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) + .subscribe_with_request(transactions, accounts, commitment, event_type_filter.as_ref()) .await?; - // 创建通道,使用配置中的通道大小 - let (mut tx, mut rx) = mpsc::channel::(self.config.backpressure.channel_size); + // 用 Arc> 包装 subscribe_tx 以支持多线程共享 + let subscribe_tx = Arc::new(Mutex::new(subscribe_tx)); + *self.current_request.write().await = Some(subscribe_request); + let (control_tx, mut control_rx) = mpsc::channel(100); + *self.control_tx.lock().await = Some(control_tx); // 启动流处理任务 - let backpressure_strategy = self.config.backpressure.strategy; + let mut event_processor = self.event_processor.clone(); + event_processor.set_protocols_and_event_type_filter( + protocols, + event_type_filter, + self.config.backpressure.clone(), + Some(Arc::new(callback)), + ); let stream_handle = tokio::spawn(async move { - while let Some(message) = stream.next().await { - match message { - Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut tx, - &mut subscribe_tx, - backpressure_strategy, - ) - .await - { - error!("Error handling message: {e:?}"); + loop { + tokio::select! { + message = stream.next() => { + match message { + Some(Ok(msg)) => { + let created_at = msg.created_at; + match msg.update_oneof { + Some(UpdateOneof::Account(account)) => { + let account_pretty = AccountPretty::from(account); + log::debug!("Received account: {:?}", account_pretty); + if let Err(e) = event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::Account(account_pretty), + bot_wallet, + ) + .await + { + error!("Error processing account event: {e:?}"); + } + } + Some(UpdateOneof::BlockMeta(sut)) => { + let block_meta_pretty = + BlockMetaPretty::from((sut, created_at)); + log::debug!("Received block meta: {:?}", block_meta_pretty); + if let Err(e) = event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::BlockMeta(block_meta_pretty), + bot_wallet, + ) + .await + { + error!("Error processing block meta event: {e:?}"); + } + } + Some(UpdateOneof::Transaction(sut)) => { + let transaction_pretty = + TransactionPretty::from((sut, created_at)); + log::debug!( + "Received transaction: {} at slot {}", + transaction_pretty.signature, + transaction_pretty.slot + ); + if let Err(e) = event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::Transaction(transaction_pretty), + bot_wallet, + ) + .await + { + error!("Error processing transaction event: {e:?}"); + } + } + Some(UpdateOneof::Ping(_)) => { + // 只在需要时获取锁,并立即释放 + if let Ok(mut tx_guard) = subscribe_tx.try_lock() { + let _ = tx_guard + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: 1 }), + ..Default::default() + }) + .await; + } + log::debug!("service is ping: {}", Local::now()); + } + Some(UpdateOneof::Pong(_)) => { + log::debug!("service is pong: {}", Local::now()); + } + _ => { + log::debug!("Received other message type"); + } + } + } + Some(Err(error)) => { + error!("Stream error: {error:?}"); + break; + } + None => break, + } + } + Some(update) = control_rx.next() => { + if let Err(e) = subscribe_tx.lock().await.send(update).await { + error!("Failed to send subscription update: {}", e); break; } } - Err(error) => { - error!("Stream error: {error:?}"); - break; - } - } - } - }); - - // 即时处理交易,无批处理 - let event_processor = self.event_processor.clone(); - let event_handle = tokio::spawn(async move { - while let Some(event_pretty) = rx.next().await { - if let Err(e) = event_processor - .process_event_transaction_with_metrics( - event_pretty, - &callback, - bot_wallet, - protocols.clone(), - event_type_filter.clone(), - ) - .await - { - error!("Error processing transaction: {e:?}"); } } }); // 保存订阅句柄 - let subscription_handle = - SubscriptionHandle::new(stream_handle, event_handle, metrics_handle); + let subscription_handle = SubscriptionHandle::new(stream_handle, None, metrics_handle); let mut handle_guard = self.subscription_handle.lock().await; *handle_guard = Some(subscription_handle); Ok(()) } - /// Advanced event subscription with batch processing and backpressure handling + /// Update subscription filters at runtime without reconnection /// /// # Parameters - /// * `protocols` - List of protocols to monitor - /// * `bot_wallet` - Optional bot wallet address for filtering related transactions - /// * `transaction_filter` - Transaction filter specifying accounts to include/exclude - /// * `account_filter` - Account filter specifying accounts and owners to monitor - /// * `event_filter` - Optional event filter for further event filtering, no filtering if None - /// * `commitment` - Optional commitment level, defaults to Confirmed - /// * `callback` - Event callback function that receives parsed unified events - /// - /// # Features - /// * Batch processing for improved throughput - /// * Backpressure handling to prevent memory overflow - /// * Automatic performance monitoring (if enabled) - /// * Configurable batch size and timeout + /// * `transaction_filter` - New transaction filter to apply + /// * `account_filter` - New account filter to apply /// /// # Returns - /// Returns `AnyResult<()>`, `Ok(())` on success, error information on failure - pub async fn subscribe_events_advanced( + /// Returns `AnyResult<()>` on success, error on failure + pub async fn update_subscription( &self, - protocols: Vec, - bot_wallet: Option, transaction_filter: TransactionFilter, account_filter: AccountFilter, - event_type_filter: Option, - commitment: Option, - callback: F, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync + 'static, - { - // 如果已有活跃订阅,先停止它 - self.stop().await; + ) -> AnyResult<()> { + let mut control_sender = { + let control_guard = self.control_tx.lock().await; - let mut metrics_handle = None; - // 启动自动性能监控(如果启用) - if self.config.enable_metrics { - metrics_handle = self.metrics_manager.start_auto_monitoring().await; - } - - let transactions = self.subscription_manager.get_subscribe_request_filter( - transaction_filter.account_include, - transaction_filter.account_exclude, - transaction_filter.account_required, - event_type_filter.clone(), - ); - let accounts = self.subscription_manager.subscribe_with_account_request( - account_filter.account, - account_filter.owner, - event_type_filter.clone(), - ); - - // Subscribe to events - let (mut subscribe_tx, mut stream) = self - .subscription_manager - .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) - .await?; - - // Create channel - let (mut tx, mut rx) = mpsc::channel::(self.config.backpressure.channel_size); - - // 创建批处理器,将单个事件回调转换为批量回调 - let batch_callback = move |events: Vec>| { - for event in events { - callback(event); + if !self.active_subscription.load(Ordering::Acquire) { + return Err(anyhow!("No active subscription to update")); } + + control_guard + .as_ref() + .ok_or_else(|| anyhow!("No active subscription to update"))? + .clone() }; - let mut batch_processor = EventBatchProcessor::new( - batch_callback, - self.config.batch.batch_size, - self.config.batch.batch_timeout_ms, - ); + let mut request = self + .current_request + .read() + .await + .as_ref() + .ok_or_else(|| anyhow!("No active subscription"))? + .clone(); - // Start task to process the stream - let backpressure_strategy = self.config.backpressure.strategy; - let stream_handle = tokio::spawn(async move { - while let Some(message) = stream.next().await { - match message { - Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut tx, - &mut subscribe_tx, - backpressure_strategy, - ) - .await - { - error!("Error handling message: {e:?}"); - break; - } - } - Err(error) => { - error!("Stream error: {error:?}"); - break; - } - } - } - }); + request.transactions = self + .subscription_manager + .get_subscribe_request_filter( + transaction_filter.account_include, + transaction_filter.account_exclude, + transaction_filter.account_required, + None, + ) + .unwrap_or_default(); - // Process transactions with batch processing - let event_processor = self.event_processor.clone(); - let event_handle = tokio::spawn(async move { - while let Some(event_pretty) = rx.next().await { - if let Err(e) = event_processor - .process_event_transaction_with_batch( - event_pretty, - &mut batch_processor, - bot_wallet, - protocols.clone(), - event_type_filter.clone(), - ) - .await - { - error!("Error processing transaction: {e:?}"); - } - } + request.accounts = self + .subscription_manager + .subscribe_with_account_request(account_filter.account, account_filter.owner, None) + .unwrap_or_default(); - // 处理剩余的事件 - batch_processor.flush(); - }); + control_sender + .send(request.clone()) + .await + .map_err(|e| anyhow!("Failed to send update: {}", e))?; - // 保存订阅句柄 - let subscription_handle = - SubscriptionHandle::new(stream_handle, event_handle, metrics_handle); - let mut handle_guard = self.subscription_handle.lock().await; - *handle_guard = Some(subscription_handle); + *self.current_request.write().await = Some(request); Ok(()) } @@ -376,17 +385,9 @@ impl Clone for YellowstoneGrpc { metrics_manager: self.metrics_manager.clone(), event_processor: self.event_processor.clone(), subscription_handle: self.subscription_handle.clone(), // 共享同一个 Arc> - } - } -} - -// 实现 Clone trait 以支持模块间共享 -impl Clone for EventProcessor { - fn clone(&self) -> Self { - Self { - metrics_manager: self.metrics_manager.clone(), - config: self.config.clone(), - parser_cache: self.parser_cache.clone(), + active_subscription: self.active_subscription.clone(), + control_tx: self.control_tx.clone(), + current_request: self.current_request.clone(), } } } diff --git a/src/streaming/yellowstone_sub_system.rs b/src/streaming/yellowstone_sub_system.rs index ebb4aa9..030397d 100755 --- a/src/streaming/yellowstone_sub_system.rs +++ b/src/streaming/yellowstone_sub_system.rs @@ -1,19 +1,20 @@ use crate::{ common::AnyResult, streaming::{ - grpc::{BackpressureStrategy, EventPretty, StreamHandler}, + grpc::{EventPretty, TransactionPretty}, yellowstone_grpc::YellowstoneGrpc, }, }; -use futures::{channel::mpsc, StreamExt}; +use futures::{SinkExt, StreamExt}; use log::error; use solana_program::pubkey; use solana_sdk::{pubkey::Pubkey, transaction::VersionedTransaction}; -use solana_transaction_status::EncodedTransactionWithStatusMeta; +use solana_transaction_status::TransactionWithStatusMeta; +use yellowstone_grpc_proto::geyser::{ + subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, +}; const SYSTEM_PROGRAM_ID: Pubkey = pubkey!("11111111111111111111111111111111"); -// 根据实际并发量调整通道大小,避免背压 -const CHANNEL_SIZE: usize = 50000; // 增加到 50000 #[derive(Debug)] pub enum SystemEvent { @@ -36,7 +37,7 @@ impl YellowstoneGrpc { account_exclude: Option>, ) -> AnyResult<()> where - F: Fn(SystemEvent) + Send + Sync + 'static, + F: Fn(SystemEvent) + Send + Sync + Clone + 'static, { let addrs = vec![SYSTEM_PROGRAM_ID.to_string()]; let account_include = account_include.unwrap_or_default(); @@ -47,11 +48,10 @@ impl YellowstoneGrpc { addrs, None, ); - let (mut subscribe_tx, mut stream) = self + let (mut subscribe_tx, mut stream, _) = self .subscription_manager .subscribe_with_request(transactions, None, None, None) .await?; - let (mut tx, mut rx) = mpsc::channel::(CHANNEL_SIZE); let callback = Box::new(callback); @@ -59,16 +59,34 @@ impl YellowstoneGrpc { while let Some(message) = stream.next().await { match message { Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut tx, - &mut subscribe_tx, - BackpressureStrategy::Block, - ) - .await - { - error!("Error handling message: {e:?}"); - break; + let created_at = msg.created_at; + match msg.update_oneof { + Some(UpdateOneof::Transaction(sut)) => { + let transaction_pretty = TransactionPretty::from((sut, created_at)); + let event_pretty = EventPretty::Transaction(transaction_pretty); + if let Err(e) = Self::process_system_transaction( + event_pretty, + &*callback, + ) + .await + { + error!("Error processing transaction: {e:?}"); + } + } + Some(UpdateOneof::Ping(_)) => { + let _ = subscribe_tx + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: 1 }), + ..Default::default() + }) + .await; + } + Some(UpdateOneof::Pong(_)) => { + // Pong response, no action needed + } + _ => { + // Other message types, ignore for system subscription + } } } Err(error) => { @@ -78,12 +96,6 @@ impl YellowstoneGrpc { } } }); - - while let Some(event_pretty) = rx.next().await { - if let Err(e) = Self::process_system_transaction(event_pretty, &*callback).await { - error!("Error processing transaction: {e:?}"); - } - } Ok(()) } @@ -93,20 +105,19 @@ impl YellowstoneGrpc { { match event_pretty { EventPretty::Transaction(transaction_pretty) => { - let trade_raw: EncodedTransactionWithStatusMeta = transaction_pretty.tx; - let meta = trade_raw - .meta - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Missing transaction metadata"))?; + let trade_raw: TransactionWithStatusMeta = transaction_pretty.tx; + let meta = trade_raw.get_status_meta(); - if meta.err.is_some() { + if meta.is_none() { return Ok(()); } + let transaction = trade_raw.get_transaction(); + callback(SystemEvent::NewTransfer(TransferInfo { slot: transaction_pretty.slot, signature: transaction_pretty.signature.to_string(), - tx: trade_raw.transaction.decode(), + tx: Some(transaction), })); } _ => {}