Merge commit '74781e5cbe6f48877ea564e4ae9da6ec51282b16'

This commit is contained in:
ysq
2025-08-31 22:25:47 +08:00
50 changed files with 3886 additions and 3114 deletions
+2 -1
View File
@@ -20,4 +20,5 @@ Cargo.lock
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
.cargo/
.cargo/
.claude/
+8 -1
View File
@@ -33,7 +33,6 @@ serde-big-array = "0.5.1"
futures = "0.3.31"
futures-util = "0.3.31"
base64 = "0.22.1"
bs58 = "0.5.1"
rand = "0.9.0"
bincode = "1.3.3"
anyhow = "1.0.90"
@@ -52,6 +51,7 @@ thiserror = "2.0.11"
async-trait = "0.1.86"
lazy_static = "1.5.0"
once_cell = "1.20.3"
dashmap = "6.0.1"
prost = "0.13.5"
prost-types = "0.13.5"
num_enum = "0.7.3"
@@ -64,3 +64,10 @@ borsh-derive = "1.5.5"
indicatif = "0.18.0"
maplit = "1.0.2"
env_logger = "0.11.8"
crossbeam = "0.8.4"
crossbeam-queue = "0.3.12"
parking_lot = "0.12.1"
wide = "0.7"
[dev-dependencies]
criterion = { version = "0.5", features = ["html_reports"] }
+102 -2
View File
@@ -24,11 +24,12 @@ A lightweight Rust library for real-time event streaming from Solana DEX trading
11. **Performance Monitoring**: Built-in performance metrics monitoring, including event processing speed, etc.
12. **Memory Optimization**: Object pooling and caching mechanisms to reduce memory allocations
13. **Flexible Configuration System**: Support for custom batch sizes, backpressure strategies, channel sizes, and other parameters
14. **Preset Configurations**: Provides high-performance, low-latency, ordered processing, and other preset configurations
15. **Backpressure Handling**: Supports blocking, dropping, retrying, ordered, and other backpressure strategies
14. **Preset Configurations**: Provides high-throughput and low-latency preset configurations optimized for different use cases
15. **Backpressure Handling**: Supports blocking and dropping backpressure strategies
16. **Runtime Configuration Updates**: Supports dynamic configuration parameter updates at runtime
17. **Full Function Performance Monitoring**: All subscribe_events functions support performance monitoring, automatically collecting and reporting performance metrics
18. **Graceful Shutdown**: Support for programmatic stop() method for clean shutdown
19. **Dynamic Subscription Management**: Runtime filter updates without reconnection, enabling adaptive monitoring strategies
## Installation
@@ -55,6 +56,65 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" }
solana-streamer-sdk = "0.3.10"
```
## Configuration System
### Preset Configurations
The library provides three preset configurations optimized for different use cases:
#### 1. High Throughput Configuration (`high_throughput()`)
Optimized for high-concurrency scenarios, prioritizing throughput over latency:
```rust
let config = StreamClientConfig::high_throughput();
// Or use convenience methods
let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?;
let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?;
```
**Features:**
- **Backpressure Strategy**: Drop - drops messages during high load to avoid blocking
- **Buffer Size**: 5,000 permits to handle burst traffic
- **Use Case**: Scenarios where you need to process large volumes of data and can tolerate occasional message drops during peak loads
#### 2. Low Latency Configuration (`low_latency()`)
Optimized for real-time scenarios, prioritizing latency over throughput:
```rust
let config = StreamClientConfig::low_latency();
// Or use convenience methods
let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?;
let shred = ShredStreamGrpc::new_low_latency(endpoint).await?;
```
**Features:**
- **Backpressure Strategy**: Block - ensures no data loss
- **Buffer Size**: 4000 permits for balanced throughput and latency
- **Immediate Processing**: No buffering, processes events immediately
- **Use Case**: Scenarios where every millisecond counts and you cannot afford to lose any events, such as trading applications or real-time monitoring
### Custom Configuration
You can also create custom configurations:
```rust
let config = StreamClientConfig {
connection: ConnectionConfig {
connect_timeout: 30,
request_timeout: 120,
max_decoding_message_size: 20 * 1024 * 1024, // 20MB
},
backpressure: BackpressureConfig {
permits: 2000,
strategy: BackpressureStrategy::Block,
},
enable_metrics: true,
};
```
## Usage Examples
### Quick Start - Parse Transaction Events
@@ -72,6 +132,20 @@ This example demonstrates:
The example uses a predefined transaction signature and shows how to extract protocol-specific events from the transaction data.
### Dynamic Subscription Management Example
Test runtime filter updates without reconnection:
```bash
cargo run --example dynamic_subscription
```
This example demonstrates:
- Creating initial subscriptions with specific protocol filters
- Updating subscription filters at runtime without reconnection
- Single subscription enforcement and proper error handling
- Clean shutdown and resource management
### Advanced Usage - Complete Example
```rust
@@ -470,6 +544,32 @@ let event_type_filter = Some(EventTypeFilter {
});
```
## Dynamic Subscription Management
Update subscription filters at runtime without reconnecting to the stream.
```rust
// Update filters on existing subscription
grpc.update_subscription(
TransactionFilter {
account_include: vec!["new_program_id".to_string()],
account_exclude: vec![],
account_required: vec![],
},
AccountFilter {
account: vec![],
owner: vec![],
},
).await?;
```
- **No Reconnection**: Filter changes apply immediately without closing the stream
- **Atomic Updates**: Both transaction and account filters updated together
- **Single Subscription**: One active subscription per client instance
- **Compatible**: Works with both immediate and advanced subscription methods
Note: Multiple subscription attempts on the same client return an error.
## Supported Protocols
- **PumpFun**: Primary meme coin trading platform
+62 -3
View File
@@ -24,8 +24,8 @@
11. **性能监控**: 内置性能指标监控,包括事件处理速度等
12. **内存优化**: 对象池和缓存机制减少内存分配
13. **灵活配置系统**: 支持自定义批处理大小、背压策略、通道大小等参数
14. **预设配置**: 提供高性能、低延迟、有序处理等预设配置
15. **背压处理**: 支持阻塞、丢弃、重试、有序等多种背压策略
14. **预设配置**: 提供高吞吐量、低延迟等预设配置,针对不同使用场景优化
15. **背压处理**: 支持阻塞、丢弃背压策略
16. **运行时配置更新**: 支持在运行时动态更新配置参数
17. **全函数性能监控**: 所有subscribe_events函数都支持性能监控,自动收集和报告性能指标
18. **优雅关闭**: 支持编程式 stop() 方法进行干净的关闭
@@ -55,6 +55,65 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" }
solana-streamer-sdk = "0.3.10"
```
## 配置系统
### 预设配置
库提供了三种预设配置,针对不同的使用场景进行了优化:
#### 1. 高吞吐量配置 (`high_throughput()`)
专为高并发场景优化,优先考虑吞吐量而非延迟:
```rust
let config = StreamClientConfig::high_throughput();
// 或者使用便捷方法
let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?;
let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?;
```
**特性:**
- **背压策略**: Drop(丢弃策略)- 在高负载时丢弃消息以避免阻塞
- **缓冲区大小**: 5,000 个许可证,处理突发流量
- **适用场景**: 需要处理大量数据且可以容忍在峰值负载时偶尔丢失消息的场景
#### 2. 低延迟配置 (`low_latency()`)
专为实时场景优化,优先考虑延迟而非吞吐量:
```rust
let config = StreamClientConfig::low_latency();
// 或者使用便捷方法
let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?;
let shred = ShredStreamGrpc::new_low_latency(endpoint).await?;
```
**特性:**
- **背压策略**: Block(阻塞策略)- 确保不丢失任何数据
- **缓冲区大小**: 4000 个许可证,平衡吞吐量和延迟
- **立即处理**: 不进行缓冲,立即处理事件
- **适用场景**: 每毫秒都很重要且不能丢失任何事件的场景,如交易应用或实时监控
### 自定义配置
您也可以创建自定义配置:
```rust
let config = StreamClientConfig {
connection: ConnectionConfig {
connect_timeout: 30,
request_timeout: 120,
max_decoding_message_size: 20 * 1024 * 1024, // 20MB
},
backpressure: BackpressureConfig {
permits: 2000,
strategy: BackpressureStrategy::Block,
},
enable_metrics: true,
};
```
## 使用示例
### 快速开始 - 解析交易事件
@@ -502,7 +561,7 @@ let event_type_filter = Some(EventTypeFilter {
- **Yellowstone gRPC 客户端**: 针对 Solana 事件流优化
- **ShredStream 客户端**: 替代流实现
- **异步处理**: 非阻塞事件处理
- **高性能处理**: 优化的事件处理机制
## 项目结构
+468
View File
@@ -0,0 +1,468 @@
use anyhow::Result;
use solana_streamer_sdk::streaming::yellowstone_grpc::{AccountFilter, TransactionFilter, YellowstoneGrpc};
use solana_streamer_sdk::streaming::event_parser::Protocol;
use solana_streamer_sdk::streaming::event_parser::common::filter::EventTypeFilter;
use solana_streamer_sdk::streaming::event_parser::common::types::EventType;
use solana_sdk::signature::{Keypair, Signer};
use std::sync::{Arc, Mutex};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::time::sleep;
const PUMPFUN_PROGRAM_ID: &str = "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P";
const RAYDIUM_CPMM_PROGRAM_ID: &str = "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C";
const GRPC_ENDPOINT: &str = "https://solana-yellowstone-grpc.publicnode.com:443";
const API_KEY: Option<&str> = None;
const MONITORING_DURATION_SECS: u64 = 10;
/// Demonstrates dynamic subscription updates and filter changes in real-time
#[tokio::main]
async fn main() -> Result<()> {
env_logger::init();
println!("Connecting to Yellowstone gRPC at {}", GRPC_ENDPOINT);
let client = Arc::new(YellowstoneGrpc::new(
GRPC_ENDPOINT.to_string(),
API_KEY.map(|s| s.to_string())
)?);
let event_counter = Arc::new(AtomicU64::new(0));
let counter = event_counter.clone();
let callback = move |event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
let count = counter.fetch_add(1, Ordering::Relaxed);
let protocol = match event.event_type() {
EventType::PumpFunBuy | EventType::PumpFunSell => "PumpFun",
EventType::RaydiumCpmmSwapBaseInput | EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmm",
_ => "Unknown"
};
println!("Event #{}: {:11} - {:.8}...", count + 1, protocol, event.signature());
};
println!("\n=== Phase 1: PumpFun only ===");
let pumpfun_filter = TransactionFilter {
account_include: vec![PUMPFUN_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
};
let account_filter = AccountFilter {
account: vec![],
owner: vec![],
};
let trade_event_filter = EventTypeFilter {
include: vec![
EventType::PumpFunBuy,
EventType::PumpFunSell,
EventType::RaydiumCpmmSwapBaseInput,
EventType::RaydiumCpmmSwapBaseOutput,
],
};
if let Err(e) = client.subscribe_events_immediate(
vec![Protocol::PumpFun, Protocol::RaydiumCpmm],
None,
pumpfun_filter,
account_filter,
Some(trade_event_filter),
None,
callback,
).await {
println!("Failed to create subscription: {}", e);
return Ok(());
}
println!("Subscribed to PumpFun transactions with trade event filters, monitoring for {}s...", MONITORING_DURATION_SECS);
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
let phase1_count = event_counter.load(Ordering::Relaxed);
println!("Phase 1: {} events", phase1_count);
println!("\n=== Phase 2: PumpFun + RaydiumCpmm ===");
let multi_protocol_filter = TransactionFilter {
account_include: vec![
PUMPFUN_PROGRAM_ID.to_string(),
RAYDIUM_CPMM_PROGRAM_ID.to_string(),
],
account_exclude: vec![],
account_required: vec![],
};
if let Err(e) = client.update_subscription(
multi_protocol_filter,
AccountFilter {
account: vec![],
owner: vec![],
},
).await {
println!("Failed to update subscription: {}", e);
return Ok(());
}
println!("Updated to PumpFun + RaydiumCpmm transactions, monitoring for {}s...", MONITORING_DURATION_SECS);
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
let phase2_count = event_counter.load(Ordering::Relaxed);
println!("Phase 2: {} events", phase2_count - phase1_count);
println!("\n=== Phase 3: RaydiumCpmm only ===");
let raydium_cpmm_filter = TransactionFilter {
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
};
if let Err(e) = client.update_subscription(
raydium_cpmm_filter,
AccountFilter {
account: vec![],
owner: vec![],
},
).await {
println!("Failed to update subscription: {}", e);
return Ok(());
}
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
println!("Updated to RaydiumCpmm transactions only, monitoring for {}s...", MONITORING_DURATION_SECS);
let phase3_count = event_counter.load(Ordering::Relaxed);
println!("Phase 3: {} events", phase3_count - phase2_count);
println!("\n=== Phase 4: Back to PumpFun only ===");
let pumpfun_only_filter = TransactionFilter {
account_include: vec![PUMPFUN_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
};
if let Err(e) = client.update_subscription(
pumpfun_only_filter,
AccountFilter {
account: vec![],
owner: vec![],
},
).await {
println!("Failed to update subscription: {}", e);
return Ok(());
}
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
println!("Updated to PumpFun transactions only, monitoring for {}s...", MONITORING_DURATION_SECS);
let phase4_count = event_counter.load(Ordering::Relaxed);
println!("Phase 4: {} events", phase4_count - phase3_count);
println!("\n=== Phase 5: All events ===");
let empty_filter = TransactionFilter {
account_include: vec![],
account_exclude: vec![],
account_required: vec![],
};
if let Err(e) = client.update_subscription(
empty_filter,
AccountFilter {
account: vec![],
owner: vec![],
},
).await {
println!("Failed to update subscription: {}", e);
return Ok(());
}
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
println!("Updated to all transactions (no filters), monitoring for {}s...", MONITORING_DURATION_SECS);
let phase5_count = event_counter.load(Ordering::Relaxed);
println!("Phase 5: {} events", phase5_count - phase4_count);
println!("\n=== Phase 6: Silence ===");
let random_keypair_1 = Keypair::new();
let random_keypair_2 = Keypair::new();
let random_pubkey_1 = random_keypair_1.pubkey();
let random_pubkey_2 = random_keypair_2.pubkey();
let silence_filter = TransactionFilter {
account_include: vec![],
account_exclude: vec![],
account_required: vec![
random_pubkey_1.to_string(),
random_pubkey_2.to_string(),
],
};
if let Err(e) = client.update_subscription(
silence_filter,
AccountFilter {
account: vec![],
owner: vec![],
},
).await {
println!("Failed to update subscription: {}", e);
return Ok(());
}
println!("Updated to random addresses (expecting silence), monitoring for 3s...");
let before_silence = event_counter.load(Ordering::Relaxed);
let start_time = Instant::now();
let last_event_time = Arc::new(Mutex::new(start_time));
let last_event_time_clone = last_event_time.clone();
let mut last_count = before_silence;
for _ in 0..6 {
sleep(Duration::from_millis(500)).await;
let current_count = event_counter.load(Ordering::Relaxed);
if current_count > last_count {
if let Ok(mut time) = last_event_time_clone.lock() {
*time = Instant::now();
}
last_count = current_count;
}
}
let final_count = event_counter.load(Ordering::Relaxed);
let events_during_silence = final_count - before_silence;
if events_during_silence == 0 {
println!("Phase 6: 0 events (immediate filter application)");
} else if let Ok(last_time) = last_event_time.lock() {
let propagation_time = last_time.duration_since(start_time);
println!("Phase 6: {} events during propagation, filter took {}ms",
events_during_silence, propagation_time.as_millis());
}
println!("\n=== Phase 7: Shutdown ===");
let shutdown_client = Arc::new(YellowstoneGrpc::new(
GRPC_ENDPOINT.to_string(),
API_KEY.map(|s| s.to_string())
)?);
let shutdown_event_counter = Arc::new(AtomicU64::new(0));
let shutdown_counter = shutdown_event_counter.clone();
let shutdown_callback = move |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
shutdown_counter.fetch_add(1, Ordering::Relaxed);
};
if let Err(e) = shutdown_client.subscribe_events_immediate(
vec![Protocol::PumpFun, Protocol::RaydiumCpmm],
None,
TransactionFilter {
account_include: vec![],
account_exclude: vec![],
account_required: vec![],
},
AccountFilter {
account: vec![],
owner: vec![],
},
None,
None,
shutdown_callback,
).await {
println!("Failed to subscribe shutdown client: {}", e);
return Ok(());
}
sleep(Duration::from_millis(1000)).await;
let pre_stop_count = shutdown_event_counter.load(Ordering::Relaxed);
println!("Received {} events before stop", pre_stop_count);
let stop_time = Instant::now();
shutdown_client.stop().await;
let shutdown_duration = stop_time.elapsed();
println!("stop() completed in {:.1}ms", shutdown_duration.as_millis());
let post_stop_count = shutdown_event_counter.load(Ordering::Relaxed);
let during_stop = post_stop_count - pre_stop_count;
if during_stop > 0 {
println!(" {} events received during stop()", during_stop);
}
let last_event_time = Arc::new(Mutex::new(stop_time));
let last_event_time_clone = last_event_time.clone();
let mut last_count = post_stop_count;
for _ in 0..20 {
sleep(Duration::from_millis(100)).await;
let current_count = shutdown_event_counter.load(Ordering::Relaxed);
if current_count > last_count {
if let Ok(mut time) = last_event_time_clone.lock() {
*time = Instant::now();
}
last_count = current_count;
}
}
let final_count = shutdown_event_counter.load(Ordering::Relaxed);
let after_stop = final_count - post_stop_count;
if after_stop == 0 {
println!("Phase 7: Clean shutdown - no events after stop()");
} else if let Ok(last_time) = last_event_time.lock() {
let post_stop_duration = last_time.duration_since(stop_time);
let silence_duration = Instant::now().duration_since(*last_time);
println!("Phase 7: {} events arrived up to {}ms after stop(), then silent for {}ms",
after_stop, post_stop_duration.as_millis(), silence_duration.as_millis());
}
println!("\n=== Subscription enforcement ===");
let test_callback = |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {};
match client.subscribe_events_immediate(
vec![Protocol::RaydiumCpmm],
None,
TransactionFilter {
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
},
AccountFilter {
account: vec![],
owner: vec![],
},
None,
None,
test_callback,
).await {
Ok(_) => println!("ERROR: Same client created second subscription"),
Err(e) if e.to_string().contains("Already subscribed") => {
println!("✓ Single subscription enforcement working");
},
Err(e) => println!("Unexpected error: {}", e),
}
let client2 = Arc::new(YellowstoneGrpc::new(
GRPC_ENDPOINT.to_string(),
API_KEY.map(|s| s.to_string())
)?);
let client2_counter = Arc::new(AtomicU64::new(0));
let counter2 = client2_counter.clone();
let client2_callback = move |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
counter2.fetch_add(1, Ordering::Relaxed);
};
match client2.subscribe_events_immediate(
vec![Protocol::RaydiumCpmm],
None,
TransactionFilter {
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
},
AccountFilter {
account: vec![],
owner: vec![],
},
None,
None,
client2_callback,
).await {
Ok(_) => {
sleep(Duration::from_millis(500)).await;
let count = client2_counter.load(Ordering::Relaxed);
println!("✓ Second client: {} events", count);
client2.stop().await;
},
Err(e) => println!("ERROR: Second client failed: {}", e),
}
println!("\n=== Advanced subscription enforcement ===");
let test_callback_advanced = |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {};
let client3 = Arc::new(YellowstoneGrpc::new(
GRPC_ENDPOINT.to_string(),
API_KEY.map(|s| s.to_string())
)?);
// First subscription should succeed
match client3.subscribe_events_immediate(
vec![Protocol::RaydiumCpmm],
None,
TransactionFilter {
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
},
AccountFilter {
account: vec![],
owner: vec![],
},
None,
None,
test_callback_advanced,
).await {
Ok(_) => {
// Second subscription attempt on same client should fail
match client3.subscribe_events_immediate(
vec![Protocol::RaydiumCpmm],
None,
TransactionFilter {
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
},
AccountFilter {
account: vec![],
owner: vec![],
},
None,
None,
|_| {},
).await {
Ok(_) => println!("ERROR: Same client created second advanced subscription"),
Err(e) if e.to_string().contains("Already subscribed") => {
println!("✓ Advanced single subscription enforcement working");
},
Err(e) => println!("Unexpected error: {}", e),
}
},
Err(e) => println!("ERROR: First advanced subscription failed: {}", e),
}
// Test that a second client can subscribe using advanced method
let client4 = Arc::new(YellowstoneGrpc::new(
GRPC_ENDPOINT.to_string(),
API_KEY.map(|s| s.to_string())
)?);
let client4_counter = Arc::new(AtomicU64::new(0));
let counter4 = client4_counter.clone();
let client4_callback = move |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
counter4.fetch_add(1, Ordering::Relaxed);
};
match client4.subscribe_events_immediate(
vec![Protocol::RaydiumCpmm],
None,
TransactionFilter {
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
account_exclude: vec![],
account_required: vec![],
},
AccountFilter {
account: vec![],
owner: vec![],
},
None,
None,
client4_callback,
).await {
Ok(_) => {
sleep(Duration::from_millis(500)).await;
let count = client4_counter.load(Ordering::Relaxed);
println!("✓ Second client (advanced): {} events", count);
client4.stop().await;
},
Err(e) => println!("ERROR: Second client (advanced) failed: {}", e),
}
client3.stop().await;
client.stop().await;
Ok(())
}
+15 -20
View File
@@ -1,5 +1,7 @@
use anyhow::Result;
use solana_sdk::commitment_config::CommitmentConfig;
use solana_streamer_sdk::streaming::event_parser::UnifiedEvent;
use solana_streamer_sdk::streaming::event_parser::{
protocols::MutilEventParser, EventParser, Protocol,
};
@@ -10,7 +12,7 @@ use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<()> {
let signatures = vec![
"42agNk1heHabNAVRzEKqQEt5adGkQzRYf9M1Q81uBJPCCHyP4cyCA1RNkgxXrtEAWeeGcytyh2TsnkBDgqnHeq4z",
"5sDWrTTkE69CNc6nrAX7SqPS7FiajJTg8TMog3Gve7KjVfrqYn8YZcX1kAoyKok976S4RTnK1EdCV8hRiDWg68Aj",
];
// Validate signature format
let mut valid_signatures = Vec::new();
@@ -52,7 +54,7 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> {
.get_transaction_with_config(
&signature,
solana_client::rpc_config::RpcTransactionConfig {
encoding: Some(UiTransactionEncoding::Binary),
encoding: Some(UiTransactionEncoding::Base64),
commitment: Some(CommitmentConfig::confirmed()),
max_supported_transaction_version: Some(0),
},
@@ -98,30 +100,23 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> {
Protocol::RaydiumAmmV4,
];
let parser: Arc<dyn EventParser> = Arc::new(MutilEventParser::new(protocols, None));
let start_time = std::time::Instant::now();
let events = parser
.parse_transaction(
transaction.transaction.clone(),
&signature.to_string(),
Some(transaction.slot),
None,
0,
None,
parser
.parse_encoded_confirmed_transaction_with_status_meta(
signature,
transaction,
Arc::new(move |event: &Box<dyn UnifiedEvent>| {
println!("{:?}\n", event);
}),
)
.await
.unwrap_or_else(|_e| vec![]);
let end_time = std::time::Instant::now();
let duration = end_time.duration_since(start_time);
println!("Parsing time: {:?}", duration);
for event in events {
println!("{:?}\n", event);
}
.await?;
}
Err(e) => {
println!("Failed to get transaction: {}", e);
}
}
println!("Press Ctrl+C to exit example...");
tokio::signal::ctrl_c().await?;
Ok(())
}
+2 -2
View File
@@ -1,3 +1,3 @@
pub mod streaming;
pub mod common;
pub mod protos;
pub mod common;
pub mod streaming;
+8 -4
View File
@@ -61,7 +61,7 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
println!("Subscribing to Yellowstone gRPC events...");
// Create low-latency configuration
let mut config = ClientConfig::low_latency();
let mut config: ClientConfig = ClientConfig::low_latency();
// Enable performance monitoring, has performance overhead, disabled by default
config.enable_metrics = true;
let grpc = YellowstoneGrpc::new_with_config(
@@ -112,7 +112,7 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
// No event filtering, includes all events
let event_type_filter = None;
// Only include PumpSwapBuy events and PumpSwapSell events
// let event_type_filter = EventTypeFilter { include: vec![EventType::PumpSwapBuy, EventType::PumpSwapSell] };
// let event_type_filter = Some(EventTypeFilter { include: vec![EventType::PumpFunBuy] });
println!("Starting to listen for events, press Ctrl+C to stop...");
println!("Monitoring programs: {:?}", account_include);
@@ -188,11 +188,15 @@ async fn test_shreds() -> Result<(), Box<dyn std::error::Error>> {
fn create_event_callback() -> impl Fn(Box<dyn UnifiedEvent>) {
|event: Box<dyn UnifiedEvent>| {
println!("🎉 Event received! Type: {:?}, ID: {}", event.event_type(), event.id());
println!(
"🎉 Event received! Type: {:?}, transaction_index: {:?}",
event.event_type(),
event.transaction_index()
);
match_event!(event, {
// -------------------------- block meta -----------------------
BlockMetaEvent => |e: BlockMetaEvent| {
println!("BlockMetaEvent: {e:?}");
println!("BlockMetaEvent: {:?}", e.metadata.program_handle_time_consuming_us);
},
// -------------------------- bonk -----------------------
BonkPoolCreateEvent => |e: BonkPoolCreateEvent| {
-131
View File
@@ -1,131 +0,0 @@
use crate::streaming::event_parser::UnifiedEvent;
/// 通用批处理事件收集器
pub struct EventBatchProcessor<F>
where
F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
{
pub(crate) callback: F,
batch: Vec<Box<dyn UnifiedEvent>>,
batch_size: usize,
timeout_ms: u64,
last_flush_time: std::time::Instant,
}
impl<F> EventBatchProcessor<F>
where
F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
{
/// 创建新的批处理器
pub fn new(callback: F, batch_size: usize, timeout_ms: u64) -> Self {
Self {
callback,
batch: Vec::with_capacity(batch_size),
batch_size,
timeout_ms,
last_flush_time: std::time::Instant::now(),
}
}
/// 添加事件到批次
pub fn add_event(&mut self, event: Box<dyn UnifiedEvent>) {
log::debug!("Adding event to batch: {} (type: {:?})", event.id(), event.event_type());
self.batch.push(event);
// 检查是否需要刷新批次
if self.batch.len() >= self.batch_size || self.should_flush_by_timeout() {
log::debug!("Flushing batch: size={}, timeout={}", self.batch.len(), self.should_flush_by_timeout());
self.flush();
}
}
/// 强制刷新当前批次
pub fn flush(&mut self) {
if !self.batch.is_empty() {
let events = std::mem::replace(&mut self.batch, Vec::with_capacity(self.batch_size));
log::debug!("Flushing {} events from batch processor", events.len());
// 添加调试信息(仅在debug模式下)
if log::log_enabled!(log::Level::Debug) {
for (i, event) in events.iter().enumerate() {
log::debug!("Event {}: Type={:?}, ID={}", i, event.event_type(), event.id());
}
}
// 执行回调并捕获可能的错误
log::debug!("Executing batch callback with {} events", events.len());
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
(self.callback)(events);
})) {
Ok(_) => {
log::debug!("Batch callback executed successfully");
}
Err(e) => {
log::error!("Batch callback panicked: {:?}", e);
}
}
self.last_flush_time = std::time::Instant::now();
} else {
log::debug!("No events to flush");
}
}
/// 获取当前批次大小
pub fn current_batch_size(&self) -> usize {
self.batch.len()
}
/// 检查是否应该基于超时刷新
fn should_flush_by_timeout(&self) -> bool {
self.last_flush_time.elapsed().as_millis() >= self.timeout_ms as u128
}
/// 检查批次是否已满
pub fn is_batch_full(&self) -> bool {
self.batch.len() >= self.batch_size
}
/// 检查是否需要刷新(大小或超时)
pub fn should_flush(&self) -> bool {
self.is_batch_full() || self.should_flush_by_timeout()
}
}
/// 简单的事件批处理器,用于将单个事件回调转换为批量回调
pub struct SimpleEventBatchProcessor<F>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
callback: F,
}
impl<F> SimpleEventBatchProcessor<F>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
pub fn new(callback: F) -> Self {
Self { callback }
}
/// 将批量事件拆分为单个事件处理
pub fn process_batch(&self, events: Vec<Box<dyn UnifiedEvent>>) {
for event in events {
(self.callback)(event);
}
}
}
/// 批处理器包装器,用于将单个事件回调适配为批量处理
pub fn create_batch_callback_adapter<F>(
single_event_callback: F,
) -> impl FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
move |events: Vec<Box<dyn UnifiedEvent>>| {
for event in events {
single_event_callback(event);
}
}
}
+37 -56
View File
@@ -1,14 +1,12 @@
use super::constants::*;
/// 背压处理策略
/// Backpressure handling strategy
#[derive(Debug, Clone, Copy)]
pub enum BackpressureStrategy {
/// 阻塞等待(默认)
/// Block and wait (default)
Block,
/// 丢弃消息
/// Drop messages
Drop,
/// 重试有限次数后丢弃
Retry { max_attempts: usize, wait_ms: u64 },
}
impl Default for BackpressureStrategy {
@@ -17,50 +15,29 @@ impl Default for BackpressureStrategy {
}
}
/// 批处理配置
#[derive(Debug, Clone)]
pub struct BatchConfig {
/// 批处理大小(默认:100
pub batch_size: usize,
/// 批处理超时时间(毫秒,默认:5ms)
pub batch_timeout_ms: u64,
/// 是否启用批处理(默认:true)
pub enabled: bool,
}
impl Default for BatchConfig {
fn default() -> Self {
Self {
batch_size: DEFAULT_BATCH_SIZE,
batch_timeout_ms: DEFAULT_BATCH_TIMEOUT_MS,
enabled: true,
}
}
}
/// 背压配置
/// Backpressure configuration
#[derive(Debug, Clone)]
pub struct BackpressureConfig {
/// 通道大小(默认:1000
pub channel_size: usize,
/// 背压处理策略(默认:Block
/// Channel size (default: 1000)
pub permits: usize,
/// Backpressure handling strategy (default: Block)
pub strategy: BackpressureStrategy,
}
impl Default for BackpressureConfig {
fn default() -> Self {
Self { channel_size: DEFAULT_CHANNEL_SIZE, strategy: BackpressureStrategy::default() }
Self { permits: 1, strategy: BackpressureStrategy::default() }
}
}
/// 连接配置
/// Connection configuration
#[derive(Debug, Clone)]
pub struct ConnectionConfig {
/// 连接超时时间(秒,默认:10
/// Connection timeout in seconds (default: 10)
pub connect_timeout: u64,
/// 请求超时时间(秒,默认:60
/// Request timeout in seconds (default: 60)
pub request_timeout: u64,
/// 最大解码消息大小(字节,默认:10MB
/// Maximum decoding message size in bytes (default: 10MB)
pub max_decoding_message_size: usize,
}
@@ -74,16 +51,14 @@ impl Default for ConnectionConfig {
}
}
/// 通用客户端配置
/// Common client configuration
#[derive(Debug, Clone)]
pub struct StreamClientConfig {
/// 连接配置
/// Connection configuration
pub connection: ConnectionConfig,
/// 批处理配置
pub batch: BatchConfig,
/// 背压配置
/// Backpressure configuration
pub backpressure: BackpressureConfig,
/// 是否启用性能监控(默认:false
/// Whether performance monitoring is enabled (default: false)
pub enable_metrics: bool,
}
@@ -91,7 +66,6 @@ impl Default for StreamClientConfig {
fn default() -> Self {
Self {
connection: ConnectionConfig::default(),
batch: BatchConfig::default(),
backpressure: BackpressureConfig::default(),
enable_metrics: false,
}
@@ -99,33 +73,40 @@ impl Default for StreamClientConfig {
}
impl StreamClientConfig {
/// 创建高性能配置(适合高并发场景)
pub fn high_performance() -> Self {
/// Creates a high-throughput configuration optimized for high-concurrency scenarios.
///
/// This configuration prioritizes throughput over latency by:
/// - Implementing a drop strategy for backpressure to avoid blocking
/// - Setting a large permit buffer (5,000) to handle burst traffic
///
/// Ideal for scenarios where you need to process large volumes of data
/// and can tolerate occasional message drops during peak loads.
pub fn high_throughput() -> Self {
Self {
connection: ConnectionConfig::default(),
batch: BatchConfig { batch_size: 200, batch_timeout_ms: 5, enabled: true },
backpressure: BackpressureConfig {
channel_size: 20000,
permits: 20000,
strategy: BackpressureStrategy::Drop,
},
enable_metrics: false,
}
}
/// 创建低延迟配置(适合实时场景)
/// Creates a low-latency configuration optimized for real-time scenarios.
///
/// This configuration prioritizes latency over throughput by:
/// - Processing events immediately without buffering
/// - Implementing a blocking backpressure strategy to ensure no data loss
/// - Setting optimal permits (4000) for balanced throughput and latency
///
/// Ideal for scenarios where every millisecond counts and you cannot
/// afford to lose any events, such as trading applications or real-time monitoring.
pub fn low_latency() -> Self {
Self {
connection: ConnectionConfig::default(),
batch: BatchConfig {
batch_size: 10,
batch_timeout_ms: 1,
enabled: false, // 禁用批处理,即时处理
},
backpressure: BackpressureConfig {
channel_size: 1000,
strategy: BackpressureStrategy::Block,
},
backpressure: BackpressureConfig { permits: 4000, strategy: BackpressureStrategy::Block },
enable_metrics: false,
}
}
}
+1 -3
View File
@@ -5,10 +5,8 @@ pub const DEFAULT_CONNECT_TIMEOUT: u64 = 10;
pub const DEFAULT_REQUEST_TIMEOUT: u64 = 60;
pub const DEFAULT_CHANNEL_SIZE: usize = 1000;
pub const DEFAULT_MAX_DECODING_MESSAGE_SIZE: usize = 1024 * 1024 * 10;
pub const DEFAULT_BATCH_SIZE: usize = 100;
pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5;
// 性能监控相关常量
pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5;
pub const DEFAULT_METRICS_PRINT_INTERVAL_SECONDS: u64 = 10;
pub const SLOW_PROCESSING_THRESHOLD_MS: f64 = 10.0;
pub const SLOW_PROCESSING_THRESHOLD_US: f64 = 3000.0;
+434
View File
@@ -0,0 +1,434 @@
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Instant;
use crossbeam_queue::SegQueue;
use solana_sdk::pubkey::Pubkey;
use crate::common::AnyResult;
use crate::streaming::common::BackpressureStrategy;
use crate::streaming::common::{
MetricsEventType, MetricsManager, StreamClientConfig as ClientConfig,
};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
use crate::streaming::event_parser::core::traits::get_high_perf_clock;
use crate::streaming::event_parser::EventParser;
use crate::streaming::event_parser::{
core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
};
use crate::streaming::grpc::{BackpressureConfig, EventPretty};
use crate::streaming::shred::TransactionWithSlot;
use once_cell::sync::OnceCell;
/// High-performance Event processor using SegQueue for all strategies
pub struct EventProcessor {
pub(crate) metrics_manager: MetricsManager,
pub(crate) config: ClientConfig,
pub(crate) parser_cache: OnceCell<Arc<dyn EventParser>>,
pub(crate) protocols: Vec<Protocol>,
pub(crate) event_type_filter: Option<EventTypeFilter>,
pub(crate) callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
pub(crate) backpressure_config: BackpressureConfig,
/// High-performance lockfree queue for gRPC events
pub(crate) grpc_queue: Arc<SegQueue<(EventPretty, Option<Pubkey>)>>,
/// High-performance lockfree queue for shred events
pub(crate) shred_queue: Arc<SegQueue<(TransactionWithSlot, Option<Pubkey>)>>,
/// Fast O(1) counter for Drop strategy (avoids expensive SegQueue::len())
pub(crate) grpc_pending_count: Arc<AtomicUsize>,
pub(crate) shred_pending_count: Arc<AtomicUsize>,
/// Processing thread control
pub(crate) processing_shutdown: Arc<AtomicBool>,
}
impl EventProcessor {
/// Create a new high-performance event processor
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
let backpressure_config = config.backpressure.clone();
let grpc_queue = Arc::new(SegQueue::new());
let shred_queue = Arc::new(SegQueue::new());
let grpc_pending_count = Arc::new(AtomicUsize::new(0));
let shred_pending_count = Arc::new(AtomicUsize::new(0));
let processing_shutdown = Arc::new(AtomicBool::new(false));
Self {
metrics_manager,
config,
parser_cache: OnceCell::new(),
protocols: vec![],
event_type_filter: None,
backpressure_config,
callback: None,
grpc_queue,
shred_queue,
grpc_pending_count,
shred_pending_count,
processing_shutdown,
}
}
pub fn set_protocols_and_event_type_filter(
&mut self,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
backpressure_config: BackpressureConfig,
callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
) {
self.protocols = protocols;
self.event_type_filter = event_type_filter;
// Check if Block processing thread should be started (before moving backpressure_config)
let should_start_block_processing = true;
// matches!(backpressure_config.strategy, BackpressureStrategy::Block);
self.backpressure_config = backpressure_config;
self.callback = callback;
// Use stored values to initialize parser_cache
let protocols_ref = &self.protocols;
let event_type_filter_ref = self.event_type_filter.as_ref();
self.parser_cache.get_or_init(|| {
Arc::new(MutilEventParser::new(protocols_ref.clone(), event_type_filter_ref.cloned()))
});
// Start Block processing thread if using Block strategy
if should_start_block_processing {
self.start_block_processing_thread();
}
}
pub fn get_parser(&self) -> Arc<dyn EventParser> {
self.parser_cache.get().unwrap().clone()
}
/// Create adapter callback
fn create_adapter_callback(&self) -> Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync> {
let callback = self.callback.clone().unwrap();
let metrics_manager = self.metrics_manager.clone();
Arc::new(move |event: Box<dyn UnifiedEvent>| {
let processing_time_us = event.program_handle_time_consuming_us() as f64;
callback(event);
metrics_manager.update_metrics(MetricsEventType::Transaction, 1, processing_time_us);
})
}
pub async fn process_grpc_event_transaction_with_metrics(
&self,
event_pretty: EventPretty,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
self.apply_backpressure_control(event_pretty, bot_wallet).await
}
/// Apply backpressure control strategy
async fn apply_backpressure_control(
&self,
event_pretty: EventPretty,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
match self.backpressure_config.strategy {
BackpressureStrategy::Block => {
// Block strategy: async wait if queue is full (backpressure control)
loop {
let current_pending = self.grpc_pending_count.load(Ordering::Relaxed);
if current_pending < self.backpressure_config.permits {
self.grpc_queue.push((event_pretty, bot_wallet));
self.grpc_pending_count.fetch_add(1, Ordering::Relaxed);
break;
}
// Async yield to avoid blocking gRPC data source
tokio::task::yield_now().await;
}
Ok(())
}
BackpressureStrategy::Drop => {
// Drop strategy: Use O(1) atomic counter instead of expensive O(n) len()
// If pending count >= permits, DROP the event immediately
let current_pending = self.grpc_pending_count.load(Ordering::Relaxed);
if current_pending >= self.backpressure_config.permits {
self.metrics_manager.increment_dropped_events();
Ok(())
} else {
self.grpc_pending_count.fetch_add(1, Ordering::Relaxed);
let processor = self.clone();
tokio::spawn(async move {
match processor
.process_grpc_event_transaction(event_pretty, bot_wallet)
.await
{
Ok(_) => {
processor.grpc_pending_count.fetch_sub(1, Ordering::Relaxed);
}
Err(e) => {
log::error!("Error in async gRPC processing: {}", e);
}
}
});
Ok(())
}
}
}
}
async fn process_grpc_event_transaction(
&self,
event_pretty: EventPretty,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
if self.callback.is_none() {
return Ok(());
}
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_account_process_count();
let account_event = AccountEventParser::parse_account_event(
&self.protocols,
account_pretty,
self.event_type_filter.as_ref(),
);
if let Some(event) = account_event {
let processing_time_us = event.program_handle_time_consuming_us() as f64;
self.invoke_callback(event);
self.update_metrics(MetricsEventType::Account, 1, processing_time_us);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_tx_process_count();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature;
let tx = transaction_pretty.tx;
let block_time = transaction_pretty.block_time;
let program_received_time_us = transaction_pretty.program_received_time_us;
let transaction_index = transaction_pretty.transaction_index;
// Use cache to get parser
let parser = self.get_parser();
let adapter_callback = self.create_adapter_callback();
parser
.parse_transaction_owned(
tx,
signature,
Some(slot),
block_time,
program_received_time_us,
bot_wallet,
transaction_index,
adapter_callback,
)
.await?;
}
EventPretty::BlockMeta(block_meta_pretty) => {
self.metrics_manager.add_block_meta_process_count();
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
let block_meta_event = CommonEventParser::generate_block_meta_event(
block_meta_pretty.slot,
&block_meta_pretty.block_hash,
block_time_ms,
block_meta_pretty.program_received_time_us,
);
let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64;
self.invoke_callback(block_meta_event);
self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us);
}
}
Ok(())
}
pub fn invoke_callback(&self, event: Box<dyn UnifiedEvent>) {
if let Some(callback) = self.callback.as_ref() {
callback(event);
}
}
/// Process a single transaction immediately
pub async fn process_shred_transaction_immediate(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
self.process_shred_transaction(transaction_with_slot, bot_wallet).await
}
/// Process shred transaction with backpressure control and performance monitoring
pub async fn process_shred_transaction_with_metrics(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
// Backpressure control logic
self.apply_shred_backpressure_control(transaction_with_slot, bot_wallet).await
}
/// Apply shred backpressure control strategy
async fn apply_shred_backpressure_control(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
match self.backpressure_config.strategy {
BackpressureStrategy::Block => {
// Block strategy: async wait if queue is full (backpressure control)
loop {
let current_pending = self.shred_pending_count.load(Ordering::Relaxed);
if current_pending < self.backpressure_config.permits {
self.shred_queue.push((transaction_with_slot, bot_wallet));
self.shred_pending_count.fetch_add(1, Ordering::Relaxed);
break;
}
// Async yield to avoid blocking shred data source
tokio::task::yield_now().await;
}
Ok(())
}
BackpressureStrategy::Drop => {
// Drop strategy: Use O(1) atomic counter instead of expensive O(n) len()
let current_pending = self.shred_pending_count.load(Ordering::Relaxed);
if current_pending >= self.backpressure_config.permits {
self.metrics_manager.increment_dropped_events();
Ok(())
} else {
self.shred_pending_count.fetch_add(1, Ordering::Relaxed);
let processor = self.clone();
tokio::spawn(async move {
match processor
.process_shred_transaction(transaction_with_slot, bot_wallet)
.await
{
Ok(_) => {
processor.shred_pending_count.fetch_sub(1, Ordering::Relaxed);
}
Err(e) => {
log::error!("Error in async shred processing: {}", e);
}
}
});
Ok(())
}
}
}
}
pub async fn process_shred_transaction(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
if self.callback.is_none() {
return Ok(());
}
self.metrics_manager.add_tx_process_count();
let tx = transaction_with_slot.transaction;
let slot = transaction_with_slot.slot;
let signature = tx.signatures[0];
let program_received_time_us = transaction_with_slot.program_received_time_us;
// Use cache to get parser
let parser = self.get_parser();
let adapter_callback = self.create_adapter_callback();
parser
.parse_versioned_transaction_owned(
tx,
signature,
Some(slot),
None,
program_received_time_us,
bot_wallet,
None,
&[],
adapter_callback,
)
.await?;
Ok(())
}
fn update_metrics(&self, ty: MetricsEventType, count: u64, time_us: f64) {
self.metrics_manager.update_metrics(ty, count, time_us);
}
/// Start dedicated processing threads for all strategies
fn start_block_processing_thread(&self) {
// Reset shutdown flag
self.processing_shutdown.store(false, Ordering::Relaxed);
let grpc_queue = Arc::clone(&self.grpc_queue);
let shred_queue = Arc::clone(&self.shred_queue);
let grpc_pending_count = Arc::clone(&self.grpc_pending_count);
let shred_pending_count = Arc::clone(&self.shred_pending_count);
let shutdown_flag = Arc::clone(&self.processing_shutdown);
let shutdown_flag_clone = Arc::clone(&self.processing_shutdown);
let processor = self.clone();
let processor_clone = self.clone();
// 1. 专用线程 + 2. Busy-wait + 4. 无锁处理
std::thread::spawn(move || {
// 创建blocking runtime for async processing
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
while !shutdown_flag.load(Ordering::Relaxed) {
if let Some((event_pretty, bot_wallet)) = grpc_queue.pop() {
// Decrement pending counter when consuming from queue
grpc_pending_count.fetch_sub(1, Ordering::Relaxed);
// Process event in blocking runtime
if let Err(e) = rt.block_on(
processor.process_grpc_event_transaction(event_pretty, bot_wallet),
) {
println!("Error processing gRPC event: {}", e);
}
} else {
// 2. 优化忙等待: 使用轻量级休眠减少CPU占用
std::thread::yield_now();
}
}
});
// Shred处理也使用相同的低延迟优化
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
while !shutdown_flag_clone.load(Ordering::Relaxed) {
if let Some((transaction_with_slot, bot_wallet)) = shred_queue.pop() {
// Decrement pending counter when consuming from queue
shred_pending_count.fetch_sub(1, Ordering::Relaxed);
// Process transaction in blocking runtime
if let Err(e) = rt.block_on(
processor_clone
.process_shred_transaction(transaction_with_slot, bot_wallet),
) {
log::error!("Error processing shred transaction: {}", e);
}
} else {
// 优化忙等待: 使用轻量级休眠减少CPU占用
std::thread::yield_now();
}
}
});
}
/// Stop processing threads
pub fn stop_processing(&self) {
self.processing_shutdown.store(true, Ordering::Relaxed);
}
}
// Implement Clone trait to support sharing between modules
impl Clone for EventProcessor {
fn clone(&self) -> Self {
Self {
metrics_manager: self.metrics_manager.clone(),
config: self.config.clone(),
parser_cache: self.parser_cache.clone(),
protocols: self.protocols.clone(),
event_type_filter: self.event_type_filter.clone(),
backpressure_config: self.backpressure_config.clone(),
callback: self.callback.clone(),
grpc_queue: self.grpc_queue.clone(),
shred_queue: self.shred_queue.clone(),
grpc_pending_count: self.grpc_pending_count.clone(),
shred_pending_count: self.shred_pending_count.clone(),
processing_shutdown: self.processing_shutdown.clone(),
}
}
}
+557 -205
View File
@@ -1,305 +1,656 @@
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use tokio::sync::Mutex;
use super::config::StreamClientConfig;
use super::constants::*;
/// 单个事件类型的指标
/// Event type enumeration
#[derive(Debug, Clone, Copy)]
pub enum EventType {
Transaction = 0,
Account = 1,
BlockMeta = 2,
}
/// Compatibility alias
pub type MetricsEventType = EventType;
impl EventType {
#[inline]
const fn as_index(self) -> usize {
self as usize
}
const fn name(self) -> &'static str {
match self {
EventType::Transaction => "TX",
EventType::Account => "Account",
EventType::BlockMeta => "Block Meta",
}
}
// Compatibility constants
pub const TX: EventType = EventType::Transaction;
}
/// High-performance atomic event metrics
#[derive(Debug)]
struct AtomicEventMetrics {
process_count: AtomicU64,
events_processed: AtomicU64,
events_in_window: AtomicU64,
window_start_nanos: AtomicU64,
events_per_second_bits: AtomicU64, // Bit representation of f64
}
impl AtomicEventMetrics {
fn new(now_nanos: u64) -> Self {
Self {
process_count: AtomicU64::new(0),
events_processed: AtomicU64::new(0),
events_in_window: AtomicU64::new(0),
window_start_nanos: AtomicU64::new(now_nanos),
events_per_second_bits: AtomicU64::new(0),
}
}
/// Atomically increment process count
#[inline]
fn add_process_count(&self) {
self.process_count.fetch_add(1, Ordering::Relaxed);
}
/// Atomically increment event processing count
#[inline]
fn add_events_processed(&self, count: u64) {
self.events_processed.fetch_add(count, Ordering::Relaxed);
self.events_in_window.fetch_add(count, Ordering::Relaxed);
}
/// Get current count (non-blocking)
#[inline]
fn get_counts(&self) -> (u64, u64, u64) {
(
self.process_count.load(Ordering::Relaxed),
self.events_processed.load(Ordering::Relaxed),
self.events_in_window.load(Ordering::Relaxed),
)
}
/// Atomically update events per second
#[inline]
fn update_events_per_second(&self, eps: f64) {
self.events_per_second_bits.store(eps.to_bits(), Ordering::Relaxed);
}
/// Get events per second
#[inline]
fn get_events_per_second(&self) -> f64 {
f64::from_bits(self.events_per_second_bits.load(Ordering::Relaxed))
}
/// Reset window count
#[inline]
fn reset_window(&self, new_start_nanos: u64) {
self.events_in_window.store(0, Ordering::Relaxed);
self.window_start_nanos.store(new_start_nanos, Ordering::Relaxed);
}
#[inline]
fn get_window_start(&self) -> u64 {
self.window_start_nanos.load(Ordering::Relaxed)
}
}
/// High-performance atomic processing time statistics
#[derive(Debug)]
struct AtomicProcessingTimeStats {
min_time_bits: AtomicU64,
max_time_bits: AtomicU64,
min_time_timestamp_nanos: AtomicU64, // Timestamp of min value update (nanoseconds)
max_time_timestamp_nanos: AtomicU64, // Timestamp of max value update (nanoseconds)
total_time_us: AtomicU64, // Store integer part of microseconds
total_events: AtomicU64,
}
impl AtomicProcessingTimeStats {
fn new() -> Self {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
Self {
min_time_bits: AtomicU64::new(f64::INFINITY.to_bits()),
max_time_bits: AtomicU64::new(0),
min_time_timestamp_nanos: AtomicU64::new(now_nanos),
max_time_timestamp_nanos: AtomicU64::new(now_nanos),
total_time_us: AtomicU64::new(0),
total_events: AtomicU64::new(0),
}
}
/// Atomically update processing time statistics
#[inline]
fn update(&self, time_us: f64, event_count: u64) {
let time_bits = time_us.to_bits();
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
// Update minimum value, check time difference and reset if over 10 seconds
let mut current_min = self.min_time_bits.load(Ordering::Relaxed);
let min_timestamp = self.min_time_timestamp_nanos.load(Ordering::Relaxed);
// Check if min value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds)
let min_time_diff_nanos = now_nanos.saturating_sub(min_timestamp);
if min_time_diff_nanos > 10_000_000_000 {
// Over 10 seconds, reset min value
self.min_time_bits.store(f64::INFINITY.to_bits(), Ordering::Relaxed);
self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
current_min = f64::INFINITY.to_bits();
}
// If current time is less than min value, update min value and timestamp
while time_bits < current_min {
match self.min_time_bits.compare_exchange_weak(
current_min,
time_bits,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => {
// Successfully updated min value, also update timestamp
self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
break;
}
Err(x) => current_min = x,
}
}
// Update maximum value, check time difference and reset if over 10 seconds
let mut current_max = self.max_time_bits.load(Ordering::Relaxed);
let max_timestamp = self.max_time_timestamp_nanos.load(Ordering::Relaxed);
// Check if max value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds)
let time_diff_nanos = now_nanos.saturating_sub(max_timestamp);
if time_diff_nanos > 10_000_000_000 {
// Over 10 seconds, reset max value
self.max_time_bits.store(0, Ordering::Relaxed);
self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
current_max = 0;
}
// If current time is greater than max value, update max value and timestamp
while time_bits > current_max {
match self.max_time_bits.compare_exchange_weak(
current_max,
time_bits,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => {
// Successfully updated max value, also update timestamp
self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
break;
}
Err(x) => current_max = x,
}
}
// Update cumulative values (convert microseconds to integers to avoid floating point accumulation issues)
let total_time_us_int = (time_us * event_count as f64) as u64;
self.total_time_us.fetch_add(total_time_us_int, Ordering::Relaxed);
self.total_events.fetch_add(event_count, Ordering::Relaxed);
}
/// Get statistics (non-blocking)
#[inline]
fn get_stats(&self) -> ProcessingTimeStats {
let min_bits = self.min_time_bits.load(Ordering::Relaxed);
let max_bits = self.max_time_bits.load(Ordering::Relaxed);
let total_time_us_int = self.total_time_us.load(Ordering::Relaxed);
let total_events = self.total_events.load(Ordering::Relaxed);
let min_time = f64::from_bits(min_bits);
let max_time = f64::from_bits(max_bits);
let avg_time =
if total_events > 0 { total_time_us_int as f64 / total_events as f64 } else { 0.0 };
ProcessingTimeStats {
min_us: if min_time == f64::INFINITY { 0.0 } else { min_time },
max_us: max_time,
avg_us: avg_time,
}
}
}
/// Processing time statistics result
#[derive(Debug, Clone)]
pub struct EventMetrics {
pub struct ProcessingTimeStats {
pub min_us: f64,
pub max_us: f64,
pub avg_us: f64,
}
/// Event metrics snapshot
#[derive(Debug, Clone)]
pub struct EventMetricsSnapshot {
pub process_count: u64,
pub events_processed: u64,
pub events_per_second: f64,
pub events_in_window: u64,
pub window_start_time: std::time::Instant,
}
impl EventMetrics {
fn new(now: std::time::Instant) -> Self {
Self {
process_count: 0,
events_processed: 0,
events_per_second: 0.0,
events_in_window: 0,
window_start_time: now,
}
}
}
/// 通用性能监控指标
/// Compatibility structure - complete performance metrics
#[derive(Debug, Clone)]
pub struct PerformanceMetrics {
pub start_time: std::time::Instant,
pub event_metrics: [EventMetrics; 3], // [Tx, Account, BlockMeta]
pub average_processing_time_ms: f64,
pub min_processing_time_ms: f64,
pub max_processing_time_ms: f64,
pub last_update_time: std::time::Instant,
}
impl Default for PerformanceMetrics {
fn default() -> Self {
Self::new()
}
}
pub enum MetricsEventType {
Tx,
Account,
BlockMeta,
}
impl MetricsEventType {
fn as_index(&self) -> usize {
match self {
MetricsEventType::Tx => 0,
MetricsEventType::Account => 1,
MetricsEventType::BlockMeta => 2,
}
}
pub uptime: std::time::Duration,
pub tx_metrics: EventMetricsSnapshot,
pub account_metrics: EventMetricsSnapshot,
pub block_meta_metrics: EventMetricsSnapshot,
pub processing_stats: ProcessingTimeStats,
pub dropped_events_count: u64,
}
impl PerformanceMetrics {
/// Create default performance metrics (compatibility method)
pub fn new() -> Self {
let now = std::time::Instant::now();
let default_metrics =
EventMetricsSnapshot { process_count: 0, events_processed: 0, events_per_second: 0.0 };
let default_stats = ProcessingTimeStats { min_us: 0.0, max_us: 0.0, avg_us: 0.0 };
Self {
start_time: now,
event_metrics: [EventMetrics::new(now), EventMetrics::new(now), EventMetrics::new(now)],
average_processing_time_ms: 0.0,
min_processing_time_ms: 0.0,
max_processing_time_ms: 0.0,
last_update_time: now,
uptime: std::time::Duration::ZERO,
tx_metrics: default_metrics.clone(),
account_metrics: default_metrics.clone(),
block_meta_metrics: default_metrics,
processing_stats: default_stats,
dropped_events_count: 0,
}
}
}
/// High-performance metrics system
#[derive(Debug)]
pub struct HighPerformanceMetrics {
start_nanos: u64,
event_metrics: [AtomicEventMetrics; 3],
processing_stats: AtomicProcessingTimeStats,
// 丢弃事件指标
dropped_events_count: AtomicU64,
}
impl HighPerformanceMetrics {
fn new() -> Self {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
Self {
start_nanos: now_nanos,
event_metrics: [
AtomicEventMetrics::new(now_nanos),
AtomicEventMetrics::new(now_nanos),
AtomicEventMetrics::new(now_nanos),
],
processing_stats: AtomicProcessingTimeStats::new(),
// 初始化丢弃事件指标
dropped_events_count: AtomicU64::new(0),
}
}
/// 更新时间窗口指标
fn update_window_metrics(
&mut self,
event_type: &MetricsEventType,
now: std::time::Instant,
window_duration: std::time::Duration,
) {
/// 获取运行时长(秒)
#[inline]
pub fn get_uptime_seconds(&self) -> f64 {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
(now_nanos - self.start_nanos) as f64 / 1_000_000_000.0
}
/// 获取事件指标快照
#[inline]
pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot {
let index = event_type.as_index();
let event_metric = &mut self.event_metrics[index];
let (process_count, events_processed, _) = self.event_metrics[index].get_counts();
let events_per_second = self.calculate_real_time_eps(event_type);
if now.duration_since(event_metric.window_start_time) >= window_duration {
let window_seconds = now.duration_since(event_metric.window_start_time).as_secs_f64();
// 修复:正确计算每秒事件数,避免除零错误
event_metric.events_per_second = if window_seconds > 0.001 {
// 避免极小的时间差
event_metric.events_in_window as f64 / window_seconds
} else {
0.0 // 时间太短时设为0,而不是事件总数
};
// 重置窗口
event_metric.events_in_window = 0;
event_metric.window_start_time = now;
}
EventMetricsSnapshot { process_count, events_processed, events_per_second }
}
/// 计算实时每秒事件数(用于显示)
fn calculate_real_time_events_per_second(
&self,
event_type: &MetricsEventType,
now: std::time::Instant,
) -> f64 {
/// 获取处理时间统计
#[inline]
pub fn get_processing_stats(&self) -> ProcessingTimeStats {
self.processing_stats.get_stats()
}
/// 获取丢弃事件计数
#[inline]
pub fn get_dropped_events_count(&self) -> u64 {
self.dropped_events_count.load(Ordering::Relaxed)
}
/// 计算实时每秒事件数(非阻塞)
fn calculate_real_time_eps(&self, event_type: EventType) -> f64 {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
let index = event_type.as_index();
let event_metric = &self.event_metrics[index];
let current_window_duration =
now.duration_since(event_metric.window_start_time).as_secs_f64();
let window_start = event_metric.get_window_start();
let current_window_duration_secs =
(now_nanos.saturating_sub(window_start)) as f64 / 1_000_000_000.0;
let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed);
// 如果当前窗口有足够的时间和事件,使用当前窗口的数据
if current_window_duration > 1.0 && event_metric.events_in_window > 0 {
event_metric.events_in_window as f64 / current_window_duration
// 优先级1: 当前窗口实时数据(≥2秒且有事件)
if current_window_duration_secs >= 2.0 && events_in_window > 0 {
return events_in_window as f64 / current_window_duration_secs;
}
// 如果当前窗口时间太短或没有事件,使用上一个完整窗口的值
else if event_metric.events_per_second > 0.0 {
event_metric.events_per_second
// 优先级2: 上一个窗口的结果
let stored_eps = event_metric.get_events_per_second();
if stored_eps > 0.0 {
return stored_eps;
}
// 如果都没有,计算总体平均值
else {
let total_duration = now.duration_since(self.start_time).as_secs_f64();
if total_duration > 1.0 && event_metric.events_processed > 0 {
event_metric.events_processed as f64 / total_duration
} else {
0.0
// 优先级3: 总体平均值(≥3秒运行时间)
let total_duration_secs = self.get_uptime_seconds();
let total_events = event_metric.events_processed.load(Ordering::Relaxed);
if total_duration_secs >= 3.0 && total_events > 0 {
return total_events as f64 / total_duration_secs;
}
0.0
}
/// 更新窗口指标(后台任务调用)
fn update_window_metrics(&self, event_type: EventType, window_duration_nanos: u64) {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
let index = event_type.as_index();
let event_metric = &self.event_metrics[index];
let window_start = event_metric.get_window_start();
if now_nanos.saturating_sub(window_start) >= window_duration_nanos {
let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed);
let window_duration_secs = window_duration_nanos as f64 / 1_000_000_000.0;
if window_duration_secs > 0.001 && events_in_window > 0 {
let eps = events_in_window as f64 / window_duration_secs;
event_metric.update_events_per_second(eps);
}
event_metric.reset_window(now_nanos);
}
}
}
/// 通用性能监控管理器
/// 高性能指标管理器
pub struct MetricsManager {
metrics: Arc<Mutex<PerformanceMetrics>>,
config: Arc<StreamClientConfig>,
metrics: Arc<HighPerformanceMetrics>,
enable_metrics: bool,
stream_name: String,
background_task_running: AtomicBool,
}
impl MetricsManager {
/// 创建新的性能监控管理器
pub fn new(
metrics: Arc<Mutex<PerformanceMetrics>>,
config: Arc<StreamClientConfig>,
stream_name: String,
) -> Self {
Self { metrics, config, stream_name }
/// 创建新的指标管理器
pub fn new(enable_metrics: bool, stream_name: String) -> Self {
let manager = Self {
metrics: Arc::new(HighPerformanceMetrics::new()),
enable_metrics,
stream_name,
background_task_running: AtomicBool::new(false),
};
// 启动后台任务
manager.start_background_tasks();
manager
}
/// 获取性能指标
pub async fn get_metrics(&self) -> PerformanceMetrics {
let metrics = self.metrics.lock().await;
metrics.clone()
/// 启动后台任务
fn start_background_tasks(&self) {
if self
.background_task_running
.compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
if !self.enable_metrics {
return;
}
let metrics = self.metrics.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(std::time::Duration::from_millis(500));
loop {
interval.tick().await;
let window_duration_nanos = DEFAULT_METRICS_WINDOW_SECONDS * 1_000_000_000;
// 更新所有事件类型的窗口指标
metrics.update_window_metrics(EventType::Transaction, window_duration_nanos);
metrics.update_window_metrics(EventType::Account, window_duration_nanos);
metrics.update_window_metrics(EventType::BlockMeta, window_duration_nanos);
}
});
}
}
/// 打印性能指标
pub async fn print_metrics(&self) {
let metrics = self.get_metrics().await;
let event_names = ["TX", "Account", "Block Meta"];
let event_types =
[MetricsEventType::Tx, MetricsEventType::Account, MetricsEventType::BlockMeta];
let now = std::time::Instant::now();
/// 记录处理次数(非阻塞)
#[inline]
pub fn record_process(&self, event_type: EventType) {
if self.enable_metrics {
self.metrics.event_metrics[event_type.as_index()].add_process_count();
}
}
/// 记录事件处理(非阻塞)
#[inline]
pub fn record_events(&self, event_type: EventType, count: u64, processing_time_us: f64) {
if !self.enable_metrics {
return;
}
// 原子更新事件计数
self.metrics.event_metrics[event_type.as_index()].add_events_processed(count);
// 原子更新处理时间统计
self.metrics.processing_stats.update(processing_time_us, count);
}
/// 记录慢处理操作
#[inline]
pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) {
if processing_time_us > SLOW_PROCESSING_THRESHOLD_US {
log::debug!(
"{} slow processing: {:.2}us for {} events",
self.stream_name,
processing_time_us,
event_count,
);
}
}
/// 获取运行时长
pub fn get_uptime(&self) -> std::time::Duration {
std::time::Duration::from_secs_f64(self.metrics.get_uptime_seconds())
}
/// 获取事件指标
pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot {
self.metrics.get_event_metrics(event_type)
}
/// 获取处理时间统计
pub fn get_processing_stats(&self) -> ProcessingTimeStats {
self.metrics.get_processing_stats()
}
/// 获取丢弃事件计数
pub fn get_dropped_events_count(&self) -> u64 {
self.metrics.get_dropped_events_count()
}
/// 打印性能指标(非阻塞)
pub fn print_metrics(&self) {
println!("\n📊 {} Performance Metrics", self.stream_name);
println!(" Run Time: {:?}", metrics.start_time.elapsed());
// 打印表格头部
println!(" Run Time: {:?}", self.get_uptime());
// 打印丢弃事件指标
let dropped_count = self.get_dropped_events_count();
if dropped_count > 0 {
println!("\n⚠️ Dropped Events: {}", dropped_count);
}
// 打印事件指标表格
println!("┌─────────────┬──────────────┬──────────────────┬─────────────────┐");
println!("│ Event Type │ Process Count│ Events Processed │ Events/Second │");
println!("├─────────────┼──────────────┼──────────────────┼─────────────────┤");
// 打印每种事件类型的数据
for (i, name) in event_names.iter().enumerate() {
let event_metric = &metrics.event_metrics[i];
// 使用实时计算的每秒事件数,而不是窗口更新的值
let real_time_eps = metrics.calculate_real_time_events_per_second(&event_types[i], now);
for event_type in [EventType::Transaction, EventType::Account, EventType::BlockMeta] {
let metrics = self.get_event_metrics(event_type);
println!(
"{:11}{:12}{:16}{:13.2}",
name,
event_metric.process_count,
event_metric.events_processed,
real_time_eps
event_type.name(),
metrics.process_count,
metrics.events_processed,
metrics.events_per_second
);
}
println!("└─────────────┴──────────────┴──────────────────┴─────────────────┘");
// 打印处理时间统计表格
let stats = self.get_processing_stats();
println!("\n⏱️ Processing Time Statistics");
println!("┌─────────────────────┬─────────────┐");
println!("│ Metric │ Value (ms) │");
println!("├─────────────────────┼─────────────┤");
println!("│ Average │ {:9.2}", metrics.average_processing_time_ms);
println!("│ Minimum {:9.2}", metrics.min_processing_time_ms);
println!("│ Maximum {:9.2}", metrics.max_processing_time_ms);
println!("└─────────────────────┴─────────────┘");
println!("┌───────────────────────┬─────────────┐");
println!("│ Metric │ Value (us) │");
println!("├───────────────────────┼─────────────┤");
println!("│ Average {:9.2}", stats.avg_us);
println!("│ Minimum within 10s{:9.2}", stats.min_us);
println!("│ Maximum within 10s{:9.2}", stats.max_us);
println!("└───────────────────────┴─────────────┘");
println!();
}
/// 启动自动性能监控任务
pub async fn start_auto_monitoring(&self) -> Option<tokio::task::JoinHandle<()>> {
// 检查是否启用性能监控
if !self.config.enable_metrics {
return None; // 如果未启用性能监控,不启动监控任务
if !self.enable_metrics {
return None;
}
let metrics_manager = self.clone();
let manager = self.clone();
let handle = tokio::spawn(async move {
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(
let mut interval = tokio::time::interval(std::time::Duration::from_secs(
DEFAULT_METRICS_PRINT_INTERVAL_SECONDS,
));
loop {
interval.tick().await;
metrics_manager.print_metrics().await;
manager.print_metrics();
}
});
Some(handle)
}
/// 更新处理次数
pub async fn add_process_count(&self, event_type: MetricsEventType) {
if !self.config.enable_metrics {
return;
// === 兼容性方法 ===
/// 兼容性构造函数
pub fn new_with_metrics(
_metrics: Arc<std::sync::RwLock<PerformanceMetrics>>,
enable_metrics: bool,
stream_name: String,
) -> Self {
Self::new(enable_metrics, stream_name)
}
/// 获取完整的性能指标(兼容性方法)
pub fn get_metrics(&self) -> PerformanceMetrics {
PerformanceMetrics {
uptime: self.get_uptime(),
tx_metrics: self.get_event_metrics(EventType::Transaction),
account_metrics: self.get_event_metrics(EventType::Account),
block_meta_metrics: self.get_event_metrics(EventType::BlockMeta),
processing_stats: self.get_processing_stats(),
dropped_events_count: self.metrics.get_dropped_events_count(),
}
let mut metrics = self.metrics.lock().await;
metrics.event_metrics[event_type.as_index()].process_count += 1;
}
// 保持向后兼容方法
pub async fn add_tx_process_count(&self) {
self.add_process_count(MetricsEventType::Tx).await;
/// 兼容方法 - 添加交易处理计数
#[inline]
pub fn add_tx_process_count(&self) {
self.record_process(EventType::Transaction);
}
pub async fn add_account_process_count(&self) {
self.add_process_count(MetricsEventType::Account).await;
/// 兼容性方法 - 添加账户处理计数
#[inline]
pub fn add_account_process_count(&self) {
self.record_process(EventType::Account);
}
pub async fn add_block_meta_process_count(&self) {
self.add_process_count(MetricsEventType::BlockMeta).await;
/// 兼容性方法 - 添加区块元数据处理计数
#[inline]
pub fn add_block_meta_process_count(&self) {
self.record_process(EventType::BlockMeta);
}
/// 更新性能指标
pub async fn update_metrics(
/// 兼容性方法 - 更新指标
#[inline]
pub fn update_metrics(
&self,
event_type: MetricsEventType,
events_processed: u64,
processing_time_ms: f64,
processing_time_us: f64,
) {
// 检查是否启用性能监控
if !self.config.enable_metrics {
self.record_events(event_type, events_processed, processing_time_us);
self.log_slow_processing(processing_time_us, events_processed as usize);
}
/// 增加丢弃事件计数
#[inline]
pub fn increment_dropped_events(&self) {
if !self.enable_metrics {
return;
}
let mut metrics = self.metrics.lock().await;
let now = std::time::Instant::now();
let index = event_type.as_index();
// 原子地增加丢弃事件计数
let new_count = self.metrics.dropped_events_count.fetch_add(1, Ordering::Relaxed) + 1;
// 更新事件计数
metrics.event_metrics[index].events_processed += events_processed;
metrics.event_metrics[index].events_in_window += events_processed;
metrics.last_update_time = now;
// 更新处理时间统计
if processing_time_ms < metrics.min_processing_time_ms
|| metrics.min_processing_time_ms == 0.0
{
metrics.min_processing_time_ms = processing_time_ms;
// 每丢弃1000个事件记录一次警告日志
if new_count % 1000 == 0 {
log::debug!("{} dropped events count reached: {}", self.stream_name, new_count);
}
if processing_time_ms > metrics.max_processing_time_ms {
metrics.max_processing_time_ms = processing_time_ms;
}
// 计算平均处理时间 - 使用增量更新避免重复计算
let total_events = metrics.event_metrics[index].events_processed;
if total_events > 0 {
let total_events_f64 = total_events as f64;
let old_total = (total_events_f64 - events_processed as f64).max(0.0);
metrics.average_processing_time_ms = if old_total > 0.0 {
(metrics.average_processing_time_ms * old_total
+ processing_time_ms * events_processed as f64)
/ total_events_f64
} else {
processing_time_ms
};
}
// 更新时间窗口指标
let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS);
metrics.update_window_metrics(&event_type, now, window_duration);
}
/// 记录慢处理操作
pub fn log_slow_processing(&self, processing_time_ms: f64, event_count: usize) {
if processing_time_ms > SLOW_PROCESSING_THRESHOLD_MS {
log::warn!(
"{} slow processing: {processing_time_ms}ms for {event_count} events",
self.stream_name
/// 批量增加丢弃事件计数
#[inline]
pub fn increment_dropped_events_by(&self, count: u64) {
if !self.enable_metrics || count == 0 {
return;
}
// 原子地增加丢弃事件计数
let new_count =
self.metrics.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count;
// 记录批量丢弃事件的日志
if count > 1 {
log::debug!(
"{} dropped batch of {} events, total dropped: {}",
self.stream_name,
count,
new_count
);
}
// 每丢弃1000个事件记录一次警告日志
if new_count % 1000 == 0 || (new_count / 1000) != ((new_count - count) / 1000) {
log::debug!("{} dropped events count reached: {}", self.stream_name, new_count);
}
}
}
@@ -307,8 +658,9 @@ impl Clone for MetricsManager {
fn clone(&self) -> Self {
Self {
metrics: self.metrics.clone(),
config: self.config.clone(),
enable_metrics: self.enable_metrics,
stream_name: self.stream_name.clone(),
background_task_running: AtomicBool::new(false), // 新实例不自动启动后台任务
}
}
}
+4 -2
View File
@@ -1,13 +1,15 @@
// 公用模块 - 包含流处理相关的通用功能
pub mod config;
pub mod metrics;
pub mod batch;
pub mod constants;
pub mod subscription;
pub mod event_processor;
pub mod simd_utils;
// 重新导出主要类型
pub use config::*;
pub use metrics::*;
pub use batch::*;
pub use constants::*;
pub use subscription::*;
pub use event_processor::*;
pub use simd_utils::*;
+295
View File
@@ -0,0 +1,295 @@
use wide::*;
/// SIMD-accelerated data parsing utilities
pub struct SimdUtils;
impl SimdUtils {
/// SIMD-accelerated byte array comparison
/// For arrays with length >= 16, uses SIMD instructions for fast comparison
#[inline(always)]
pub fn fast_bytes_equal(a: &[u8], b: &[u8]) -> bool {
if a.len() != b.len() {
return false;
}
let len = a.len();
// For small arrays, use standard comparison directly
if len < 16 {
return a == b;
}
// Use SIMD to process 16-byte chunks
let chunks = len / 16;
let remainder = len % 16;
// Process complete 16-byte chunks
for i in 0..chunks {
let offset = i * 16;
let chunk_a = u8x16::from(&a[offset..offset + 16]);
let chunk_b = u8x16::from(&b[offset..offset + 16]);
if !chunk_a.cmp_eq(chunk_b).all() {
return false;
}
}
// Process remaining bytes
if remainder > 0 {
let start = chunks * 16;
return &a[start..] == &b[start..];
}
true
}
/// Fast discriminator matching, specifically for instruction discriminator comparison
#[inline(always)]
pub fn fast_discriminator_match(data: &[u8], discriminator: &[u8]) -> bool {
if data.len() < discriminator.len() {
return false;
}
let disc_len = discriminator.len();
// Optimize for common discriminator lengths
match disc_len {
1 => data[0] == discriminator[0],
2 => {
let data_u16 = u16::from_le_bytes([data[0], data[1]]);
let disc_u16 = u16::from_le_bytes([discriminator[0], discriminator[1]]);
data_u16 == disc_u16
}
4 => {
let data_u32 = u32::from_le_bytes([data[0], data[1], data[2], data[3]]);
let disc_u32 = u32::from_le_bytes([
discriminator[0],
discriminator[1],
discriminator[2],
discriminator[3],
]);
data_u32 == disc_u32
}
8 => {
let data_u64 = u64::from_le_bytes([
data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7],
]);
let disc_u64 = u64::from_le_bytes([
discriminator[0],
discriminator[1],
discriminator[2],
discriminator[3],
discriminator[4],
discriminator[5],
discriminator[6],
discriminator[7],
]);
data_u64 == disc_u64
}
16 => {
// Use SIMD to process 16-byte discriminators
let data_chunk = u8x16::from(&data[..16]);
let disc_chunk = u8x16::from(discriminator);
data_chunk.cmp_eq(disc_chunk).all()
}
_ => {
// For other lengths, use generic SIMD comparison
Self::fast_bytes_equal(&data[..disc_len], discriminator)
}
}
}
/// SIMD-accelerated memory search to find specific patterns in data
#[inline(always)]
pub fn find_pattern_simd(haystack: &[u8], needle: &[u8]) -> Option<usize> {
if needle.is_empty() || haystack.len() < needle.len() {
return None;
}
let needle_len = needle.len();
let haystack_len = haystack.len();
// For single-byte search, use optimized method
if needle_len == 1 {
let target = needle[0];
return haystack.iter().position(|&b| b == target);
}
// For multi-byte search, use SIMD acceleration
if needle_len <= 16 && haystack_len >= 16 {
let first_byte = needle[0];
let chunks = (haystack_len - needle_len + 1) / 16;
for chunk_idx in 0..chunks {
let start = chunk_idx * 16;
let end = std::cmp::min(start + 16, haystack_len - needle_len + 1);
// Use SIMD to find first byte matches
let chunk = &haystack[start..start + 16];
let target_vec = u8x16::splat(first_byte);
let chunk_vec = u8x16::from(chunk);
let matches = chunk_vec.cmp_eq(target_vec);
// Check each match position
let matches_array: [u8; 16] = matches.into();
for i in 0..16 {
if start + i >= end {
break;
}
if matches_array[i] != 0 && start + i + needle_len <= haystack_len {
if Self::fast_bytes_equal(
&haystack[start + i..start + i + needle_len],
needle,
) {
return Some(start + i);
}
}
}
}
// Process remaining part
let remaining_start = chunks * 16;
for i in remaining_start..=(haystack_len - needle_len) {
if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) {
return Some(i);
}
}
} else {
// Fallback to standard search
for i in 0..=(haystack_len - needle_len) {
if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) {
return Some(i);
}
}
}
None
}
/// SIMD-accelerated data validation to check if data conforms to specific format
#[inline(always)]
pub fn validate_data_format(data: &[u8], min_length: usize) -> bool {
if data.len() < min_length {
return false;
}
true
}
/// Fast checksum calculation (maintains API consistency)
#[inline(always)]
pub fn fast_checksum(data: &[u8]) -> u32 {
// Simplified implementation, directly sum all bytes
data.iter().map(|&b| b as u32).sum()
}
/// SIMD-accelerated data copy (for large data blocks)
#[inline(always)]
pub fn fast_copy(src: &[u8], dst: &mut [u8]) {
if src.len() != dst.len() {
panic!("Source and destination must have the same length");
}
let len = src.len();
if len >= 32 {
// Use 32-byte SIMD copy
let chunks = len / 32;
for i in 0..chunks {
let start = i * 32;
let src_chunk1 = u8x16::from(&src[start..start + 16]);
let src_chunk2 = u8x16::from(&src[start + 16..start + 32]);
let chunk1_array: [u8; 16] = src_chunk1.into();
let chunk2_array: [u8; 16] = src_chunk2.into();
dst[start..start + 16].copy_from_slice(&chunk1_array);
dst[start + 16..start + 32].copy_from_slice(&chunk2_array);
}
// Process remaining bytes
let remaining_start = chunks * 32;
dst[remaining_start..].copy_from_slice(&src[remaining_start..]);
} else {
// For small data, use standard copy
dst.copy_from_slice(src);
}
}
/// SIMD-accelerated account indices validation
/// Validates that all indices in the account index array are less than the total account count
#[inline(always)]
pub fn validate_account_indices_simd(indices: &[u8], account_count: usize) -> bool {
if indices.is_empty() {
return true;
}
let max_valid_index = account_count as u8;
// For small arrays, use standard comparison directly
if indices.len() < 16 {
return indices.iter().all(|&idx| idx < max_valid_index);
}
// Use SIMD for batch loading and comparison
let chunks = indices.len() / 16;
let remainder = indices.len() % 16;
// Process complete 16-byte chunks
for i in 0..chunks {
let start = i * 16;
let indices_chunk = u8x16::from(&indices[start..start + 16]);
// Convert SIMD vector to array for fast batch checking
let indices_array: [u8; 16] = indices_chunk.into();
// Use unrolled loop for fast comparison, compiler will optimize this
if indices_array[0] >= max_valid_index
|| indices_array[1] >= max_valid_index
|| indices_array[2] >= max_valid_index
|| indices_array[3] >= max_valid_index
|| indices_array[4] >= max_valid_index
|| indices_array[5] >= max_valid_index
|| indices_array[6] >= max_valid_index
|| indices_array[7] >= max_valid_index
|| indices_array[8] >= max_valid_index
|| indices_array[9] >= max_valid_index
|| indices_array[10] >= max_valid_index
|| indices_array[11] >= max_valid_index
|| indices_array[12] >= max_valid_index
|| indices_array[13] >= max_valid_index
|| indices_array[14] >= max_valid_index
|| indices_array[15] >= max_valid_index
{
return false;
}
}
// Process remaining bytes
if remainder > 0 {
let remaining_start = chunks * 16;
return indices[remaining_start..].iter().all(|&idx| idx < max_valid_index);
}
true
}
/// SIMD-accelerated instruction data validation
/// Validates basic format and length requirements of instruction data
#[inline(always)]
pub fn validate_instruction_data_simd(
data: &[u8],
min_length: usize,
discriminator_length: usize,
) -> bool {
// Basic length check
if data.len() < min_length || data.len() < discriminator_length {
return false;
}
// Use existing data format validation
Self::validate_data_format(data, min_length)
}
}
+8 -4
View File
@@ -3,7 +3,7 @@ use tokio::task::JoinHandle;
/// Subscription handle for managing and stopping subscriptions
pub struct SubscriptionHandle {
stream_handle: JoinHandle<()>,
event_handle: JoinHandle<()>,
event_handle: Option<JoinHandle<()>>,
metrics_handle: Option<JoinHandle<()>>,
}
@@ -11,7 +11,7 @@ impl SubscriptionHandle {
/// Create a new subscription handle
pub fn new(
stream_handle: JoinHandle<()>,
event_handle: JoinHandle<()>,
event_handle: Option<JoinHandle<()>>,
metrics_handle: Option<JoinHandle<()>>,
) -> Self {
Self { stream_handle, event_handle, metrics_handle }
@@ -20,7 +20,9 @@ impl SubscriptionHandle {
/// Stop subscription and abort all related tasks
pub fn stop(self) {
self.stream_handle.abort();
self.event_handle.abort();
if let Some(handle) = self.event_handle {
handle.abort();
}
if let Some(handle) = self.metrics_handle {
handle.abort();
}
@@ -29,7 +31,9 @@ impl SubscriptionHandle {
/// Asynchronously wait for all tasks to complete
pub async fn join(self) -> Result<(), tokio::task::JoinError> {
let _ = self.stream_handle.await;
let _ = self.event_handle.await;
if let Some(handle) = self.event_handle {
let _ = handle.await;
}
if let Some(handle) = self.metrics_handle {
let _ = handle.await;
}
+1 -1
View File
@@ -2,7 +2,7 @@ use crate::streaming::event_parser::common::{
types::EventType, ACCOUNT_EVENT_TYPES, BLOCK_EVENT_TYPES,
};
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Default)]
pub struct EventTypeFilter {
pub include: Vec<EventType>,
}
+22 -15
View File
@@ -8,10 +8,6 @@ macro_rules! impl_unified_event {
// 带有自定义ID表达式的版本
($struct_name:ident, $($field:ident),*) => {
impl $crate::streaming::event_parser::core::traits::UnifiedEvent for $struct_name {
fn id(&self) -> &str {
&self.metadata.id
}
fn event_type(&self) -> $crate::streaming::event_parser::common::types::EventType {
self.metadata.event_type.clone()
}
@@ -24,16 +20,16 @@ macro_rules! impl_unified_event {
self.metadata.slot
}
fn program_received_time_ms(&self) -> i64 {
self.metadata.program_received_time_ms
fn program_received_time_us(&self) -> i64 {
self.metadata.program_received_time_us
}
fn program_handle_time_consuming_ms(&self) -> i64 {
self.metadata.program_handle_time_consuming_ms
fn program_handle_time_consuming_us(&self) -> i64 {
self.metadata.program_handle_time_consuming_us
}
fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64) {
self.metadata.program_handle_time_consuming_ms = program_handle_time_consuming_ms;
fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64) {
self.metadata.program_handle_time_consuming_us = program_handle_time_consuming_us;
}
fn as_any(&self) -> &dyn std::any::Any {
@@ -48,7 +44,7 @@ macro_rules! impl_unified_event {
Box::new(self.clone())
}
fn merge(&mut self, other: Box<dyn $crate::streaming::event_parser::core::traits::UnifiedEvent>) {
fn merge(&mut self, other: &dyn $crate::streaming::event_parser::core::traits::UnifiedEvent) {
if let Some(_e) = other.as_any().downcast_ref::<$struct_name>() {
$(
self.$field = _e.$field.clone();
@@ -56,12 +52,23 @@ macro_rules! impl_unified_event {
}
}
fn set_transfer_datas(&mut self, transfer_datas: Vec<$crate::streaming::event_parser::common::types::TransferData>, swap_data: Option<$crate::streaming::event_parser::common::types::SwapData>) {
self.metadata.set_transfer_datas(transfer_datas, swap_data);
fn set_swap_data(&mut self, swap_data: $crate::streaming::event_parser::common::types::SwapData) {
self.metadata.set_swap_data(swap_data);
}
fn index(&self) -> String {
self.metadata.index.clone()
fn swap_data_is_parsed(&self) -> bool {
self.metadata.swap_data.is_some()
}
fn instruction_outer_index(&self) -> i64 {
self.metadata.instruction_outer_index
}
fn instruction_inner_index(&self) -> Option<i64> {
self.metadata.instruction_inner_index
}
fn transaction_index(&self) -> Option<u64> {
self.metadata.transaction_index
}
}
};
+247 -348
View File
@@ -1,26 +1,24 @@
use borsh::{BorshDeserialize, BorshSerialize};
use crossbeam_queue::ArrayQueue;
use serde::{Deserialize, Serialize};
use solana_sdk::pubkey::Pubkey;
use solana_transaction_status::UiInstruction;
use std::{
hash::{DefaultHasher, Hash, Hasher},
str::FromStr,
sync::Arc,
};
use tokio::sync::Mutex;
use std::{borrow::Cow, fmt, str::FromStr, sync::Arc};
use crate::{
match_event,
streaming::event_parser::{
protocols::{
bonk::BonkTradeEvent,
pumpfun::PumpFunTradeEvent,
pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent},
raydium_amm_v4::RaydiumAmmV4SwapEvent,
raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event},
raydium_cpmm::RaydiumCpmmSwapEvent,
streaming::{
common::SimdUtils,
event_parser::{
protocols::{
bonk::BonkTradeEvent,
pumpfun::PumpFunTradeEvent,
pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent},
raydium_amm_v4::RaydiumAmmV4SwapEvent,
raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event},
raydium_cpmm::RaydiumCpmmSwapEvent,
},
UnifiedEvent,
},
UnifiedEvent,
},
};
@@ -30,7 +28,7 @@ const TRANSFER_DATA_POOL_SIZE: usize = 2000;
/// Event metadata object pool
pub struct EventMetadataPool {
pool: Arc<Mutex<Vec<EventMetadata>>>,
pool: Arc<ArrayQueue<EventMetadata>>,
}
impl Default for EventMetadataPool {
@@ -41,55 +39,22 @@ impl Default for EventMetadataPool {
impl EventMetadataPool {
pub fn new() -> Self {
Self { pool: Arc::new(Mutex::new(Vec::with_capacity(EVENT_METADATA_POOL_SIZE))) }
Self { pool: Arc::new(ArrayQueue::new(EVENT_METADATA_POOL_SIZE)) }
}
pub async fn acquire(&self) -> Option<EventMetadata> {
let mut pool = self.pool.lock().await;
pool.pop()
pub fn acquire(&self) -> Option<EventMetadata> {
self.pool.pop()
}
pub async fn release(&self, metadata: EventMetadata) {
let mut pool = self.pool.lock().await;
if pool.len() < EVENT_METADATA_POOL_SIZE {
pool.push(metadata);
}
}
}
/// Transfer data object pool
pub struct TransferDataPool {
pool: Arc<Mutex<Vec<TransferData>>>,
}
impl Default for TransferDataPool {
fn default() -> Self {
Self::new()
}
}
impl TransferDataPool {
pub fn new() -> Self {
Self { pool: Arc::new(Mutex::new(Vec::with_capacity(TRANSFER_DATA_POOL_SIZE))) }
}
pub async fn acquire(&self) -> Option<TransferData> {
let mut pool = self.pool.lock().await;
pool.pop()
}
pub async fn release(&self, transfer_data: TransferData) {
let mut pool = self.pool.lock().await;
if pool.len() < TRANSFER_DATA_POOL_SIZE {
pool.push(transfer_data);
}
pub fn release(&self, metadata: EventMetadata) {
// 如果队列已满,push 会失败,但不会阻塞
let _ = self.pool.push(metadata);
}
}
// Global object pool instances
lazy_static::lazy_static! {
pub static ref EVENT_METADATA_POOL: EventMetadataPool = EventMetadataPool::new();
pub static ref TRANSFER_DATA_POOL: TransferDataPool = TransferDataPool::new();
}
#[derive(
@@ -199,70 +164,69 @@ pub const ACCOUNT_EVENT_TYPES: &[EventType] = &[
];
pub const BLOCK_EVENT_TYPES: &[EventType] = &[EventType::BlockMeta];
impl EventType {
#[allow(clippy::inherent_to_string)]
pub fn to_string(&self) -> String {
impl fmt::Display for EventType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
EventType::PumpSwapBuy => "PumpSwapBuy".to_string(),
EventType::PumpSwapSell => "PumpSwapSell".to_string(),
EventType::PumpSwapCreatePool => "PumpSwapCreatePool".to_string(),
EventType::PumpSwapDeposit => "PumpSwapDeposit".to_string(),
EventType::PumpSwapWithdraw => "PumpSwapWithdraw".to_string(),
EventType::PumpFunCreateToken => "PumpFunCreateToken".to_string(),
EventType::PumpFunBuy => "PumpFunBuy".to_string(),
EventType::PumpFunSell => "PumpFunSell".to_string(),
EventType::PumpFunMigrate => "PumpFunMigrate".to_string(),
EventType::BonkBuyExactIn => "BonkBuyExactIn".to_string(),
EventType::BonkBuyExactOut => "BonkBuyExactOut".to_string(),
EventType::BonkSellExactIn => "BonkSellExactIn".to_string(),
EventType::BonkSellExactOut => "BonkSellExactOut".to_string(),
EventType::BonkInitialize => "BonkInitialize".to_string(),
EventType::BonkInitializeV2 => "BonkInitializeV2".to_string(),
EventType::BonkMigrateToAmm => "BonkMigrateToAmm".to_string(),
EventType::BonkMigrateToCpswap => "BonkMigrateToCpswap".to_string(),
EventType::AccountPumpFunBondingCurve => "AccountPumpFunBondingCurve".to_string(),
EventType::AccountPumpFunGlobal => "AccountPumpFunGlobal".to_string(),
EventType::AccountPumpSwapGlobalConfig => "AccountPumpSwapGlobalConfig".to_string(),
EventType::AccountPumpSwapPool => "AccountPumpSwapPool".to_string(),
EventType::AccountBonkPoolState => "AccountBonkPoolState".to_string(),
EventType::AccountBonkGlobalConfig => "AccountBonkGlobalConfig".to_string(),
EventType::AccountBonkPlatformConfig => "AccountBonkPlatformConfig".to_string(),
EventType::AccountBonkVestingRecord => "AccountBonkVestingRecord".to_string(),
EventType::RaydiumCpmmSwapBaseInput => "RaydiumCpmmSwapBaseInput".to_string(),
EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmmSwapBaseOutput".to_string(),
EventType::RaydiumCpmmDeposit => "RaydiumCpmmDeposit".to_string(),
EventType::RaydiumCpmmInitialize => "RaydiumCpmmInitialize".to_string(),
EventType::RaydiumCpmmWithdraw => "RaydiumCpmmWithdraw".to_string(),
EventType::RaydiumClmmSwap => "RaydiumClmmSwap".to_string(),
EventType::RaydiumClmmSwapV2 => "RaydiumClmmSwapV2".to_string(),
EventType::RaydiumClmmClosePosition => "RaydiumClmmClosePosition".to_string(),
EventType::PumpSwapBuy => write!(f, "PumpSwapBuy"),
EventType::PumpSwapSell => write!(f, "PumpSwapSell"),
EventType::PumpSwapCreatePool => write!(f, "PumpSwapCreatePool"),
EventType::PumpSwapDeposit => write!(f, "PumpSwapDeposit"),
EventType::PumpSwapWithdraw => write!(f, "PumpSwapWithdraw"),
EventType::PumpFunCreateToken => write!(f, "PumpFunCreateToken"),
EventType::PumpFunBuy => write!(f, "PumpFunBuy"),
EventType::PumpFunSell => write!(f, "PumpFunSell"),
EventType::PumpFunMigrate => write!(f, "PumpFunMigrate"),
EventType::BonkBuyExactIn => write!(f, "BonkBuyExactIn"),
EventType::BonkBuyExactOut => write!(f, "BonkBuyExactOut"),
EventType::BonkSellExactIn => write!(f, "BonkSellExactIn"),
EventType::BonkSellExactOut => write!(f, "BonkSellExactOut"),
EventType::BonkInitialize => write!(f, "BonkInitialize"),
EventType::BonkInitializeV2 => write!(f, "BonkInitializeV2"),
EventType::BonkMigrateToAmm => write!(f, "BonkMigrateToAmm"),
EventType::BonkMigrateToCpswap => write!(f, "BonkMigrateToCpswap"),
EventType::RaydiumCpmmSwapBaseInput => write!(f, "RaydiumCpmmSwapBaseInput"),
EventType::RaydiumCpmmSwapBaseOutput => write!(f, "RaydiumCpmmSwapBaseOutput"),
EventType::RaydiumCpmmDeposit => write!(f, "RaydiumCpmmDeposit"),
EventType::RaydiumCpmmInitialize => write!(f, "RaydiumCpmmInitialize"),
EventType::RaydiumCpmmWithdraw => write!(f, "RaydiumCpmmWithdraw"),
EventType::RaydiumClmmSwap => write!(f, "RaydiumClmmSwap"),
EventType::RaydiumClmmSwapV2 => write!(f, "RaydiumClmmSwapV2"),
EventType::RaydiumClmmClosePosition => write!(f, "RaydiumClmmClosePosition"),
EventType::RaydiumClmmDecreaseLiquidityV2 => {
"RaydiumClmmDecreaseLiquidityV2".to_string()
write!(f, "RaydiumClmmDecreaseLiquidityV2")
}
EventType::RaydiumClmmCreatePool => "RaydiumClmmCreatePool".to_string(),
EventType::RaydiumClmmCreatePool => write!(f, "RaydiumClmmCreatePool"),
EventType::RaydiumClmmIncreaseLiquidityV2 => {
"RaydiumClmmIncreaseLiquidityV2".to_string()
write!(f, "RaydiumClmmIncreaseLiquidityV2")
}
EventType::RaydiumClmmOpenPositionWithToken22Nft => {
"RaydiumClmmOpenPositionWithToken22Nft".to_string()
write!(f, "RaydiumClmmOpenPositionWithToken22Nft")
}
EventType::RaydiumClmmOpenPositionV2 => "RaydiumClmmOpenPositionV2".to_string(),
EventType::RaydiumAmmV4SwapBaseIn => "RaydiumAmmV4SwapBaseIn".to_string(),
EventType::RaydiumAmmV4SwapBaseOut => "RaydiumAmmV4SwapBaseOut".to_string(),
EventType::RaydiumAmmV4Deposit => "RaydiumAmmV4Deposit".to_string(),
EventType::RaydiumAmmV4Initialize2 => "RaydiumAmmV4Initialize2".to_string(),
EventType::RaydiumAmmV4Withdraw => "RaydiumAmmV4Withdraw".to_string(),
EventType::RaydiumAmmV4WithdrawPnl => "RaydiumAmmV4WithdrawPnl".to_string(),
EventType::AccountRaydiumAmmV4AmmInfo => "AccountRaydiumAmmV4AmmInfo".to_string(),
EventType::AccountRaydiumClmmAmmConfig => "AccountRaydiumClmmAmmConfig".to_string(),
EventType::AccountRaydiumClmmPoolState => "AccountRaydiumClmmPoolState".to_string(),
EventType::RaydiumClmmOpenPositionV2 => write!(f, "RaydiumClmmOpenPositionV2"),
EventType::RaydiumAmmV4SwapBaseIn => write!(f, "RaydiumAmmV4SwapBaseIn"),
EventType::RaydiumAmmV4SwapBaseOut => write!(f, "RaydiumAmmV4SwapBaseOut"),
EventType::RaydiumAmmV4Deposit => write!(f, "RaydiumAmmV4Deposit"),
EventType::RaydiumAmmV4Initialize2 => write!(f, "RaydiumAmmV4Initialize2"),
EventType::RaydiumAmmV4Withdraw => write!(f, "RaydiumAmmV4Withdraw"),
EventType::RaydiumAmmV4WithdrawPnl => write!(f, "RaydiumAmmV4WithdrawPnl"),
EventType::AccountRaydiumAmmV4AmmInfo => write!(f, "AccountRaydiumAmmV4AmmInfo"),
EventType::AccountPumpSwapGlobalConfig => write!(f, "AccountPumpSwapGlobalConfig"),
EventType::AccountPumpSwapPool => write!(f, "AccountPumpSwapPool"),
EventType::AccountBonkPoolState => write!(f, "AccountBonkPoolState"),
EventType::AccountBonkGlobalConfig => write!(f, "AccountBonkGlobalConfig"),
EventType::AccountBonkPlatformConfig => write!(f, "AccountBonkPlatformConfig"),
EventType::AccountBonkVestingRecord => write!(f, "AccountBonkVestingRecord"),
EventType::AccountPumpFunBondingCurve => write!(f, "AccountPumpFunBondingCurve"),
EventType::AccountPumpFunGlobal => write!(f, "AccountPumpFunGlobal"),
EventType::AccountRaydiumClmmAmmConfig => write!(f, "AccountRaydiumClmmAmmConfig"),
EventType::AccountRaydiumClmmPoolState => write!(f, "AccountRaydiumClmmPoolState"),
EventType::AccountRaydiumClmmTickArrayState => {
"AccountRaydiumClmmTickArrayState".to_string()
write!(f, "AccountRaydiumClmmTickArrayState")
}
EventType::AccountRaydiumCpmmAmmConfig => "AccountRaydiumCpmmAmmConfig".to_string(),
EventType::AccountRaydiumCpmmPoolState => "AccountRaydiumCpmmPoolState".to_string(),
EventType::BlockMeta => "BlockMeta".to_string(),
EventType::Unknown => "Unknown".to_string(),
EventType::AccountRaydiumCpmmAmmConfig => write!(f, "AccountRaydiumCpmmAmmConfig"),
EventType::AccountRaydiumCpmmPoolState => write!(f, "AccountRaydiumCpmmPoolState"),
EventType::BlockMeta => write!(f, "BlockMeta"),
EventType::Unknown => write!(f, "Unknown"),
}
}
}
@@ -310,20 +274,6 @@ impl ProtocolInfo {
}
}
/// Transfer data
#[derive(
Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize,
)]
pub struct TransferData {
pub token_program: Pubkey,
pub source: Pubkey,
pub destination: Pubkey,
pub authority: Option<Pubkey>,
pub amount: u64,
pub decimals: Option<u8>,
pub mint: Option<Pubkey>,
}
#[derive(
Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize,
)]
@@ -340,285 +290,234 @@ pub struct SwapData {
Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize,
)]
pub struct EventMetadata {
pub id: String,
pub signature: String,
pub signature: Cow<'static, str>,
pub slot: u64,
pub transaction_index: Option<u64>, // 新增:交易在slot中的索引
pub block_time: i64,
pub block_time_ms: i64,
pub program_received_time_ms: i64,
pub program_handle_time_consuming_ms: i64,
pub program_received_time_us: i64,
pub program_handle_time_consuming_us: i64,
pub protocol: ProtocolType,
pub event_type: EventType,
pub program_id: Pubkey,
pub transfer_datas: Vec<TransferData>,
pub swap_data: Option<SwapData>,
pub index: String,
pub instruction_outer_index: i64,
pub instruction_inner_index: Option<i64>,
}
impl EventMetadata {
#[allow(clippy::too_many_arguments)]
pub fn new(
id: String,
signature: String,
signature: Cow<'static, str>,
slot: u64,
block_time: i64,
block_time_ms: i64,
protocol: ProtocolType,
event_type: EventType,
program_id: Pubkey,
index: String,
program_received_time_ms: i64,
instruction_outer_index: i64,
instruction_inner_index: Option<i64>,
program_received_time_us: i64,
transaction_index: Option<u64>,
) -> Self {
Self {
id,
signature,
slot,
block_time,
block_time_ms,
program_received_time_ms,
program_handle_time_consuming_ms: 0,
program_received_time_us,
program_handle_time_consuming_us: 0,
protocol,
event_type,
program_id,
transfer_datas: Vec::with_capacity(4), // Pre-allocate capacity
swap_data: None,
index,
instruction_outer_index,
instruction_inner_index,
transaction_index,
}
}
pub fn set_id(&mut self, id: String) {
let _id = format!("{}-{}-{}", self.signature, self.event_type.to_string(), id);
let mut hasher = DefaultHasher::new();
_id.hash(&mut hasher);
let hash_value = hasher.finish();
self.id = format!("{:x}", hash_value);
}
pub fn set_transfer_datas(
&mut self,
transfer_datas: Vec<TransferData>,
swap_data: Option<SwapData>,
) {
self.transfer_datas = transfer_datas;
self.swap_data = swap_data;
pub fn set_swap_data(&mut self, swap_data: SwapData) {
self.swap_data = Some(swap_data);
}
/// Recycle EventMetadata to object pool
pub async fn recycle(self) {
EVENT_METADATA_POOL.release(self).await;
pub fn recycle(self) {
EVENT_METADATA_POOL.release(self);
}
}
/// Parse token transfer data from next instructions
pub fn parse_transfer_datas_from_next_instructions(
event: Box<dyn UnifiedEvent>,
inner_instruction: &solana_transaction_status::UiInnerInstructions,
current_index: i8,
accounts: &[Pubkey],
) -> (Vec<TransferData>, Option<SwapData>) {
let mut transfer_datas = vec![];
// Get the next two instructions after the current instruction
let next_instructions: Vec<&UiInstruction> =
inner_instruction.instructions.iter().skip((current_index + 1) as usize).collect();
let system_programs = vec![
// Token Program
lazy_static::lazy_static! {
static ref SOL_MINT: Pubkey = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap();
static ref SYSTEM_PROGRAMS: [Pubkey; 3] = [
Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(),
// Token 2022 Program
Pubkey::from_str("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb").unwrap(),
// System Program
Pubkey::from_str("11111111111111111111111111111111").unwrap(),
];
for instruction in next_instructions {
if let UiInstruction::Compiled(compiled) = instruction {
if !system_programs.contains(&accounts[compiled.program_id_index as usize]) {
break;
}
if let Ok(data) = bs58::decode(compiled.data.clone()).into_vec() {
// Token Program: transferChecked
// Token 2022 Program: transferChecked
if data[0] == 12 {
let account_pubkeys: Vec<Pubkey> =
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
if account_pubkeys.len() < 4 {
continue;
}
let (source, mint, destination, authority) = (
account_pubkeys[0],
account_pubkeys[1],
account_pubkeys[2],
account_pubkeys[3],
);
let amount = u64::from_le_bytes(data[1..9].try_into().unwrap());
let decimals = data[9];
let token_program = accounts[compiled.program_id_index as usize];
transfer_datas.push(TransferData {
amount,
decimals: Some(decimals),
mint: Some(mint),
source,
destination,
authority: Some(authority),
token_program,
});
}
// Token Program: transfer
else if data[0] == 3 {
let account_pubkeys: Vec<Pubkey> =
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
if account_pubkeys.len() < 3 {
continue;
}
let (source, destination, authority) =
(account_pubkeys[0], account_pubkeys[1], account_pubkeys[2]);
let amount = u64::from_le_bytes(data[1..9].try_into().unwrap());
let token_program = accounts[compiled.program_id_index as usize];
transfer_datas.push(TransferData {
amount,
decimals: None,
mint: None,
source,
destination,
authority: Some(authority),
token_program,
});
}
//System Program: transfer
else if data[0] == 2 {
let account_pubkeys: Vec<Pubkey> =
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
if account_pubkeys.len() < 2 {
continue;
}
let (source, destination) = (account_pubkeys[0], account_pubkeys[1]);
let amount = u64::from_le_bytes(data[4..12].try_into().unwrap());
let token_program = accounts[compiled.program_id_index as usize];
transfer_datas.push(TransferData {
amount,
decimals: None,
mint: None,
source,
destination,
authority: None,
token_program,
});
}
}
}
}
let mut swap_data: SwapData = SwapData {
}
/// Parse token transfer data from next instructions
pub fn parse_swap_data_from_next_instructions(
event: &dyn UnifiedEvent,
inner_instruction: &solana_transaction_status::InnerInstructions,
current_index: i8,
accounts: &[Pubkey],
) -> Option<SwapData> {
let mut swap_data = SwapData {
from_mint: Pubkey::default(),
to_mint: Pubkey::default(),
from_amount: 0,
to_amount: 0,
description: None,
};
let sol_mint = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap();
if transfer_datas.len() > 0 {
let mut user: Option<Pubkey> = None;
let mut from_mint: Option<Pubkey> = None;
let mut to_mint: Option<Pubkey> = None;
let mut user_from_token: Option<Pubkey> = None;
let mut user_to_token: Option<Pubkey> = None;
let mut from_vault: Option<Pubkey> = None;
let mut to_vault: Option<Pubkey> = None;
match_event!(event, {
BonkTradeEvent => |e: BonkTradeEvent| {
user = Some(e.payer);
from_mint = Some(e.base_token_mint);
to_mint = Some(e.quote_token_mint);
user_from_token = Some(e.user_base_token);
user_to_token = Some(e.user_quote_token);
from_vault = Some(e.base_vault);
to_vault = Some(e.quote_vault);
},
PumpFunTradeEvent => |e: PumpFunTradeEvent| {
swap_data.from_mint = if e.is_buy {
sol_mint
} else {
e.mint
};
swap_data.to_mint = if e.is_buy {
e.mint
} else {
sol_mint
};
},
PumpSwapBuyEvent => |e: PumpSwapBuyEvent| {
swap_data.from_mint = e.quote_mint;
swap_data.to_mint = e.base_mint;
},
PumpSwapSellEvent => |e: PumpSwapSellEvent| {
swap_data.from_mint = e.base_mint;
swap_data.to_mint = e.quote_mint;
},
RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| {
user = Some(e.payer);
from_mint = Some(e.input_token_mint);
to_mint = Some(e.output_token_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| {
user = Some(e.payer);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string());
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| {
user = Some(e.payer);
from_mint = Some(e.input_vault_mint);
to_mint = Some(e.output_vault_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| {
user = Some(e.user_source_owner);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string());
user_from_token = Some(e.user_source_token_account);
user_to_token = Some(e.user_destination_token_account);
from_vault = Some(e.pool_pc_token_account);
to_vault = Some(e.pool_coin_token_account);
},
});
for transfer_data in transfer_datas.clone() {
if transfer_data.source == user_to_token.unwrap_or_default()
&& transfer_data.destination == to_vault.unwrap_or_default()
{
swap_data.from_mint = to_mint.unwrap_or_default();
swap_data.from_amount = transfer_data.amount;
} else if transfer_data.source == from_vault.unwrap_or_default()
&& transfer_data.destination == user_from_token.unwrap_or_default()
{
swap_data.to_mint = from_mint.unwrap_or_default();
swap_data.to_amount = transfer_data.amount;
} else if transfer_data.source == user_from_token.unwrap_or_default()
&& transfer_data.destination == from_vault.unwrap_or_default()
{
swap_data.from_mint = from_mint.unwrap_or_default();
swap_data.from_amount = transfer_data.amount;
} else if transfer_data.source == to_vault.unwrap_or_default()
&& transfer_data.destination == user_to_token.unwrap_or_default()
{
swap_data.to_mint = to_mint.unwrap_or_default();
swap_data.to_amount = transfer_data.amount;
// 先根据 event 取出关键信息
let mut user: Option<Pubkey> = None;
let mut from_mint: Option<Pubkey> = None;
let mut to_mint: Option<Pubkey> = None;
let mut user_from_token: Option<Pubkey> = None;
let mut user_to_token: Option<Pubkey> = None;
let mut from_vault: Option<Pubkey> = None;
let mut to_vault: Option<Pubkey> = None;
match_event!(&*event, {
BonkTradeEvent => |e: BonkTradeEvent| {
user = Some(e.payer);
from_mint = Some(e.base_token_mint);
to_mint = Some(e.quote_token_mint);
user_from_token = Some(e.user_base_token);
user_to_token = Some(e.user_quote_token);
from_vault = Some(e.base_vault);
to_vault = Some(e.quote_vault);
},
PumpFunTradeEvent => |e: PumpFunTradeEvent| {
swap_data.from_mint = if e.is_buy { *SOL_MINT } else { e.mint };
swap_data.to_mint = if e.is_buy { e.mint } else { *SOL_MINT };
},
PumpSwapBuyEvent => |e: PumpSwapBuyEvent| {
swap_data.from_mint = e.quote_mint;
swap_data.to_mint = e.base_mint;
},
PumpSwapSellEvent => |e: PumpSwapSellEvent| {
swap_data.from_mint = e.base_mint;
swap_data.to_mint = e.quote_mint;
},
RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| {
user = Some(e.payer);
from_mint = Some(e.input_token_mint);
to_mint = Some(e.output_token_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| {
user = Some(e.payer);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string());
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| {
user = Some(e.payer);
from_mint = Some(e.input_vault_mint);
to_mint = Some(e.output_vault_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| {
user = Some(e.user_source_owner);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string());
user_from_token = Some(e.user_source_token_account);
user_to_token = Some(e.user_destination_token_account);
from_vault = Some(e.pool_pc_token_account);
to_vault = Some(e.pool_coin_token_account);
},
});
let user_to_token = user_to_token.unwrap_or_default();
let user_from_token = user_from_token.unwrap_or_default();
let to_vault = to_vault.unwrap_or_default();
let from_vault = from_vault.unwrap_or_default();
let to_mint = to_mint.unwrap_or_default();
let from_mint = from_mint.unwrap_or_default();
// 单次循环完成提取和判断
for instruction in inner_instruction.instructions.iter().skip((current_index + 1) as usize) {
let compiled = &instruction.instruction;
let program_id = accounts[compiled.program_id_index as usize];
if !SYSTEM_PROGRAMS.contains(&program_id) {
break;
}
let data = &compiled.data;
// 使用 SIMD 验证数据格式
if !SimdUtils::validate_data_format(data, 8) {
continue;
}
let get_pubkey = |i: usize| accounts[compiled.accounts[i] as usize];
let (source, destination, amount) = match data[0] {
12 if compiled.accounts.len() >= 4 => {
let amt = u64::from_le_bytes(data[1..9].try_into().unwrap());
(get_pubkey(0), get_pubkey(2), amt)
}
3 if compiled.accounts.len() >= 3 => {
let amt = u64::from_le_bytes(data[1..9].try_into().unwrap());
(get_pubkey(0), get_pubkey(1), amt)
}
2 if compiled.accounts.len() >= 2 => {
let amt = u64::from_le_bytes(data[4..12].try_into().unwrap());
(get_pubkey(0), get_pubkey(1), amt)
}
_ => continue,
};
match (source, destination) {
(s, d) if s == user_to_token && d == to_vault => {
swap_data.from_mint = to_mint;
swap_data.from_amount = amount;
}
(s, d) if s == from_vault && d == user_from_token => {
swap_data.to_mint = from_mint;
swap_data.to_amount = amount;
}
(s, d) if s == user_from_token && d == from_vault => {
swap_data.from_mint = from_mint;
swap_data.from_amount = amount;
}
(s, d) if s == to_vault && d == user_to_token => {
swap_data.to_mint = to_mint;
swap_data.to_amount = amount;
}
(s, d) if s == user_from_token && d == to_vault => {
swap_data.from_mint = from_mint;
swap_data.from_amount = amount;
}
(s, d) if s == from_vault && d == user_to_token => {
swap_data.to_mint = to_mint;
swap_data.to_amount = amount;
}
_ => {}
}
if swap_data.from_mint != Pubkey::default() && swap_data.to_mint != Pubkey::default() {
break;
}
if swap_data.from_amount != 0 && swap_data.to_amount != 0 {
break;
}
}
if swap_data.from_mint != Pubkey::default()
|| swap_data.to_mint != Pubkey::default()
|| swap_data.from_amount != 0
|| swap_data.to_amount != 0
{
(transfer_datas, Some(swap_data))
Some(swap_data)
} else {
(transfer_datas, None)
None
}
}
@@ -1,5 +1,3 @@
use base64::engine::general_purpose;
use base64::Engine;
use std::time::{SystemTime, UNIX_EPOCH};
/// 获取当前时间戳
@@ -7,16 +5,6 @@ pub fn current_timestamp() -> i64 {
SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards").as_secs() as i64
}
/// 从base64字符串解码数据
pub fn decode_base64(data: &str) -> Result<Vec<u8>, base64::DecodeError> {
general_purpose::STANDARD.decode(data)
}
/// 将数据编码为base64字符串
pub fn encode_base64(data: &[u8]) -> String {
general_purpose::STANDARD.encode(data)
}
/// 从字节数组中提取鉴别器和剩余数据
pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8])> {
if data.len() < length {
@@ -25,15 +13,6 @@ pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8]
Some((&data[..length], &data[length..]))
}
/// 检查鉴别器是否匹配 - 优化版本
pub fn discriminator_matches(data: &str, expected: &str) -> bool {
if data.len() < expected.len() {
return false;
}
// 使用字节比较而不是字符串比较,更高效
data.as_bytes().starts_with(expected.as_bytes())
}
/// 从日志中提取程序数据
pub fn extract_program_data(log: &str) -> Option<&str> {
const PROGRAM_DATA_PREFIX: &str = "Program data: ";
@@ -1,11 +1,13 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::OnceLock;
use solana_sdk::pubkey::Pubkey;
use crate::streaming::common::SimdUtils;
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::common::{EventMetadata, EventType, ProtocolType};
use crate::streaming::event_parser::core::traits::UnifiedEvent;
use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock};
use crate::streaming::event_parser::protocols::bonk::parser::BONK_PROGRAM_ID;
use crate::streaming::event_parser::protocols::pumpfun::parser::PUMPFUN_PROGRAM_ID;
use crate::streaming::event_parser::protocols::pumpswap::parser::PUMPSWAP_PROGRAM_ID;
@@ -35,7 +37,10 @@ static PROTOCOL_CONFIGS_CACHE: OnceLock<HashMap<Protocol, Vec<AccountEventParseC
pub struct AccountEventParser {}
impl AccountEventParser {
pub fn configs(protocols: Vec<Protocol>, event_type_filter: Option<EventTypeFilter>) -> Vec<AccountEventParseConfig> {
pub fn configs(
protocols: &[Protocol],
event_type_filter: Option<&EventTypeFilter>,
) -> Vec<AccountEventParseConfig> {
let protocols_map = PROTOCOL_CONFIGS_CACHE.get_or_init(|| {
let mut map: HashMap<Protocol, Vec<AccountEventParseConfig>> = HashMap::new();
map.insert(Protocol::PumpSwap, vec![
@@ -145,43 +150,49 @@ impl AccountEventParser {
});
let mut configs = vec![];
let empty_vec = vec![];
for protocol in protocols {
let protocol_configs = protocols_map.get(&protocol).unwrap_or(&vec![]).clone();
let filtered_configs: Vec<AccountEventParseConfig> = protocol_configs.into_iter().filter(|config| {
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
}).collect();
let protocol_configs = protocols_map.get(protocol).unwrap_or(&empty_vec);
let filtered_configs: Vec<AccountEventParseConfig> = protocol_configs
.iter()
.filter(|config| {
event_type_filter
.map(|filter| filter.include.contains(&config.event_type))
.unwrap_or(true)
})
.cloned()
.collect();
configs.extend(filtered_configs);
}
configs
}
pub fn parse_account_event(
protocols: Vec<Protocol>,
protocols: &[Protocol],
account: AccountPretty,
program_received_time_ms: i64,
event_type_filter: Option<EventTypeFilter>,
event_type_filter: Option<&EventTypeFilter>,
) -> Option<Box<dyn UnifiedEvent>> {
let configs = Self::configs(protocols, event_type_filter);
for config in configs {
if account.owner == config.program_id.to_string()
&& account.data[..config.account_discriminator.len()]
== *config.account_discriminator
if account.owner == config.program_id
&& SimdUtils::fast_discriminator_match(&account.data, config.account_discriminator)
{
let signature_str = Cow::Owned(account.signature.to_string());
let event = (config.account_parser)(
&account,
EventMetadata {
slot: account.slot,
signature: account.signature.clone(),
signature: signature_str,
protocol: config.protocol_type,
event_type: config.event_type,
program_id: config.program_id,
program_received_time_ms,
program_received_time_us: account.program_received_time_us,
..Default::default()
},
);
if let Some(mut event) = event {
event.set_program_handle_time_consuming_ms(
chrono::Utc::now().timestamp_millis() - program_received_time_ms,
event.set_program_handle_time_consuming_us(
get_high_perf_clock().elapsed_micros_since(account.program_received_time_us),
);
return Some(event);
}
@@ -1,4 +1,4 @@
use crate::streaming::event_parser::core::traits::UnifiedEvent;
use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock};
use crate::streaming::event_parser::protocols::block::block_meta_event::BlockMetaEvent;
pub struct CommonEventParser {}
@@ -8,8 +8,17 @@ impl CommonEventParser {
slot: u64,
block_hash: &str,
block_time_ms: i64,
program_received_time_us: i64,
) -> Box<dyn UnifiedEvent> {
let block_meta_event = BlockMetaEvent::new(slot, block_hash.to_string(), block_time_ms);
let mut block_meta_event = BlockMetaEvent::new(
slot,
block_hash.to_string(),
block_time_ms,
program_received_time_us,
);
block_meta_event.set_program_handle_time_consuming_us(
get_high_perf_clock().elapsed_micros_since(program_received_time_us),
);
Box::new(block_meta_event)
}
}
@@ -0,0 +1,243 @@
use solana_sdk::pubkey::Pubkey;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use dashmap::DashMap;
use std::collections::BTreeSet;
const MAX_SLOTS: usize = 1000;
const CLEANUP_BATCH_SIZE: usize = 100;
/// Slot-based trader addresses, completely lock-free
#[derive(Default)]
struct SlotAddresses {
/// Developer addresses for this slot
dev_addresses: BTreeSet<Pubkey>,
/// Bonk developer addresses for this slot
bonk_dev_addresses: BTreeSet<Pubkey>,
}
/// High-performance global state with lock-free slot-based storage
pub struct GlobalState {
/// Slot -> trader addresses mapping (lock-free concurrent hashmap)
slot_data: DashMap<u64, SlotAddresses>,
/// Current slot count for capacity management
slot_count: AtomicUsize,
/// Generation counter to handle cleanup races
generation: AtomicU64,
}
impl GlobalState {
/// Create a new high-performance global state instance
pub fn new() -> Self {
Self {
slot_data: DashMap::new(),
slot_count: AtomicUsize::new(0),
generation: AtomicU64::new(0),
}
}
/// Lock-free capacity management - cleanup old slots when limit exceeded
fn maybe_cleanup(&self) {
let current_count = self.slot_count.load(Ordering::Relaxed);
if current_count <= MAX_SLOTS {
return;
}
// Use CAS to ensure only one thread performs cleanup
let gen = self.generation.load(Ordering::Relaxed);
if self.generation.compare_exchange_weak(gen, gen + 1, Ordering::Acquire, Ordering::Relaxed).is_err() {
return; // Another thread is cleaning up
}
// Collect oldest slots (BTreeMap naturally orders by key)
let mut slots_to_remove: Vec<u64> = self.slot_data.iter()
.map(|entry| *entry.key())
.collect();
if slots_to_remove.len() <= MAX_SLOTS {
return; // Race condition, already cleaned up
}
slots_to_remove.sort_unstable();
slots_to_remove.truncate(CLEANUP_BATCH_SIZE);
// Remove old slots atomically
for slot in slots_to_remove {
self.slot_data.remove(&slot);
self.slot_count.fetch_sub(1, Ordering::Relaxed);
}
}
/// Add developer address for a specific slot (lock-free)
pub fn add_dev_address(&self, slot: u64, address: Pubkey) {
self.maybe_cleanup();
self.slot_data.entry(slot)
.and_modify(|addresses| {
addresses.dev_addresses.insert(address);
})
.or_insert_with(|| {
self.slot_count.fetch_add(1, Ordering::Relaxed);
let mut slot_addr = SlotAddresses::default();
slot_addr.dev_addresses.insert(address);
slot_addr
});
}
/// Add Bonk developer address for a specific slot (lock-free)
pub fn add_bonk_dev_address(&self, slot: u64, address: Pubkey) {
self.maybe_cleanup();
self.slot_data.entry(slot)
.and_modify(|addresses| {
addresses.bonk_dev_addresses.insert(address);
})
.or_insert_with(|| {
self.slot_count.fetch_add(1, Ordering::Relaxed);
let mut slot_addr = SlotAddresses::default();
slot_addr.bonk_dev_addresses.insert(address);
slot_addr
});
}
/// High-performance: Check if address is a developer address in specific slot (O(log m))
pub fn is_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool {
self.slot_data.get(&slot)
.map(|entry| entry.dev_addresses.contains(address))
.unwrap_or(false)
}
/// High-performance: Check if address is a Bonk developer address in specific slot (O(log m))
pub fn is_bonk_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool {
self.slot_data.get(&slot)
.map(|entry| entry.bonk_dev_addresses.contains(address))
.unwrap_or(false)
}
/// Check if address is a developer address in any slot (lock-free scan, slower)
pub fn is_dev_address(&self, address: &Pubkey) -> bool {
self.slot_data.iter().any(|entry| entry.dev_addresses.contains(address))
}
/// Check if address is a Bonk developer address in any slot (lock-free scan, slower)
pub fn is_bonk_dev_address(&self, address: &Pubkey) -> bool {
self.slot_data.iter().any(|entry| entry.bonk_dev_addresses.contains(address))
}
/// Get all developer addresses from all slots (lock-free aggregation)
pub fn get_dev_addresses(&self) -> Vec<Pubkey> {
let mut all_addresses = BTreeSet::new();
for entry in self.slot_data.iter() {
for addr in &entry.dev_addresses {
all_addresses.insert(*addr);
}
}
all_addresses.into_iter().collect()
}
/// Get all Bonk developer addresses from all slots (lock-free aggregation)
pub fn get_bonk_dev_addresses(&self) -> Vec<Pubkey> {
let mut all_addresses = BTreeSet::new();
for entry in self.slot_data.iter() {
for addr in &entry.bonk_dev_addresses {
all_addresses.insert(*addr);
}
}
all_addresses.into_iter().collect()
}
/// Get developer addresses for a specific slot
pub fn get_dev_addresses_for_slot(&self, slot: u64) -> Vec<Pubkey> {
self.slot_data.get(&slot)
.map(|entry| entry.dev_addresses.iter().copied().collect())
.unwrap_or_default()
}
/// Get Bonk developer addresses for a specific slot
pub fn get_bonk_dev_addresses_for_slot(&self, slot: u64) -> Vec<Pubkey> {
self.slot_data.get(&slot)
.map(|entry| entry.bonk_dev_addresses.iter().copied().collect())
.unwrap_or_default()
}
/// Get current slot count
pub fn get_slot_count(&self) -> usize {
self.slot_count.load(Ordering::Relaxed)
}
/// Clear all data (lock-free)
pub fn clear_all_data(&self) {
self.slot_data.clear();
self.slot_count.store(0, Ordering::Relaxed);
self.generation.store(0, Ordering::Relaxed);
}
}
impl Default for GlobalState {
fn default() -> Self {
Self::new()
}
}
/// Global state instance
static GLOBAL_STATE: once_cell::sync::Lazy<GlobalState> =
once_cell::sync::Lazy::new(GlobalState::new);
/// Get global state instance
pub fn get_global_state() -> &'static GlobalState {
&GLOBAL_STATE
}
/// Convenience function: Add developer address for a specific slot
pub fn add_dev_address(slot: u64, address: Pubkey) {
get_global_state().add_dev_address(slot, address);
}
/// Convenience function: Check if address is a developer address
pub fn is_dev_address(address: &Pubkey) -> bool {
get_global_state().is_dev_address(address)
}
/// Convenience function: Add Bonk developer address for a specific slot
pub fn add_bonk_dev_address(slot: u64, address: Pubkey) {
get_global_state().add_bonk_dev_address(slot, address);
}
/// Convenience function: Check if address is a Bonk developer address
pub fn is_bonk_dev_address(address: &Pubkey) -> bool {
get_global_state().is_bonk_dev_address(address)
}
/// Convenience function: Get all developer addresses
pub fn get_dev_addresses() -> Vec<Pubkey> {
get_global_state().get_dev_addresses()
}
/// Convenience function: Get all Bonk developer addresses
pub fn get_bonk_dev_addresses() -> Vec<Pubkey> {
get_global_state().get_bonk_dev_addresses()
}
/// Convenience function: Get developer addresses for a specific slot
pub fn get_dev_addresses_for_slot(slot: u64) -> Vec<Pubkey> {
get_global_state().get_dev_addresses_for_slot(slot)
}
/// Convenience function: Get Bonk developer addresses for a specific slot
pub fn get_bonk_dev_addresses_for_slot(slot: u64) -> Vec<Pubkey> {
get_global_state().get_bonk_dev_addresses_for_slot(slot)
}
/// Convenience function: Get current slot count
pub fn get_slot_count() -> usize {
get_global_state().get_slot_count()
}
/// High-performance: Check if address is a developer address in specific slot
pub fn is_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool {
get_global_state().is_dev_address_in_slot(slot, address)
}
/// High-performance: Check if address is a Bonk developer address in specific slot
pub fn is_bonk_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool {
get_global_state().is_bonk_dev_address_in_slot(slot, address)
}
+96
View File
@@ -0,0 +1,96 @@
/// Macro to generate boilerplate EventParser implementation for protocol parsers
///
/// This macro eliminates the repetitive code where each parser simply delegates
/// all EventParser trait methods to its inner GenericEventParser.
///
/// Usage:
/// ```rust
/// impl_event_parser_delegate!(MyEventParser);
/// ```
///
/// This will generate the complete EventParser implementation that delegates
/// all methods to `self.inner`.
#[macro_export]
macro_rules! impl_event_parser_delegate {
($parser_type:ty) => {
#[async_trait::async_trait]
impl $crate::streaming::event_parser::core::traits::EventParser for $parser_type {
fn instruction_configs(
&self,
) -> std::collections::HashMap<
Vec<u8>,
Vec<$crate::streaming::event_parser::core::traits::GenericEventParseConfig>,
> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &solana_sdk::instruction::CompiledInstruction,
signature: solana_sdk::signature::Signature,
slot: u64,
block_time: Option<prost_types::Timestamp>,
program_received_time_us: i64,
outer_index: i64,
inner_index: Option<i64>,
transaction_index: Option<u64>,
config: &GenericEventParseConfig,
) -> Vec<Box<dyn $crate::streaming::event_parser::core::traits::UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_us,
outer_index,
inner_index,
transaction_index,
config,
)
}
fn parse_events_from_instruction(
&self,
instruction: &solana_sdk::instruction::CompiledInstruction,
accounts: &[solana_sdk::pubkey::Pubkey],
signature: solana_sdk::signature::Signature,
slot: u64,
block_time: Option<prost_types::Timestamp>,
program_received_time_us: i64,
outer_index: i64,
inner_index: Option<i64>,
bot_wallet: Option<solana_sdk::pubkey::Pubkey>,
transaction_index: Option<u64>,
inner_instructions: Option<&solana_transaction_status::InnerInstructions>,
callback: std::sync::Arc<
dyn for<'a> Fn(&'a Box<dyn $crate::streaming::event_parser::core::traits::UnifiedEvent>)
+ Send
+ Sync,
>,
) -> anyhow::Result<()> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_us,
outer_index,
inner_index,
bot_wallet,
transaction_index,
inner_instructions,
callback,
)
}
fn should_handle(&self, program_id: &solana_sdk::pubkey::Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<solana_sdk::pubkey::Pubkey> {
self.inner.supported_program_ids()
}
}
};
}
+2
View File
@@ -1,4 +1,6 @@
pub mod common_event_parser;
pub mod traits;
pub mod account_event_parser;
pub mod macros;
pub mod global_state;
pub use traits::{EventParser, UnifiedEvent};
File diff suppressed because it is too large Load Diff
@@ -1,3 +1,5 @@
use std::borrow::Cow;
use crate::impl_unified_event;
use crate::streaming::event_parser::common::{types::EventType, EventMetadata};
use borsh::BorshDeserialize;
@@ -13,18 +15,24 @@ pub struct BlockMetaEvent {
}
impl BlockMetaEvent {
pub fn new(slot: u64, block_hash: String, block_time_ms: i64) -> Self {
pub fn new(
slot: u64,
block_hash: String,
block_time_ms: i64,
program_received_time_us: i64,
) -> Self {
let metadata = EventMetadata::new(
format!("block_{}_{}", slot, block_hash),
"".to_string(),
Cow::Borrowed(""),
slot,
block_time_ms / 1000,
block_time_ms,
crate::streaming::event_parser::common::types::ProtocolType::Common,
EventType::BlockMeta,
solana_sdk::pubkey::Pubkey::default(),
"".to_string(),
chrono::Utc::now().timestamp_millis(),
0,
None,
program_received_time_us,
None,
);
Self { metadata, slot, block_hash }
}
@@ -314,8 +314,12 @@ impl_unified_event!(BonkPlatformConfigAccountEvent,);
/// Event discriminator constants
pub mod discriminators {
// Event discriminators
pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae";
// pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
pub const TRADE_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238];
// pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae";
pub const POOL_CREATE_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 151, 215, 226, 9, 118, 161, 115, 174];
// Instruction discriminators
pub const BUY_EXACT_IN: &[u8] = &[250, 234, 13, 123, 213, 156, 19, 236];
@@ -1,14 +1,16 @@
use std::collections::HashMap;
use solana_sdk::pubkey::Pubkey;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::{
common::{utils::*, EventMetadata, EventType, ProtocolType},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::bonk::{
bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection, VestingParams
use crate::{
impl_event_parser_delegate,
streaming::event_parser::{
common::{utils::*, EventMetadata, EventType, ProtocolType},
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::bonk::{
bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators,
AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent,
BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams,
TradeDirection, VestingParams,
},
},
};
@@ -88,7 +90,7 @@ impl BonkEventParser {
GenericEventParseConfig {
program_id: BONK_PROGRAM_ID,
protocol_type: ProtocolType::Bonk,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::MIGRATE_TO_AMM,
event_type: EventType::BonkMigrateToAmm,
inner_instruction_parser: None,
@@ -97,7 +99,7 @@ impl BonkEventParser {
GenericEventParseConfig {
program_id: BONK_PROGRAM_ID,
protocol_type: ProtocolType::Bonk,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::MIGRATE_TO_CP_SWAP,
event_type: EventType::BonkMigrateToCpswap,
inner_instruction_parser: None,
@@ -116,8 +118,6 @@ impl BonkEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = bonk_pool_create_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkPoolCreateEvent { metadata, ..event }))
} else {
None
@@ -130,8 +130,6 @@ impl BonkEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = bonk_trade_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, event.pool_state));
if metadata.event_type == EventType::BonkBuyExactIn
|| metadata.event_type == EventType::BonkBuyExactOut
{
@@ -164,9 +162,6 @@ impl BonkEventParser {
let minimum_amount_out = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_in,
@@ -205,9 +200,6 @@ impl BonkEventParser {
let maximum_amount_in = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_out,
@@ -246,9 +238,6 @@ impl BonkEventParser {
let minimum_amount_out = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_in,
@@ -287,9 +276,6 @@ impl BonkEventParser {
let maximum_amount_in = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_out,
@@ -330,9 +316,6 @@ impl BonkEventParser {
let curve_param = Self::parse_curve_params(data, &mut offset)?;
let vesting_param = Self::parse_vesting_params(data, &mut offset)?;
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkPoolCreateEvent {
metadata,
payer: accounts[0],
@@ -367,9 +350,6 @@ impl BonkEventParser {
let vesting_param = Self::parse_vesting_params(data, &mut offset)?;
let amm_fee_on = data[offset];
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkPoolCreateEvent {
metadata,
payer: accounts[0],
@@ -512,9 +492,6 @@ impl BonkEventParser {
let quote_lot_size = u64::from_le_bytes(data[8..16].try_into().unwrap());
let market_vault_signer_nonce = data[16];
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkMigrateToAmmEvent {
metadata,
base_lot_size,
@@ -562,9 +539,6 @@ impl BonkEventParser {
accounts: &[Pubkey],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkMigrateToCpswapEvent {
metadata,
payer: accounts[0],
@@ -601,59 +575,4 @@ impl BonkEventParser {
}
}
#[async_trait::async_trait]
impl EventParser for BonkEventParser {
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
self.inner.inner_instruction_configs()
}
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}
impl_event_parser_delegate!(BonkEventParser);
@@ -1,13 +1,10 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::{
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
EventParserFactory, Protocol,
use crate::{
impl_event_parser_delegate,
streaming::event_parser::{
common::filter::EventTypeFilter,
core::traits::{GenericEventParseConfig, GenericEventParser},
EventParserFactory, Protocol,
},
};
pub struct MutilEventParser {
@@ -21,20 +18,22 @@ impl MutilEventParser {
for protocol in protocols {
let parse = EventParserFactory::create_parser(protocol);
// Merge inner_instruction_configs, append configurations to existing Vec
for (key, configs) in parse.inner_instruction_configs() {
let filtered_configs: Vec<GenericEventParseConfig> = configs.into_iter().filter(|config| {
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
}).collect();
inner.inner_instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs);
}
// Merge instruction_configs, append configurations to existing Vec
for (key, configs) in parse.instruction_configs() {
let filtered_configs: Vec<GenericEventParseConfig> = configs.into_iter().filter(|config| {
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
}).collect();
inner.instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs);
let filtered_configs: Vec<GenericEventParseConfig> = configs
.into_iter()
.filter(|config| {
event_type_filter
.as_ref()
.map(|filter| filter.include.contains(&config.event_type))
.unwrap_or(true)
})
.collect();
inner
.instruction_configs
.entry(key)
.or_insert_with(Vec::new)
.extend(filtered_configs);
}
// Append program_ids (this is already appending)
@@ -44,59 +43,4 @@ impl MutilEventParser {
}
}
#[async_trait::async_trait]
impl EventParser for MutilEventParser {
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
self.inner.inner_instruction_configs()
}
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}
impl_event_parser_delegate!(MutilEventParser);
@@ -255,9 +255,15 @@ impl_unified_event!(PumpFunGlobalAccountEvent,);
/// 事件鉴别器常量
pub mod discriminators {
// 事件鉴别器
pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376";
pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94";
// pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376";
pub const CREATE_TOKEN_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 27, 114, 169, 77, 222, 235, 99, 118];
// pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
pub const TRADE_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238];
// pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94";
pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 189, 233, 93, 185, 92, 148, 234, 148];
// 指令鉴别器
pub const CREATE_TOKEN_IX: &[u8] = &[24, 30, 200, 40, 5, 28, 7, 119];
@@ -1,16 +1,15 @@
use std::collections::HashMap;
use solana_sdk::pubkey::Pubkey;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::{
common::{EventMetadata, EventType, ProtocolType},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::pumpfun::{
discriminators, pumpfun_create_token_event_log_decode, pumpfun_migrate_event_log_decode,
pumpfun_trade_event_log_decode, PumpFunCreateTokenEvent, PumpFunMigrateEvent,
PumpFunTradeEvent,
use crate::{
impl_event_parser_delegate,
streaming::event_parser::{
common::{EventMetadata, EventType, ProtocolType},
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::pumpfun::{
discriminators, pumpfun_create_token_event_log_decode,
pumpfun_migrate_event_log_decode, pumpfun_trade_event_log_decode,
PumpFunCreateTokenEvent, PumpFunMigrateEvent, PumpFunTradeEvent,
},
},
};
@@ -82,8 +81,6 @@ impl PumpFunEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pumpfun_migrate_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, event.user, event.mint));
Some(Box::new(PumpFunMigrateEvent { metadata, ..event }))
} else {
None
@@ -96,11 +93,6 @@ impl PumpFunEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pumpfun_create_token_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.name, event.symbol, event.mint
));
Some(Box::new(PumpFunCreateTokenEvent { metadata, ..event }))
} else {
None
@@ -113,11 +105,6 @@ impl PumpFunEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pumpfun_trade_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.mint, event.user, event.is_buy
));
Some(Box::new(PumpFunTradeEvent { metadata, ..event }))
} else {
None
@@ -152,9 +139,6 @@ impl PumpFunEventParser {
Pubkey::default()
};
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, name, symbol, accounts[0]));
Some(Box::new(PumpFunCreateTokenEvent {
metadata,
name: name.to_string(),
@@ -181,8 +165,6 @@ impl PumpFunEventParser {
}
let amount = u64::from_le_bytes(data[0..8].try_into().unwrap());
let max_sol_cost = u64::from_le_bytes(data[8..16].try_into().unwrap());
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], true));
Some(Box::new(PumpFunTradeEvent {
metadata,
global: accounts[0],
@@ -217,9 +199,6 @@ impl PumpFunEventParser {
}
let amount = u64::from_le_bytes(data[0..8].try_into().unwrap());
let min_sol_output = u64::from_le_bytes(data[8..16].try_into().unwrap());
let mut metadata = metadata;
metadata
.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], false));
Some(Box::new(PumpFunTradeEvent {
metadata,
global: accounts[0],
@@ -252,8 +231,6 @@ impl PumpFunEventParser {
if accounts.len() < 24 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[5], accounts[2]));
Some(Box::new(PumpFunMigrateEvent {
metadata,
global: accounts[0],
@@ -285,59 +262,4 @@ impl PumpFunEventParser {
}
}
#[async_trait::async_trait]
impl EventParser for PumpFunEventParser {
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
self.inner.inner_instruction_configs()
}
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}
impl_event_parser_delegate!(PumpFunEventParser);
@@ -392,11 +392,21 @@ impl_unified_event!(PumpSwapPoolAccountEvent,);
/// 事件鉴别器常量
pub mod discriminators {
// 事件鉴别器
pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777";
pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a";
pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774";
pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90";
pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0";
// pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777";
pub const BUY_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 103, 244, 82, 31, 44, 245, 119, 119];
// pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a";
pub const SELL_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 62, 47, 55, 10, 165, 3, 220, 42];
// pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774";
pub const CREATE_POOL_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 177, 49, 12, 210, 160, 118, 167, 116];
// pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90";
pub const DEPOSIT_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 120, 248, 61, 83, 31, 142, 107, 144];
// pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0";
pub const WITHDRAW_EVENT: &[u8] =
&[228, 69, 165, 46, 81, 203, 154, 29, 22, 9, 133, 26, 160, 44, 71, 192];
// 指令鉴别器
pub const BUY_IX: &[u8] = &[102, 6, 61, 18, 1, 218, 235, 234];
@@ -1,17 +1,16 @@
use std::collections::HashMap;
use solana_sdk::pubkey::Pubkey;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::pumpswap::{
discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode,
pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode,
pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent,
PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent,
use crate::{
impl_event_parser_delegate,
streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::pumpswap::{
discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode,
pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode,
pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent,
PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent,
},
},
};
@@ -92,11 +91,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_buy_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.user, event.pool, event.base_amount_out
));
Some(Box::new(PumpSwapBuyEvent { metadata, ..event }))
} else {
None
@@ -109,11 +103,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_sell_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.user, event.pool, event.base_amount_in
));
Some(Box::new(PumpSwapSellEvent { metadata, ..event }))
} else {
None
@@ -126,11 +115,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_create_pool_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.creator, event.base_amount_in
));
Some(Box::new(PumpSwapCreatePoolEvent { metadata, ..event }))
} else {
None
@@ -143,11 +127,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_deposit_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.user, event.lp_token_amount_out
));
Some(Box::new(PumpSwapDepositEvent { metadata, ..event }))
} else {
None
@@ -160,11 +139,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_withdraw_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.user, event.lp_token_amount_in
));
Some(Box::new(PumpSwapWithdrawEvent { metadata, ..event }))
} else {
None
@@ -184,12 +158,6 @@ impl PumpSwapEventParser {
let base_amount_out = read_u64_le(data, 0)?;
let max_quote_amount_in = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[1], accounts[0], base_amount_out
));
Some(Box::new(PumpSwapBuyEvent {
metadata,
base_amount_out,
@@ -225,12 +193,6 @@ impl PumpSwapEventParser {
let base_amount_in = read_u64_le(data, 0)?;
let min_quote_amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[1], accounts[0], base_amount_in
));
Some(Box::new(PumpSwapSellEvent {
metadata,
base_amount_in,
@@ -272,12 +234,6 @@ impl PumpSwapEventParser {
Pubkey::default()
};
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], base_amount_in
));
Some(Box::new(PumpSwapCreatePoolEvent {
metadata,
index,
@@ -312,12 +268,6 @@ impl PumpSwapEventParser {
let max_base_amount_in = u64::from_le_bytes(data[8..16].try_into().ok()?);
let max_quote_amount_in = u64::from_le_bytes(data[16..24].try_into().ok()?);
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], lp_token_amount_out
));
Some(Box::new(PumpSwapDepositEvent {
metadata,
lp_token_amount_out,
@@ -350,12 +300,6 @@ impl PumpSwapEventParser {
let min_base_amount_out = u64::from_le_bytes(data[8..16].try_into().ok()?);
let min_quote_amount_out = u64::from_le_bytes(data[16..24].try_into().ok()?);
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], lp_token_amount_in
));
Some(Box::new(PumpSwapWithdrawEvent {
metadata,
lp_token_amount_in,
@@ -375,59 +319,4 @@ impl PumpSwapEventParser {
}
}
#[async_trait::async_trait]
impl EventParser for PumpSwapEventParser {
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
self.inner.inner_instruction_configs()
}
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}
impl_event_parser_delegate!(PumpSwapEventParser);
@@ -1,15 +1,14 @@
use std::collections::HashMap;
use solana_sdk::pubkey::Pubkey;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::raydium_amm_v4::{
discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event,
RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent,
use crate::{
impl_event_parser_delegate,
streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::raydium_amm_v4::{
discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event,
RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent,
},
},
};
@@ -35,7 +34,7 @@ impl RaydiumAmmV4EventParser {
GenericEventParseConfig {
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumAmmV4,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::SWAP_BASE_IN,
event_type: EventType::RaydiumAmmV4SwapBaseIn,
inner_instruction_parser: None,
@@ -44,7 +43,7 @@ impl RaydiumAmmV4EventParser {
GenericEventParseConfig {
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumAmmV4,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::SWAP_BASE_OUT,
event_type: EventType::RaydiumAmmV4SwapBaseOut,
inner_instruction_parser: None,
@@ -53,7 +52,7 @@ impl RaydiumAmmV4EventParser {
GenericEventParseConfig {
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumAmmV4,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::DEPOSIT,
event_type: EventType::RaydiumAmmV4Deposit,
inner_instruction_parser: None,
@@ -62,7 +61,7 @@ impl RaydiumAmmV4EventParser {
GenericEventParseConfig {
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumAmmV4,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::INITIALIZE2,
event_type: EventType::RaydiumAmmV4Initialize2,
inner_instruction_parser: None,
@@ -71,7 +70,7 @@ impl RaydiumAmmV4EventParser {
GenericEventParseConfig {
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumAmmV4,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::WITHDRAW,
event_type: EventType::RaydiumAmmV4Withdraw,
inner_instruction_parser: None,
@@ -80,7 +79,7 @@ impl RaydiumAmmV4EventParser {
GenericEventParseConfig {
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumAmmV4,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::WITHDRAW_PNL,
event_type: EventType::RaydiumAmmV4WithdrawPnl,
inner_instruction_parser: None,
@@ -103,12 +102,6 @@ impl RaydiumAmmV4EventParser {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4WithdrawPnlEvent {
metadata,
token_program: accounts[0],
@@ -142,12 +135,6 @@ impl RaydiumAmmV4EventParser {
}
let amount = read_u64_le(data, 0)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4WithdrawEvent {
metadata,
amount,
@@ -191,12 +178,6 @@ impl RaydiumAmmV4EventParser {
let init_pc_amount = read_u64_le(data, 9)?;
let init_coin_amount = read_u64_le(data, 17)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4Initialize2Event {
metadata,
nonce,
@@ -241,12 +222,6 @@ impl RaydiumAmmV4EventParser {
let max_pc_amount = read_u64_le(data, 8)?;
let base_side = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4DepositEvent {
metadata,
max_coin_amount,
@@ -282,12 +257,6 @@ impl RaydiumAmmV4EventParser {
let max_amount_in = read_u64_le(data, 0)?;
let amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
let mut accounts = accounts.to_vec();
if accounts.len() == 17 {
// 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符
@@ -335,12 +304,6 @@ impl RaydiumAmmV4EventParser {
let amount_in = read_u64_le(data, 0)?;
let minimum_amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
let mut accounts = accounts.to_vec();
if accounts.len() == 17 {
// 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符
@@ -377,59 +340,4 @@ impl RaydiumAmmV4EventParser {
}
}
#[async_trait::async_trait]
impl EventParser for RaydiumAmmV4EventParser {
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
self.inner.inner_instruction_configs()
}
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}
impl_event_parser_delegate!(RaydiumAmmV4EventParser);
@@ -1,20 +1,19 @@
use std::collections::HashMap;
use solana_sdk::pubkey::Pubkey;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::{
common::{
read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata,
EventType, ProtocolType,
},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::raydium_clmm::{
discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent,
RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event,
RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent,
RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event,
use crate::{
impl_event_parser_delegate,
streaming::event_parser::{
common::{
read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata,
EventType, ProtocolType,
},
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::raydium_clmm::{
discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent,
RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event,
RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent,
RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event,
},
},
};
@@ -40,7 +39,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::SWAP,
event_type: EventType::RaydiumClmmSwap,
inner_instruction_parser: None,
@@ -49,7 +48,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::SWAP_V2,
event_type: EventType::RaydiumClmmSwapV2,
inner_instruction_parser: None,
@@ -58,7 +57,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::CLOSE_POSITION,
event_type: EventType::RaydiumClmmClosePosition,
inner_instruction_parser: None,
@@ -67,7 +66,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::DECREASE_LIQUIDITY_V2,
event_type: EventType::RaydiumClmmDecreaseLiquidityV2,
inner_instruction_parser: None,
@@ -76,7 +75,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::CREATE_POOL,
event_type: EventType::RaydiumClmmCreatePool,
inner_instruction_parser: None,
@@ -85,7 +84,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::INCREASE_LIQUIDITY_V2,
event_type: EventType::RaydiumClmmIncreaseLiquidityV2,
inner_instruction_parser: None,
@@ -94,7 +93,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::OPEN_POSITION_WITH_TOKEN_22_NFT,
event_type: EventType::RaydiumClmmOpenPositionWithToken22Nft,
inner_instruction_parser: None,
@@ -103,7 +102,7 @@ impl RaydiumClmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CLMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumClmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::OPEN_POSITION_V2,
event_type: EventType::RaydiumClmmOpenPositionV2,
inner_instruction_parser: None,
@@ -125,8 +124,6 @@ impl RaydiumClmmEventParser {
if data.len() < 51 || accounts.len() < 22 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmOpenPositionV2Event {
metadata,
tick_lower_index: read_i32_le(data, 0)?,
@@ -173,8 +170,6 @@ impl RaydiumClmmEventParser {
if data.len() < 51 || accounts.len() < 20 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmOpenPositionWithToken22NftEvent {
metadata,
tick_lower_index: read_i32_le(data, 0)?,
@@ -218,8 +213,6 @@ impl RaydiumClmmEventParser {
if data.len() < 34 || accounts.len() < 15 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmIncreaseLiquidityV2Event {
metadata,
liquidity: read_u128_le(data, 0)?,
@@ -253,8 +246,6 @@ impl RaydiumClmmEventParser {
if data.len() < 24 || accounts.len() < 13 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmCreatePoolEvent {
metadata,
sqrt_price_x64: read_u128_le(data, 0)?,
@@ -284,8 +275,6 @@ impl RaydiumClmmEventParser {
if data.len() < 32 || accounts.len() < 16 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmDecreaseLiquidityV2Event {
metadata,
liquidity: read_u128_le(data, 0)?,
@@ -320,8 +309,6 @@ impl RaydiumClmmEventParser {
if accounts.len() < 6 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmClosePositionEvent {
metadata,
nft_owner: accounts[0],
@@ -348,12 +335,6 @@ impl RaydiumClmmEventParser {
let sqrt_price_limit_x64 = read_u128_le(data, 16)?;
let is_base_input = read_u8_le(data, 32)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[2], accounts[3], accounts[4]
));
Some(Box::new(RaydiumClmmSwapEvent {
metadata,
amount,
@@ -388,12 +369,6 @@ impl RaydiumClmmEventParser {
let sqrt_price_limit_x64 = read_u128_le(data, 16)?;
let is_base_input = read_u8_le(data, 32)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[2], accounts[3], accounts[4]
));
Some(Box::new(RaydiumClmmSwapV2Event {
metadata,
amount,
@@ -418,59 +393,4 @@ impl RaydiumClmmEventParser {
}
}
#[async_trait::async_trait]
impl EventParser for RaydiumClmmEventParser {
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
self.inner.inner_instruction_configs()
}
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}
impl_event_parser_delegate!(RaydiumClmmEventParser);
@@ -1,15 +1,14 @@
use std::collections::HashMap;
use solana_sdk::pubkey::Pubkey;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::raydium_cpmm::{
discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent, RaydiumCpmmSwapEvent,
RaydiumCpmmWithdrawEvent,
use crate::{
impl_event_parser_delegate,
streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::raydium_cpmm::{
discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent,
RaydiumCpmmSwapEvent, RaydiumCpmmWithdrawEvent,
},
},
};
@@ -35,7 +34,7 @@ impl RaydiumCpmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CPMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumCpmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::SWAP_BASE_IN,
event_type: EventType::RaydiumCpmmSwapBaseInput,
inner_instruction_parser: None,
@@ -44,7 +43,7 @@ impl RaydiumCpmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CPMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumCpmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::SWAP_BASE_OUT,
event_type: EventType::RaydiumCpmmSwapBaseOutput,
inner_instruction_parser: None,
@@ -53,7 +52,7 @@ impl RaydiumCpmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CPMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumCpmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::DEPOSIT,
event_type: EventType::RaydiumCpmmDeposit,
inner_instruction_parser: None,
@@ -62,7 +61,7 @@ impl RaydiumCpmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CPMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumCpmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::INITIALIZE,
event_type: EventType::RaydiumCpmmInitialize,
inner_instruction_parser: None,
@@ -71,7 +70,7 @@ impl RaydiumCpmmEventParser {
GenericEventParseConfig {
program_id: RAYDIUM_CPMM_PROGRAM_ID,
protocol_type: ProtocolType::RaydiumCpmm,
inner_instruction_discriminator: "",
inner_instruction_discriminator: &[],
instruction_discriminator: discriminators::WITHDRAW,
event_type: EventType::RaydiumCpmmWithdraw,
inner_instruction_parser: None,
@@ -93,8 +92,6 @@ impl RaydiumCpmmEventParser {
if data.len() < 24 || accounts.len() < 14 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumCpmmWithdrawEvent {
metadata,
lp_token_amount: read_u64_le(data, 0)?,
@@ -126,8 +123,6 @@ impl RaydiumCpmmEventParser {
if data.len() < 24 || accounts.len() < 20 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumCpmmInitializeEvent {
metadata,
init_amount0: read_u64_le(data, 0)?,
@@ -165,8 +160,6 @@ impl RaydiumCpmmEventParser {
if data.len() < 24 || accounts.len() < 13 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumCpmmDepositEvent {
metadata,
lp_token_amount: read_u64_le(data, 0)?,
@@ -201,12 +194,6 @@ impl RaydiumCpmmEventParser {
let amount_in = read_u64_le(data, 0)?;
let minimum_amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumCpmmSwapEvent {
metadata,
amount_in,
@@ -240,12 +227,6 @@ impl RaydiumCpmmEventParser {
let max_amount_in = read_u64_le(data, 0)?;
let amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumCpmmSwapEvent {
metadata,
max_amount_in,
@@ -268,59 +249,4 @@ impl RaydiumCpmmEventParser {
}
}
#[async_trait::async_trait]
impl EventParser for RaydiumCpmmEventParser {
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
self.inner.inner_instruction_configs()
}
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
self.inner.instruction_configs()
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
inner_instruction,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
instruction,
accounts,
signature,
slot,
block_time,
program_received_time_ms,
index,
)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}
impl_event_parser_delegate!(RaydiumCpmmEventParser);
-290
View File
@@ -1,290 +0,0 @@
use std::sync::{Arc, Mutex};
use solana_sdk::pubkey::Pubkey;
use super::types::EventPretty;
use crate::common::AnyResult;
use crate::streaming::common::{
EventBatchProcessor as EventBatchCollector, MetricsEventType, MetricsManager,
StreamClientConfig as ClientConfig,
};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
use crate::streaming::event_parser::EventParser;
use crate::streaming::event_parser::{
core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
};
/// 事件处理器
pub struct EventProcessor {
pub(crate) metrics_manager: MetricsManager,
pub(crate) config: ClientConfig,
pub(crate) parser_cache: Arc<Mutex<Option<Arc<dyn EventParser>>>>,
}
impl EventProcessor {
/// 创建新的事件处理器
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) }
}
/// 获取或创建解析器,使用缓存机制避免重复创建
fn get_or_create_parser(
&self,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> Arc<dyn EventParser> {
let mut cache = self.parser_cache.lock().unwrap();
if let Some(cached_parser) = cache.clone() {
return cached_parser.clone();
}
let parser: Arc<dyn EventParser> =
Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone()));
*cache = Some(parser.clone());
parser
}
/// 使用性能监控处理事件交易
pub async fn process_event_transaction_with_metrics<F>(
&self,
event_pretty: EventPretty,
callback: &F,
bot_wallet: Option<Pubkey>,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_account_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let account_event = AccountEventParser::parse_account_event(
protocols.clone(),
account_pretty,
program_received_time_ms,
event_type_filter,
);
if let Some(event) = account_event {
callback(event);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标(如果启用)
self.metrics_manager
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_tx_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature.to_string();
// 使用缓存获取解析器
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
let all_events = parser
.parse_transaction(
transaction_pretty.tx.clone(),
&signature,
Some(slot),
transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
seconds: ts.seconds,
nanos: ts.nanos,
}),
program_received_time_ms,
bot_wallet,
)
.await
.unwrap_or_else(|_e| vec![]);
// 保存事件数量用于日志记录
let event_count = all_events.len();
// 批量处理事件
if !all_events.is_empty() {
for event in all_events {
callback(event);
}
}
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标(如果启用)
self.metrics_manager
.update_metrics(MetricsEventType::Tx, event_count as u64, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
}
EventPretty::BlockMeta(block_meta_pretty) => {
let start_time = std::time::Instant::now();
self.metrics_manager.add_block_meta_process_count().await;
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
let block_meta_event = CommonEventParser::generate_block_meta_event(
block_meta_pretty.slot,
&block_meta_pretty.block_hash,
block_time_ms,
);
callback(block_meta_event);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标(如果启用)
self.metrics_manager
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
Ok(())
}
/// 使用批处理处理事件交易
pub async fn process_event_transaction_with_batch<F>(
&self,
event_pretty: EventPretty,
batch_processor: &mut EventBatchCollector<F>,
bot_wallet: Option<Pubkey>,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<()>
where
F: Fn(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
{
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_account_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let account_event = AccountEventParser::parse_account_event(
protocols.clone(),
account_pretty,
program_received_time_ms,
event_type_filter,
);
if let Some(event) = account_event {
(batch_processor.callback)(vec![event]);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.metrics_manager
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_tx_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature.to_string();
// 使用缓存获取解析器
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
let result = parser
.parse_transaction(
transaction_pretty.tx.clone(),
&signature,
Some(slot),
transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
seconds: ts.seconds,
nanos: ts.nanos,
}),
program_received_time_ms,
bot_wallet,
)
.await;
// 处理解析结果并使用批处理器
let total_events = match result {
Ok(events) => {
let event_count = events.len();
if !events.is_empty() {
log::debug!("Parsed {} events", event_count);
log::debug!("Adding {} events to batch processor", event_count);
for event in events {
if self.config.batch.enabled {
batch_processor.add_event(event);
} else {
// 如果批处理被禁用,直接调用回调
// 这里需要将单个事件包装成Vec来调用批处理回调
let single_event_batch = vec![event];
(batch_processor.callback)(single_event_batch);
}
}
}
event_count
}
Err(e) => {
log::warn!("Failed to parse transaction: {:?}", e);
0
}
};
// 添加调试信息
if total_events > 0 {
log::debug!(
"Total events parsed: {} for transaction {}",
total_events,
signature
);
}
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.metrics_manager
.update_metrics(MetricsEventType::Tx, total_events as u64, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, total_events);
}
EventPretty::BlockMeta(block_meta_pretty) => {
let start_time = std::time::Instant::now();
self.metrics_manager.add_block_meta_process_count().await;
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
let block_meta_event = CommonEventParser::generate_block_meta_event(
block_meta_pretty.slot,
&block_meta_pretty.block_hash,
block_time_ms,
);
(batch_processor.callback)(vec![block_meta_event]);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标(如果启用)
self.metrics_manager
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
Ok(())
}
}
+4 -14
View File
@@ -1,25 +1,15 @@
// gRPC 相关模块
pub mod connection;
pub mod types;
pub mod subscription;
pub mod stream_handler;
pub mod event_processor;
pub mod types;
// 重新导出主要类型
pub use connection::*;
pub use types::*;
pub use subscription::*;
pub use stream_handler::*;
pub use event_processor::*;
pub use types::*;
// 从公用模块重新导出
pub use crate::streaming::common::{
BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsManager, PerformanceMetrics,
StreamClientConfig as ClientConfig,
PerformanceMetrics,
MetricsManager,
EventBatchProcessor as EventBatchCollector,
BackpressureStrategy,
BatchConfig,
BackpressureConfig,
ConnectionConfig,
};
};
-134
View File
@@ -1,134 +0,0 @@
use chrono::Local;
use futures::{channel::mpsc, sink::Sink, SinkExt};
use yellowstone_grpc_proto::geyser::{
subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, SubscribeUpdate,
};
use super::types::{BlockMetaPretty, EventPretty, TransactionPretty};
use crate::common::AnyResult;
use crate::streaming::common::BackpressureStrategy;
use crate::streaming::grpc::AccountPretty;
/// 流消息处理器
pub struct StreamHandler;
impl StreamHandler {
/// 处理单个流消息
pub async fn handle_stream_message(
msg: SubscribeUpdate,
tx: &mut mpsc::Sender<EventPretty>,
subscribe_tx: &mut (impl Sink<SubscribeRequest, Error = mpsc::SendError> + Unpin),
backpressure_strategy: BackpressureStrategy,
) -> AnyResult<()> {
let created_at = msg.created_at;
match msg.update_oneof {
Some(UpdateOneof::Account(account)) => {
let account_pretty = AccountPretty::from(account);
log::debug!("Received account: {:?}", account_pretty);
Self::handle_backpressure(
tx,
EventPretty::Account(account_pretty),
backpressure_strategy,
)
.await?;
}
Some(UpdateOneof::BlockMeta(sut)) => {
let block_meta_pretty = BlockMetaPretty::from((sut, created_at));
log::debug!("Received block meta: {:?}", block_meta_pretty);
Self::handle_backpressure(
tx,
EventPretty::BlockMeta(block_meta_pretty),
backpressure_strategy,
)
.await?;
}
Some(UpdateOneof::Transaction(sut)) => {
let transaction_pretty = TransactionPretty::from((sut, created_at));
log::debug!(
"Received transaction: {} at slot {}",
transaction_pretty.signature,
transaction_pretty.slot
);
// 根据背压策略处理发送
Self::handle_backpressure(
tx,
EventPretty::Transaction(transaction_pretty),
backpressure_strategy,
)
.await?;
}
Some(UpdateOneof::Ping(_)) => {
subscribe_tx
.send(SubscribeRequest {
ping: Some(SubscribeRequestPing { id: 1 }),
..Default::default()
})
.await?;
log::debug!("service is ping: {}", Local::now());
}
Some(UpdateOneof::Pong(_)) => {
log::debug!("service is pong: {}", Local::now());
}
_ => {
log::debug!("Received other message type");
}
}
Ok(())
}
/// 处理背压策略
async fn handle_backpressure(
tx: &mut mpsc::Sender<EventPretty>,
event_pretty: EventPretty,
backpressure_strategy: BackpressureStrategy,
) -> AnyResult<()> {
match backpressure_strategy {
BackpressureStrategy::Block => {
// 阻塞等待,直到有空间
if let Err(e) = tx.send(event_pretty).await {
log::error!("Failed to send transaction to channel: {:?}", e);
return Err(anyhow::anyhow!("Channel send failed: {:?}", e));
}
}
BackpressureStrategy::Drop => {
// 尝试发送,如果失败则丢弃
if let Err(e) = tx.try_send(event_pretty) {
if e.is_full() {
log::warn!("Channel is full, dropping transaction");
} else {
log::error!("Channel is closed: {:?}", e);
return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
}
}
}
BackpressureStrategy::Retry { max_attempts, wait_ms } => {
// 重试有限次数
let mut retry_count = 0;
loop {
match tx.try_send(event_pretty.clone()) {
Ok(_) => break,
Err(e) => {
if e.is_full() {
retry_count += 1;
if retry_count >= max_attempts {
log::warn!(
"Channel is full after {} attempts, dropping transaction",
retry_count
);
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms))
.await;
} else {
log::error!("Channel is closed: {:?}", e);
return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
}
}
}
}
}
}
Ok(())
}
}
+16 -15
View File
@@ -14,7 +14,7 @@ use crate::common::AnyResult;
use crate::streaming::common::StreamClientConfig as ClientConfig;
use crate::streaming::event_parser::common::filter::EventTypeFilter;
/// 订阅管理器
/// Subscription manager
#[derive(Clone)]
pub struct SubscriptionManager {
endpoint: String,
@@ -23,12 +23,12 @@ pub struct SubscriptionManager {
}
impl SubscriptionManager {
/// 创建新的订阅管理器
/// Create a new subscription manager
pub fn new(endpoint: String, x_token: Option<String>, config: ClientConfig) -> Self {
Self { endpoint, x_token, config }
}
/// 创建 gRPC 连接
/// Create gRPC connection
pub async fn connect(&self) -> AnyResult<GeyserGrpcClient<impl Interceptor>> {
let builder = GeyserGrpcClient::build_from_shared(self.endpoint.clone())?
.x_token(self.x_token.clone())?
@@ -39,19 +39,20 @@ impl SubscriptionManager {
Ok(builder.connect().await?)
}
/// 创建订阅请求并返回流
/// Create subscription request and return stream
pub async fn subscribe_with_request(
&self,
transactions: Option<TransactionsFilterMap>,
accounts: Option<AccountsFilterMap>,
commitment: Option<CommitmentLevel>,
event_type_filter: Option<EventTypeFilter>,
event_type_filter: Option<&EventTypeFilter>,
) -> AnyResult<(
impl Sink<SubscribeRequest, Error = mpsc::SendError>,
impl Stream<Item = Result<SubscribeUpdate, Status>>,
SubscribeRequest,
)> {
let blocks_meta = if event_type_filter.is_some()
&& event_type_filter.as_ref().unwrap().include_block_event()
&& event_type_filter.unwrap().include_block_event()
{
hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} }
} else if event_type_filter.is_none() {
@@ -71,22 +72,22 @@ impl SubscriptionManager {
..Default::default()
};
let mut client = self.connect().await?;
let (sink, stream) = client.subscribe_with_request(Some(subscribe_request)).await?;
Ok((sink, stream))
let (sink, stream) = client.subscribe_with_request(Some(subscribe_request.clone())).await?;
Ok((sink, stream, subscribe_request))
}
/// 创建账户订阅请求并返回流
/// Create account subscription request and return stream
pub fn subscribe_with_account_request(
&self,
account: Vec<String>,
owner: Vec<String>,
event_type_filter: Option<EventTypeFilter>,
event_type_filter: Option<&EventTypeFilter>,
) -> Option<AccountsFilterMap> {
if account.len() == 0 && owner.len() == 0 {
return None;
}
if event_type_filter.is_some()
&& !event_type_filter.as_ref().unwrap().include_account_event()
&& !event_type_filter.unwrap().include_account_event()
{
return None;
}
@@ -103,16 +104,16 @@ impl SubscriptionManager {
Some(accounts)
}
/// 生成订阅请求过滤器
/// Generate subscription request filter
pub fn get_subscribe_request_filter(
&self,
account_include: Vec<String>,
account_exclude: Vec<String>,
account_required: Vec<String>,
event_type_filter: Option<EventTypeFilter>,
event_type_filter: Option<&EventTypeFilter>,
) -> Option<TransactionsFilterMap> {
if event_type_filter.is_some()
&& !event_type_filter.as_ref().unwrap().include_transaction_event()
&& !event_type_filter.unwrap().include_transaction_event()
{
return None;
}
@@ -131,7 +132,7 @@ impl SubscriptionManager {
Some(transactions)
}
/// 获取配置
/// Get configuration
pub fn get_config(&self) -> &ClientConfig {
&self.config
}
+32 -22
View File
@@ -1,5 +1,5 @@
use solana_sdk::signature::Signature;
use solana_transaction_status::{EncodedTransactionWithStatusMeta, UiTransactionEncoding};
use solana_sdk::{pubkey::Pubkey, signature::Signature};
use solana_transaction_status::TransactionWithStatusMeta;
use std::{collections::HashMap, fmt};
use yellowstone_grpc_proto::{
geyser::{
@@ -22,13 +22,14 @@ pub enum EventPretty {
#[derive(Clone)]
pub struct AccountPretty {
pub slot: u64,
pub signature: String,
pub pubkey: String,
pub signature: Signature,
pub pubkey: Pubkey,
pub executable: bool,
pub lamports: u64,
pub owner: String,
pub owner: Pubkey,
pub rent_epoch: u64,
pub data: Vec<u8>,
pub program_received_time_us: i64,
}
impl fmt::Debug for AccountPretty {
@@ -51,6 +52,7 @@ pub struct BlockMetaPretty {
pub slot: u64,
pub block_hash: String,
pub block_time: Option<Timestamp>,
pub program_received_time_us: i64,
}
impl fmt::Debug for BlockMetaPretty {
@@ -59,6 +61,7 @@ impl fmt::Debug for BlockMetaPretty {
.field("slot", &self.slot)
.field("block_hash", &self.block_hash)
.field("block_time", &self.block_time)
.field("program_received_time_us", &self.program_received_time_us)
.finish()
}
}
@@ -66,28 +69,23 @@ impl fmt::Debug for BlockMetaPretty {
#[derive(Clone)]
pub struct TransactionPretty {
pub slot: u64,
pub transaction_index: Option<u64>, // 新增:交易在slot中的索引
pub block_hash: String,
pub block_time: Option<Timestamp>,
pub signature: Signature,
pub is_vote: bool,
pub tx: EncodedTransactionWithStatusMeta,
pub tx: TransactionWithStatusMeta,
pub program_received_time_us: i64,
}
impl fmt::Debug for TransactionPretty {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
struct TxWrap<'a>(&'a EncodedTransactionWithStatusMeta);
impl<'a> fmt::Debug for TxWrap<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let serialized = serde_json::to_string(self.0).expect("failed to serialize");
fmt::Display::fmt(&serialized, f)
}
}
f.debug_struct("TransactionPretty")
.field("slot", &self.slot)
.field("transaction_index", &self.transaction_index)
.field("signature", &self.signature)
.field("is_vote", &self.is_vote)
.field("tx", &TxWrap(&self.tx))
.field("program_received_time_us", &self.program_received_time_us)
.finish()
}
}
@@ -97,13 +95,18 @@ impl From<SubscribeUpdateAccount> for AccountPretty {
let account_info = account.account.unwrap();
Self {
slot: account.slot,
signature: bs58::encode(&account_info.txn_signature.unwrap_or_default()).into_string(),
pubkey: bs58::encode(&account_info.pubkey).into_string(),
signature: if let Some(txn_signature) = account_info.txn_signature {
Signature::try_from(txn_signature.as_slice()).expect("valid signature")
} else {
Signature::default()
},
pubkey: Pubkey::try_from(account_info.pubkey.as_slice()).expect("valid pubkey"),
executable: account_info.executable,
lamports: account_info.lamports,
owner: bs58::encode(&account_info.owner).into_string(),
owner: Pubkey::try_from(account_info.owner.as_slice()).expect("valid pubkey"),
rent_epoch: account_info.rent_epoch,
data: account_info.data,
program_received_time_us: chrono::Utc::now().timestamp_micros(),
}
}
}
@@ -115,7 +118,12 @@ impl From<(SubscribeUpdateBlockMeta, Option<Timestamp>)> for BlockMetaPretty {
Option<Timestamp>,
),
) -> Self {
Self { block_hash: blockhash.to_string(), block_time, slot }
Self {
block_hash: blockhash.to_string(),
block_time,
slot,
program_received_time_us: chrono::Utc::now().timestamp_micros(),
}
}
}
@@ -127,16 +135,18 @@ impl From<(SubscribeUpdateTransaction, Option<Timestamp>)> for TransactionPretty
),
) -> Self {
let tx = transaction.expect("should be defined");
// 根据用户说明,交易索引在 transaction.index 中
let transaction_index = tx.index;
Self {
slot,
transaction_index: Some(transaction_index), // 提取交易索引
block_time,
block_hash: "".to_string(),
signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"),
is_vote: tx.is_vote,
tx: yellowstone_grpc_proto::convert_from::create_tx_with_meta(tx)
.expect("valid tx with meta")
.encode(UiTransactionEncoding::Base64, Some(u8::MAX), true)
.expect("failed to encode"),
.expect("valid tx with meta"),
program_received_time_us: chrono::Utc::now().timestamp_micros(),
}
}
}
+22 -14
View File
@@ -1,4 +1,5 @@
use std::sync::Arc;
use std::sync::RwLock;
use tokio::sync::Mutex;
use tonic::transport::Channel;
@@ -13,7 +14,7 @@ use crate::streaming::common::{
pub struct ShredStreamGrpc {
pub shredstream_client: Arc<ShredstreamProxyClient<Channel>>,
pub config: StreamClientConfig,
pub metrics: Arc<Mutex<PerformanceMetrics>>,
pub metrics: Arc<RwLock<PerformanceMetrics>>,
pub metrics_manager: MetricsManager,
pub subscription_handle: Arc<Mutex<Option<SubscriptionHandle>>>,
}
@@ -27,31 +28,38 @@ impl ShredStreamGrpc {
/// 创建客户端,使用自定义配置
pub async fn new_with_config(endpoint: String, config: StreamClientConfig) -> AnyResult<Self> {
let shredstream_client = ShredstreamProxyClient::connect(endpoint.clone()).await?;
let metrics = Arc::new(Mutex::new(PerformanceMetrics::new()));
let config_arc = Arc::new(config.clone());
let metrics = Arc::new(RwLock::new(PerformanceMetrics::new()));
let metrics_manager =
MetricsManager::new(metrics.clone(), config_arc, "ShredStream".to_string());
let metrics_manager = MetricsManager::new(config.enable_metrics, "ShredStream".to_string());
Ok(Self {
shredstream_client: Arc::new(shredstream_client),
config,
metrics,
metrics: metrics.clone(),
metrics_manager,
subscription_handle: Arc::new(Mutex::new(None)),
})
}
/// 创建高性能客户端(适合高并发场景)
pub async fn new_high_performance(endpoint: String) -> AnyResult<Self> {
Self::new_with_config(endpoint, StreamClientConfig::high_performance()).await
/// Creates a new ShredStreamClient with high-throughput configuration.
///
/// This is a convenience method that creates a client optimized for high-concurrency scenarios
/// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()`
/// for detailed configuration information.
pub async fn new_high_throughput(endpoint: String) -> AnyResult<Self> {
Self::new_with_config(endpoint, StreamClientConfig::high_throughput()).await
}
/// 创建低延迟客户端(适合实时场景)
/// Creates a new ShredStreamClient with low-latency configuration.
///
/// This is a convenience method that creates a client optimized for real-time scenarios
/// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()`
/// for detailed configuration information.
pub async fn new_low_latency(endpoint: String) -> AnyResult<Self> {
Self::new_with_config(endpoint, StreamClientConfig::low_latency()).await
}
/// 获取当前配置
pub fn get_config(&self) -> &StreamClientConfig {
&self.config
@@ -63,8 +71,8 @@ impl ShredStreamGrpc {
}
/// 获取性能指标
pub async fn get_metrics(&self) -> PerformanceMetrics {
self.metrics_manager.get_metrics().await
pub fn get_metrics(&self) -> PerformanceMetrics {
self.metrics_manager.get_metrics()
}
/// 启用或禁用性能监控
@@ -73,8 +81,8 @@ impl ShredStreamGrpc {
}
/// 打印性能指标
pub async fn print_metrics(&self) {
self.metrics_manager.print_metrics().await;
pub fn print_metrics(&self) {
self.metrics_manager.print_metrics();
}
/// 启动自动性能监控任务
-170
View File
@@ -1,170 +0,0 @@
use solana_sdk::pubkey::Pubkey;
use std::sync::{Arc, Mutex};
use crate::common::AnyResult;
use crate::streaming::common::{
EventBatchProcessor, MetricsEventType, MetricsManager, StreamClientConfig,
};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::protocols::MutilEventParser;
use crate::streaming::event_parser::{EventParser, Protocol, UnifiedEvent};
use crate::streaming::shred::TransactionWithSlot;
/// ShredStream 事件处理器
pub struct ShredEventProcessor {
pub(crate) metrics_manager: MetricsManager,
pub(crate) config: StreamClientConfig,
pub(crate) parser_cache: Arc<Mutex<Option<Arc<dyn EventParser>>>>,
}
impl ShredEventProcessor {
/// 创建新的事件处理器
pub fn new(metrics_manager: MetricsManager, config: StreamClientConfig) -> Self {
Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) }
}
/// 获取或创建解析器,使用缓存机制避免重复创建
fn get_or_create_parser(
&self,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> Arc<dyn EventParser> {
let mut cache = self.parser_cache.lock().unwrap();
if let Some(cached_parser) = cache.clone() {
return cached_parser.clone();
}
let parser: Arc<dyn EventParser> =
Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone()));
*cache = Some(parser.clone());
parser
}
/// 即时处理单个交易
pub async fn process_transaction_immediate<F>(
&self,
transaction_with_slot: TransactionWithSlot,
protocols: Vec<Protocol>,
bot_wallet: Option<Pubkey>,
event_type_filter: Option<EventTypeFilter>,
callback: &F,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
let start_time = std::time::Instant::now();
self.metrics_manager.add_tx_process_count().await;
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_with_slot.slot;
let versioned_tx = transaction_with_slot.transaction;
let signature = versioned_tx.signatures[0];
// 获取缓存的解析器
let parser = self.get_or_create_parser(protocols, event_type_filter);
let all_events = parser
.parse_versioned_transaction(
&versioned_tx,
&signature.to_string(),
Some(slot),
None,
program_received_time_ms,
bot_wallet,
)
.await
.unwrap_or_else(|_e| vec![]);
// 保存事件数量用于日志记录
let event_count = all_events.len();
// 即时处理事件
for event in all_events {
callback(event);
}
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.update_metrics(event_count as u64, processing_time_ms).await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
Ok(())
}
/// 批处理模式处理单个交易
pub async fn process_transaction_with_batch<F>(
&self,
transaction_with_slot: TransactionWithSlot,
protocols: Vec<Protocol>,
bot_wallet: Option<Pubkey>,
batch_processor: &mut EventBatchProcessor<F>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<()>
where
F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
{
let start_time = std::time::Instant::now();
self.metrics_manager.add_tx_process_count().await;
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_with_slot.slot;
let versioned_tx = transaction_with_slot.transaction;
let signature = versioned_tx.signatures[0];
// 获取缓存的解析器
let parser = self.get_or_create_parser(protocols, event_type_filter);
let all_events = parser
.parse_versioned_transaction(
&versioned_tx,
&signature.to_string(),
Some(slot),
None,
program_received_time_ms,
bot_wallet,
)
.await
.unwrap_or_else(|_e| vec![]);
// 保存事件数量用于日志记录
let event_count = all_events.len();
// 使用批处理器处理事件
for event in all_events {
batch_processor.add_event(event);
}
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.update_metrics(event_count as u64, processing_time_ms).await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
Ok(())
}
/// 更新性能指标
async fn update_metrics(&self, events_processed: u64, processing_time_ms: f64) {
// 使用统一的指标管理器,这里假设 ShredStream 主要处理交易事件
self.metrics_manager
.update_metrics(MetricsEventType::Tx, events_processed, processing_time_ms)
.await;
}
}
// 实现 Clone trait 以支持模块间共享
impl Clone for ShredEventProcessor {
fn clone(&self) -> Self {
Self {
metrics_manager: self.metrics_manager.clone(),
config: self.config.clone(),
parser_cache: self.parser_cache.clone(),
}
}
}
+2 -6
View File
@@ -1,17 +1,13 @@
// ShredStream 相关模块
pub mod connection;
pub mod types;
pub mod stream_handler;
pub mod event_processor;
// 重新导出主要类型
pub use connection::*;
pub use types::*;
pub use stream_handler::*;
pub use event_processor::*;
// 从公用模块重新导出
pub use crate::streaming::common::{
BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, EventBatchProcessor,
MetricsEventType, MetricsManager, PerformanceMetrics, StreamClientConfig,
BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsEventType, MetricsManager,
PerformanceMetrics, StreamClientConfig,
};
-116
View File
@@ -1,116 +0,0 @@
use futures::{channel::mpsc, StreamExt};
use log::error;
use solana_entry::entry::Entry;
use tokio::task::JoinHandle;
use crate::common::AnyResult;
use crate::protos::shredstream::{
shredstream_proxy_client::ShredstreamProxyClient, SubscribeEntriesRequest,
};
use crate::streaming::shred::TransactionWithSlot;
/// ShredStream 流处理器
pub struct ShredStreamHandler;
impl ShredStreamHandler {
/// 启动 ShredStream 流处理任务
///
/// # 参数
/// * `client` - ShredStream 客户端
/// * `tx` - 事务发送通道
/// * `channel_size` - 通道缓冲区大小
///
/// # 返回值
/// 返回 ShredStream 流处理任务句柄和事务接收通道
pub async fn start_stream_processing(
mut client: ShredstreamProxyClient<tonic::transport::Channel>,
channel_size: usize,
) -> AnyResult<(JoinHandle<()>, mpsc::Receiver<TransactionWithSlot>)> {
let request = tonic::Request::new(SubscribeEntriesRequest {});
let stream = client.subscribe_entries(request).await?.into_inner();
let (tx, rx) = mpsc::channel::<TransactionWithSlot>(channel_size);
let stream_task = tokio::spawn(Self::process_stream_messages(stream, tx));
Ok((stream_task, rx))
}
/// 处理流消息
///
/// # 参数
/// * `stream` - ShredStream 数据流
/// * `tx` - 事务发送通道
async fn process_stream_messages(
mut stream: tonic::codec::Streaming<crate::protos::shredstream::Entry>,
mut tx: mpsc::Sender<TransactionWithSlot>,
) {
while let Some(message) = stream.next().await {
match message {
Ok(msg) => {
if let Err(e) = Self::handle_stream_message(msg, &mut tx).await {
error!("Error handling stream message: {e:?}");
continue;
}
}
Err(error) => {
error!("Stream error: {error:?}");
break;
}
}
}
}
/// 处理单个流消息
///
/// # 参数
/// * `msg` - ShredStream 消息
/// * `tx` - 事务发送通道
async fn handle_stream_message(
msg: crate::protos::shredstream::Entry,
tx: &mut mpsc::Sender<TransactionWithSlot>,
) -> AnyResult<()> {
if let Ok(entries) = bincode::deserialize::<Vec<Entry>>(&msg.entries) {
for entry in entries {
for transaction in entry.transactions {
let transaction_with_slot =
TransactionWithSlot::new(transaction.clone(), msg.slot);
if let Err(e) = tx.try_send(transaction_with_slot) {
// 如果通道满了,记录警告但不中断处理
if e.is_full() {
log::warn!("Transaction channel is full, dropping transaction");
} else {
// 通道已关闭,返回错误
return Err(e.into());
}
}
}
}
}
Ok(())
}
/// 启动事务处理任务
///
/// # 参数
/// * `rx` - 事务接收通道
/// * `processor` - 事务处理器
pub fn start_transaction_processing<F>(
mut rx: mpsc::Receiver<TransactionWithSlot>,
processor: F,
) -> JoinHandle<()>
where
F: Fn(TransactionWithSlot) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+ Send
+ Sync
+ 'static,
{
tokio::spawn(async move {
while let Some(transaction_with_slot) = rx.next().await {
if let Err(e) = processor(transaction_with_slot) {
error!("Error processing transaction: {e:?}");
}
}
})
}
}
+7 -7
View File
@@ -5,16 +5,16 @@ use solana_sdk::transaction::VersionedTransaction;
pub struct TransactionWithSlot {
pub transaction: VersionedTransaction,
pub slot: u64,
pub program_received_time_us: i64,
}
impl TransactionWithSlot {
/// 创建新的带槽位的交易
pub fn new(transaction: VersionedTransaction, slot: u64) -> Self {
Self { transaction, slot }
}
/// 获取交易签名
pub fn signature(&self) -> String {
self.transaction.signatures[0].to_string()
pub fn new(
transaction: VersionedTransaction,
slot: u64,
program_received_time_us: i64,
) -> Self {
Self { transaction, slot, program_received_time_us }
}
}
+57 -110
View File
@@ -1,10 +1,16 @@
use std::sync::Arc;
use futures::StreamExt;
use solana_sdk::pubkey::Pubkey;
use crate::common::AnyResult;
use crate::streaming::common::{EventBatchProcessor, SubscriptionHandle};
use crate::protos::shredstream::SubscribeEntriesRequest;
use crate::streaming::common::{EventProcessor, SubscriptionHandle};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::{Protocol, UnifiedEvent};
use crate::streaming::shred::{ShredEventProcessor, ShredStreamHandler, TransactionWithSlot};
use crate::streaming::shred::TransactionWithSlot;
use log::error;
use solana_entry::entry::Entry;
use super::ShredStreamGrpc;
@@ -29,120 +35,61 @@ impl ShredStreamGrpc {
metrics_handle = self.metrics_manager.start_auto_monitoring().await;
}
// 启动流处理
let client = (*self.shredstream_client).clone();
let (stream_task, rx) = ShredStreamHandler::start_stream_processing(
client,
self.config.backpressure.channel_size,
)
.await?;
// 创建事件处理
let mut event_processor =
EventProcessor::new(self.metrics_manager.clone(), self.config.clone());
event_processor.set_protocols_and_event_type_filter(
protocols,
event_type_filter,
self.config.backpressure.clone(),
Some(Arc::new(callback)),
);
// 根据配置选择处理模式并获取事件处理任务句柄
let event_handle = if self.config.batch.enabled {
// 批处理模式
self.process_with_batch(rx, protocols, bot_wallet, event_type_filter, callback).await?
} else {
// 即时处理模式
self.process_immediate(rx, protocols, bot_wallet, event_type_filter, callback).await?
};
// 启动流处理
let mut client = (*self.shredstream_client).clone();
let request = tonic::Request::new(SubscribeEntriesRequest {});
let mut stream = client.subscribe_entries(request).await?.into_inner();
let event_processor_clone = event_processor.clone();
let stream_task = tokio::spawn(async move {
while let Some(message) = stream.next().await {
match message {
Ok(msg) => {
if let Ok(entries) = bincode::deserialize::<Vec<Entry>>(&msg.entries) {
for entry in entries {
for transaction in entry.transactions {
let transaction_with_slot = TransactionWithSlot::new(
transaction.clone(),
msg.slot,
chrono::Utc::now().timestamp_micros(),
);
// 直接处理,背压控制在 EventProcessor 内部处理
if let Err(e) = event_processor_clone
.process_shred_transaction_with_metrics(
transaction_with_slot,
bot_wallet,
)
.await
{
error!("Error handling message: {e:?}");
}
}
}
}
continue;
}
Err(error) => {
error!("Stream error: {error:?}");
break;
}
}
}
});
// 保存订阅句柄
let subscription_handle = SubscriptionHandle::new(stream_task, event_handle, metrics_handle);
let subscription_handle = SubscriptionHandle::new(stream_task, None, metrics_handle);
let mut handle_guard = self.subscription_handle.lock().await;
*handle_guard = Some(subscription_handle);
Ok(())
}
/// 批处理模式
async fn process_with_batch<F>(
&self,
mut rx: futures::channel::mpsc::Receiver<TransactionWithSlot>,
protocols: Vec<Protocol>,
bot_wallet: Option<Pubkey>,
event_type_filter: Option<EventTypeFilter>,
callback: F,
) -> AnyResult<tokio::task::JoinHandle<()>>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
use futures::StreamExt;
// 创建批处理器,将单个事件回调转换为批量回调
let batch_callback = move |events: Vec<Box<dyn UnifiedEvent>>| {
for event in events {
callback(event);
}
};
let mut batch_processor = EventBatchProcessor::new(
batch_callback,
self.config.batch.batch_size,
self.config.batch.batch_timeout_ms,
);
// 创建事件处理器
let event_processor =
ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone());
let event_handle = tokio::spawn(async move {
while let Some(transaction_with_slot) = rx.next().await {
if let Err(e) = event_processor
.process_transaction_with_batch(
transaction_with_slot,
protocols.clone(),
bot_wallet,
&mut batch_processor,
event_type_filter.clone(),
)
.await
{
log::error!("Error processing transaction: {e:?}");
}
}
// 处理剩余的事件
batch_processor.flush();
});
Ok(event_handle)
}
/// 即时处理模式
async fn process_immediate<F>(
&self,
mut rx: futures::channel::mpsc::Receiver<TransactionWithSlot>,
protocols: Vec<Protocol>,
bot_wallet: Option<Pubkey>,
event_type_filter: Option<EventTypeFilter>,
callback: F,
) -> AnyResult<tokio::task::JoinHandle<()>>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
use futures::StreamExt;
// 创建事件处理器
let event_processor =
ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone());
let event_handle = tokio::spawn(async move {
while let Some(transaction_with_slot) = rx.next().await {
if let Err(e) = event_processor
.process_transaction_immediate(
transaction_with_slot,
protocols.clone(),
bot_wallet,
event_type_filter.clone(),
&callback,
)
.await
{
log::error!("Error processing transaction: {e:?}");
}
}
});
Ok(event_handle)
}
}
+200 -199
View File
@@ -1,19 +1,26 @@
use futures::{channel::mpsc, StreamExt};
use log::error;
use solana_sdk::pubkey::Pubkey;
use std::sync::Arc;
use tokio::sync::Mutex;
use yellowstone_grpc_proto::geyser::CommitmentLevel;
use crate::common::AnyResult;
use crate::streaming::common::{
EventBatchProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle,
EventProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle,
};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::{Protocol, UnifiedEvent};
use crate::streaming::grpc::{EventPretty, EventProcessor, StreamHandler, SubscriptionManager};
use crate::streaming::grpc::{
AccountPretty, BlockMetaPretty, EventPretty, SubscriptionManager, TransactionPretty,
};
use anyhow::anyhow;
use chrono::Local;
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use log::error;
use solana_sdk::pubkey::Pubkey;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, RwLock};
use tokio::sync::Mutex;
use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestPing};
/// 交易过滤器
#[derive(Debug, Clone)]
pub struct TransactionFilter {
pub account_include: Vec<String>,
pub account_exclude: Vec<String>,
@@ -21,6 +28,7 @@ pub struct TransactionFilter {
}
/// 账户过滤器
#[derive(Debug, Clone)]
pub struct AccountFilter {
pub account: Vec<String>,
pub owner: Vec<String>,
@@ -30,11 +38,15 @@ pub struct YellowstoneGrpc {
pub endpoint: String,
pub x_token: Option<String>,
pub config: StreamClientConfig,
pub metrics: Arc<Mutex<PerformanceMetrics>>,
pub metrics: Arc<RwLock<PerformanceMetrics>>,
pub subscription_manager: SubscriptionManager,
pub metrics_manager: MetricsManager,
pub event_processor: EventProcessor,
pub subscription_handle: Arc<Mutex<Option<SubscriptionHandle>>>,
// Dynamic subscription management fields
pub active_subscription: Arc<AtomicBool>,
pub control_tx: Arc<tokio::sync::Mutex<Option<mpsc::Sender<SubscribeRequest>>>>,
pub current_request: Arc<tokio::sync::RwLock<Option<SubscribeRequest>>>,
}
impl YellowstoneGrpc {
@@ -50,43 +62,50 @@ impl YellowstoneGrpc {
config: StreamClientConfig,
) -> AnyResult<Self> {
let _ = rustls::crypto::ring::default_provider().install_default().ok();
let metrics = Arc::new(Mutex::new(PerformanceMetrics::new()));
let config_arc = Arc::new(config.clone());
let metrics = Arc::new(RwLock::new(PerformanceMetrics::new()));
let subscription_manager =
SubscriptionManager::new(endpoint.clone(), x_token.clone(), config.clone());
let metrics_manager =
MetricsManager::new(metrics.clone(), config_arc.clone(), "YellowstoneGrpc".to_string());
let metrics_manager = MetricsManager::new_with_metrics(
metrics.clone(),
config.enable_metrics,
"YellowstoneGrpc".to_string(),
);
let event_processor = EventProcessor::new(metrics_manager.clone(), config.clone());
Ok(Self {
endpoint,
x_token,
config,
metrics,
metrics: metrics.clone(),
subscription_manager,
metrics_manager,
event_processor,
subscription_handle: Arc::new(Mutex::new(None)),
active_subscription: Arc::new(AtomicBool::new(false)),
control_tx: Arc::new(tokio::sync::Mutex::new(None)),
current_request: Arc::new(tokio::sync::RwLock::new(None)),
})
}
/// 创建高性能客户端
pub fn new_high_performance(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
Self::new_with_config(endpoint, x_token, StreamClientConfig::high_performance())
/// Creates a new YellowstoneGrpcClient with high-throughput configuration.
///
/// This is a convenience method that creates a client optimized for high-concurrency scenarios
/// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()`
/// for detailed configuration information.
pub fn new_high_throughput(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
Self::new_with_config(endpoint, x_token, StreamClientConfig::high_throughput())
}
/// 创建低延迟客户端
/// Creates a new YellowstoneGrpcClient with low-latency configuration.
///
/// This is a convenience method that creates a client optimized for real-time scenarios
/// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()`
/// for detailed configuration information.
pub fn new_low_latency(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
Self::new_with_config(endpoint, x_token, StreamClientConfig::low_latency())
}
/// 创建即时处理客户端
pub fn new_immediate(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
let mut config = StreamClientConfig::low_latency();
config.enable_metrics = false;
Self::new_with_config(endpoint, x_token, config)
}
/// 获取配置
pub fn get_config(&self) -> &StreamClientConfig {
@@ -99,13 +118,13 @@ impl YellowstoneGrpc {
}
/// 获取性能指标
pub async fn get_metrics(&self) -> PerformanceMetrics {
self.metrics_manager.get_metrics().await
pub fn get_metrics(&self) -> PerformanceMetrics {
self.metrics_manager.get_metrics()
}
/// 打印性能指标
pub async fn print_metrics(&self) {
self.metrics_manager.print_metrics().await;
pub fn print_metrics(&self) {
self.metrics_manager.print_metrics();
}
/// 启用或禁用性能监控
@@ -119,6 +138,9 @@ impl YellowstoneGrpc {
if let Some(handle) = handle_guard.take() {
handle.stop();
}
*self.control_tx.lock().await = None;
*self.current_request.write().await = None;
self.active_subscription.store(false, Ordering::Release);
}
/// Simplified immediate event subscription (recommended for simple scenarios)
@@ -147,8 +169,13 @@ impl YellowstoneGrpc {
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
// 如果已有活跃订阅,先停止它
self.stop().await;
if self
.active_subscription
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
return Err(anyhow!("Already subscribed. Use update_subscription() to modify filters"));
}
let mut metrics_handle = None;
// 启动自动性能监控(如果启用)
@@ -160,205 +187,187 @@ impl YellowstoneGrpc {
transaction_filter.account_include,
transaction_filter.account_exclude,
transaction_filter.account_required,
event_type_filter.clone(),
event_type_filter.as_ref(),
);
let accounts = self.subscription_manager.subscribe_with_account_request(
account_filter.account,
account_filter.owner,
event_type_filter.clone(),
event_type_filter.as_ref(),
);
// 订阅事件
let (mut subscribe_tx, mut stream) = self
let (mut subscribe_tx, mut stream, subscribe_request) = self
.subscription_manager
.subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone())
.subscribe_with_request(transactions, accounts, commitment, event_type_filter.as_ref())
.await?;
// 创建通道,使用配置中的通道大小
let (mut tx, mut rx) = mpsc::channel::<EventPretty>(self.config.backpressure.channel_size);
// 用 Arc<Mutex<>> 包装 subscribe_tx 以支持多线程共享
let subscribe_tx = Arc::new(Mutex::new(subscribe_tx));
*self.current_request.write().await = Some(subscribe_request);
let (control_tx, mut control_rx) = mpsc::channel(100);
*self.control_tx.lock().await = Some(control_tx);
// 启动流处理任务
let backpressure_strategy = self.config.backpressure.strategy;
let mut event_processor = self.event_processor.clone();
event_processor.set_protocols_and_event_type_filter(
protocols,
event_type_filter,
self.config.backpressure.clone(),
Some(Arc::new(callback)),
);
let stream_handle = tokio::spawn(async move {
while let Some(message) = stream.next().await {
match message {
Ok(msg) => {
if let Err(e) = StreamHandler::handle_stream_message(
msg,
&mut tx,
&mut subscribe_tx,
backpressure_strategy,
)
.await
{
error!("Error handling message: {e:?}");
loop {
tokio::select! {
message = stream.next() => {
match message {
Some(Ok(msg)) => {
let created_at = msg.created_at;
match msg.update_oneof {
Some(UpdateOneof::Account(account)) => {
let account_pretty = AccountPretty::from(account);
log::debug!("Received account: {:?}", account_pretty);
if let Err(e) = event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::Account(account_pretty),
bot_wallet,
)
.await
{
error!("Error processing account event: {e:?}");
}
}
Some(UpdateOneof::BlockMeta(sut)) => {
let block_meta_pretty =
BlockMetaPretty::from((sut, created_at));
log::debug!("Received block meta: {:?}", block_meta_pretty);
if let Err(e) = event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::BlockMeta(block_meta_pretty),
bot_wallet,
)
.await
{
error!("Error processing block meta event: {e:?}");
}
}
Some(UpdateOneof::Transaction(sut)) => {
let transaction_pretty =
TransactionPretty::from((sut, created_at));
log::debug!(
"Received transaction: {} at slot {}",
transaction_pretty.signature,
transaction_pretty.slot
);
if let Err(e) = event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::Transaction(transaction_pretty),
bot_wallet,
)
.await
{
error!("Error processing transaction event: {e:?}");
}
}
Some(UpdateOneof::Ping(_)) => {
// 只在需要时获取锁,并立即释放
if let Ok(mut tx_guard) = subscribe_tx.try_lock() {
let _ = tx_guard
.send(SubscribeRequest {
ping: Some(SubscribeRequestPing { id: 1 }),
..Default::default()
})
.await;
}
log::debug!("service is ping: {}", Local::now());
}
Some(UpdateOneof::Pong(_)) => {
log::debug!("service is pong: {}", Local::now());
}
_ => {
log::debug!("Received other message type");
}
}
}
Some(Err(error)) => {
error!("Stream error: {error:?}");
break;
}
None => break,
}
}
Some(update) = control_rx.next() => {
if let Err(e) = subscribe_tx.lock().await.send(update).await {
error!("Failed to send subscription update: {}", e);
break;
}
}
Err(error) => {
error!("Stream error: {error:?}");
break;
}
}
}
});
// 即时处理交易,无批处理
let event_processor = self.event_processor.clone();
let event_handle = tokio::spawn(async move {
while let Some(event_pretty) = rx.next().await {
if let Err(e) = event_processor
.process_event_transaction_with_metrics(
event_pretty,
&callback,
bot_wallet,
protocols.clone(),
event_type_filter.clone(),
)
.await
{
error!("Error processing transaction: {e:?}");
}
}
});
// 保存订阅句柄
let subscription_handle =
SubscriptionHandle::new(stream_handle, event_handle, metrics_handle);
let subscription_handle = SubscriptionHandle::new(stream_handle, None, metrics_handle);
let mut handle_guard = self.subscription_handle.lock().await;
*handle_guard = Some(subscription_handle);
Ok(())
}
/// Advanced event subscription with batch processing and backpressure handling
/// Update subscription filters at runtime without reconnection
///
/// # Parameters
/// * `protocols` - List of protocols to monitor
/// * `bot_wallet` - Optional bot wallet address for filtering related transactions
/// * `transaction_filter` - Transaction filter specifying accounts to include/exclude
/// * `account_filter` - Account filter specifying accounts and owners to monitor
/// * `event_filter` - Optional event filter for further event filtering, no filtering if None
/// * `commitment` - Optional commitment level, defaults to Confirmed
/// * `callback` - Event callback function that receives parsed unified events
///
/// # Features
/// * Batch processing for improved throughput
/// * Backpressure handling to prevent memory overflow
/// * Automatic performance monitoring (if enabled)
/// * Configurable batch size and timeout
/// * `transaction_filter` - New transaction filter to apply
/// * `account_filter` - New account filter to apply
///
/// # Returns
/// Returns `AnyResult<()>`, `Ok(())` on success, error information on failure
pub async fn subscribe_events_advanced<F>(
/// Returns `AnyResult<()>` on success, error on failure
pub async fn update_subscription(
&self,
protocols: Vec<Protocol>,
bot_wallet: Option<Pubkey>,
transaction_filter: TransactionFilter,
account_filter: AccountFilter,
event_type_filter: Option<EventTypeFilter>,
commitment: Option<CommitmentLevel>,
callback: F,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
// 如果已有活跃订阅,先停止它
self.stop().await;
) -> AnyResult<()> {
let mut control_sender = {
let control_guard = self.control_tx.lock().await;
let mut metrics_handle = None;
// 启动自动性能监控(如果启用)
if self.config.enable_metrics {
metrics_handle = self.metrics_manager.start_auto_monitoring().await;
}
let transactions = self.subscription_manager.get_subscribe_request_filter(
transaction_filter.account_include,
transaction_filter.account_exclude,
transaction_filter.account_required,
event_type_filter.clone(),
);
let accounts = self.subscription_manager.subscribe_with_account_request(
account_filter.account,
account_filter.owner,
event_type_filter.clone(),
);
// Subscribe to events
let (mut subscribe_tx, mut stream) = self
.subscription_manager
.subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone())
.await?;
// Create channel
let (mut tx, mut rx) = mpsc::channel::<EventPretty>(self.config.backpressure.channel_size);
// 创建批处理器,将单个事件回调转换为批量回调
let batch_callback = move |events: Vec<Box<dyn UnifiedEvent>>| {
for event in events {
callback(event);
if !self.active_subscription.load(Ordering::Acquire) {
return Err(anyhow!("No active subscription to update"));
}
control_guard
.as_ref()
.ok_or_else(|| anyhow!("No active subscription to update"))?
.clone()
};
let mut batch_processor = EventBatchProcessor::new(
batch_callback,
self.config.batch.batch_size,
self.config.batch.batch_timeout_ms,
);
let mut request = self
.current_request
.read()
.await
.as_ref()
.ok_or_else(|| anyhow!("No active subscription"))?
.clone();
// Start task to process the stream
let backpressure_strategy = self.config.backpressure.strategy;
let stream_handle = tokio::spawn(async move {
while let Some(message) = stream.next().await {
match message {
Ok(msg) => {
if let Err(e) = StreamHandler::handle_stream_message(
msg,
&mut tx,
&mut subscribe_tx,
backpressure_strategy,
)
.await
{
error!("Error handling message: {e:?}");
break;
}
}
Err(error) => {
error!("Stream error: {error:?}");
break;
}
}
}
});
request.transactions = self
.subscription_manager
.get_subscribe_request_filter(
transaction_filter.account_include,
transaction_filter.account_exclude,
transaction_filter.account_required,
None,
)
.unwrap_or_default();
// Process transactions with batch processing
let event_processor = self.event_processor.clone();
let event_handle = tokio::spawn(async move {
while let Some(event_pretty) = rx.next().await {
if let Err(e) = event_processor
.process_event_transaction_with_batch(
event_pretty,
&mut batch_processor,
bot_wallet,
protocols.clone(),
event_type_filter.clone(),
)
.await
{
error!("Error processing transaction: {e:?}");
}
}
request.accounts = self
.subscription_manager
.subscribe_with_account_request(account_filter.account, account_filter.owner, None)
.unwrap_or_default();
// 处理剩余的事件
batch_processor.flush();
});
control_sender
.send(request.clone())
.await
.map_err(|e| anyhow!("Failed to send update: {}", e))?;
// 保存订阅句柄
let subscription_handle =
SubscriptionHandle::new(stream_handle, event_handle, metrics_handle);
let mut handle_guard = self.subscription_handle.lock().await;
*handle_guard = Some(subscription_handle);
*self.current_request.write().await = Some(request);
Ok(())
}
@@ -376,17 +385,9 @@ impl Clone for YellowstoneGrpc {
metrics_manager: self.metrics_manager.clone(),
event_processor: self.event_processor.clone(),
subscription_handle: self.subscription_handle.clone(), // 共享同一个 Arc<Mutex<>>
}
}
}
// 实现 Clone trait 以支持模块间共享
impl Clone for EventProcessor {
fn clone(&self) -> Self {
Self {
metrics_manager: self.metrics_manager.clone(),
config: self.config.clone(),
parser_cache: self.parser_cache.clone(),
active_subscription: self.active_subscription.clone(),
control_tx: self.control_tx.clone(),
current_request: self.current_request.clone(),
}
}
}
+42 -31
View File
@@ -1,19 +1,20 @@
use crate::{
common::AnyResult,
streaming::{
grpc::{BackpressureStrategy, EventPretty, StreamHandler},
grpc::{EventPretty, TransactionPretty},
yellowstone_grpc::YellowstoneGrpc,
},
};
use futures::{channel::mpsc, StreamExt};
use futures::{SinkExt, StreamExt};
use log::error;
use solana_program::pubkey;
use solana_sdk::{pubkey::Pubkey, transaction::VersionedTransaction};
use solana_transaction_status::EncodedTransactionWithStatusMeta;
use solana_transaction_status::TransactionWithStatusMeta;
use yellowstone_grpc_proto::geyser::{
subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing,
};
const SYSTEM_PROGRAM_ID: Pubkey = pubkey!("11111111111111111111111111111111");
// 根据实际并发量调整通道大小,避免背压
const CHANNEL_SIZE: usize = 50000; // 增加到 50000
#[derive(Debug)]
pub enum SystemEvent {
@@ -36,7 +37,7 @@ impl YellowstoneGrpc {
account_exclude: Option<Vec<String>>,
) -> AnyResult<()>
where
F: Fn(SystemEvent) + Send + Sync + 'static,
F: Fn(SystemEvent) + Send + Sync + Clone + 'static,
{
let addrs = vec![SYSTEM_PROGRAM_ID.to_string()];
let account_include = account_include.unwrap_or_default();
@@ -47,11 +48,10 @@ impl YellowstoneGrpc {
addrs,
None,
);
let (mut subscribe_tx, mut stream) = self
let (mut subscribe_tx, mut stream, _) = self
.subscription_manager
.subscribe_with_request(transactions, None, None, None)
.await?;
let (mut tx, mut rx) = mpsc::channel::<EventPretty>(CHANNEL_SIZE);
let callback = Box::new(callback);
@@ -59,16 +59,34 @@ impl YellowstoneGrpc {
while let Some(message) = stream.next().await {
match message {
Ok(msg) => {
if let Err(e) = StreamHandler::handle_stream_message(
msg,
&mut tx,
&mut subscribe_tx,
BackpressureStrategy::Block,
)
.await
{
error!("Error handling message: {e:?}");
break;
let created_at = msg.created_at;
match msg.update_oneof {
Some(UpdateOneof::Transaction(sut)) => {
let transaction_pretty = TransactionPretty::from((sut, created_at));
let event_pretty = EventPretty::Transaction(transaction_pretty);
if let Err(e) = Self::process_system_transaction(
event_pretty,
&*callback,
)
.await
{
error!("Error processing transaction: {e:?}");
}
}
Some(UpdateOneof::Ping(_)) => {
let _ = subscribe_tx
.send(SubscribeRequest {
ping: Some(SubscribeRequestPing { id: 1 }),
..Default::default()
})
.await;
}
Some(UpdateOneof::Pong(_)) => {
// Pong response, no action needed
}
_ => {
// Other message types, ignore for system subscription
}
}
}
Err(error) => {
@@ -78,12 +96,6 @@ impl YellowstoneGrpc {
}
}
});
while let Some(event_pretty) = rx.next().await {
if let Err(e) = Self::process_system_transaction(event_pretty, &*callback).await {
error!("Error processing transaction: {e:?}");
}
}
Ok(())
}
@@ -93,20 +105,19 @@ impl YellowstoneGrpc {
{
match event_pretty {
EventPretty::Transaction(transaction_pretty) => {
let trade_raw: EncodedTransactionWithStatusMeta = transaction_pretty.tx;
let meta = trade_raw
.meta
.as_ref()
.ok_or_else(|| anyhow::anyhow!("Missing transaction metadata"))?;
let trade_raw: TransactionWithStatusMeta = transaction_pretty.tx;
let meta = trade_raw.get_status_meta();
if meta.err.is_some() {
if meta.is_none() {
return Ok(());
}
let transaction = trade_raw.get_transaction();
callback(SystemEvent::NewTransfer(TransferInfo {
slot: transaction_pretty.slot,
signature: transaction_pretty.signature.to_string(),
tx: trade_raw.transaction.decode(),
tx: Some(transaction),
}));
}
_ => {}