mirror of
https://github.com/0xfnzero/solana-streamer.git
synced 2026-08-09 07:07:44 +00:00
Merge commit '74781e5cbe6f48877ea564e4ae9da6ec51282b16'
This commit is contained in:
+2
-1
@@ -20,4 +20,5 @@ Cargo.lock
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
.cargo/
|
||||
.cargo/
|
||||
.claude/
|
||||
+8
-1
@@ -33,7 +33,6 @@ serde-big-array = "0.5.1"
|
||||
futures = "0.3.31"
|
||||
futures-util = "0.3.31"
|
||||
base64 = "0.22.1"
|
||||
bs58 = "0.5.1"
|
||||
rand = "0.9.0"
|
||||
bincode = "1.3.3"
|
||||
anyhow = "1.0.90"
|
||||
@@ -52,6 +51,7 @@ thiserror = "2.0.11"
|
||||
async-trait = "0.1.86"
|
||||
lazy_static = "1.5.0"
|
||||
once_cell = "1.20.3"
|
||||
dashmap = "6.0.1"
|
||||
prost = "0.13.5"
|
||||
prost-types = "0.13.5"
|
||||
num_enum = "0.7.3"
|
||||
@@ -64,3 +64,10 @@ borsh-derive = "1.5.5"
|
||||
indicatif = "0.18.0"
|
||||
maplit = "1.0.2"
|
||||
env_logger = "0.11.8"
|
||||
crossbeam = "0.8.4"
|
||||
crossbeam-queue = "0.3.12"
|
||||
parking_lot = "0.12.1"
|
||||
wide = "0.7"
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { version = "0.5", features = ["html_reports"] }
|
||||
@@ -24,11 +24,12 @@ A lightweight Rust library for real-time event streaming from Solana DEX trading
|
||||
11. **Performance Monitoring**: Built-in performance metrics monitoring, including event processing speed, etc.
|
||||
12. **Memory Optimization**: Object pooling and caching mechanisms to reduce memory allocations
|
||||
13. **Flexible Configuration System**: Support for custom batch sizes, backpressure strategies, channel sizes, and other parameters
|
||||
14. **Preset Configurations**: Provides high-performance, low-latency, ordered processing, and other preset configurations
|
||||
15. **Backpressure Handling**: Supports blocking, dropping, retrying, ordered, and other backpressure strategies
|
||||
14. **Preset Configurations**: Provides high-throughput and low-latency preset configurations optimized for different use cases
|
||||
15. **Backpressure Handling**: Supports blocking and dropping backpressure strategies
|
||||
16. **Runtime Configuration Updates**: Supports dynamic configuration parameter updates at runtime
|
||||
17. **Full Function Performance Monitoring**: All subscribe_events functions support performance monitoring, automatically collecting and reporting performance metrics
|
||||
18. **Graceful Shutdown**: Support for programmatic stop() method for clean shutdown
|
||||
19. **Dynamic Subscription Management**: Runtime filter updates without reconnection, enabling adaptive monitoring strategies
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -55,6 +56,65 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" }
|
||||
solana-streamer-sdk = "0.3.10"
|
||||
```
|
||||
|
||||
## Configuration System
|
||||
|
||||
### Preset Configurations
|
||||
|
||||
The library provides three preset configurations optimized for different use cases:
|
||||
|
||||
#### 1. High Throughput Configuration (`high_throughput()`)
|
||||
|
||||
Optimized for high-concurrency scenarios, prioritizing throughput over latency:
|
||||
|
||||
```rust
|
||||
let config = StreamClientConfig::high_throughput();
|
||||
// Or use convenience methods
|
||||
let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?;
|
||||
let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?;
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- **Backpressure Strategy**: Drop - drops messages during high load to avoid blocking
|
||||
- **Buffer Size**: 5,000 permits to handle burst traffic
|
||||
- **Use Case**: Scenarios where you need to process large volumes of data and can tolerate occasional message drops during peak loads
|
||||
|
||||
#### 2. Low Latency Configuration (`low_latency()`)
|
||||
|
||||
Optimized for real-time scenarios, prioritizing latency over throughput:
|
||||
|
||||
```rust
|
||||
let config = StreamClientConfig::low_latency();
|
||||
// Or use convenience methods
|
||||
let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?;
|
||||
let shred = ShredStreamGrpc::new_low_latency(endpoint).await?;
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- **Backpressure Strategy**: Block - ensures no data loss
|
||||
- **Buffer Size**: 4000 permits for balanced throughput and latency
|
||||
- **Immediate Processing**: No buffering, processes events immediately
|
||||
- **Use Case**: Scenarios where every millisecond counts and you cannot afford to lose any events, such as trading applications or real-time monitoring
|
||||
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
You can also create custom configurations:
|
||||
|
||||
```rust
|
||||
let config = StreamClientConfig {
|
||||
connection: ConnectionConfig {
|
||||
connect_timeout: 30,
|
||||
request_timeout: 120,
|
||||
max_decoding_message_size: 20 * 1024 * 1024, // 20MB
|
||||
},
|
||||
backpressure: BackpressureConfig {
|
||||
permits: 2000,
|
||||
strategy: BackpressureStrategy::Block,
|
||||
},
|
||||
enable_metrics: true,
|
||||
};
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Quick Start - Parse Transaction Events
|
||||
@@ -72,6 +132,20 @@ This example demonstrates:
|
||||
|
||||
The example uses a predefined transaction signature and shows how to extract protocol-specific events from the transaction data.
|
||||
|
||||
### Dynamic Subscription Management Example
|
||||
|
||||
Test runtime filter updates without reconnection:
|
||||
|
||||
```bash
|
||||
cargo run --example dynamic_subscription
|
||||
```
|
||||
|
||||
This example demonstrates:
|
||||
- Creating initial subscriptions with specific protocol filters
|
||||
- Updating subscription filters at runtime without reconnection
|
||||
- Single subscription enforcement and proper error handling
|
||||
- Clean shutdown and resource management
|
||||
|
||||
### Advanced Usage - Complete Example
|
||||
|
||||
```rust
|
||||
@@ -470,6 +544,32 @@ let event_type_filter = Some(EventTypeFilter {
|
||||
});
|
||||
```
|
||||
|
||||
## Dynamic Subscription Management
|
||||
|
||||
Update subscription filters at runtime without reconnecting to the stream.
|
||||
|
||||
```rust
|
||||
// Update filters on existing subscription
|
||||
grpc.update_subscription(
|
||||
TransactionFilter {
|
||||
account_include: vec!["new_program_id".to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
},
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
).await?;
|
||||
```
|
||||
|
||||
- **No Reconnection**: Filter changes apply immediately without closing the stream
|
||||
- **Atomic Updates**: Both transaction and account filters updated together
|
||||
- **Single Subscription**: One active subscription per client instance
|
||||
- **Compatible**: Works with both immediate and advanced subscription methods
|
||||
|
||||
Note: Multiple subscription attempts on the same client return an error.
|
||||
|
||||
## Supported Protocols
|
||||
|
||||
- **PumpFun**: Primary meme coin trading platform
|
||||
|
||||
+62
-3
@@ -24,8 +24,8 @@
|
||||
11. **性能监控**: 内置性能指标监控,包括事件处理速度等
|
||||
12. **内存优化**: 对象池和缓存机制减少内存分配
|
||||
13. **灵活配置系统**: 支持自定义批处理大小、背压策略、通道大小等参数
|
||||
14. **预设配置**: 提供高性能、低延迟、有序处理等预设配置
|
||||
15. **背压处理**: 支持阻塞、丢弃、重试、有序等多种背压策略
|
||||
14. **预设配置**: 提供高吞吐量、低延迟等预设配置,针对不同使用场景优化
|
||||
15. **背压处理**: 支持阻塞、丢弃等背压策略
|
||||
16. **运行时配置更新**: 支持在运行时动态更新配置参数
|
||||
17. **全函数性能监控**: 所有subscribe_events函数都支持性能监控,自动收集和报告性能指标
|
||||
18. **优雅关闭**: 支持编程式 stop() 方法进行干净的关闭
|
||||
@@ -55,6 +55,65 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" }
|
||||
solana-streamer-sdk = "0.3.10"
|
||||
```
|
||||
|
||||
## 配置系统
|
||||
|
||||
### 预设配置
|
||||
|
||||
库提供了三种预设配置,针对不同的使用场景进行了优化:
|
||||
|
||||
#### 1. 高吞吐量配置 (`high_throughput()`)
|
||||
|
||||
专为高并发场景优化,优先考虑吞吐量而非延迟:
|
||||
|
||||
```rust
|
||||
let config = StreamClientConfig::high_throughput();
|
||||
// 或者使用便捷方法
|
||||
let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?;
|
||||
let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?;
|
||||
```
|
||||
|
||||
**特性:**
|
||||
- **背压策略**: Drop(丢弃策略)- 在高负载时丢弃消息以避免阻塞
|
||||
- **缓冲区大小**: 5,000 个许可证,处理突发流量
|
||||
- **适用场景**: 需要处理大量数据且可以容忍在峰值负载时偶尔丢失消息的场景
|
||||
|
||||
#### 2. 低延迟配置 (`low_latency()`)
|
||||
|
||||
专为实时场景优化,优先考虑延迟而非吞吐量:
|
||||
|
||||
```rust
|
||||
let config = StreamClientConfig::low_latency();
|
||||
// 或者使用便捷方法
|
||||
let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?;
|
||||
let shred = ShredStreamGrpc::new_low_latency(endpoint).await?;
|
||||
```
|
||||
|
||||
**特性:**
|
||||
- **背压策略**: Block(阻塞策略)- 确保不丢失任何数据
|
||||
- **缓冲区大小**: 4000 个许可证,平衡吞吐量和延迟
|
||||
- **立即处理**: 不进行缓冲,立即处理事件
|
||||
- **适用场景**: 每毫秒都很重要且不能丢失任何事件的场景,如交易应用或实时监控
|
||||
|
||||
|
||||
### 自定义配置
|
||||
|
||||
您也可以创建自定义配置:
|
||||
|
||||
```rust
|
||||
let config = StreamClientConfig {
|
||||
connection: ConnectionConfig {
|
||||
connect_timeout: 30,
|
||||
request_timeout: 120,
|
||||
max_decoding_message_size: 20 * 1024 * 1024, // 20MB
|
||||
},
|
||||
backpressure: BackpressureConfig {
|
||||
permits: 2000,
|
||||
strategy: BackpressureStrategy::Block,
|
||||
},
|
||||
enable_metrics: true,
|
||||
};
|
||||
```
|
||||
|
||||
## 使用示例
|
||||
|
||||
### 快速开始 - 解析交易事件
|
||||
@@ -502,7 +561,7 @@ let event_type_filter = Some(EventTypeFilter {
|
||||
|
||||
- **Yellowstone gRPC 客户端**: 针对 Solana 事件流优化
|
||||
- **ShredStream 客户端**: 替代流实现
|
||||
- **异步处理**: 非阻塞事件处理
|
||||
- **高性能处理**: 优化的事件处理机制
|
||||
|
||||
## 项目结构
|
||||
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
use anyhow::Result;
|
||||
use solana_streamer_sdk::streaming::yellowstone_grpc::{AccountFilter, TransactionFilter, YellowstoneGrpc};
|
||||
use solana_streamer_sdk::streaming::event_parser::Protocol;
|
||||
use solana_streamer_sdk::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use solana_streamer_sdk::streaming::event_parser::common::types::EventType;
|
||||
use solana_sdk::signature::{Keypair, Signer};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
const PUMPFUN_PROGRAM_ID: &str = "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P";
|
||||
const RAYDIUM_CPMM_PROGRAM_ID: &str = "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C";
|
||||
|
||||
const GRPC_ENDPOINT: &str = "https://solana-yellowstone-grpc.publicnode.com:443";
|
||||
const API_KEY: Option<&str> = None;
|
||||
const MONITORING_DURATION_SECS: u64 = 10;
|
||||
|
||||
/// Demonstrates dynamic subscription updates and filter changes in real-time
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
env_logger::init();
|
||||
|
||||
println!("Connecting to Yellowstone gRPC at {}", GRPC_ENDPOINT);
|
||||
let client = Arc::new(YellowstoneGrpc::new(
|
||||
GRPC_ENDPOINT.to_string(),
|
||||
API_KEY.map(|s| s.to_string())
|
||||
)?);
|
||||
|
||||
let event_counter = Arc::new(AtomicU64::new(0));
|
||||
let counter = event_counter.clone();
|
||||
|
||||
let callback = move |event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
|
||||
let count = counter.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let protocol = match event.event_type() {
|
||||
EventType::PumpFunBuy | EventType::PumpFunSell => "PumpFun",
|
||||
EventType::RaydiumCpmmSwapBaseInput | EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmm",
|
||||
_ => "Unknown"
|
||||
};
|
||||
|
||||
println!("Event #{}: {:11} - {:.8}...", count + 1, protocol, event.signature());
|
||||
};
|
||||
|
||||
println!("\n=== Phase 1: PumpFun only ===");
|
||||
let pumpfun_filter = TransactionFilter {
|
||||
account_include: vec![PUMPFUN_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
};
|
||||
|
||||
let account_filter = AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
};
|
||||
let trade_event_filter = EventTypeFilter {
|
||||
include: vec![
|
||||
EventType::PumpFunBuy,
|
||||
EventType::PumpFunSell,
|
||||
EventType::RaydiumCpmmSwapBaseInput,
|
||||
EventType::RaydiumCpmmSwapBaseOutput,
|
||||
],
|
||||
};
|
||||
|
||||
if let Err(e) = client.subscribe_events_immediate(
|
||||
vec![Protocol::PumpFun, Protocol::RaydiumCpmm],
|
||||
None,
|
||||
pumpfun_filter,
|
||||
account_filter,
|
||||
Some(trade_event_filter),
|
||||
None,
|
||||
callback,
|
||||
).await {
|
||||
println!("Failed to create subscription: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("Subscribed to PumpFun transactions with trade event filters, monitoring for {}s...", MONITORING_DURATION_SECS);
|
||||
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
|
||||
let phase1_count = event_counter.load(Ordering::Relaxed);
|
||||
println!("Phase 1: {} events", phase1_count);
|
||||
|
||||
println!("\n=== Phase 2: PumpFun + RaydiumCpmm ===");
|
||||
let multi_protocol_filter = TransactionFilter {
|
||||
account_include: vec![
|
||||
PUMPFUN_PROGRAM_ID.to_string(),
|
||||
RAYDIUM_CPMM_PROGRAM_ID.to_string(),
|
||||
],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
};
|
||||
|
||||
if let Err(e) = client.update_subscription(
|
||||
multi_protocol_filter,
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
).await {
|
||||
println!("Failed to update subscription: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("Updated to PumpFun + RaydiumCpmm transactions, monitoring for {}s...", MONITORING_DURATION_SECS);
|
||||
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
|
||||
let phase2_count = event_counter.load(Ordering::Relaxed);
|
||||
println!("Phase 2: {} events", phase2_count - phase1_count);
|
||||
|
||||
println!("\n=== Phase 3: RaydiumCpmm only ===");
|
||||
let raydium_cpmm_filter = TransactionFilter {
|
||||
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
};
|
||||
|
||||
if let Err(e) = client.update_subscription(
|
||||
raydium_cpmm_filter,
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
).await {
|
||||
println!("Failed to update subscription: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
|
||||
println!("Updated to RaydiumCpmm transactions only, monitoring for {}s...", MONITORING_DURATION_SECS);
|
||||
let phase3_count = event_counter.load(Ordering::Relaxed);
|
||||
println!("Phase 3: {} events", phase3_count - phase2_count);
|
||||
|
||||
println!("\n=== Phase 4: Back to PumpFun only ===");
|
||||
let pumpfun_only_filter = TransactionFilter {
|
||||
account_include: vec![PUMPFUN_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
};
|
||||
|
||||
if let Err(e) = client.update_subscription(
|
||||
pumpfun_only_filter,
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
).await {
|
||||
println!("Failed to update subscription: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
|
||||
println!("Updated to PumpFun transactions only, monitoring for {}s...", MONITORING_DURATION_SECS);
|
||||
let phase4_count = event_counter.load(Ordering::Relaxed);
|
||||
println!("Phase 4: {} events", phase4_count - phase3_count);
|
||||
|
||||
println!("\n=== Phase 5: All events ===");
|
||||
let empty_filter = TransactionFilter {
|
||||
account_include: vec![],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
};
|
||||
|
||||
if let Err(e) = client.update_subscription(
|
||||
empty_filter,
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
).await {
|
||||
println!("Failed to update subscription: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await;
|
||||
println!("Updated to all transactions (no filters), monitoring for {}s...", MONITORING_DURATION_SECS);
|
||||
let phase5_count = event_counter.load(Ordering::Relaxed);
|
||||
println!("Phase 5: {} events", phase5_count - phase4_count);
|
||||
|
||||
println!("\n=== Phase 6: Silence ===");
|
||||
|
||||
let random_keypair_1 = Keypair::new();
|
||||
let random_keypair_2 = Keypair::new();
|
||||
let random_pubkey_1 = random_keypair_1.pubkey();
|
||||
let random_pubkey_2 = random_keypair_2.pubkey();
|
||||
|
||||
let silence_filter = TransactionFilter {
|
||||
account_include: vec![],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![
|
||||
random_pubkey_1.to_string(),
|
||||
random_pubkey_2.to_string(),
|
||||
],
|
||||
};
|
||||
|
||||
if let Err(e) = client.update_subscription(
|
||||
silence_filter,
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
).await {
|
||||
println!("Failed to update subscription: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("Updated to random addresses (expecting silence), monitoring for 3s...");
|
||||
let before_silence = event_counter.load(Ordering::Relaxed);
|
||||
let start_time = Instant::now();
|
||||
let last_event_time = Arc::new(Mutex::new(start_time));
|
||||
let last_event_time_clone = last_event_time.clone();
|
||||
|
||||
let mut last_count = before_silence;
|
||||
for _ in 0..6 {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
let current_count = event_counter.load(Ordering::Relaxed);
|
||||
if current_count > last_count {
|
||||
if let Ok(mut time) = last_event_time_clone.lock() {
|
||||
*time = Instant::now();
|
||||
}
|
||||
last_count = current_count;
|
||||
}
|
||||
}
|
||||
|
||||
let final_count = event_counter.load(Ordering::Relaxed);
|
||||
let events_during_silence = final_count - before_silence;
|
||||
|
||||
if events_during_silence == 0 {
|
||||
println!("Phase 6: 0 events (immediate filter application)");
|
||||
} else if let Ok(last_time) = last_event_time.lock() {
|
||||
let propagation_time = last_time.duration_since(start_time);
|
||||
println!("Phase 6: {} events during propagation, filter took {}ms",
|
||||
events_during_silence, propagation_time.as_millis());
|
||||
}
|
||||
|
||||
println!("\n=== Phase 7: Shutdown ===");
|
||||
|
||||
let shutdown_client = Arc::new(YellowstoneGrpc::new(
|
||||
GRPC_ENDPOINT.to_string(),
|
||||
API_KEY.map(|s| s.to_string())
|
||||
)?);
|
||||
|
||||
let shutdown_event_counter = Arc::new(AtomicU64::new(0));
|
||||
let shutdown_counter = shutdown_event_counter.clone();
|
||||
let shutdown_callback = move |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
|
||||
shutdown_counter.fetch_add(1, Ordering::Relaxed);
|
||||
};
|
||||
|
||||
if let Err(e) = shutdown_client.subscribe_events_immediate(
|
||||
vec![Protocol::PumpFun, Protocol::RaydiumCpmm],
|
||||
None,
|
||||
TransactionFilter {
|
||||
account_include: vec![],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
},
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
None,
|
||||
None,
|
||||
shutdown_callback,
|
||||
).await {
|
||||
println!("Failed to subscribe shutdown client: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
sleep(Duration::from_millis(1000)).await;
|
||||
let pre_stop_count = shutdown_event_counter.load(Ordering::Relaxed);
|
||||
println!("Received {} events before stop", pre_stop_count);
|
||||
|
||||
let stop_time = Instant::now();
|
||||
shutdown_client.stop().await;
|
||||
let shutdown_duration = stop_time.elapsed();
|
||||
println!("stop() completed in {:.1}ms", shutdown_duration.as_millis());
|
||||
|
||||
let post_stop_count = shutdown_event_counter.load(Ordering::Relaxed);
|
||||
let during_stop = post_stop_count - pre_stop_count;
|
||||
if during_stop > 0 {
|
||||
println!(" {} events received during stop()", during_stop);
|
||||
}
|
||||
|
||||
let last_event_time = Arc::new(Mutex::new(stop_time));
|
||||
let last_event_time_clone = last_event_time.clone();
|
||||
let mut last_count = post_stop_count;
|
||||
|
||||
for _ in 0..20 {
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
let current_count = shutdown_event_counter.load(Ordering::Relaxed);
|
||||
if current_count > last_count {
|
||||
if let Ok(mut time) = last_event_time_clone.lock() {
|
||||
*time = Instant::now();
|
||||
}
|
||||
last_count = current_count;
|
||||
}
|
||||
}
|
||||
|
||||
let final_count = shutdown_event_counter.load(Ordering::Relaxed);
|
||||
let after_stop = final_count - post_stop_count;
|
||||
|
||||
if after_stop == 0 {
|
||||
println!("Phase 7: Clean shutdown - no events after stop()");
|
||||
} else if let Ok(last_time) = last_event_time.lock() {
|
||||
let post_stop_duration = last_time.duration_since(stop_time);
|
||||
let silence_duration = Instant::now().duration_since(*last_time);
|
||||
println!("Phase 7: {} events arrived up to {}ms after stop(), then silent for {}ms",
|
||||
after_stop, post_stop_duration.as_millis(), silence_duration.as_millis());
|
||||
}
|
||||
|
||||
println!("\n=== Subscription enforcement ===");
|
||||
|
||||
let test_callback = |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {};
|
||||
|
||||
match client.subscribe_events_immediate(
|
||||
vec![Protocol::RaydiumCpmm],
|
||||
None,
|
||||
TransactionFilter {
|
||||
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
},
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
None,
|
||||
None,
|
||||
test_callback,
|
||||
).await {
|
||||
Ok(_) => println!("ERROR: Same client created second subscription"),
|
||||
Err(e) if e.to_string().contains("Already subscribed") => {
|
||||
println!("✓ Single subscription enforcement working");
|
||||
},
|
||||
Err(e) => println!("Unexpected error: {}", e),
|
||||
}
|
||||
|
||||
let client2 = Arc::new(YellowstoneGrpc::new(
|
||||
GRPC_ENDPOINT.to_string(),
|
||||
API_KEY.map(|s| s.to_string())
|
||||
)?);
|
||||
|
||||
let client2_counter = Arc::new(AtomicU64::new(0));
|
||||
let counter2 = client2_counter.clone();
|
||||
let client2_callback = move |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
|
||||
counter2.fetch_add(1, Ordering::Relaxed);
|
||||
};
|
||||
|
||||
match client2.subscribe_events_immediate(
|
||||
vec![Protocol::RaydiumCpmm],
|
||||
None,
|
||||
TransactionFilter {
|
||||
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
},
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
None,
|
||||
None,
|
||||
client2_callback,
|
||||
).await {
|
||||
Ok(_) => {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
let count = client2_counter.load(Ordering::Relaxed);
|
||||
println!("✓ Second client: {} events", count);
|
||||
client2.stop().await;
|
||||
},
|
||||
Err(e) => println!("ERROR: Second client failed: {}", e),
|
||||
}
|
||||
|
||||
println!("\n=== Advanced subscription enforcement ===");
|
||||
|
||||
let test_callback_advanced = |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {};
|
||||
|
||||
let client3 = Arc::new(YellowstoneGrpc::new(
|
||||
GRPC_ENDPOINT.to_string(),
|
||||
API_KEY.map(|s| s.to_string())
|
||||
)?);
|
||||
|
||||
// First subscription should succeed
|
||||
match client3.subscribe_events_immediate(
|
||||
vec![Protocol::RaydiumCpmm],
|
||||
None,
|
||||
TransactionFilter {
|
||||
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
},
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
None,
|
||||
None,
|
||||
test_callback_advanced,
|
||||
).await {
|
||||
Ok(_) => {
|
||||
// Second subscription attempt on same client should fail
|
||||
match client3.subscribe_events_immediate(
|
||||
vec![Protocol::RaydiumCpmm],
|
||||
None,
|
||||
TransactionFilter {
|
||||
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
},
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
None,
|
||||
None,
|
||||
|_| {},
|
||||
).await {
|
||||
Ok(_) => println!("ERROR: Same client created second advanced subscription"),
|
||||
Err(e) if e.to_string().contains("Already subscribed") => {
|
||||
println!("✓ Advanced single subscription enforcement working");
|
||||
},
|
||||
Err(e) => println!("Unexpected error: {}", e),
|
||||
}
|
||||
},
|
||||
Err(e) => println!("ERROR: First advanced subscription failed: {}", e),
|
||||
}
|
||||
|
||||
// Test that a second client can subscribe using advanced method
|
||||
let client4 = Arc::new(YellowstoneGrpc::new(
|
||||
GRPC_ENDPOINT.to_string(),
|
||||
API_KEY.map(|s| s.to_string())
|
||||
)?);
|
||||
|
||||
let client4_counter = Arc::new(AtomicU64::new(0));
|
||||
let counter4 = client4_counter.clone();
|
||||
let client4_callback = move |_event: Box<dyn solana_streamer_sdk::streaming::event_parser::UnifiedEvent>| {
|
||||
counter4.fetch_add(1, Ordering::Relaxed);
|
||||
};
|
||||
|
||||
match client4.subscribe_events_immediate(
|
||||
vec![Protocol::RaydiumCpmm],
|
||||
None,
|
||||
TransactionFilter {
|
||||
account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()],
|
||||
account_exclude: vec![],
|
||||
account_required: vec![],
|
||||
},
|
||||
AccountFilter {
|
||||
account: vec![],
|
||||
owner: vec![],
|
||||
},
|
||||
None,
|
||||
None,
|
||||
client4_callback,
|
||||
).await {
|
||||
Ok(_) => {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
let count = client4_counter.load(Ordering::Relaxed);
|
||||
println!("✓ Second client (advanced): {} events", count);
|
||||
client4.stop().await;
|
||||
},
|
||||
Err(e) => println!("ERROR: Second client (advanced) failed: {}", e),
|
||||
}
|
||||
|
||||
client3.stop().await;
|
||||
|
||||
client.stop().await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
+15
-20
@@ -1,5 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use solana_sdk::commitment_config::CommitmentConfig;
|
||||
|
||||
use solana_streamer_sdk::streaming::event_parser::UnifiedEvent;
|
||||
use solana_streamer_sdk::streaming::event_parser::{
|
||||
protocols::MutilEventParser, EventParser, Protocol,
|
||||
};
|
||||
@@ -10,7 +12,7 @@ use std::sync::Arc;
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let signatures = vec![
|
||||
"42agNk1heHabNAVRzEKqQEt5adGkQzRYf9M1Q81uBJPCCHyP4cyCA1RNkgxXrtEAWeeGcytyh2TsnkBDgqnHeq4z",
|
||||
"5sDWrTTkE69CNc6nrAX7SqPS7FiajJTg8TMog3Gve7KjVfrqYn8YZcX1kAoyKok976S4RTnK1EdCV8hRiDWg68Aj",
|
||||
];
|
||||
// Validate signature format
|
||||
let mut valid_signatures = Vec::new();
|
||||
@@ -52,7 +54,7 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> {
|
||||
.get_transaction_with_config(
|
||||
&signature,
|
||||
solana_client::rpc_config::RpcTransactionConfig {
|
||||
encoding: Some(UiTransactionEncoding::Binary),
|
||||
encoding: Some(UiTransactionEncoding::Base64),
|
||||
commitment: Some(CommitmentConfig::confirmed()),
|
||||
max_supported_transaction_version: Some(0),
|
||||
},
|
||||
@@ -98,30 +100,23 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> {
|
||||
Protocol::RaydiumAmmV4,
|
||||
];
|
||||
let parser: Arc<dyn EventParser> = Arc::new(MutilEventParser::new(protocols, None));
|
||||
let start_time = std::time::Instant::now();
|
||||
let events = parser
|
||||
.parse_transaction(
|
||||
transaction.transaction.clone(),
|
||||
&signature.to_string(),
|
||||
Some(transaction.slot),
|
||||
None,
|
||||
0,
|
||||
None,
|
||||
parser
|
||||
.parse_encoded_confirmed_transaction_with_status_meta(
|
||||
signature,
|
||||
transaction,
|
||||
Arc::new(move |event: &Box<dyn UnifiedEvent>| {
|
||||
println!("{:?}\n", event);
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_e| vec![]);
|
||||
|
||||
let end_time = std::time::Instant::now();
|
||||
let duration = end_time.duration_since(start_time);
|
||||
println!("Parsing time: {:?}", duration);
|
||||
for event in events {
|
||||
println!("{:?}\n", event);
|
||||
}
|
||||
.await?;
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Failed to get transaction: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
println!("Press Ctrl+C to exit example...");
|
||||
tokio::signal::ctrl_c().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,3 +1,3 @@
|
||||
pub mod streaming;
|
||||
pub mod common;
|
||||
pub mod protos;
|
||||
pub mod common;
|
||||
pub mod streaming;
|
||||
|
||||
+8
-4
@@ -61,7 +61,7 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
|
||||
println!("Subscribing to Yellowstone gRPC events...");
|
||||
|
||||
// Create low-latency configuration
|
||||
let mut config = ClientConfig::low_latency();
|
||||
let mut config: ClientConfig = ClientConfig::low_latency();
|
||||
// Enable performance monitoring, has performance overhead, disabled by default
|
||||
config.enable_metrics = true;
|
||||
let grpc = YellowstoneGrpc::new_with_config(
|
||||
@@ -112,7 +112,7 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// No event filtering, includes all events
|
||||
let event_type_filter = None;
|
||||
// Only include PumpSwapBuy events and PumpSwapSell events
|
||||
// let event_type_filter = EventTypeFilter { include: vec![EventType::PumpSwapBuy, EventType::PumpSwapSell] };
|
||||
// let event_type_filter = Some(EventTypeFilter { include: vec![EventType::PumpFunBuy] });
|
||||
|
||||
println!("Starting to listen for events, press Ctrl+C to stop...");
|
||||
println!("Monitoring programs: {:?}", account_include);
|
||||
@@ -188,11 +188,15 @@ async fn test_shreds() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
fn create_event_callback() -> impl Fn(Box<dyn UnifiedEvent>) {
|
||||
|event: Box<dyn UnifiedEvent>| {
|
||||
println!("🎉 Event received! Type: {:?}, ID: {}", event.event_type(), event.id());
|
||||
println!(
|
||||
"🎉 Event received! Type: {:?}, transaction_index: {:?}",
|
||||
event.event_type(),
|
||||
event.transaction_index()
|
||||
);
|
||||
match_event!(event, {
|
||||
// -------------------------- block meta -----------------------
|
||||
BlockMetaEvent => |e: BlockMetaEvent| {
|
||||
println!("BlockMetaEvent: {e:?}");
|
||||
println!("BlockMetaEvent: {:?}", e.metadata.program_handle_time_consuming_us);
|
||||
},
|
||||
// -------------------------- bonk -----------------------
|
||||
BonkPoolCreateEvent => |e: BonkPoolCreateEvent| {
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
use crate::streaming::event_parser::UnifiedEvent;
|
||||
|
||||
/// 通用批处理事件收集器
|
||||
pub struct EventBatchProcessor<F>
|
||||
where
|
||||
F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
|
||||
{
|
||||
pub(crate) callback: F,
|
||||
batch: Vec<Box<dyn UnifiedEvent>>,
|
||||
batch_size: usize,
|
||||
timeout_ms: u64,
|
||||
last_flush_time: std::time::Instant,
|
||||
}
|
||||
|
||||
impl<F> EventBatchProcessor<F>
|
||||
where
|
||||
F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
|
||||
{
|
||||
/// 创建新的批处理器
|
||||
pub fn new(callback: F, batch_size: usize, timeout_ms: u64) -> Self {
|
||||
Self {
|
||||
callback,
|
||||
batch: Vec::with_capacity(batch_size),
|
||||
batch_size,
|
||||
timeout_ms,
|
||||
last_flush_time: std::time::Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 添加事件到批次
|
||||
pub fn add_event(&mut self, event: Box<dyn UnifiedEvent>) {
|
||||
log::debug!("Adding event to batch: {} (type: {:?})", event.id(), event.event_type());
|
||||
self.batch.push(event);
|
||||
|
||||
// 检查是否需要刷新批次
|
||||
if self.batch.len() >= self.batch_size || self.should_flush_by_timeout() {
|
||||
log::debug!("Flushing batch: size={}, timeout={}", self.batch.len(), self.should_flush_by_timeout());
|
||||
self.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/// 强制刷新当前批次
|
||||
pub fn flush(&mut self) {
|
||||
if !self.batch.is_empty() {
|
||||
let events = std::mem::replace(&mut self.batch, Vec::with_capacity(self.batch_size));
|
||||
log::debug!("Flushing {} events from batch processor", events.len());
|
||||
|
||||
// 添加调试信息(仅在debug模式下)
|
||||
if log::log_enabled!(log::Level::Debug) {
|
||||
for (i, event) in events.iter().enumerate() {
|
||||
log::debug!("Event {}: Type={:?}, ID={}", i, event.event_type(), event.id());
|
||||
}
|
||||
}
|
||||
|
||||
// 执行回调并捕获可能的错误
|
||||
log::debug!("Executing batch callback with {} events", events.len());
|
||||
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
(self.callback)(events);
|
||||
})) {
|
||||
Ok(_) => {
|
||||
log::debug!("Batch callback executed successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Batch callback panicked: {:?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
self.last_flush_time = std::time::Instant::now();
|
||||
} else {
|
||||
log::debug!("No events to flush");
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取当前批次大小
|
||||
pub fn current_batch_size(&self) -> usize {
|
||||
self.batch.len()
|
||||
}
|
||||
|
||||
/// 检查是否应该基于超时刷新
|
||||
fn should_flush_by_timeout(&self) -> bool {
|
||||
self.last_flush_time.elapsed().as_millis() >= self.timeout_ms as u128
|
||||
}
|
||||
|
||||
/// 检查批次是否已满
|
||||
pub fn is_batch_full(&self) -> bool {
|
||||
self.batch.len() >= self.batch_size
|
||||
}
|
||||
|
||||
/// 检查是否需要刷新(大小或超时)
|
||||
pub fn should_flush(&self) -> bool {
|
||||
self.is_batch_full() || self.should_flush_by_timeout()
|
||||
}
|
||||
}
|
||||
|
||||
/// 简单的事件批处理器,用于将单个事件回调转换为批量回调
|
||||
pub struct SimpleEventBatchProcessor<F>
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
|
||||
{
|
||||
callback: F,
|
||||
}
|
||||
|
||||
impl<F> SimpleEventBatchProcessor<F>
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
|
||||
{
|
||||
pub fn new(callback: F) -> Self {
|
||||
Self { callback }
|
||||
}
|
||||
|
||||
/// 将批量事件拆分为单个事件处理
|
||||
pub fn process_batch(&self, events: Vec<Box<dyn UnifiedEvent>>) {
|
||||
for event in events {
|
||||
(self.callback)(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 批处理器包装器,用于将单个事件回调适配为批量处理
|
||||
pub fn create_batch_callback_adapter<F>(
|
||||
single_event_callback: F,
|
||||
) -> impl FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
|
||||
{
|
||||
move |events: Vec<Box<dyn UnifiedEvent>>| {
|
||||
for event in events {
|
||||
single_event_callback(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
use super::constants::*;
|
||||
|
||||
/// 背压处理策略
|
||||
/// Backpressure handling strategy
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum BackpressureStrategy {
|
||||
/// 阻塞等待(默认)
|
||||
/// Block and wait (default)
|
||||
Block,
|
||||
/// 丢弃消息
|
||||
/// Drop messages
|
||||
Drop,
|
||||
/// 重试有限次数后丢弃
|
||||
Retry { max_attempts: usize, wait_ms: u64 },
|
||||
}
|
||||
|
||||
impl Default for BackpressureStrategy {
|
||||
@@ -17,50 +15,29 @@ impl Default for BackpressureStrategy {
|
||||
}
|
||||
}
|
||||
|
||||
/// 批处理配置
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BatchConfig {
|
||||
/// 批处理大小(默认:100)
|
||||
pub batch_size: usize,
|
||||
/// 批处理超时时间(毫秒,默认:5ms)
|
||||
pub batch_timeout_ms: u64,
|
||||
/// 是否启用批处理(默认:true)
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for BatchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
batch_size: DEFAULT_BATCH_SIZE,
|
||||
batch_timeout_ms: DEFAULT_BATCH_TIMEOUT_MS,
|
||||
enabled: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 背压配置
|
||||
/// Backpressure configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackpressureConfig {
|
||||
/// 通道大小(默认:1000)
|
||||
pub channel_size: usize,
|
||||
/// 背压处理策略(默认:Block)
|
||||
/// Channel size (default: 1000)
|
||||
pub permits: usize,
|
||||
/// Backpressure handling strategy (default: Block)
|
||||
pub strategy: BackpressureStrategy,
|
||||
}
|
||||
|
||||
impl Default for BackpressureConfig {
|
||||
fn default() -> Self {
|
||||
Self { channel_size: DEFAULT_CHANNEL_SIZE, strategy: BackpressureStrategy::default() }
|
||||
Self { permits: 1, strategy: BackpressureStrategy::default() }
|
||||
}
|
||||
}
|
||||
|
||||
/// 连接配置
|
||||
/// Connection configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ConnectionConfig {
|
||||
/// 连接超时时间(秒,默认:10)
|
||||
/// Connection timeout in seconds (default: 10)
|
||||
pub connect_timeout: u64,
|
||||
/// 请求超时时间(秒,默认:60)
|
||||
/// Request timeout in seconds (default: 60)
|
||||
pub request_timeout: u64,
|
||||
/// 最大解码消息大小(字节,默认:10MB)
|
||||
/// Maximum decoding message size in bytes (default: 10MB)
|
||||
pub max_decoding_message_size: usize,
|
||||
}
|
||||
|
||||
@@ -74,16 +51,14 @@ impl Default for ConnectionConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// 通用客户端配置
|
||||
/// Common client configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StreamClientConfig {
|
||||
/// 连接配置
|
||||
/// Connection configuration
|
||||
pub connection: ConnectionConfig,
|
||||
/// 批处理配置
|
||||
pub batch: BatchConfig,
|
||||
/// 背压配置
|
||||
/// Backpressure configuration
|
||||
pub backpressure: BackpressureConfig,
|
||||
/// 是否启用性能监控(默认:false)
|
||||
/// Whether performance monitoring is enabled (default: false)
|
||||
pub enable_metrics: bool,
|
||||
}
|
||||
|
||||
@@ -91,7 +66,6 @@ impl Default for StreamClientConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
connection: ConnectionConfig::default(),
|
||||
batch: BatchConfig::default(),
|
||||
backpressure: BackpressureConfig::default(),
|
||||
enable_metrics: false,
|
||||
}
|
||||
@@ -99,33 +73,40 @@ impl Default for StreamClientConfig {
|
||||
}
|
||||
|
||||
impl StreamClientConfig {
|
||||
/// 创建高性能配置(适合高并发场景)
|
||||
pub fn high_performance() -> Self {
|
||||
/// Creates a high-throughput configuration optimized for high-concurrency scenarios.
|
||||
///
|
||||
/// This configuration prioritizes throughput over latency by:
|
||||
/// - Implementing a drop strategy for backpressure to avoid blocking
|
||||
/// - Setting a large permit buffer (5,000) to handle burst traffic
|
||||
///
|
||||
/// Ideal for scenarios where you need to process large volumes of data
|
||||
/// and can tolerate occasional message drops during peak loads.
|
||||
pub fn high_throughput() -> Self {
|
||||
Self {
|
||||
connection: ConnectionConfig::default(),
|
||||
batch: BatchConfig { batch_size: 200, batch_timeout_ms: 5, enabled: true },
|
||||
backpressure: BackpressureConfig {
|
||||
channel_size: 20000,
|
||||
permits: 20000,
|
||||
strategy: BackpressureStrategy::Drop,
|
||||
},
|
||||
enable_metrics: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建低延迟配置(适合实时场景)
|
||||
/// Creates a low-latency configuration optimized for real-time scenarios.
|
||||
///
|
||||
/// This configuration prioritizes latency over throughput by:
|
||||
/// - Processing events immediately without buffering
|
||||
/// - Implementing a blocking backpressure strategy to ensure no data loss
|
||||
/// - Setting optimal permits (4000) for balanced throughput and latency
|
||||
///
|
||||
/// Ideal for scenarios where every millisecond counts and you cannot
|
||||
/// afford to lose any events, such as trading applications or real-time monitoring.
|
||||
pub fn low_latency() -> Self {
|
||||
Self {
|
||||
connection: ConnectionConfig::default(),
|
||||
batch: BatchConfig {
|
||||
batch_size: 10,
|
||||
batch_timeout_ms: 1,
|
||||
enabled: false, // 禁用批处理,即时处理
|
||||
},
|
||||
backpressure: BackpressureConfig {
|
||||
channel_size: 1000,
|
||||
strategy: BackpressureStrategy::Block,
|
||||
},
|
||||
backpressure: BackpressureConfig { permits: 4000, strategy: BackpressureStrategy::Block },
|
||||
enable_metrics: false,
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -5,10 +5,8 @@ pub const DEFAULT_CONNECT_TIMEOUT: u64 = 10;
|
||||
pub const DEFAULT_REQUEST_TIMEOUT: u64 = 60;
|
||||
pub const DEFAULT_CHANNEL_SIZE: usize = 1000;
|
||||
pub const DEFAULT_MAX_DECODING_MESSAGE_SIZE: usize = 1024 * 1024 * 10;
|
||||
pub const DEFAULT_BATCH_SIZE: usize = 100;
|
||||
pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5;
|
||||
|
||||
// 性能监控相关常量
|
||||
pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5;
|
||||
pub const DEFAULT_METRICS_PRINT_INTERVAL_SECONDS: u64 = 10;
|
||||
pub const SLOW_PROCESSING_THRESHOLD_MS: f64 = 10.0;
|
||||
pub const SLOW_PROCESSING_THRESHOLD_US: f64 = 3000.0;
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use crossbeam_queue::SegQueue;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use crate::common::AnyResult;
|
||||
use crate::streaming::common::BackpressureStrategy;
|
||||
use crate::streaming::common::{
|
||||
MetricsEventType, MetricsManager, StreamClientConfig as ClientConfig,
|
||||
};
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
|
||||
use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
|
||||
use crate::streaming::event_parser::core::traits::get_high_perf_clock;
|
||||
use crate::streaming::event_parser::EventParser;
|
||||
use crate::streaming::event_parser::{
|
||||
core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
|
||||
};
|
||||
use crate::streaming::grpc::{BackpressureConfig, EventPretty};
|
||||
use crate::streaming::shred::TransactionWithSlot;
|
||||
use once_cell::sync::OnceCell;
|
||||
|
||||
/// High-performance Event processor using SegQueue for all strategies
|
||||
pub struct EventProcessor {
|
||||
pub(crate) metrics_manager: MetricsManager,
|
||||
pub(crate) config: ClientConfig,
|
||||
pub(crate) parser_cache: OnceCell<Arc<dyn EventParser>>,
|
||||
pub(crate) protocols: Vec<Protocol>,
|
||||
pub(crate) event_type_filter: Option<EventTypeFilter>,
|
||||
pub(crate) callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
|
||||
pub(crate) backpressure_config: BackpressureConfig,
|
||||
/// High-performance lockfree queue for gRPC events
|
||||
pub(crate) grpc_queue: Arc<SegQueue<(EventPretty, Option<Pubkey>)>>,
|
||||
/// High-performance lockfree queue for shred events
|
||||
pub(crate) shred_queue: Arc<SegQueue<(TransactionWithSlot, Option<Pubkey>)>>,
|
||||
/// Fast O(1) counter for Drop strategy (avoids expensive SegQueue::len())
|
||||
pub(crate) grpc_pending_count: Arc<AtomicUsize>,
|
||||
pub(crate) shred_pending_count: Arc<AtomicUsize>,
|
||||
/// Processing thread control
|
||||
pub(crate) processing_shutdown: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl EventProcessor {
|
||||
/// Create a new high-performance event processor
|
||||
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
|
||||
let backpressure_config = config.backpressure.clone();
|
||||
let grpc_queue = Arc::new(SegQueue::new());
|
||||
let shred_queue = Arc::new(SegQueue::new());
|
||||
let grpc_pending_count = Arc::new(AtomicUsize::new(0));
|
||||
let shred_pending_count = Arc::new(AtomicUsize::new(0));
|
||||
let processing_shutdown = Arc::new(AtomicBool::new(false));
|
||||
|
||||
Self {
|
||||
metrics_manager,
|
||||
config,
|
||||
parser_cache: OnceCell::new(),
|
||||
protocols: vec![],
|
||||
event_type_filter: None,
|
||||
backpressure_config,
|
||||
callback: None,
|
||||
grpc_queue,
|
||||
shred_queue,
|
||||
grpc_pending_count,
|
||||
shred_pending_count,
|
||||
processing_shutdown,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_protocols_and_event_type_filter(
|
||||
&mut self,
|
||||
protocols: Vec<Protocol>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
backpressure_config: BackpressureConfig,
|
||||
callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
|
||||
) {
|
||||
self.protocols = protocols;
|
||||
self.event_type_filter = event_type_filter;
|
||||
|
||||
// Check if Block processing thread should be started (before moving backpressure_config)
|
||||
let should_start_block_processing = true;
|
||||
// matches!(backpressure_config.strategy, BackpressureStrategy::Block);
|
||||
|
||||
self.backpressure_config = backpressure_config;
|
||||
self.callback = callback;
|
||||
// Use stored values to initialize parser_cache
|
||||
let protocols_ref = &self.protocols;
|
||||
let event_type_filter_ref = self.event_type_filter.as_ref();
|
||||
self.parser_cache.get_or_init(|| {
|
||||
Arc::new(MutilEventParser::new(protocols_ref.clone(), event_type_filter_ref.cloned()))
|
||||
});
|
||||
|
||||
// Start Block processing thread if using Block strategy
|
||||
if should_start_block_processing {
|
||||
self.start_block_processing_thread();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_parser(&self) -> Arc<dyn EventParser> {
|
||||
self.parser_cache.get().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Create adapter callback
|
||||
fn create_adapter_callback(&self) -> Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync> {
|
||||
let callback = self.callback.clone().unwrap();
|
||||
let metrics_manager = self.metrics_manager.clone();
|
||||
|
||||
Arc::new(move |event: Box<dyn UnifiedEvent>| {
|
||||
let processing_time_us = event.program_handle_time_consuming_us() as f64;
|
||||
callback(event);
|
||||
metrics_manager.update_metrics(MetricsEventType::Transaction, 1, processing_time_us);
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn process_grpc_event_transaction_with_metrics(
|
||||
&self,
|
||||
event_pretty: EventPretty,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
) -> AnyResult<()> {
|
||||
self.apply_backpressure_control(event_pretty, bot_wallet).await
|
||||
}
|
||||
|
||||
/// Apply backpressure control strategy
|
||||
async fn apply_backpressure_control(
|
||||
&self,
|
||||
event_pretty: EventPretty,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
) -> AnyResult<()> {
|
||||
match self.backpressure_config.strategy {
|
||||
BackpressureStrategy::Block => {
|
||||
// Block strategy: async wait if queue is full (backpressure control)
|
||||
loop {
|
||||
let current_pending = self.grpc_pending_count.load(Ordering::Relaxed);
|
||||
if current_pending < self.backpressure_config.permits {
|
||||
self.grpc_queue.push((event_pretty, bot_wallet));
|
||||
self.grpc_pending_count.fetch_add(1, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
// Async yield to avoid blocking gRPC data source
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
BackpressureStrategy::Drop => {
|
||||
// Drop strategy: Use O(1) atomic counter instead of expensive O(n) len()
|
||||
// If pending count >= permits, DROP the event immediately
|
||||
let current_pending = self.grpc_pending_count.load(Ordering::Relaxed);
|
||||
if current_pending >= self.backpressure_config.permits {
|
||||
self.metrics_manager.increment_dropped_events();
|
||||
Ok(())
|
||||
} else {
|
||||
self.grpc_pending_count.fetch_add(1, Ordering::Relaxed);
|
||||
let processor = self.clone();
|
||||
tokio::spawn(async move {
|
||||
match processor
|
||||
.process_grpc_event_transaction(event_pretty, bot_wallet)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
processor.grpc_pending_count.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Error in async gRPC processing: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_grpc_event_transaction(
|
||||
&self,
|
||||
event_pretty: EventPretty,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
) -> AnyResult<()> {
|
||||
if self.callback.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
match event_pretty {
|
||||
EventPretty::Account(account_pretty) => {
|
||||
self.metrics_manager.add_account_process_count();
|
||||
let account_event = AccountEventParser::parse_account_event(
|
||||
&self.protocols,
|
||||
account_pretty,
|
||||
self.event_type_filter.as_ref(),
|
||||
);
|
||||
if let Some(event) = account_event {
|
||||
let processing_time_us = event.program_handle_time_consuming_us() as f64;
|
||||
self.invoke_callback(event);
|
||||
self.update_metrics(MetricsEventType::Account, 1, processing_time_us);
|
||||
}
|
||||
}
|
||||
EventPretty::Transaction(transaction_pretty) => {
|
||||
self.metrics_manager.add_tx_process_count();
|
||||
let slot = transaction_pretty.slot;
|
||||
let signature = transaction_pretty.signature;
|
||||
let tx = transaction_pretty.tx;
|
||||
let block_time = transaction_pretty.block_time;
|
||||
let program_received_time_us = transaction_pretty.program_received_time_us;
|
||||
let transaction_index = transaction_pretty.transaction_index;
|
||||
// Use cache to get parser
|
||||
let parser = self.get_parser();
|
||||
let adapter_callback = self.create_adapter_callback();
|
||||
parser
|
||||
.parse_transaction_owned(
|
||||
tx,
|
||||
signature,
|
||||
Some(slot),
|
||||
block_time,
|
||||
program_received_time_us,
|
||||
bot_wallet,
|
||||
transaction_index,
|
||||
adapter_callback,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
EventPretty::BlockMeta(block_meta_pretty) => {
|
||||
self.metrics_manager.add_block_meta_process_count();
|
||||
let block_time_ms = block_meta_pretty
|
||||
.block_time
|
||||
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
|
||||
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
|
||||
let block_meta_event = CommonEventParser::generate_block_meta_event(
|
||||
block_meta_pretty.slot,
|
||||
&block_meta_pretty.block_hash,
|
||||
block_time_ms,
|
||||
block_meta_pretty.program_received_time_us,
|
||||
);
|
||||
let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64;
|
||||
self.invoke_callback(block_meta_event);
|
||||
self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn invoke_callback(&self, event: Box<dyn UnifiedEvent>) {
|
||||
if let Some(callback) = self.callback.as_ref() {
|
||||
callback(event);
|
||||
}
|
||||
}
|
||||
|
||||
/// Process a single transaction immediately
|
||||
pub async fn process_shred_transaction_immediate(
|
||||
&self,
|
||||
transaction_with_slot: TransactionWithSlot,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
) -> AnyResult<()> {
|
||||
self.process_shred_transaction(transaction_with_slot, bot_wallet).await
|
||||
}
|
||||
|
||||
/// Process shred transaction with backpressure control and performance monitoring
|
||||
pub async fn process_shred_transaction_with_metrics(
|
||||
&self,
|
||||
transaction_with_slot: TransactionWithSlot,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
) -> AnyResult<()> {
|
||||
// Backpressure control logic
|
||||
self.apply_shred_backpressure_control(transaction_with_slot, bot_wallet).await
|
||||
}
|
||||
|
||||
/// Apply shred backpressure control strategy
|
||||
async fn apply_shred_backpressure_control(
|
||||
&self,
|
||||
transaction_with_slot: TransactionWithSlot,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
) -> AnyResult<()> {
|
||||
match self.backpressure_config.strategy {
|
||||
BackpressureStrategy::Block => {
|
||||
// Block strategy: async wait if queue is full (backpressure control)
|
||||
loop {
|
||||
let current_pending = self.shred_pending_count.load(Ordering::Relaxed);
|
||||
if current_pending < self.backpressure_config.permits {
|
||||
self.shred_queue.push((transaction_with_slot, bot_wallet));
|
||||
self.shred_pending_count.fetch_add(1, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
// Async yield to avoid blocking shred data source
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
BackpressureStrategy::Drop => {
|
||||
// Drop strategy: Use O(1) atomic counter instead of expensive O(n) len()
|
||||
let current_pending = self.shred_pending_count.load(Ordering::Relaxed);
|
||||
if current_pending >= self.backpressure_config.permits {
|
||||
self.metrics_manager.increment_dropped_events();
|
||||
Ok(())
|
||||
} else {
|
||||
self.shred_pending_count.fetch_add(1, Ordering::Relaxed);
|
||||
let processor = self.clone();
|
||||
tokio::spawn(async move {
|
||||
match processor
|
||||
.process_shred_transaction(transaction_with_slot, bot_wallet)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
processor.shred_pending_count.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Error in async shred processing: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn process_shred_transaction(
|
||||
&self,
|
||||
transaction_with_slot: TransactionWithSlot,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
) -> AnyResult<()> {
|
||||
if self.callback.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
self.metrics_manager.add_tx_process_count();
|
||||
let tx = transaction_with_slot.transaction;
|
||||
|
||||
let slot = transaction_with_slot.slot;
|
||||
let signature = tx.signatures[0];
|
||||
let program_received_time_us = transaction_with_slot.program_received_time_us;
|
||||
// Use cache to get parser
|
||||
let parser = self.get_parser();
|
||||
let adapter_callback = self.create_adapter_callback();
|
||||
parser
|
||||
.parse_versioned_transaction_owned(
|
||||
tx,
|
||||
signature,
|
||||
Some(slot),
|
||||
None,
|
||||
program_received_time_us,
|
||||
bot_wallet,
|
||||
None,
|
||||
&[],
|
||||
adapter_callback,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn update_metrics(&self, ty: MetricsEventType, count: u64, time_us: f64) {
|
||||
self.metrics_manager.update_metrics(ty, count, time_us);
|
||||
}
|
||||
|
||||
/// Start dedicated processing threads for all strategies
|
||||
fn start_block_processing_thread(&self) {
|
||||
// Reset shutdown flag
|
||||
self.processing_shutdown.store(false, Ordering::Relaxed);
|
||||
|
||||
let grpc_queue = Arc::clone(&self.grpc_queue);
|
||||
let shred_queue = Arc::clone(&self.shred_queue);
|
||||
let grpc_pending_count = Arc::clone(&self.grpc_pending_count);
|
||||
let shred_pending_count = Arc::clone(&self.shred_pending_count);
|
||||
let shutdown_flag = Arc::clone(&self.processing_shutdown);
|
||||
let shutdown_flag_clone = Arc::clone(&self.processing_shutdown);
|
||||
let processor = self.clone();
|
||||
let processor_clone = self.clone();
|
||||
// 1. 专用线程 + 2. Busy-wait + 4. 无锁处理
|
||||
std::thread::spawn(move || {
|
||||
// 创建blocking runtime for async processing
|
||||
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
|
||||
while !shutdown_flag.load(Ordering::Relaxed) {
|
||||
if let Some((event_pretty, bot_wallet)) = grpc_queue.pop() {
|
||||
// Decrement pending counter when consuming from queue
|
||||
grpc_pending_count.fetch_sub(1, Ordering::Relaxed);
|
||||
// Process event in blocking runtime
|
||||
if let Err(e) = rt.block_on(
|
||||
processor.process_grpc_event_transaction(event_pretty, bot_wallet),
|
||||
) {
|
||||
println!("Error processing gRPC event: {}", e);
|
||||
}
|
||||
} else {
|
||||
// 2. 优化忙等待: 使用轻量级休眠减少CPU占用
|
||||
std::thread::yield_now();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Shred处理也使用相同的低延迟优化
|
||||
std::thread::spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
|
||||
|
||||
while !shutdown_flag_clone.load(Ordering::Relaxed) {
|
||||
if let Some((transaction_with_slot, bot_wallet)) = shred_queue.pop() {
|
||||
// Decrement pending counter when consuming from queue
|
||||
shred_pending_count.fetch_sub(1, Ordering::Relaxed);
|
||||
// Process transaction in blocking runtime
|
||||
if let Err(e) = rt.block_on(
|
||||
processor_clone
|
||||
.process_shred_transaction(transaction_with_slot, bot_wallet),
|
||||
) {
|
||||
log::error!("Error processing shred transaction: {}", e);
|
||||
}
|
||||
} else {
|
||||
// 优化忙等待: 使用轻量级休眠减少CPU占用
|
||||
std::thread::yield_now();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Stop processing threads
|
||||
pub fn stop_processing(&self) {
|
||||
self.processing_shutdown.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
// Implement Clone trait to support sharing between modules
|
||||
impl Clone for EventProcessor {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
metrics_manager: self.metrics_manager.clone(),
|
||||
config: self.config.clone(),
|
||||
parser_cache: self.parser_cache.clone(),
|
||||
protocols: self.protocols.clone(),
|
||||
event_type_filter: self.event_type_filter.clone(),
|
||||
backpressure_config: self.backpressure_config.clone(),
|
||||
callback: self.callback.clone(),
|
||||
grpc_queue: self.grpc_queue.clone(),
|
||||
shred_queue: self.shred_queue.clone(),
|
||||
grpc_pending_count: self.grpc_pending_count.clone(),
|
||||
shred_pending_count: self.shred_pending_count.clone(),
|
||||
processing_shutdown: self.processing_shutdown.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
+557
-205
@@ -1,305 +1,656 @@
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use super::config::StreamClientConfig;
|
||||
use super::constants::*;
|
||||
|
||||
/// 单个事件类型的指标
|
||||
/// Event type enumeration
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum EventType {
|
||||
Transaction = 0,
|
||||
Account = 1,
|
||||
BlockMeta = 2,
|
||||
}
|
||||
|
||||
/// Compatibility alias
|
||||
pub type MetricsEventType = EventType;
|
||||
|
||||
impl EventType {
|
||||
#[inline]
|
||||
const fn as_index(self) -> usize {
|
||||
self as usize
|
||||
}
|
||||
|
||||
const fn name(self) -> &'static str {
|
||||
match self {
|
||||
EventType::Transaction => "TX",
|
||||
EventType::Account => "Account",
|
||||
EventType::BlockMeta => "Block Meta",
|
||||
}
|
||||
}
|
||||
|
||||
// Compatibility constants
|
||||
pub const TX: EventType = EventType::Transaction;
|
||||
}
|
||||
|
||||
/// High-performance atomic event metrics
|
||||
#[derive(Debug)]
|
||||
struct AtomicEventMetrics {
|
||||
process_count: AtomicU64,
|
||||
events_processed: AtomicU64,
|
||||
events_in_window: AtomicU64,
|
||||
window_start_nanos: AtomicU64,
|
||||
events_per_second_bits: AtomicU64, // Bit representation of f64
|
||||
}
|
||||
|
||||
impl AtomicEventMetrics {
|
||||
fn new(now_nanos: u64) -> Self {
|
||||
Self {
|
||||
process_count: AtomicU64::new(0),
|
||||
events_processed: AtomicU64::new(0),
|
||||
events_in_window: AtomicU64::new(0),
|
||||
window_start_nanos: AtomicU64::new(now_nanos),
|
||||
events_per_second_bits: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically increment process count
|
||||
#[inline]
|
||||
fn add_process_count(&self) {
|
||||
self.process_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Atomically increment event processing count
|
||||
#[inline]
|
||||
fn add_events_processed(&self, count: u64) {
|
||||
self.events_processed.fetch_add(count, Ordering::Relaxed);
|
||||
self.events_in_window.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Get current count (non-blocking)
|
||||
#[inline]
|
||||
fn get_counts(&self) -> (u64, u64, u64) {
|
||||
(
|
||||
self.process_count.load(Ordering::Relaxed),
|
||||
self.events_processed.load(Ordering::Relaxed),
|
||||
self.events_in_window.load(Ordering::Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
/// Atomically update events per second
|
||||
#[inline]
|
||||
fn update_events_per_second(&self, eps: f64) {
|
||||
self.events_per_second_bits.store(eps.to_bits(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Get events per second
|
||||
#[inline]
|
||||
fn get_events_per_second(&self) -> f64 {
|
||||
f64::from_bits(self.events_per_second_bits.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Reset window count
|
||||
#[inline]
|
||||
fn reset_window(&self, new_start_nanos: u64) {
|
||||
self.events_in_window.store(0, Ordering::Relaxed);
|
||||
self.window_start_nanos.store(new_start_nanos, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_window_start(&self) -> u64 {
|
||||
self.window_start_nanos.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
/// High-performance atomic processing time statistics
|
||||
#[derive(Debug)]
|
||||
struct AtomicProcessingTimeStats {
|
||||
min_time_bits: AtomicU64,
|
||||
max_time_bits: AtomicU64,
|
||||
min_time_timestamp_nanos: AtomicU64, // Timestamp of min value update (nanoseconds)
|
||||
max_time_timestamp_nanos: AtomicU64, // Timestamp of max value update (nanoseconds)
|
||||
total_time_us: AtomicU64, // Store integer part of microseconds
|
||||
total_events: AtomicU64,
|
||||
}
|
||||
|
||||
impl AtomicProcessingTimeStats {
|
||||
fn new() -> Self {
|
||||
let now_nanos =
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
as u64;
|
||||
|
||||
Self {
|
||||
min_time_bits: AtomicU64::new(f64::INFINITY.to_bits()),
|
||||
max_time_bits: AtomicU64::new(0),
|
||||
min_time_timestamp_nanos: AtomicU64::new(now_nanos),
|
||||
max_time_timestamp_nanos: AtomicU64::new(now_nanos),
|
||||
total_time_us: AtomicU64::new(0),
|
||||
total_events: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically update processing time statistics
|
||||
#[inline]
|
||||
fn update(&self, time_us: f64, event_count: u64) {
|
||||
let time_bits = time_us.to_bits();
|
||||
let now_nanos =
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
as u64;
|
||||
|
||||
// Update minimum value, check time difference and reset if over 10 seconds
|
||||
let mut current_min = self.min_time_bits.load(Ordering::Relaxed);
|
||||
let min_timestamp = self.min_time_timestamp_nanos.load(Ordering::Relaxed);
|
||||
|
||||
// Check if min value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds)
|
||||
let min_time_diff_nanos = now_nanos.saturating_sub(min_timestamp);
|
||||
if min_time_diff_nanos > 10_000_000_000 {
|
||||
// Over 10 seconds, reset min value
|
||||
self.min_time_bits.store(f64::INFINITY.to_bits(), Ordering::Relaxed);
|
||||
self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
|
||||
current_min = f64::INFINITY.to_bits();
|
||||
}
|
||||
|
||||
// If current time is less than min value, update min value and timestamp
|
||||
while time_bits < current_min {
|
||||
match self.min_time_bits.compare_exchange_weak(
|
||||
current_min,
|
||||
time_bits,
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => {
|
||||
// Successfully updated min value, also update timestamp
|
||||
self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
Err(x) => current_min = x,
|
||||
}
|
||||
}
|
||||
|
||||
// Update maximum value, check time difference and reset if over 10 seconds
|
||||
let mut current_max = self.max_time_bits.load(Ordering::Relaxed);
|
||||
let max_timestamp = self.max_time_timestamp_nanos.load(Ordering::Relaxed);
|
||||
|
||||
// Check if max value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds)
|
||||
let time_diff_nanos = now_nanos.saturating_sub(max_timestamp);
|
||||
if time_diff_nanos > 10_000_000_000 {
|
||||
// Over 10 seconds, reset max value
|
||||
self.max_time_bits.store(0, Ordering::Relaxed);
|
||||
self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
|
||||
current_max = 0;
|
||||
}
|
||||
|
||||
// If current time is greater than max value, update max value and timestamp
|
||||
while time_bits > current_max {
|
||||
match self.max_time_bits.compare_exchange_weak(
|
||||
current_max,
|
||||
time_bits,
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => {
|
||||
// Successfully updated max value, also update timestamp
|
||||
self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
Err(x) => current_max = x,
|
||||
}
|
||||
}
|
||||
|
||||
// Update cumulative values (convert microseconds to integers to avoid floating point accumulation issues)
|
||||
let total_time_us_int = (time_us * event_count as f64) as u64;
|
||||
self.total_time_us.fetch_add(total_time_us_int, Ordering::Relaxed);
|
||||
self.total_events.fetch_add(event_count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Get statistics (non-blocking)
|
||||
#[inline]
|
||||
fn get_stats(&self) -> ProcessingTimeStats {
|
||||
let min_bits = self.min_time_bits.load(Ordering::Relaxed);
|
||||
let max_bits = self.max_time_bits.load(Ordering::Relaxed);
|
||||
let total_time_us_int = self.total_time_us.load(Ordering::Relaxed);
|
||||
let total_events = self.total_events.load(Ordering::Relaxed);
|
||||
|
||||
let min_time = f64::from_bits(min_bits);
|
||||
let max_time = f64::from_bits(max_bits);
|
||||
let avg_time =
|
||||
if total_events > 0 { total_time_us_int as f64 / total_events as f64 } else { 0.0 };
|
||||
|
||||
ProcessingTimeStats {
|
||||
min_us: if min_time == f64::INFINITY { 0.0 } else { min_time },
|
||||
max_us: max_time,
|
||||
avg_us: avg_time,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Processing time statistics result
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EventMetrics {
|
||||
pub struct ProcessingTimeStats {
|
||||
pub min_us: f64,
|
||||
pub max_us: f64,
|
||||
pub avg_us: f64,
|
||||
}
|
||||
|
||||
/// Event metrics snapshot
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EventMetricsSnapshot {
|
||||
pub process_count: u64,
|
||||
pub events_processed: u64,
|
||||
pub events_per_second: f64,
|
||||
pub events_in_window: u64,
|
||||
pub window_start_time: std::time::Instant,
|
||||
}
|
||||
|
||||
impl EventMetrics {
|
||||
fn new(now: std::time::Instant) -> Self {
|
||||
Self {
|
||||
process_count: 0,
|
||||
events_processed: 0,
|
||||
events_per_second: 0.0,
|
||||
events_in_window: 0,
|
||||
window_start_time: now,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 通用性能监控指标
|
||||
/// Compatibility structure - complete performance metrics
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PerformanceMetrics {
|
||||
pub start_time: std::time::Instant,
|
||||
pub event_metrics: [EventMetrics; 3], // [Tx, Account, BlockMeta]
|
||||
pub average_processing_time_ms: f64,
|
||||
pub min_processing_time_ms: f64,
|
||||
pub max_processing_time_ms: f64,
|
||||
pub last_update_time: std::time::Instant,
|
||||
}
|
||||
|
||||
impl Default for PerformanceMetrics {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub enum MetricsEventType {
|
||||
Tx,
|
||||
Account,
|
||||
BlockMeta,
|
||||
}
|
||||
|
||||
impl MetricsEventType {
|
||||
fn as_index(&self) -> usize {
|
||||
match self {
|
||||
MetricsEventType::Tx => 0,
|
||||
MetricsEventType::Account => 1,
|
||||
MetricsEventType::BlockMeta => 2,
|
||||
}
|
||||
}
|
||||
pub uptime: std::time::Duration,
|
||||
pub tx_metrics: EventMetricsSnapshot,
|
||||
pub account_metrics: EventMetricsSnapshot,
|
||||
pub block_meta_metrics: EventMetricsSnapshot,
|
||||
pub processing_stats: ProcessingTimeStats,
|
||||
pub dropped_events_count: u64,
|
||||
}
|
||||
|
||||
impl PerformanceMetrics {
|
||||
/// Create default performance metrics (compatibility method)
|
||||
pub fn new() -> Self {
|
||||
let now = std::time::Instant::now();
|
||||
let default_metrics =
|
||||
EventMetricsSnapshot { process_count: 0, events_processed: 0, events_per_second: 0.0 };
|
||||
let default_stats = ProcessingTimeStats { min_us: 0.0, max_us: 0.0, avg_us: 0.0 };
|
||||
|
||||
Self {
|
||||
start_time: now,
|
||||
event_metrics: [EventMetrics::new(now), EventMetrics::new(now), EventMetrics::new(now)],
|
||||
average_processing_time_ms: 0.0,
|
||||
min_processing_time_ms: 0.0,
|
||||
max_processing_time_ms: 0.0,
|
||||
last_update_time: now,
|
||||
uptime: std::time::Duration::ZERO,
|
||||
tx_metrics: default_metrics.clone(),
|
||||
account_metrics: default_metrics.clone(),
|
||||
block_meta_metrics: default_metrics,
|
||||
processing_stats: default_stats,
|
||||
dropped_events_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// High-performance metrics system
|
||||
#[derive(Debug)]
|
||||
pub struct HighPerformanceMetrics {
|
||||
start_nanos: u64,
|
||||
event_metrics: [AtomicEventMetrics; 3],
|
||||
processing_stats: AtomicProcessingTimeStats,
|
||||
// 丢弃事件指标
|
||||
dropped_events_count: AtomicU64,
|
||||
}
|
||||
|
||||
impl HighPerformanceMetrics {
|
||||
fn new() -> Self {
|
||||
let now_nanos =
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
as u64;
|
||||
|
||||
Self {
|
||||
start_nanos: now_nanos,
|
||||
event_metrics: [
|
||||
AtomicEventMetrics::new(now_nanos),
|
||||
AtomicEventMetrics::new(now_nanos),
|
||||
AtomicEventMetrics::new(now_nanos),
|
||||
],
|
||||
processing_stats: AtomicProcessingTimeStats::new(),
|
||||
// 初始化丢弃事件指标
|
||||
dropped_events_count: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// 更新时间窗口指标
|
||||
fn update_window_metrics(
|
||||
&mut self,
|
||||
event_type: &MetricsEventType,
|
||||
now: std::time::Instant,
|
||||
window_duration: std::time::Duration,
|
||||
) {
|
||||
/// 获取运行时长(秒)
|
||||
#[inline]
|
||||
pub fn get_uptime_seconds(&self) -> f64 {
|
||||
let now_nanos =
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
as u64;
|
||||
(now_nanos - self.start_nanos) as f64 / 1_000_000_000.0
|
||||
}
|
||||
|
||||
/// 获取事件指标快照
|
||||
#[inline]
|
||||
pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot {
|
||||
let index = event_type.as_index();
|
||||
let event_metric = &mut self.event_metrics[index];
|
||||
let (process_count, events_processed, _) = self.event_metrics[index].get_counts();
|
||||
let events_per_second = self.calculate_real_time_eps(event_type);
|
||||
|
||||
if now.duration_since(event_metric.window_start_time) >= window_duration {
|
||||
let window_seconds = now.duration_since(event_metric.window_start_time).as_secs_f64();
|
||||
// 修复:正确计算每秒事件数,避免除零错误
|
||||
event_metric.events_per_second = if window_seconds > 0.001 {
|
||||
// 避免极小的时间差
|
||||
event_metric.events_in_window as f64 / window_seconds
|
||||
} else {
|
||||
0.0 // 时间太短时设为0,而不是事件总数
|
||||
};
|
||||
|
||||
// 重置窗口
|
||||
event_metric.events_in_window = 0;
|
||||
event_metric.window_start_time = now;
|
||||
}
|
||||
EventMetricsSnapshot { process_count, events_processed, events_per_second }
|
||||
}
|
||||
|
||||
/// 计算实时每秒事件数(用于显示)
|
||||
fn calculate_real_time_events_per_second(
|
||||
&self,
|
||||
event_type: &MetricsEventType,
|
||||
now: std::time::Instant,
|
||||
) -> f64 {
|
||||
/// 获取处理时间统计
|
||||
#[inline]
|
||||
pub fn get_processing_stats(&self) -> ProcessingTimeStats {
|
||||
self.processing_stats.get_stats()
|
||||
}
|
||||
|
||||
/// 获取丢弃事件计数
|
||||
#[inline]
|
||||
pub fn get_dropped_events_count(&self) -> u64 {
|
||||
self.dropped_events_count.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// 计算实时每秒事件数(非阻塞)
|
||||
fn calculate_real_time_eps(&self, event_type: EventType) -> f64 {
|
||||
let now_nanos =
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
as u64;
|
||||
|
||||
let index = event_type.as_index();
|
||||
let event_metric = &self.event_metrics[index];
|
||||
|
||||
let current_window_duration =
|
||||
now.duration_since(event_metric.window_start_time).as_secs_f64();
|
||||
let window_start = event_metric.get_window_start();
|
||||
let current_window_duration_secs =
|
||||
(now_nanos.saturating_sub(window_start)) as f64 / 1_000_000_000.0;
|
||||
let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed);
|
||||
|
||||
// 如果当前窗口有足够的时间和事件,使用当前窗口的数据
|
||||
if current_window_duration > 1.0 && event_metric.events_in_window > 0 {
|
||||
event_metric.events_in_window as f64 / current_window_duration
|
||||
// 优先级1: 当前窗口实时数据(≥2秒且有事件)
|
||||
if current_window_duration_secs >= 2.0 && events_in_window > 0 {
|
||||
return events_in_window as f64 / current_window_duration_secs;
|
||||
}
|
||||
// 如果当前窗口时间太短或没有事件,使用上一个完整窗口的值
|
||||
else if event_metric.events_per_second > 0.0 {
|
||||
event_metric.events_per_second
|
||||
|
||||
// 优先级2: 上一个窗口的结果
|
||||
let stored_eps = event_metric.get_events_per_second();
|
||||
if stored_eps > 0.0 {
|
||||
return stored_eps;
|
||||
}
|
||||
// 如果都没有,计算总体平均值
|
||||
else {
|
||||
let total_duration = now.duration_since(self.start_time).as_secs_f64();
|
||||
if total_duration > 1.0 && event_metric.events_processed > 0 {
|
||||
event_metric.events_processed as f64 / total_duration
|
||||
} else {
|
||||
0.0
|
||||
|
||||
// 优先级3: 总体平均值(≥3秒运行时间)
|
||||
let total_duration_secs = self.get_uptime_seconds();
|
||||
let total_events = event_metric.events_processed.load(Ordering::Relaxed);
|
||||
if total_duration_secs >= 3.0 && total_events > 0 {
|
||||
return total_events as f64 / total_duration_secs;
|
||||
}
|
||||
|
||||
0.0
|
||||
}
|
||||
|
||||
/// 更新窗口指标(后台任务调用)
|
||||
fn update_window_metrics(&self, event_type: EventType, window_duration_nanos: u64) {
|
||||
let now_nanos =
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
as u64;
|
||||
|
||||
let index = event_type.as_index();
|
||||
let event_metric = &self.event_metrics[index];
|
||||
|
||||
let window_start = event_metric.get_window_start();
|
||||
if now_nanos.saturating_sub(window_start) >= window_duration_nanos {
|
||||
let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed);
|
||||
let window_duration_secs = window_duration_nanos as f64 / 1_000_000_000.0;
|
||||
|
||||
if window_duration_secs > 0.001 && events_in_window > 0 {
|
||||
let eps = events_in_window as f64 / window_duration_secs;
|
||||
event_metric.update_events_per_second(eps);
|
||||
}
|
||||
|
||||
event_metric.reset_window(now_nanos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 通用性能监控管理器
|
||||
/// 高性能指标管理器
|
||||
pub struct MetricsManager {
|
||||
metrics: Arc<Mutex<PerformanceMetrics>>,
|
||||
config: Arc<StreamClientConfig>,
|
||||
metrics: Arc<HighPerformanceMetrics>,
|
||||
enable_metrics: bool,
|
||||
stream_name: String,
|
||||
background_task_running: AtomicBool,
|
||||
}
|
||||
|
||||
impl MetricsManager {
|
||||
/// 创建新的性能监控管理器
|
||||
pub fn new(
|
||||
metrics: Arc<Mutex<PerformanceMetrics>>,
|
||||
config: Arc<StreamClientConfig>,
|
||||
stream_name: String,
|
||||
) -> Self {
|
||||
Self { metrics, config, stream_name }
|
||||
/// 创建新的指标管理器
|
||||
pub fn new(enable_metrics: bool, stream_name: String) -> Self {
|
||||
let manager = Self {
|
||||
metrics: Arc::new(HighPerformanceMetrics::new()),
|
||||
enable_metrics,
|
||||
stream_name,
|
||||
background_task_running: AtomicBool::new(false),
|
||||
};
|
||||
|
||||
// 启动后台任务
|
||||
manager.start_background_tasks();
|
||||
manager
|
||||
}
|
||||
|
||||
/// 获取性能指标
|
||||
pub async fn get_metrics(&self) -> PerformanceMetrics {
|
||||
let metrics = self.metrics.lock().await;
|
||||
metrics.clone()
|
||||
/// 启动后台任务
|
||||
fn start_background_tasks(&self) {
|
||||
if self
|
||||
.background_task_running
|
||||
.compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
if !self.enable_metrics {
|
||||
return;
|
||||
}
|
||||
|
||||
let metrics = self.metrics.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_millis(500));
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
let window_duration_nanos = DEFAULT_METRICS_WINDOW_SECONDS * 1_000_000_000;
|
||||
|
||||
// 更新所有事件类型的窗口指标
|
||||
metrics.update_window_metrics(EventType::Transaction, window_duration_nanos);
|
||||
metrics.update_window_metrics(EventType::Account, window_duration_nanos);
|
||||
metrics.update_window_metrics(EventType::BlockMeta, window_duration_nanos);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// 打印性能指标
|
||||
pub async fn print_metrics(&self) {
|
||||
let metrics = self.get_metrics().await;
|
||||
let event_names = ["TX", "Account", "Block Meta"];
|
||||
let event_types =
|
||||
[MetricsEventType::Tx, MetricsEventType::Account, MetricsEventType::BlockMeta];
|
||||
let now = std::time::Instant::now();
|
||||
/// 记录处理次数(非阻塞)
|
||||
#[inline]
|
||||
pub fn record_process(&self, event_type: EventType) {
|
||||
if self.enable_metrics {
|
||||
self.metrics.event_metrics[event_type.as_index()].add_process_count();
|
||||
}
|
||||
}
|
||||
|
||||
/// 记录事件处理(非阻塞)
|
||||
#[inline]
|
||||
pub fn record_events(&self, event_type: EventType, count: u64, processing_time_us: f64) {
|
||||
if !self.enable_metrics {
|
||||
return;
|
||||
}
|
||||
|
||||
// 原子更新事件计数
|
||||
self.metrics.event_metrics[event_type.as_index()].add_events_processed(count);
|
||||
|
||||
// 原子更新处理时间统计
|
||||
self.metrics.processing_stats.update(processing_time_us, count);
|
||||
}
|
||||
|
||||
/// 记录慢处理操作
|
||||
#[inline]
|
||||
pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) {
|
||||
if processing_time_us > SLOW_PROCESSING_THRESHOLD_US {
|
||||
log::debug!(
|
||||
"{} slow processing: {:.2}us for {} events",
|
||||
self.stream_name,
|
||||
processing_time_us,
|
||||
event_count,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取运行时长
|
||||
pub fn get_uptime(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs_f64(self.metrics.get_uptime_seconds())
|
||||
}
|
||||
|
||||
/// 获取事件指标
|
||||
pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot {
|
||||
self.metrics.get_event_metrics(event_type)
|
||||
}
|
||||
|
||||
/// 获取处理时间统计
|
||||
pub fn get_processing_stats(&self) -> ProcessingTimeStats {
|
||||
self.metrics.get_processing_stats()
|
||||
}
|
||||
|
||||
/// 获取丢弃事件计数
|
||||
pub fn get_dropped_events_count(&self) -> u64 {
|
||||
self.metrics.get_dropped_events_count()
|
||||
}
|
||||
|
||||
/// 打印性能指标(非阻塞)
|
||||
pub fn print_metrics(&self) {
|
||||
println!("\n📊 {} Performance Metrics", self.stream_name);
|
||||
println!(" Run Time: {:?}", metrics.start_time.elapsed());
|
||||
|
||||
// 打印表格头部
|
||||
println!(" Run Time: {:?}", self.get_uptime());
|
||||
|
||||
// 打印丢弃事件指标
|
||||
let dropped_count = self.get_dropped_events_count();
|
||||
if dropped_count > 0 {
|
||||
println!("\n⚠️ Dropped Events: {}", dropped_count);
|
||||
}
|
||||
|
||||
// 打印事件指标表格
|
||||
println!("┌─────────────┬──────────────┬──────────────────┬─────────────────┐");
|
||||
println!("│ Event Type │ Process Count│ Events Processed │ Events/Second │");
|
||||
println!("├─────────────┼──────────────┼──────────────────┼─────────────────┤");
|
||||
|
||||
// 打印每种事件类型的数据
|
||||
for (i, name) in event_names.iter().enumerate() {
|
||||
let event_metric = &metrics.event_metrics[i];
|
||||
// 使用实时计算的每秒事件数,而不是窗口更新的值
|
||||
let real_time_eps = metrics.calculate_real_time_events_per_second(&event_types[i], now);
|
||||
|
||||
for event_type in [EventType::Transaction, EventType::Account, EventType::BlockMeta] {
|
||||
let metrics = self.get_event_metrics(event_type);
|
||||
println!(
|
||||
"│ {:11} │ {:12} │ {:16} │ {:13.2} │",
|
||||
name,
|
||||
event_metric.process_count,
|
||||
event_metric.events_processed,
|
||||
real_time_eps
|
||||
event_type.name(),
|
||||
metrics.process_count,
|
||||
metrics.events_processed,
|
||||
metrics.events_per_second
|
||||
);
|
||||
}
|
||||
|
||||
println!("└─────────────┴──────────────┴──────────────────┴─────────────────┘");
|
||||
|
||||
// 打印处理时间统计表格
|
||||
let stats = self.get_processing_stats();
|
||||
println!("\n⏱️ Processing Time Statistics");
|
||||
println!("┌─────────────────────┬─────────────┐");
|
||||
println!("│ Metric │ Value (ms) │");
|
||||
println!("├─────────────────────┼─────────────┤");
|
||||
println!("│ Average │ {:9.2} │", metrics.average_processing_time_ms);
|
||||
println!("│ Minimum │ {:9.2} │", metrics.min_processing_time_ms);
|
||||
println!("│ Maximum │ {:9.2} │", metrics.max_processing_time_ms);
|
||||
println!("└─────────────────────┴─────────────┘");
|
||||
println!("┌───────────────────────┬─────────────┐");
|
||||
println!("│ Metric │ Value (us) │");
|
||||
println!("├───────────────────────┼─────────────┤");
|
||||
println!("│ Average │ {:9.2} │", stats.avg_us);
|
||||
println!("│ Minimum within 10s │ {:9.2} │", stats.min_us);
|
||||
println!("│ Maximum within 10s │ {:9.2} │", stats.max_us);
|
||||
println!("└───────────────────────┴─────────────┘");
|
||||
|
||||
println!();
|
||||
}
|
||||
|
||||
/// 启动自动性能监控任务
|
||||
pub async fn start_auto_monitoring(&self) -> Option<tokio::task::JoinHandle<()>> {
|
||||
// 检查是否启用性能监控
|
||||
if !self.config.enable_metrics {
|
||||
return None; // 如果未启用性能监控,不启动监控任务
|
||||
if !self.enable_metrics {
|
||||
return None;
|
||||
}
|
||||
|
||||
let metrics_manager = self.clone();
|
||||
let manager = self.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(
|
||||
DEFAULT_METRICS_PRINT_INTERVAL_SECONDS,
|
||||
));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
metrics_manager.print_metrics().await;
|
||||
manager.print_metrics();
|
||||
}
|
||||
});
|
||||
Some(handle)
|
||||
}
|
||||
|
||||
/// 更新处理次数
|
||||
pub async fn add_process_count(&self, event_type: MetricsEventType) {
|
||||
if !self.config.enable_metrics {
|
||||
return;
|
||||
// === 兼容性方法 ===
|
||||
|
||||
/// 兼容性构造函数
|
||||
pub fn new_with_metrics(
|
||||
_metrics: Arc<std::sync::RwLock<PerformanceMetrics>>,
|
||||
enable_metrics: bool,
|
||||
stream_name: String,
|
||||
) -> Self {
|
||||
Self::new(enable_metrics, stream_name)
|
||||
}
|
||||
|
||||
/// 获取完整的性能指标(兼容性方法)
|
||||
pub fn get_metrics(&self) -> PerformanceMetrics {
|
||||
PerformanceMetrics {
|
||||
uptime: self.get_uptime(),
|
||||
tx_metrics: self.get_event_metrics(EventType::Transaction),
|
||||
account_metrics: self.get_event_metrics(EventType::Account),
|
||||
block_meta_metrics: self.get_event_metrics(EventType::BlockMeta),
|
||||
processing_stats: self.get_processing_stats(),
|
||||
dropped_events_count: self.metrics.get_dropped_events_count(),
|
||||
}
|
||||
let mut metrics = self.metrics.lock().await;
|
||||
metrics.event_metrics[event_type.as_index()].process_count += 1;
|
||||
}
|
||||
|
||||
// 保持向后兼容的方法
|
||||
pub async fn add_tx_process_count(&self) {
|
||||
self.add_process_count(MetricsEventType::Tx).await;
|
||||
/// 兼容性方法 - 添加交易处理计数
|
||||
#[inline]
|
||||
pub fn add_tx_process_count(&self) {
|
||||
self.record_process(EventType::Transaction);
|
||||
}
|
||||
|
||||
pub async fn add_account_process_count(&self) {
|
||||
self.add_process_count(MetricsEventType::Account).await;
|
||||
/// 兼容性方法 - 添加账户处理计数
|
||||
#[inline]
|
||||
pub fn add_account_process_count(&self) {
|
||||
self.record_process(EventType::Account);
|
||||
}
|
||||
|
||||
pub async fn add_block_meta_process_count(&self) {
|
||||
self.add_process_count(MetricsEventType::BlockMeta).await;
|
||||
/// 兼容性方法 - 添加区块元数据处理计数
|
||||
#[inline]
|
||||
pub fn add_block_meta_process_count(&self) {
|
||||
self.record_process(EventType::BlockMeta);
|
||||
}
|
||||
|
||||
/// 更新性能指标
|
||||
pub async fn update_metrics(
|
||||
/// 兼容性方法 - 更新指标
|
||||
#[inline]
|
||||
pub fn update_metrics(
|
||||
&self,
|
||||
event_type: MetricsEventType,
|
||||
events_processed: u64,
|
||||
processing_time_ms: f64,
|
||||
processing_time_us: f64,
|
||||
) {
|
||||
// 检查是否启用性能监控
|
||||
if !self.config.enable_metrics {
|
||||
self.record_events(event_type, events_processed, processing_time_us);
|
||||
self.log_slow_processing(processing_time_us, events_processed as usize);
|
||||
}
|
||||
|
||||
/// 增加丢弃事件计数
|
||||
#[inline]
|
||||
pub fn increment_dropped_events(&self) {
|
||||
if !self.enable_metrics {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut metrics = self.metrics.lock().await;
|
||||
let now = std::time::Instant::now();
|
||||
let index = event_type.as_index();
|
||||
// 原子地增加丢弃事件计数
|
||||
let new_count = self.metrics.dropped_events_count.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
|
||||
// 更新事件计数
|
||||
metrics.event_metrics[index].events_processed += events_processed;
|
||||
metrics.event_metrics[index].events_in_window += events_processed;
|
||||
|
||||
metrics.last_update_time = now;
|
||||
|
||||
// 更新处理时间统计
|
||||
if processing_time_ms < metrics.min_processing_time_ms
|
||||
|| metrics.min_processing_time_ms == 0.0
|
||||
{
|
||||
metrics.min_processing_time_ms = processing_time_ms;
|
||||
// 每丢弃1000个事件记录一次警告日志
|
||||
if new_count % 1000 == 0 {
|
||||
log::debug!("{} dropped events count reached: {}", self.stream_name, new_count);
|
||||
}
|
||||
if processing_time_ms > metrics.max_processing_time_ms {
|
||||
metrics.max_processing_time_ms = processing_time_ms;
|
||||
}
|
||||
|
||||
// 计算平均处理时间 - 使用增量更新避免重复计算
|
||||
let total_events = metrics.event_metrics[index].events_processed;
|
||||
if total_events > 0 {
|
||||
let total_events_f64 = total_events as f64;
|
||||
let old_total = (total_events_f64 - events_processed as f64).max(0.0);
|
||||
|
||||
metrics.average_processing_time_ms = if old_total > 0.0 {
|
||||
(metrics.average_processing_time_ms * old_total
|
||||
+ processing_time_ms * events_processed as f64)
|
||||
/ total_events_f64
|
||||
} else {
|
||||
processing_time_ms
|
||||
};
|
||||
}
|
||||
|
||||
// 更新时间窗口指标
|
||||
let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS);
|
||||
metrics.update_window_metrics(&event_type, now, window_duration);
|
||||
}
|
||||
|
||||
/// 记录慢处理操作
|
||||
pub fn log_slow_processing(&self, processing_time_ms: f64, event_count: usize) {
|
||||
if processing_time_ms > SLOW_PROCESSING_THRESHOLD_MS {
|
||||
log::warn!(
|
||||
"{} slow processing: {processing_time_ms}ms for {event_count} events",
|
||||
self.stream_name
|
||||
/// 批量增加丢弃事件计数
|
||||
#[inline]
|
||||
pub fn increment_dropped_events_by(&self, count: u64) {
|
||||
if !self.enable_metrics || count == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
// 原子地增加丢弃事件计数
|
||||
let new_count =
|
||||
self.metrics.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count;
|
||||
|
||||
// 记录批量丢弃事件的日志
|
||||
if count > 1 {
|
||||
log::debug!(
|
||||
"{} dropped batch of {} events, total dropped: {}",
|
||||
self.stream_name,
|
||||
count,
|
||||
new_count
|
||||
);
|
||||
}
|
||||
|
||||
// 每丢弃1000个事件记录一次警告日志
|
||||
if new_count % 1000 == 0 || (new_count / 1000) != ((new_count - count) / 1000) {
|
||||
log::debug!("{} dropped events count reached: {}", self.stream_name, new_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,8 +658,9 @@ impl Clone for MetricsManager {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
metrics: self.metrics.clone(),
|
||||
config: self.config.clone(),
|
||||
enable_metrics: self.enable_metrics,
|
||||
stream_name: self.stream_name.clone(),
|
||||
background_task_running: AtomicBool::new(false), // 新实例不自动启动后台任务
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
// 公用模块 - 包含流处理相关的通用功能
|
||||
pub mod config;
|
||||
pub mod metrics;
|
||||
pub mod batch;
|
||||
pub mod constants;
|
||||
pub mod subscription;
|
||||
pub mod event_processor;
|
||||
pub mod simd_utils;
|
||||
|
||||
// 重新导出主要类型
|
||||
pub use config::*;
|
||||
pub use metrics::*;
|
||||
pub use batch::*;
|
||||
pub use constants::*;
|
||||
pub use subscription::*;
|
||||
pub use event_processor::*;
|
||||
pub use simd_utils::*;
|
||||
@@ -0,0 +1,295 @@
|
||||
use wide::*;
|
||||
|
||||
/// SIMD-accelerated data parsing utilities
|
||||
pub struct SimdUtils;
|
||||
|
||||
impl SimdUtils {
|
||||
/// SIMD-accelerated byte array comparison
|
||||
/// For arrays with length >= 16, uses SIMD instructions for fast comparison
|
||||
#[inline(always)]
|
||||
pub fn fast_bytes_equal(a: &[u8], b: &[u8]) -> bool {
|
||||
if a.len() != b.len() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let len = a.len();
|
||||
|
||||
// For small arrays, use standard comparison directly
|
||||
if len < 16 {
|
||||
return a == b;
|
||||
}
|
||||
|
||||
// Use SIMD to process 16-byte chunks
|
||||
let chunks = len / 16;
|
||||
let remainder = len % 16;
|
||||
|
||||
// Process complete 16-byte chunks
|
||||
for i in 0..chunks {
|
||||
let offset = i * 16;
|
||||
let chunk_a = u8x16::from(&a[offset..offset + 16]);
|
||||
let chunk_b = u8x16::from(&b[offset..offset + 16]);
|
||||
|
||||
if !chunk_a.cmp_eq(chunk_b).all() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Process remaining bytes
|
||||
if remainder > 0 {
|
||||
let start = chunks * 16;
|
||||
return &a[start..] == &b[start..];
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
/// Fast discriminator matching, specifically for instruction discriminator comparison
|
||||
#[inline(always)]
|
||||
pub fn fast_discriminator_match(data: &[u8], discriminator: &[u8]) -> bool {
|
||||
if data.len() < discriminator.len() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let disc_len = discriminator.len();
|
||||
|
||||
// Optimize for common discriminator lengths
|
||||
match disc_len {
|
||||
1 => data[0] == discriminator[0],
|
||||
2 => {
|
||||
let data_u16 = u16::from_le_bytes([data[0], data[1]]);
|
||||
let disc_u16 = u16::from_le_bytes([discriminator[0], discriminator[1]]);
|
||||
data_u16 == disc_u16
|
||||
}
|
||||
4 => {
|
||||
let data_u32 = u32::from_le_bytes([data[0], data[1], data[2], data[3]]);
|
||||
let disc_u32 = u32::from_le_bytes([
|
||||
discriminator[0],
|
||||
discriminator[1],
|
||||
discriminator[2],
|
||||
discriminator[3],
|
||||
]);
|
||||
data_u32 == disc_u32
|
||||
}
|
||||
8 => {
|
||||
let data_u64 = u64::from_le_bytes([
|
||||
data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7],
|
||||
]);
|
||||
let disc_u64 = u64::from_le_bytes([
|
||||
discriminator[0],
|
||||
discriminator[1],
|
||||
discriminator[2],
|
||||
discriminator[3],
|
||||
discriminator[4],
|
||||
discriminator[5],
|
||||
discriminator[6],
|
||||
discriminator[7],
|
||||
]);
|
||||
data_u64 == disc_u64
|
||||
}
|
||||
16 => {
|
||||
// Use SIMD to process 16-byte discriminators
|
||||
let data_chunk = u8x16::from(&data[..16]);
|
||||
let disc_chunk = u8x16::from(discriminator);
|
||||
data_chunk.cmp_eq(disc_chunk).all()
|
||||
}
|
||||
_ => {
|
||||
// For other lengths, use generic SIMD comparison
|
||||
Self::fast_bytes_equal(&data[..disc_len], discriminator)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SIMD-accelerated memory search to find specific patterns in data
|
||||
#[inline(always)]
|
||||
pub fn find_pattern_simd(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
if needle.is_empty() || haystack.len() < needle.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let needle_len = needle.len();
|
||||
let haystack_len = haystack.len();
|
||||
|
||||
// For single-byte search, use optimized method
|
||||
if needle_len == 1 {
|
||||
let target = needle[0];
|
||||
return haystack.iter().position(|&b| b == target);
|
||||
}
|
||||
|
||||
// For multi-byte search, use SIMD acceleration
|
||||
if needle_len <= 16 && haystack_len >= 16 {
|
||||
let first_byte = needle[0];
|
||||
let chunks = (haystack_len - needle_len + 1) / 16;
|
||||
|
||||
for chunk_idx in 0..chunks {
|
||||
let start = chunk_idx * 16;
|
||||
let end = std::cmp::min(start + 16, haystack_len - needle_len + 1);
|
||||
|
||||
// Use SIMD to find first byte matches
|
||||
let chunk = &haystack[start..start + 16];
|
||||
let target_vec = u8x16::splat(first_byte);
|
||||
let chunk_vec = u8x16::from(chunk);
|
||||
let matches = chunk_vec.cmp_eq(target_vec);
|
||||
|
||||
// Check each match position
|
||||
let matches_array: [u8; 16] = matches.into();
|
||||
for i in 0..16 {
|
||||
if start + i >= end {
|
||||
break;
|
||||
}
|
||||
|
||||
if matches_array[i] != 0 && start + i + needle_len <= haystack_len {
|
||||
if Self::fast_bytes_equal(
|
||||
&haystack[start + i..start + i + needle_len],
|
||||
needle,
|
||||
) {
|
||||
return Some(start + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process remaining part
|
||||
let remaining_start = chunks * 16;
|
||||
for i in remaining_start..=(haystack_len - needle_len) {
|
||||
if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Fallback to standard search
|
||||
for i in 0..=(haystack_len - needle_len) {
|
||||
if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// SIMD-accelerated data validation to check if data conforms to specific format
|
||||
#[inline(always)]
|
||||
pub fn validate_data_format(data: &[u8], min_length: usize) -> bool {
|
||||
if data.len() < min_length {
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
/// Fast checksum calculation (maintains API consistency)
|
||||
#[inline(always)]
|
||||
pub fn fast_checksum(data: &[u8]) -> u32 {
|
||||
// Simplified implementation, directly sum all bytes
|
||||
data.iter().map(|&b| b as u32).sum()
|
||||
}
|
||||
|
||||
/// SIMD-accelerated data copy (for large data blocks)
|
||||
#[inline(always)]
|
||||
pub fn fast_copy(src: &[u8], dst: &mut [u8]) {
|
||||
if src.len() != dst.len() {
|
||||
panic!("Source and destination must have the same length");
|
||||
}
|
||||
|
||||
let len = src.len();
|
||||
|
||||
if len >= 32 {
|
||||
// Use 32-byte SIMD copy
|
||||
let chunks = len / 32;
|
||||
|
||||
for i in 0..chunks {
|
||||
let start = i * 32;
|
||||
let src_chunk1 = u8x16::from(&src[start..start + 16]);
|
||||
let src_chunk2 = u8x16::from(&src[start + 16..start + 32]);
|
||||
|
||||
let chunk1_array: [u8; 16] = src_chunk1.into();
|
||||
let chunk2_array: [u8; 16] = src_chunk2.into();
|
||||
|
||||
dst[start..start + 16].copy_from_slice(&chunk1_array);
|
||||
dst[start + 16..start + 32].copy_from_slice(&chunk2_array);
|
||||
}
|
||||
|
||||
// Process remaining bytes
|
||||
let remaining_start = chunks * 32;
|
||||
dst[remaining_start..].copy_from_slice(&src[remaining_start..]);
|
||||
} else {
|
||||
// For small data, use standard copy
|
||||
dst.copy_from_slice(src);
|
||||
}
|
||||
}
|
||||
|
||||
/// SIMD-accelerated account indices validation
|
||||
/// Validates that all indices in the account index array are less than the total account count
|
||||
#[inline(always)]
|
||||
pub fn validate_account_indices_simd(indices: &[u8], account_count: usize) -> bool {
|
||||
if indices.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let max_valid_index = account_count as u8;
|
||||
|
||||
// For small arrays, use standard comparison directly
|
||||
if indices.len() < 16 {
|
||||
return indices.iter().all(|&idx| idx < max_valid_index);
|
||||
}
|
||||
|
||||
// Use SIMD for batch loading and comparison
|
||||
let chunks = indices.len() / 16;
|
||||
let remainder = indices.len() % 16;
|
||||
|
||||
// Process complete 16-byte chunks
|
||||
for i in 0..chunks {
|
||||
let start = i * 16;
|
||||
let indices_chunk = u8x16::from(&indices[start..start + 16]);
|
||||
|
||||
// Convert SIMD vector to array for fast batch checking
|
||||
let indices_array: [u8; 16] = indices_chunk.into();
|
||||
|
||||
// Use unrolled loop for fast comparison, compiler will optimize this
|
||||
if indices_array[0] >= max_valid_index
|
||||
|| indices_array[1] >= max_valid_index
|
||||
|| indices_array[2] >= max_valid_index
|
||||
|| indices_array[3] >= max_valid_index
|
||||
|| indices_array[4] >= max_valid_index
|
||||
|| indices_array[5] >= max_valid_index
|
||||
|| indices_array[6] >= max_valid_index
|
||||
|| indices_array[7] >= max_valid_index
|
||||
|| indices_array[8] >= max_valid_index
|
||||
|| indices_array[9] >= max_valid_index
|
||||
|| indices_array[10] >= max_valid_index
|
||||
|| indices_array[11] >= max_valid_index
|
||||
|| indices_array[12] >= max_valid_index
|
||||
|| indices_array[13] >= max_valid_index
|
||||
|| indices_array[14] >= max_valid_index
|
||||
|| indices_array[15] >= max_valid_index
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Process remaining bytes
|
||||
if remainder > 0 {
|
||||
let remaining_start = chunks * 16;
|
||||
return indices[remaining_start..].iter().all(|&idx| idx < max_valid_index);
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
/// SIMD-accelerated instruction data validation
|
||||
/// Validates basic format and length requirements of instruction data
|
||||
#[inline(always)]
|
||||
pub fn validate_instruction_data_simd(
|
||||
data: &[u8],
|
||||
min_length: usize,
|
||||
discriminator_length: usize,
|
||||
) -> bool {
|
||||
// Basic length check
|
||||
if data.len() < min_length || data.len() < discriminator_length {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Use existing data format validation
|
||||
Self::validate_data_format(data, min_length)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ use tokio::task::JoinHandle;
|
||||
/// Subscription handle for managing and stopping subscriptions
|
||||
pub struct SubscriptionHandle {
|
||||
stream_handle: JoinHandle<()>,
|
||||
event_handle: JoinHandle<()>,
|
||||
event_handle: Option<JoinHandle<()>>,
|
||||
metrics_handle: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ impl SubscriptionHandle {
|
||||
/// Create a new subscription handle
|
||||
pub fn new(
|
||||
stream_handle: JoinHandle<()>,
|
||||
event_handle: JoinHandle<()>,
|
||||
event_handle: Option<JoinHandle<()>>,
|
||||
metrics_handle: Option<JoinHandle<()>>,
|
||||
) -> Self {
|
||||
Self { stream_handle, event_handle, metrics_handle }
|
||||
@@ -20,7 +20,9 @@ impl SubscriptionHandle {
|
||||
/// Stop subscription and abort all related tasks
|
||||
pub fn stop(self) {
|
||||
self.stream_handle.abort();
|
||||
self.event_handle.abort();
|
||||
if let Some(handle) = self.event_handle {
|
||||
handle.abort();
|
||||
}
|
||||
if let Some(handle) = self.metrics_handle {
|
||||
handle.abort();
|
||||
}
|
||||
@@ -29,7 +31,9 @@ impl SubscriptionHandle {
|
||||
/// Asynchronously wait for all tasks to complete
|
||||
pub async fn join(self) -> Result<(), tokio::task::JoinError> {
|
||||
let _ = self.stream_handle.await;
|
||||
let _ = self.event_handle.await;
|
||||
if let Some(handle) = self.event_handle {
|
||||
let _ = handle.await;
|
||||
}
|
||||
if let Some(handle) = self.metrics_handle {
|
||||
let _ = handle.await;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use crate::streaming::event_parser::common::{
|
||||
types::EventType, ACCOUNT_EVENT_TYPES, BLOCK_EVENT_TYPES,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct EventTypeFilter {
|
||||
pub include: Vec<EventType>,
|
||||
}
|
||||
|
||||
@@ -8,10 +8,6 @@ macro_rules! impl_unified_event {
|
||||
// 带有自定义ID表达式的版本
|
||||
($struct_name:ident, $($field:ident),*) => {
|
||||
impl $crate::streaming::event_parser::core::traits::UnifiedEvent for $struct_name {
|
||||
fn id(&self) -> &str {
|
||||
&self.metadata.id
|
||||
}
|
||||
|
||||
fn event_type(&self) -> $crate::streaming::event_parser::common::types::EventType {
|
||||
self.metadata.event_type.clone()
|
||||
}
|
||||
@@ -24,16 +20,16 @@ macro_rules! impl_unified_event {
|
||||
self.metadata.slot
|
||||
}
|
||||
|
||||
fn program_received_time_ms(&self) -> i64 {
|
||||
self.metadata.program_received_time_ms
|
||||
fn program_received_time_us(&self) -> i64 {
|
||||
self.metadata.program_received_time_us
|
||||
}
|
||||
|
||||
fn program_handle_time_consuming_ms(&self) -> i64 {
|
||||
self.metadata.program_handle_time_consuming_ms
|
||||
fn program_handle_time_consuming_us(&self) -> i64 {
|
||||
self.metadata.program_handle_time_consuming_us
|
||||
}
|
||||
|
||||
fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64) {
|
||||
self.metadata.program_handle_time_consuming_ms = program_handle_time_consuming_ms;
|
||||
fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64) {
|
||||
self.metadata.program_handle_time_consuming_us = program_handle_time_consuming_us;
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
@@ -48,7 +44,7 @@ macro_rules! impl_unified_event {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
fn merge(&mut self, other: Box<dyn $crate::streaming::event_parser::core::traits::UnifiedEvent>) {
|
||||
fn merge(&mut self, other: &dyn $crate::streaming::event_parser::core::traits::UnifiedEvent) {
|
||||
if let Some(_e) = other.as_any().downcast_ref::<$struct_name>() {
|
||||
$(
|
||||
self.$field = _e.$field.clone();
|
||||
@@ -56,12 +52,23 @@ macro_rules! impl_unified_event {
|
||||
}
|
||||
}
|
||||
|
||||
fn set_transfer_datas(&mut self, transfer_datas: Vec<$crate::streaming::event_parser::common::types::TransferData>, swap_data: Option<$crate::streaming::event_parser::common::types::SwapData>) {
|
||||
self.metadata.set_transfer_datas(transfer_datas, swap_data);
|
||||
fn set_swap_data(&mut self, swap_data: $crate::streaming::event_parser::common::types::SwapData) {
|
||||
self.metadata.set_swap_data(swap_data);
|
||||
}
|
||||
|
||||
fn index(&self) -> String {
|
||||
self.metadata.index.clone()
|
||||
fn swap_data_is_parsed(&self) -> bool {
|
||||
self.metadata.swap_data.is_some()
|
||||
}
|
||||
|
||||
fn instruction_outer_index(&self) -> i64 {
|
||||
self.metadata.instruction_outer_index
|
||||
}
|
||||
|
||||
fn instruction_inner_index(&self) -> Option<i64> {
|
||||
self.metadata.instruction_inner_index
|
||||
}
|
||||
fn transaction_index(&self) -> Option<u64> {
|
||||
self.metadata.transaction_index
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,26 +1,24 @@
|
||||
use borsh::{BorshDeserialize, BorshSerialize};
|
||||
use crossbeam_queue::ArrayQueue;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use solana_transaction_status::UiInstruction;
|
||||
use std::{
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
str::FromStr,
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::sync::Mutex;
|
||||
use std::{borrow::Cow, fmt, str::FromStr, sync::Arc};
|
||||
|
||||
use crate::{
|
||||
match_event,
|
||||
streaming::event_parser::{
|
||||
protocols::{
|
||||
bonk::BonkTradeEvent,
|
||||
pumpfun::PumpFunTradeEvent,
|
||||
pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent},
|
||||
raydium_amm_v4::RaydiumAmmV4SwapEvent,
|
||||
raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event},
|
||||
raydium_cpmm::RaydiumCpmmSwapEvent,
|
||||
streaming::{
|
||||
common::SimdUtils,
|
||||
event_parser::{
|
||||
protocols::{
|
||||
bonk::BonkTradeEvent,
|
||||
pumpfun::PumpFunTradeEvent,
|
||||
pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent},
|
||||
raydium_amm_v4::RaydiumAmmV4SwapEvent,
|
||||
raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event},
|
||||
raydium_cpmm::RaydiumCpmmSwapEvent,
|
||||
},
|
||||
UnifiedEvent,
|
||||
},
|
||||
UnifiedEvent,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -30,7 +28,7 @@ const TRANSFER_DATA_POOL_SIZE: usize = 2000;
|
||||
|
||||
/// Event metadata object pool
|
||||
pub struct EventMetadataPool {
|
||||
pool: Arc<Mutex<Vec<EventMetadata>>>,
|
||||
pool: Arc<ArrayQueue<EventMetadata>>,
|
||||
}
|
||||
|
||||
impl Default for EventMetadataPool {
|
||||
@@ -41,55 +39,22 @@ impl Default for EventMetadataPool {
|
||||
|
||||
impl EventMetadataPool {
|
||||
pub fn new() -> Self {
|
||||
Self { pool: Arc::new(Mutex::new(Vec::with_capacity(EVENT_METADATA_POOL_SIZE))) }
|
||||
Self { pool: Arc::new(ArrayQueue::new(EVENT_METADATA_POOL_SIZE)) }
|
||||
}
|
||||
|
||||
pub async fn acquire(&self) -> Option<EventMetadata> {
|
||||
let mut pool = self.pool.lock().await;
|
||||
pool.pop()
|
||||
pub fn acquire(&self) -> Option<EventMetadata> {
|
||||
self.pool.pop()
|
||||
}
|
||||
|
||||
pub async fn release(&self, metadata: EventMetadata) {
|
||||
let mut pool = self.pool.lock().await;
|
||||
if pool.len() < EVENT_METADATA_POOL_SIZE {
|
||||
pool.push(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Transfer data object pool
|
||||
pub struct TransferDataPool {
|
||||
pool: Arc<Mutex<Vec<TransferData>>>,
|
||||
}
|
||||
|
||||
impl Default for TransferDataPool {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TransferDataPool {
|
||||
pub fn new() -> Self {
|
||||
Self { pool: Arc::new(Mutex::new(Vec::with_capacity(TRANSFER_DATA_POOL_SIZE))) }
|
||||
}
|
||||
|
||||
pub async fn acquire(&self) -> Option<TransferData> {
|
||||
let mut pool = self.pool.lock().await;
|
||||
pool.pop()
|
||||
}
|
||||
|
||||
pub async fn release(&self, transfer_data: TransferData) {
|
||||
let mut pool = self.pool.lock().await;
|
||||
if pool.len() < TRANSFER_DATA_POOL_SIZE {
|
||||
pool.push(transfer_data);
|
||||
}
|
||||
pub fn release(&self, metadata: EventMetadata) {
|
||||
// 如果队列已满,push 会失败,但不会阻塞
|
||||
let _ = self.pool.push(metadata);
|
||||
}
|
||||
}
|
||||
|
||||
// Global object pool instances
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref EVENT_METADATA_POOL: EventMetadataPool = EventMetadataPool::new();
|
||||
pub static ref TRANSFER_DATA_POOL: TransferDataPool = TransferDataPool::new();
|
||||
}
|
||||
|
||||
#[derive(
|
||||
@@ -199,70 +164,69 @@ pub const ACCOUNT_EVENT_TYPES: &[EventType] = &[
|
||||
];
|
||||
pub const BLOCK_EVENT_TYPES: &[EventType] = &[EventType::BlockMeta];
|
||||
|
||||
impl EventType {
|
||||
#[allow(clippy::inherent_to_string)]
|
||||
pub fn to_string(&self) -> String {
|
||||
impl fmt::Display for EventType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
EventType::PumpSwapBuy => "PumpSwapBuy".to_string(),
|
||||
EventType::PumpSwapSell => "PumpSwapSell".to_string(),
|
||||
EventType::PumpSwapCreatePool => "PumpSwapCreatePool".to_string(),
|
||||
EventType::PumpSwapDeposit => "PumpSwapDeposit".to_string(),
|
||||
EventType::PumpSwapWithdraw => "PumpSwapWithdraw".to_string(),
|
||||
EventType::PumpFunCreateToken => "PumpFunCreateToken".to_string(),
|
||||
EventType::PumpFunBuy => "PumpFunBuy".to_string(),
|
||||
EventType::PumpFunSell => "PumpFunSell".to_string(),
|
||||
EventType::PumpFunMigrate => "PumpFunMigrate".to_string(),
|
||||
EventType::BonkBuyExactIn => "BonkBuyExactIn".to_string(),
|
||||
EventType::BonkBuyExactOut => "BonkBuyExactOut".to_string(),
|
||||
EventType::BonkSellExactIn => "BonkSellExactIn".to_string(),
|
||||
EventType::BonkSellExactOut => "BonkSellExactOut".to_string(),
|
||||
EventType::BonkInitialize => "BonkInitialize".to_string(),
|
||||
EventType::BonkInitializeV2 => "BonkInitializeV2".to_string(),
|
||||
EventType::BonkMigrateToAmm => "BonkMigrateToAmm".to_string(),
|
||||
EventType::BonkMigrateToCpswap => "BonkMigrateToCpswap".to_string(),
|
||||
EventType::AccountPumpFunBondingCurve => "AccountPumpFunBondingCurve".to_string(),
|
||||
EventType::AccountPumpFunGlobal => "AccountPumpFunGlobal".to_string(),
|
||||
EventType::AccountPumpSwapGlobalConfig => "AccountPumpSwapGlobalConfig".to_string(),
|
||||
EventType::AccountPumpSwapPool => "AccountPumpSwapPool".to_string(),
|
||||
EventType::AccountBonkPoolState => "AccountBonkPoolState".to_string(),
|
||||
EventType::AccountBonkGlobalConfig => "AccountBonkGlobalConfig".to_string(),
|
||||
EventType::AccountBonkPlatformConfig => "AccountBonkPlatformConfig".to_string(),
|
||||
EventType::AccountBonkVestingRecord => "AccountBonkVestingRecord".to_string(),
|
||||
EventType::RaydiumCpmmSwapBaseInput => "RaydiumCpmmSwapBaseInput".to_string(),
|
||||
EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmmSwapBaseOutput".to_string(),
|
||||
EventType::RaydiumCpmmDeposit => "RaydiumCpmmDeposit".to_string(),
|
||||
EventType::RaydiumCpmmInitialize => "RaydiumCpmmInitialize".to_string(),
|
||||
EventType::RaydiumCpmmWithdraw => "RaydiumCpmmWithdraw".to_string(),
|
||||
EventType::RaydiumClmmSwap => "RaydiumClmmSwap".to_string(),
|
||||
EventType::RaydiumClmmSwapV2 => "RaydiumClmmSwapV2".to_string(),
|
||||
EventType::RaydiumClmmClosePosition => "RaydiumClmmClosePosition".to_string(),
|
||||
EventType::PumpSwapBuy => write!(f, "PumpSwapBuy"),
|
||||
EventType::PumpSwapSell => write!(f, "PumpSwapSell"),
|
||||
EventType::PumpSwapCreatePool => write!(f, "PumpSwapCreatePool"),
|
||||
EventType::PumpSwapDeposit => write!(f, "PumpSwapDeposit"),
|
||||
EventType::PumpSwapWithdraw => write!(f, "PumpSwapWithdraw"),
|
||||
EventType::PumpFunCreateToken => write!(f, "PumpFunCreateToken"),
|
||||
EventType::PumpFunBuy => write!(f, "PumpFunBuy"),
|
||||
EventType::PumpFunSell => write!(f, "PumpFunSell"),
|
||||
EventType::PumpFunMigrate => write!(f, "PumpFunMigrate"),
|
||||
EventType::BonkBuyExactIn => write!(f, "BonkBuyExactIn"),
|
||||
EventType::BonkBuyExactOut => write!(f, "BonkBuyExactOut"),
|
||||
EventType::BonkSellExactIn => write!(f, "BonkSellExactIn"),
|
||||
EventType::BonkSellExactOut => write!(f, "BonkSellExactOut"),
|
||||
EventType::BonkInitialize => write!(f, "BonkInitialize"),
|
||||
EventType::BonkInitializeV2 => write!(f, "BonkInitializeV2"),
|
||||
EventType::BonkMigrateToAmm => write!(f, "BonkMigrateToAmm"),
|
||||
EventType::BonkMigrateToCpswap => write!(f, "BonkMigrateToCpswap"),
|
||||
EventType::RaydiumCpmmSwapBaseInput => write!(f, "RaydiumCpmmSwapBaseInput"),
|
||||
EventType::RaydiumCpmmSwapBaseOutput => write!(f, "RaydiumCpmmSwapBaseOutput"),
|
||||
EventType::RaydiumCpmmDeposit => write!(f, "RaydiumCpmmDeposit"),
|
||||
EventType::RaydiumCpmmInitialize => write!(f, "RaydiumCpmmInitialize"),
|
||||
EventType::RaydiumCpmmWithdraw => write!(f, "RaydiumCpmmWithdraw"),
|
||||
EventType::RaydiumClmmSwap => write!(f, "RaydiumClmmSwap"),
|
||||
EventType::RaydiumClmmSwapV2 => write!(f, "RaydiumClmmSwapV2"),
|
||||
EventType::RaydiumClmmClosePosition => write!(f, "RaydiumClmmClosePosition"),
|
||||
EventType::RaydiumClmmDecreaseLiquidityV2 => {
|
||||
"RaydiumClmmDecreaseLiquidityV2".to_string()
|
||||
write!(f, "RaydiumClmmDecreaseLiquidityV2")
|
||||
}
|
||||
EventType::RaydiumClmmCreatePool => "RaydiumClmmCreatePool".to_string(),
|
||||
EventType::RaydiumClmmCreatePool => write!(f, "RaydiumClmmCreatePool"),
|
||||
EventType::RaydiumClmmIncreaseLiquidityV2 => {
|
||||
"RaydiumClmmIncreaseLiquidityV2".to_string()
|
||||
write!(f, "RaydiumClmmIncreaseLiquidityV2")
|
||||
}
|
||||
EventType::RaydiumClmmOpenPositionWithToken22Nft => {
|
||||
"RaydiumClmmOpenPositionWithToken22Nft".to_string()
|
||||
write!(f, "RaydiumClmmOpenPositionWithToken22Nft")
|
||||
}
|
||||
EventType::RaydiumClmmOpenPositionV2 => "RaydiumClmmOpenPositionV2".to_string(),
|
||||
EventType::RaydiumAmmV4SwapBaseIn => "RaydiumAmmV4SwapBaseIn".to_string(),
|
||||
EventType::RaydiumAmmV4SwapBaseOut => "RaydiumAmmV4SwapBaseOut".to_string(),
|
||||
EventType::RaydiumAmmV4Deposit => "RaydiumAmmV4Deposit".to_string(),
|
||||
EventType::RaydiumAmmV4Initialize2 => "RaydiumAmmV4Initialize2".to_string(),
|
||||
EventType::RaydiumAmmV4Withdraw => "RaydiumAmmV4Withdraw".to_string(),
|
||||
EventType::RaydiumAmmV4WithdrawPnl => "RaydiumAmmV4WithdrawPnl".to_string(),
|
||||
EventType::AccountRaydiumAmmV4AmmInfo => "AccountRaydiumAmmV4AmmInfo".to_string(),
|
||||
EventType::AccountRaydiumClmmAmmConfig => "AccountRaydiumClmmAmmConfig".to_string(),
|
||||
EventType::AccountRaydiumClmmPoolState => "AccountRaydiumClmmPoolState".to_string(),
|
||||
EventType::RaydiumClmmOpenPositionV2 => write!(f, "RaydiumClmmOpenPositionV2"),
|
||||
EventType::RaydiumAmmV4SwapBaseIn => write!(f, "RaydiumAmmV4SwapBaseIn"),
|
||||
EventType::RaydiumAmmV4SwapBaseOut => write!(f, "RaydiumAmmV4SwapBaseOut"),
|
||||
EventType::RaydiumAmmV4Deposit => write!(f, "RaydiumAmmV4Deposit"),
|
||||
EventType::RaydiumAmmV4Initialize2 => write!(f, "RaydiumAmmV4Initialize2"),
|
||||
EventType::RaydiumAmmV4Withdraw => write!(f, "RaydiumAmmV4Withdraw"),
|
||||
EventType::RaydiumAmmV4WithdrawPnl => write!(f, "RaydiumAmmV4WithdrawPnl"),
|
||||
EventType::AccountRaydiumAmmV4AmmInfo => write!(f, "AccountRaydiumAmmV4AmmInfo"),
|
||||
EventType::AccountPumpSwapGlobalConfig => write!(f, "AccountPumpSwapGlobalConfig"),
|
||||
EventType::AccountPumpSwapPool => write!(f, "AccountPumpSwapPool"),
|
||||
EventType::AccountBonkPoolState => write!(f, "AccountBonkPoolState"),
|
||||
EventType::AccountBonkGlobalConfig => write!(f, "AccountBonkGlobalConfig"),
|
||||
EventType::AccountBonkPlatformConfig => write!(f, "AccountBonkPlatformConfig"),
|
||||
EventType::AccountBonkVestingRecord => write!(f, "AccountBonkVestingRecord"),
|
||||
EventType::AccountPumpFunBondingCurve => write!(f, "AccountPumpFunBondingCurve"),
|
||||
EventType::AccountPumpFunGlobal => write!(f, "AccountPumpFunGlobal"),
|
||||
EventType::AccountRaydiumClmmAmmConfig => write!(f, "AccountRaydiumClmmAmmConfig"),
|
||||
EventType::AccountRaydiumClmmPoolState => write!(f, "AccountRaydiumClmmPoolState"),
|
||||
EventType::AccountRaydiumClmmTickArrayState => {
|
||||
"AccountRaydiumClmmTickArrayState".to_string()
|
||||
write!(f, "AccountRaydiumClmmTickArrayState")
|
||||
}
|
||||
EventType::AccountRaydiumCpmmAmmConfig => "AccountRaydiumCpmmAmmConfig".to_string(),
|
||||
EventType::AccountRaydiumCpmmPoolState => "AccountRaydiumCpmmPoolState".to_string(),
|
||||
EventType::BlockMeta => "BlockMeta".to_string(),
|
||||
EventType::Unknown => "Unknown".to_string(),
|
||||
EventType::AccountRaydiumCpmmAmmConfig => write!(f, "AccountRaydiumCpmmAmmConfig"),
|
||||
EventType::AccountRaydiumCpmmPoolState => write!(f, "AccountRaydiumCpmmPoolState"),
|
||||
EventType::BlockMeta => write!(f, "BlockMeta"),
|
||||
EventType::Unknown => write!(f, "Unknown"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -310,20 +274,6 @@ impl ProtocolInfo {
|
||||
}
|
||||
}
|
||||
|
||||
/// Transfer data
|
||||
#[derive(
|
||||
Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize,
|
||||
)]
|
||||
pub struct TransferData {
|
||||
pub token_program: Pubkey,
|
||||
pub source: Pubkey,
|
||||
pub destination: Pubkey,
|
||||
pub authority: Option<Pubkey>,
|
||||
pub amount: u64,
|
||||
pub decimals: Option<u8>,
|
||||
pub mint: Option<Pubkey>,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize,
|
||||
)]
|
||||
@@ -340,285 +290,234 @@ pub struct SwapData {
|
||||
Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize,
|
||||
)]
|
||||
pub struct EventMetadata {
|
||||
pub id: String,
|
||||
pub signature: String,
|
||||
pub signature: Cow<'static, str>,
|
||||
pub slot: u64,
|
||||
pub transaction_index: Option<u64>, // 新增:交易在slot中的索引
|
||||
pub block_time: i64,
|
||||
pub block_time_ms: i64,
|
||||
pub program_received_time_ms: i64,
|
||||
pub program_handle_time_consuming_ms: i64,
|
||||
pub program_received_time_us: i64,
|
||||
pub program_handle_time_consuming_us: i64,
|
||||
pub protocol: ProtocolType,
|
||||
pub event_type: EventType,
|
||||
pub program_id: Pubkey,
|
||||
pub transfer_datas: Vec<TransferData>,
|
||||
pub swap_data: Option<SwapData>,
|
||||
pub index: String,
|
||||
pub instruction_outer_index: i64,
|
||||
pub instruction_inner_index: Option<i64>,
|
||||
}
|
||||
|
||||
impl EventMetadata {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
id: String,
|
||||
signature: String,
|
||||
signature: Cow<'static, str>,
|
||||
slot: u64,
|
||||
block_time: i64,
|
||||
block_time_ms: i64,
|
||||
protocol: ProtocolType,
|
||||
event_type: EventType,
|
||||
program_id: Pubkey,
|
||||
index: String,
|
||||
program_received_time_ms: i64,
|
||||
instruction_outer_index: i64,
|
||||
instruction_inner_index: Option<i64>,
|
||||
program_received_time_us: i64,
|
||||
transaction_index: Option<u64>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
block_time_ms,
|
||||
program_received_time_ms,
|
||||
program_handle_time_consuming_ms: 0,
|
||||
program_received_time_us,
|
||||
program_handle_time_consuming_us: 0,
|
||||
protocol,
|
||||
event_type,
|
||||
program_id,
|
||||
transfer_datas: Vec::with_capacity(4), // Pre-allocate capacity
|
||||
swap_data: None,
|
||||
index,
|
||||
instruction_outer_index,
|
||||
instruction_inner_index,
|
||||
transaction_index,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_id(&mut self, id: String) {
|
||||
let _id = format!("{}-{}-{}", self.signature, self.event_type.to_string(), id);
|
||||
let mut hasher = DefaultHasher::new();
|
||||
_id.hash(&mut hasher);
|
||||
let hash_value = hasher.finish();
|
||||
self.id = format!("{:x}", hash_value);
|
||||
}
|
||||
|
||||
pub fn set_transfer_datas(
|
||||
&mut self,
|
||||
transfer_datas: Vec<TransferData>,
|
||||
swap_data: Option<SwapData>,
|
||||
) {
|
||||
self.transfer_datas = transfer_datas;
|
||||
self.swap_data = swap_data;
|
||||
pub fn set_swap_data(&mut self, swap_data: SwapData) {
|
||||
self.swap_data = Some(swap_data);
|
||||
}
|
||||
|
||||
/// Recycle EventMetadata to object pool
|
||||
pub async fn recycle(self) {
|
||||
EVENT_METADATA_POOL.release(self).await;
|
||||
pub fn recycle(self) {
|
||||
EVENT_METADATA_POOL.release(self);
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse token transfer data from next instructions
|
||||
pub fn parse_transfer_datas_from_next_instructions(
|
||||
event: Box<dyn UnifiedEvent>,
|
||||
inner_instruction: &solana_transaction_status::UiInnerInstructions,
|
||||
current_index: i8,
|
||||
accounts: &[Pubkey],
|
||||
) -> (Vec<TransferData>, Option<SwapData>) {
|
||||
let mut transfer_datas = vec![];
|
||||
// Get the next two instructions after the current instruction
|
||||
let next_instructions: Vec<&UiInstruction> =
|
||||
inner_instruction.instructions.iter().skip((current_index + 1) as usize).collect();
|
||||
|
||||
let system_programs = vec![
|
||||
// Token Program
|
||||
lazy_static::lazy_static! {
|
||||
static ref SOL_MINT: Pubkey = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap();
|
||||
static ref SYSTEM_PROGRAMS: [Pubkey; 3] = [
|
||||
Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(),
|
||||
// Token 2022 Program
|
||||
Pubkey::from_str("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb").unwrap(),
|
||||
// System Program
|
||||
Pubkey::from_str("11111111111111111111111111111111").unwrap(),
|
||||
];
|
||||
for instruction in next_instructions {
|
||||
if let UiInstruction::Compiled(compiled) = instruction {
|
||||
if !system_programs.contains(&accounts[compiled.program_id_index as usize]) {
|
||||
break;
|
||||
}
|
||||
if let Ok(data) = bs58::decode(compiled.data.clone()).into_vec() {
|
||||
// Token Program: transferChecked
|
||||
// Token 2022 Program: transferChecked
|
||||
if data[0] == 12 {
|
||||
let account_pubkeys: Vec<Pubkey> =
|
||||
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
|
||||
if account_pubkeys.len() < 4 {
|
||||
continue;
|
||||
}
|
||||
let (source, mint, destination, authority) = (
|
||||
account_pubkeys[0],
|
||||
account_pubkeys[1],
|
||||
account_pubkeys[2],
|
||||
account_pubkeys[3],
|
||||
);
|
||||
let amount = u64::from_le_bytes(data[1..9].try_into().unwrap());
|
||||
let decimals = data[9];
|
||||
let token_program = accounts[compiled.program_id_index as usize];
|
||||
transfer_datas.push(TransferData {
|
||||
amount,
|
||||
decimals: Some(decimals),
|
||||
mint: Some(mint),
|
||||
source,
|
||||
destination,
|
||||
authority: Some(authority),
|
||||
token_program,
|
||||
});
|
||||
}
|
||||
// Token Program: transfer
|
||||
else if data[0] == 3 {
|
||||
let account_pubkeys: Vec<Pubkey> =
|
||||
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
|
||||
if account_pubkeys.len() < 3 {
|
||||
continue;
|
||||
}
|
||||
let (source, destination, authority) =
|
||||
(account_pubkeys[0], account_pubkeys[1], account_pubkeys[2]);
|
||||
let amount = u64::from_le_bytes(data[1..9].try_into().unwrap());
|
||||
let token_program = accounts[compiled.program_id_index as usize];
|
||||
transfer_datas.push(TransferData {
|
||||
amount,
|
||||
decimals: None,
|
||||
mint: None,
|
||||
source,
|
||||
destination,
|
||||
authority: Some(authority),
|
||||
token_program,
|
||||
});
|
||||
}
|
||||
//System Program: transfer
|
||||
else if data[0] == 2 {
|
||||
let account_pubkeys: Vec<Pubkey> =
|
||||
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
|
||||
if account_pubkeys.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
let (source, destination) = (account_pubkeys[0], account_pubkeys[1]);
|
||||
let amount = u64::from_le_bytes(data[4..12].try_into().unwrap());
|
||||
let token_program = accounts[compiled.program_id_index as usize];
|
||||
transfer_datas.push(TransferData {
|
||||
amount,
|
||||
decimals: None,
|
||||
mint: None,
|
||||
source,
|
||||
destination,
|
||||
authority: None,
|
||||
token_program,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut swap_data: SwapData = SwapData {
|
||||
}
|
||||
|
||||
/// Parse token transfer data from next instructions
|
||||
pub fn parse_swap_data_from_next_instructions(
|
||||
event: &dyn UnifiedEvent,
|
||||
inner_instruction: &solana_transaction_status::InnerInstructions,
|
||||
current_index: i8,
|
||||
accounts: &[Pubkey],
|
||||
) -> Option<SwapData> {
|
||||
let mut swap_data = SwapData {
|
||||
from_mint: Pubkey::default(),
|
||||
to_mint: Pubkey::default(),
|
||||
from_amount: 0,
|
||||
to_amount: 0,
|
||||
description: None,
|
||||
};
|
||||
let sol_mint = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap();
|
||||
if transfer_datas.len() > 0 {
|
||||
let mut user: Option<Pubkey> = None;
|
||||
let mut from_mint: Option<Pubkey> = None;
|
||||
let mut to_mint: Option<Pubkey> = None;
|
||||
let mut user_from_token: Option<Pubkey> = None;
|
||||
let mut user_to_token: Option<Pubkey> = None;
|
||||
let mut from_vault: Option<Pubkey> = None;
|
||||
let mut to_vault: Option<Pubkey> = None;
|
||||
match_event!(event, {
|
||||
BonkTradeEvent => |e: BonkTradeEvent| {
|
||||
user = Some(e.payer);
|
||||
from_mint = Some(e.base_token_mint);
|
||||
to_mint = Some(e.quote_token_mint);
|
||||
user_from_token = Some(e.user_base_token);
|
||||
user_to_token = Some(e.user_quote_token);
|
||||
from_vault = Some(e.base_vault);
|
||||
to_vault = Some(e.quote_vault);
|
||||
},
|
||||
PumpFunTradeEvent => |e: PumpFunTradeEvent| {
|
||||
swap_data.from_mint = if e.is_buy {
|
||||
sol_mint
|
||||
} else {
|
||||
e.mint
|
||||
};
|
||||
swap_data.to_mint = if e.is_buy {
|
||||
e.mint
|
||||
} else {
|
||||
sol_mint
|
||||
};
|
||||
},
|
||||
PumpSwapBuyEvent => |e: PumpSwapBuyEvent| {
|
||||
swap_data.from_mint = e.quote_mint;
|
||||
swap_data.to_mint = e.base_mint;
|
||||
},
|
||||
PumpSwapSellEvent => |e: PumpSwapSellEvent| {
|
||||
swap_data.from_mint = e.base_mint;
|
||||
swap_data.to_mint = e.quote_mint;
|
||||
},
|
||||
RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| {
|
||||
user = Some(e.payer);
|
||||
from_mint = Some(e.input_token_mint);
|
||||
to_mint = Some(e.output_token_mint);
|
||||
user_from_token = Some(e.input_token_account);
|
||||
user_to_token = Some(e.output_token_account);
|
||||
from_vault = Some(e.input_vault);
|
||||
to_vault = Some(e.output_vault);
|
||||
},
|
||||
RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| {
|
||||
user = Some(e.payer);
|
||||
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string());
|
||||
user_from_token = Some(e.input_token_account);
|
||||
user_to_token = Some(e.output_token_account);
|
||||
from_vault = Some(e.input_vault);
|
||||
to_vault = Some(e.output_vault);
|
||||
},
|
||||
RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| {
|
||||
user = Some(e.payer);
|
||||
from_mint = Some(e.input_vault_mint);
|
||||
to_mint = Some(e.output_vault_mint);
|
||||
user_from_token = Some(e.input_token_account);
|
||||
user_to_token = Some(e.output_token_account);
|
||||
from_vault = Some(e.input_vault);
|
||||
to_vault = Some(e.output_vault);
|
||||
},
|
||||
RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| {
|
||||
user = Some(e.user_source_owner);
|
||||
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string());
|
||||
user_from_token = Some(e.user_source_token_account);
|
||||
user_to_token = Some(e.user_destination_token_account);
|
||||
from_vault = Some(e.pool_pc_token_account);
|
||||
to_vault = Some(e.pool_coin_token_account);
|
||||
},
|
||||
});
|
||||
|
||||
for transfer_data in transfer_datas.clone() {
|
||||
if transfer_data.source == user_to_token.unwrap_or_default()
|
||||
&& transfer_data.destination == to_vault.unwrap_or_default()
|
||||
{
|
||||
swap_data.from_mint = to_mint.unwrap_or_default();
|
||||
swap_data.from_amount = transfer_data.amount;
|
||||
} else if transfer_data.source == from_vault.unwrap_or_default()
|
||||
&& transfer_data.destination == user_from_token.unwrap_or_default()
|
||||
{
|
||||
swap_data.to_mint = from_mint.unwrap_or_default();
|
||||
swap_data.to_amount = transfer_data.amount;
|
||||
} else if transfer_data.source == user_from_token.unwrap_or_default()
|
||||
&& transfer_data.destination == from_vault.unwrap_or_default()
|
||||
{
|
||||
swap_data.from_mint = from_mint.unwrap_or_default();
|
||||
swap_data.from_amount = transfer_data.amount;
|
||||
} else if transfer_data.source == to_vault.unwrap_or_default()
|
||||
&& transfer_data.destination == user_to_token.unwrap_or_default()
|
||||
{
|
||||
swap_data.to_mint = to_mint.unwrap_or_default();
|
||||
swap_data.to_amount = transfer_data.amount;
|
||||
// 先根据 event 取出关键信息
|
||||
let mut user: Option<Pubkey> = None;
|
||||
let mut from_mint: Option<Pubkey> = None;
|
||||
let mut to_mint: Option<Pubkey> = None;
|
||||
let mut user_from_token: Option<Pubkey> = None;
|
||||
let mut user_to_token: Option<Pubkey> = None;
|
||||
let mut from_vault: Option<Pubkey> = None;
|
||||
let mut to_vault: Option<Pubkey> = None;
|
||||
|
||||
match_event!(&*event, {
|
||||
BonkTradeEvent => |e: BonkTradeEvent| {
|
||||
user = Some(e.payer);
|
||||
from_mint = Some(e.base_token_mint);
|
||||
to_mint = Some(e.quote_token_mint);
|
||||
user_from_token = Some(e.user_base_token);
|
||||
user_to_token = Some(e.user_quote_token);
|
||||
from_vault = Some(e.base_vault);
|
||||
to_vault = Some(e.quote_vault);
|
||||
},
|
||||
PumpFunTradeEvent => |e: PumpFunTradeEvent| {
|
||||
swap_data.from_mint = if e.is_buy { *SOL_MINT } else { e.mint };
|
||||
swap_data.to_mint = if e.is_buy { e.mint } else { *SOL_MINT };
|
||||
},
|
||||
PumpSwapBuyEvent => |e: PumpSwapBuyEvent| {
|
||||
swap_data.from_mint = e.quote_mint;
|
||||
swap_data.to_mint = e.base_mint;
|
||||
},
|
||||
PumpSwapSellEvent => |e: PumpSwapSellEvent| {
|
||||
swap_data.from_mint = e.base_mint;
|
||||
swap_data.to_mint = e.quote_mint;
|
||||
},
|
||||
RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| {
|
||||
user = Some(e.payer);
|
||||
from_mint = Some(e.input_token_mint);
|
||||
to_mint = Some(e.output_token_mint);
|
||||
user_from_token = Some(e.input_token_account);
|
||||
user_to_token = Some(e.output_token_account);
|
||||
from_vault = Some(e.input_vault);
|
||||
to_vault = Some(e.output_vault);
|
||||
},
|
||||
RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| {
|
||||
user = Some(e.payer);
|
||||
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string());
|
||||
user_from_token = Some(e.input_token_account);
|
||||
user_to_token = Some(e.output_token_account);
|
||||
from_vault = Some(e.input_vault);
|
||||
to_vault = Some(e.output_vault);
|
||||
},
|
||||
RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| {
|
||||
user = Some(e.payer);
|
||||
from_mint = Some(e.input_vault_mint);
|
||||
to_mint = Some(e.output_vault_mint);
|
||||
user_from_token = Some(e.input_token_account);
|
||||
user_to_token = Some(e.output_token_account);
|
||||
from_vault = Some(e.input_vault);
|
||||
to_vault = Some(e.output_vault);
|
||||
},
|
||||
RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| {
|
||||
user = Some(e.user_source_owner);
|
||||
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string());
|
||||
user_from_token = Some(e.user_source_token_account);
|
||||
user_to_token = Some(e.user_destination_token_account);
|
||||
from_vault = Some(e.pool_pc_token_account);
|
||||
to_vault = Some(e.pool_coin_token_account);
|
||||
},
|
||||
});
|
||||
|
||||
let user_to_token = user_to_token.unwrap_or_default();
|
||||
let user_from_token = user_from_token.unwrap_or_default();
|
||||
let to_vault = to_vault.unwrap_or_default();
|
||||
let from_vault = from_vault.unwrap_or_default();
|
||||
let to_mint = to_mint.unwrap_or_default();
|
||||
let from_mint = from_mint.unwrap_or_default();
|
||||
|
||||
// 单次循环完成提取和判断
|
||||
for instruction in inner_instruction.instructions.iter().skip((current_index + 1) as usize) {
|
||||
let compiled = &instruction.instruction;
|
||||
let program_id = accounts[compiled.program_id_index as usize];
|
||||
if !SYSTEM_PROGRAMS.contains(&program_id) {
|
||||
break;
|
||||
}
|
||||
let data = &compiled.data;
|
||||
|
||||
// 使用 SIMD 验证数据格式
|
||||
if !SimdUtils::validate_data_format(data, 8) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let get_pubkey = |i: usize| accounts[compiled.accounts[i] as usize];
|
||||
let (source, destination, amount) = match data[0] {
|
||||
12 if compiled.accounts.len() >= 4 => {
|
||||
let amt = u64::from_le_bytes(data[1..9].try_into().unwrap());
|
||||
(get_pubkey(0), get_pubkey(2), amt)
|
||||
}
|
||||
3 if compiled.accounts.len() >= 3 => {
|
||||
let amt = u64::from_le_bytes(data[1..9].try_into().unwrap());
|
||||
(get_pubkey(0), get_pubkey(1), amt)
|
||||
}
|
||||
2 if compiled.accounts.len() >= 2 => {
|
||||
let amt = u64::from_le_bytes(data[4..12].try_into().unwrap());
|
||||
(get_pubkey(0), get_pubkey(1), amt)
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
match (source, destination) {
|
||||
(s, d) if s == user_to_token && d == to_vault => {
|
||||
swap_data.from_mint = to_mint;
|
||||
swap_data.from_amount = amount;
|
||||
}
|
||||
(s, d) if s == from_vault && d == user_from_token => {
|
||||
swap_data.to_mint = from_mint;
|
||||
swap_data.to_amount = amount;
|
||||
}
|
||||
(s, d) if s == user_from_token && d == from_vault => {
|
||||
swap_data.from_mint = from_mint;
|
||||
swap_data.from_amount = amount;
|
||||
}
|
||||
(s, d) if s == to_vault && d == user_to_token => {
|
||||
swap_data.to_mint = to_mint;
|
||||
swap_data.to_amount = amount;
|
||||
}
|
||||
(s, d) if s == user_from_token && d == to_vault => {
|
||||
swap_data.from_mint = from_mint;
|
||||
swap_data.from_amount = amount;
|
||||
}
|
||||
(s, d) if s == from_vault && d == user_to_token => {
|
||||
swap_data.to_mint = to_mint;
|
||||
swap_data.to_amount = amount;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if swap_data.from_mint != Pubkey::default() && swap_data.to_mint != Pubkey::default() {
|
||||
break;
|
||||
}
|
||||
if swap_data.from_amount != 0 && swap_data.to_amount != 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if swap_data.from_mint != Pubkey::default()
|
||||
|| swap_data.to_mint != Pubkey::default()
|
||||
|| swap_data.from_amount != 0
|
||||
|| swap_data.to_amount != 0
|
||||
{
|
||||
(transfer_datas, Some(swap_data))
|
||||
Some(swap_data)
|
||||
} else {
|
||||
(transfer_datas, None)
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use base64::engine::general_purpose;
|
||||
use base64::Engine;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// 获取当前时间戳
|
||||
@@ -7,16 +5,6 @@ pub fn current_timestamp() -> i64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards").as_secs() as i64
|
||||
}
|
||||
|
||||
/// 从base64字符串解码数据
|
||||
pub fn decode_base64(data: &str) -> Result<Vec<u8>, base64::DecodeError> {
|
||||
general_purpose::STANDARD.decode(data)
|
||||
}
|
||||
|
||||
/// 将数据编码为base64字符串
|
||||
pub fn encode_base64(data: &[u8]) -> String {
|
||||
general_purpose::STANDARD.encode(data)
|
||||
}
|
||||
|
||||
/// 从字节数组中提取鉴别器和剩余数据
|
||||
pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8])> {
|
||||
if data.len() < length {
|
||||
@@ -25,15 +13,6 @@ pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8]
|
||||
Some((&data[..length], &data[length..]))
|
||||
}
|
||||
|
||||
/// 检查鉴别器是否匹配 - 优化版本
|
||||
pub fn discriminator_matches(data: &str, expected: &str) -> bool {
|
||||
if data.len() < expected.len() {
|
||||
return false;
|
||||
}
|
||||
// 使用字节比较而不是字符串比较,更高效
|
||||
data.as_bytes().starts_with(expected.as_bytes())
|
||||
}
|
||||
|
||||
/// 从日志中提取程序数据
|
||||
pub fn extract_program_data(log: &str) -> Option<&str> {
|
||||
const PROGRAM_DATA_PREFIX: &str = "Program data: ";
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use crate::streaming::common::SimdUtils;
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use crate::streaming::event_parser::common::{EventMetadata, EventType, ProtocolType};
|
||||
use crate::streaming::event_parser::core::traits::UnifiedEvent;
|
||||
use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock};
|
||||
use crate::streaming::event_parser::protocols::bonk::parser::BONK_PROGRAM_ID;
|
||||
use crate::streaming::event_parser::protocols::pumpfun::parser::PUMPFUN_PROGRAM_ID;
|
||||
use crate::streaming::event_parser::protocols::pumpswap::parser::PUMPSWAP_PROGRAM_ID;
|
||||
@@ -35,7 +37,10 @@ static PROTOCOL_CONFIGS_CACHE: OnceLock<HashMap<Protocol, Vec<AccountEventParseC
|
||||
pub struct AccountEventParser {}
|
||||
|
||||
impl AccountEventParser {
|
||||
pub fn configs(protocols: Vec<Protocol>, event_type_filter: Option<EventTypeFilter>) -> Vec<AccountEventParseConfig> {
|
||||
pub fn configs(
|
||||
protocols: &[Protocol],
|
||||
event_type_filter: Option<&EventTypeFilter>,
|
||||
) -> Vec<AccountEventParseConfig> {
|
||||
let protocols_map = PROTOCOL_CONFIGS_CACHE.get_or_init(|| {
|
||||
let mut map: HashMap<Protocol, Vec<AccountEventParseConfig>> = HashMap::new();
|
||||
map.insert(Protocol::PumpSwap, vec![
|
||||
@@ -145,43 +150,49 @@ impl AccountEventParser {
|
||||
});
|
||||
|
||||
let mut configs = vec![];
|
||||
let empty_vec = vec![];
|
||||
for protocol in protocols {
|
||||
let protocol_configs = protocols_map.get(&protocol).unwrap_or(&vec![]).clone();
|
||||
let filtered_configs: Vec<AccountEventParseConfig> = protocol_configs.into_iter().filter(|config| {
|
||||
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
|
||||
}).collect();
|
||||
let protocol_configs = protocols_map.get(protocol).unwrap_or(&empty_vec);
|
||||
let filtered_configs: Vec<AccountEventParseConfig> = protocol_configs
|
||||
.iter()
|
||||
.filter(|config| {
|
||||
event_type_filter
|
||||
.map(|filter| filter.include.contains(&config.event_type))
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
configs.extend(filtered_configs);
|
||||
}
|
||||
configs
|
||||
}
|
||||
|
||||
pub fn parse_account_event(
|
||||
protocols: Vec<Protocol>,
|
||||
protocols: &[Protocol],
|
||||
account: AccountPretty,
|
||||
program_received_time_ms: i64,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
event_type_filter: Option<&EventTypeFilter>,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
let configs = Self::configs(protocols, event_type_filter);
|
||||
for config in configs {
|
||||
if account.owner == config.program_id.to_string()
|
||||
&& account.data[..config.account_discriminator.len()]
|
||||
== *config.account_discriminator
|
||||
if account.owner == config.program_id
|
||||
&& SimdUtils::fast_discriminator_match(&account.data, config.account_discriminator)
|
||||
{
|
||||
let signature_str = Cow::Owned(account.signature.to_string());
|
||||
let event = (config.account_parser)(
|
||||
&account,
|
||||
EventMetadata {
|
||||
slot: account.slot,
|
||||
signature: account.signature.clone(),
|
||||
signature: signature_str,
|
||||
protocol: config.protocol_type,
|
||||
event_type: config.event_type,
|
||||
program_id: config.program_id,
|
||||
program_received_time_ms,
|
||||
program_received_time_us: account.program_received_time_us,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
if let Some(mut event) = event {
|
||||
event.set_program_handle_time_consuming_ms(
|
||||
chrono::Utc::now().timestamp_millis() - program_received_time_ms,
|
||||
event.set_program_handle_time_consuming_us(
|
||||
get_high_perf_clock().elapsed_micros_since(account.program_received_time_us),
|
||||
);
|
||||
return Some(event);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::streaming::event_parser::core::traits::UnifiedEvent;
|
||||
use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock};
|
||||
use crate::streaming::event_parser::protocols::block::block_meta_event::BlockMetaEvent;
|
||||
|
||||
pub struct CommonEventParser {}
|
||||
@@ -8,8 +8,17 @@ impl CommonEventParser {
|
||||
slot: u64,
|
||||
block_hash: &str,
|
||||
block_time_ms: i64,
|
||||
program_received_time_us: i64,
|
||||
) -> Box<dyn UnifiedEvent> {
|
||||
let block_meta_event = BlockMetaEvent::new(slot, block_hash.to_string(), block_time_ms);
|
||||
let mut block_meta_event = BlockMetaEvent::new(
|
||||
slot,
|
||||
block_hash.to_string(),
|
||||
block_time_ms,
|
||||
program_received_time_us,
|
||||
);
|
||||
block_meta_event.set_program_handle_time_consuming_us(
|
||||
get_high_perf_clock().elapsed_micros_since(program_received_time_us),
|
||||
);
|
||||
Box::new(block_meta_event)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use dashmap::DashMap;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
const MAX_SLOTS: usize = 1000;
|
||||
const CLEANUP_BATCH_SIZE: usize = 100;
|
||||
|
||||
/// Slot-based trader addresses, completely lock-free
|
||||
#[derive(Default)]
|
||||
struct SlotAddresses {
|
||||
/// Developer addresses for this slot
|
||||
dev_addresses: BTreeSet<Pubkey>,
|
||||
/// Bonk developer addresses for this slot
|
||||
bonk_dev_addresses: BTreeSet<Pubkey>,
|
||||
}
|
||||
|
||||
/// High-performance global state with lock-free slot-based storage
|
||||
pub struct GlobalState {
|
||||
/// Slot -> trader addresses mapping (lock-free concurrent hashmap)
|
||||
slot_data: DashMap<u64, SlotAddresses>,
|
||||
/// Current slot count for capacity management
|
||||
slot_count: AtomicUsize,
|
||||
/// Generation counter to handle cleanup races
|
||||
generation: AtomicU64,
|
||||
}
|
||||
|
||||
impl GlobalState {
|
||||
/// Create a new high-performance global state instance
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
slot_data: DashMap::new(),
|
||||
slot_count: AtomicUsize::new(0),
|
||||
generation: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Lock-free capacity management - cleanup old slots when limit exceeded
|
||||
fn maybe_cleanup(&self) {
|
||||
let current_count = self.slot_count.load(Ordering::Relaxed);
|
||||
if current_count <= MAX_SLOTS {
|
||||
return;
|
||||
}
|
||||
|
||||
// Use CAS to ensure only one thread performs cleanup
|
||||
let gen = self.generation.load(Ordering::Relaxed);
|
||||
if self.generation.compare_exchange_weak(gen, gen + 1, Ordering::Acquire, Ordering::Relaxed).is_err() {
|
||||
return; // Another thread is cleaning up
|
||||
}
|
||||
|
||||
// Collect oldest slots (BTreeMap naturally orders by key)
|
||||
let mut slots_to_remove: Vec<u64> = self.slot_data.iter()
|
||||
.map(|entry| *entry.key())
|
||||
.collect();
|
||||
|
||||
if slots_to_remove.len() <= MAX_SLOTS {
|
||||
return; // Race condition, already cleaned up
|
||||
}
|
||||
|
||||
slots_to_remove.sort_unstable();
|
||||
slots_to_remove.truncate(CLEANUP_BATCH_SIZE);
|
||||
|
||||
// Remove old slots atomically
|
||||
for slot in slots_to_remove {
|
||||
self.slot_data.remove(&slot);
|
||||
self.slot_count.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Add developer address for a specific slot (lock-free)
|
||||
pub fn add_dev_address(&self, slot: u64, address: Pubkey) {
|
||||
self.maybe_cleanup();
|
||||
|
||||
self.slot_data.entry(slot)
|
||||
.and_modify(|addresses| {
|
||||
addresses.dev_addresses.insert(address);
|
||||
})
|
||||
.or_insert_with(|| {
|
||||
self.slot_count.fetch_add(1, Ordering::Relaxed);
|
||||
let mut slot_addr = SlotAddresses::default();
|
||||
slot_addr.dev_addresses.insert(address);
|
||||
slot_addr
|
||||
});
|
||||
}
|
||||
|
||||
/// Add Bonk developer address for a specific slot (lock-free)
|
||||
pub fn add_bonk_dev_address(&self, slot: u64, address: Pubkey) {
|
||||
self.maybe_cleanup();
|
||||
|
||||
self.slot_data.entry(slot)
|
||||
.and_modify(|addresses| {
|
||||
addresses.bonk_dev_addresses.insert(address);
|
||||
})
|
||||
.or_insert_with(|| {
|
||||
self.slot_count.fetch_add(1, Ordering::Relaxed);
|
||||
let mut slot_addr = SlotAddresses::default();
|
||||
slot_addr.bonk_dev_addresses.insert(address);
|
||||
slot_addr
|
||||
});
|
||||
}
|
||||
|
||||
/// High-performance: Check if address is a developer address in specific slot (O(log m))
|
||||
pub fn is_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool {
|
||||
self.slot_data.get(&slot)
|
||||
.map(|entry| entry.dev_addresses.contains(address))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// High-performance: Check if address is a Bonk developer address in specific slot (O(log m))
|
||||
pub fn is_bonk_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool {
|
||||
self.slot_data.get(&slot)
|
||||
.map(|entry| entry.bonk_dev_addresses.contains(address))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Check if address is a developer address in any slot (lock-free scan, slower)
|
||||
pub fn is_dev_address(&self, address: &Pubkey) -> bool {
|
||||
self.slot_data.iter().any(|entry| entry.dev_addresses.contains(address))
|
||||
}
|
||||
|
||||
/// Check if address is a Bonk developer address in any slot (lock-free scan, slower)
|
||||
pub fn is_bonk_dev_address(&self, address: &Pubkey) -> bool {
|
||||
self.slot_data.iter().any(|entry| entry.bonk_dev_addresses.contains(address))
|
||||
}
|
||||
|
||||
/// Get all developer addresses from all slots (lock-free aggregation)
|
||||
pub fn get_dev_addresses(&self) -> Vec<Pubkey> {
|
||||
let mut all_addresses = BTreeSet::new();
|
||||
for entry in self.slot_data.iter() {
|
||||
for addr in &entry.dev_addresses {
|
||||
all_addresses.insert(*addr);
|
||||
}
|
||||
}
|
||||
all_addresses.into_iter().collect()
|
||||
}
|
||||
|
||||
/// Get all Bonk developer addresses from all slots (lock-free aggregation)
|
||||
pub fn get_bonk_dev_addresses(&self) -> Vec<Pubkey> {
|
||||
let mut all_addresses = BTreeSet::new();
|
||||
for entry in self.slot_data.iter() {
|
||||
for addr in &entry.bonk_dev_addresses {
|
||||
all_addresses.insert(*addr);
|
||||
}
|
||||
}
|
||||
all_addresses.into_iter().collect()
|
||||
}
|
||||
|
||||
/// Get developer addresses for a specific slot
|
||||
pub fn get_dev_addresses_for_slot(&self, slot: u64) -> Vec<Pubkey> {
|
||||
self.slot_data.get(&slot)
|
||||
.map(|entry| entry.dev_addresses.iter().copied().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Get Bonk developer addresses for a specific slot
|
||||
pub fn get_bonk_dev_addresses_for_slot(&self, slot: u64) -> Vec<Pubkey> {
|
||||
self.slot_data.get(&slot)
|
||||
.map(|entry| entry.bonk_dev_addresses.iter().copied().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Get current slot count
|
||||
pub fn get_slot_count(&self) -> usize {
|
||||
self.slot_count.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Clear all data (lock-free)
|
||||
pub fn clear_all_data(&self) {
|
||||
self.slot_data.clear();
|
||||
self.slot_count.store(0, Ordering::Relaxed);
|
||||
self.generation.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GlobalState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Global state instance
|
||||
static GLOBAL_STATE: once_cell::sync::Lazy<GlobalState> =
|
||||
once_cell::sync::Lazy::new(GlobalState::new);
|
||||
|
||||
/// Get global state instance
|
||||
pub fn get_global_state() -> &'static GlobalState {
|
||||
&GLOBAL_STATE
|
||||
}
|
||||
|
||||
/// Convenience function: Add developer address for a specific slot
|
||||
pub fn add_dev_address(slot: u64, address: Pubkey) {
|
||||
get_global_state().add_dev_address(slot, address);
|
||||
}
|
||||
|
||||
/// Convenience function: Check if address is a developer address
|
||||
pub fn is_dev_address(address: &Pubkey) -> bool {
|
||||
get_global_state().is_dev_address(address)
|
||||
}
|
||||
|
||||
/// Convenience function: Add Bonk developer address for a specific slot
|
||||
pub fn add_bonk_dev_address(slot: u64, address: Pubkey) {
|
||||
get_global_state().add_bonk_dev_address(slot, address);
|
||||
}
|
||||
|
||||
/// Convenience function: Check if address is a Bonk developer address
|
||||
pub fn is_bonk_dev_address(address: &Pubkey) -> bool {
|
||||
get_global_state().is_bonk_dev_address(address)
|
||||
}
|
||||
|
||||
/// Convenience function: Get all developer addresses
|
||||
pub fn get_dev_addresses() -> Vec<Pubkey> {
|
||||
get_global_state().get_dev_addresses()
|
||||
}
|
||||
|
||||
/// Convenience function: Get all Bonk developer addresses
|
||||
pub fn get_bonk_dev_addresses() -> Vec<Pubkey> {
|
||||
get_global_state().get_bonk_dev_addresses()
|
||||
}
|
||||
|
||||
/// Convenience function: Get developer addresses for a specific slot
|
||||
pub fn get_dev_addresses_for_slot(slot: u64) -> Vec<Pubkey> {
|
||||
get_global_state().get_dev_addresses_for_slot(slot)
|
||||
}
|
||||
|
||||
/// Convenience function: Get Bonk developer addresses for a specific slot
|
||||
pub fn get_bonk_dev_addresses_for_slot(slot: u64) -> Vec<Pubkey> {
|
||||
get_global_state().get_bonk_dev_addresses_for_slot(slot)
|
||||
}
|
||||
|
||||
/// Convenience function: Get current slot count
|
||||
pub fn get_slot_count() -> usize {
|
||||
get_global_state().get_slot_count()
|
||||
}
|
||||
|
||||
/// High-performance: Check if address is a developer address in specific slot
|
||||
pub fn is_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool {
|
||||
get_global_state().is_dev_address_in_slot(slot, address)
|
||||
}
|
||||
|
||||
/// High-performance: Check if address is a Bonk developer address in specific slot
|
||||
pub fn is_bonk_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool {
|
||||
get_global_state().is_bonk_dev_address_in_slot(slot, address)
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/// Macro to generate boilerplate EventParser implementation for protocol parsers
|
||||
///
|
||||
/// This macro eliminates the repetitive code where each parser simply delegates
|
||||
/// all EventParser trait methods to its inner GenericEventParser.
|
||||
///
|
||||
/// Usage:
|
||||
/// ```rust
|
||||
/// impl_event_parser_delegate!(MyEventParser);
|
||||
/// ```
|
||||
///
|
||||
/// This will generate the complete EventParser implementation that delegates
|
||||
/// all methods to `self.inner`.
|
||||
#[macro_export]
|
||||
macro_rules! impl_event_parser_delegate {
|
||||
($parser_type:ty) => {
|
||||
#[async_trait::async_trait]
|
||||
impl $crate::streaming::event_parser::core::traits::EventParser for $parser_type {
|
||||
fn instruction_configs(
|
||||
&self,
|
||||
) -> std::collections::HashMap<
|
||||
Vec<u8>,
|
||||
Vec<$crate::streaming::event_parser::core::traits::GenericEventParseConfig>,
|
||||
> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &solana_sdk::instruction::CompiledInstruction,
|
||||
signature: solana_sdk::signature::Signature,
|
||||
slot: u64,
|
||||
block_time: Option<prost_types::Timestamp>,
|
||||
program_received_time_us: i64,
|
||||
outer_index: i64,
|
||||
inner_index: Option<i64>,
|
||||
transaction_index: Option<u64>,
|
||||
config: &GenericEventParseConfig,
|
||||
) -> Vec<Box<dyn $crate::streaming::event_parser::core::traits::UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_us,
|
||||
outer_index,
|
||||
inner_index,
|
||||
transaction_index,
|
||||
config,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &solana_sdk::instruction::CompiledInstruction,
|
||||
accounts: &[solana_sdk::pubkey::Pubkey],
|
||||
signature: solana_sdk::signature::Signature,
|
||||
slot: u64,
|
||||
block_time: Option<prost_types::Timestamp>,
|
||||
program_received_time_us: i64,
|
||||
outer_index: i64,
|
||||
inner_index: Option<i64>,
|
||||
bot_wallet: Option<solana_sdk::pubkey::Pubkey>,
|
||||
transaction_index: Option<u64>,
|
||||
inner_instructions: Option<&solana_transaction_status::InnerInstructions>,
|
||||
callback: std::sync::Arc<
|
||||
dyn for<'a> Fn(&'a Box<dyn $crate::streaming::event_parser::core::traits::UnifiedEvent>)
|
||||
+ Send
|
||||
+ Sync,
|
||||
>,
|
||||
) -> anyhow::Result<()> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_us,
|
||||
outer_index,
|
||||
inner_index,
|
||||
bot_wallet,
|
||||
transaction_index,
|
||||
inner_instructions,
|
||||
callback,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &solana_sdk::pubkey::Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<solana_sdk::pubkey::Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
pub mod common_event_parser;
|
||||
pub mod traits;
|
||||
pub mod account_event_parser;
|
||||
pub mod macros;
|
||||
pub mod global_state;
|
||||
pub use traits::{EventParser, UnifiedEvent};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,5 @@
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::impl_unified_event;
|
||||
use crate::streaming::event_parser::common::{types::EventType, EventMetadata};
|
||||
use borsh::BorshDeserialize;
|
||||
@@ -13,18 +15,24 @@ pub struct BlockMetaEvent {
|
||||
}
|
||||
|
||||
impl BlockMetaEvent {
|
||||
pub fn new(slot: u64, block_hash: String, block_time_ms: i64) -> Self {
|
||||
pub fn new(
|
||||
slot: u64,
|
||||
block_hash: String,
|
||||
block_time_ms: i64,
|
||||
program_received_time_us: i64,
|
||||
) -> Self {
|
||||
let metadata = EventMetadata::new(
|
||||
format!("block_{}_{}", slot, block_hash),
|
||||
"".to_string(),
|
||||
Cow::Borrowed(""),
|
||||
slot,
|
||||
block_time_ms / 1000,
|
||||
block_time_ms,
|
||||
crate::streaming::event_parser::common::types::ProtocolType::Common,
|
||||
EventType::BlockMeta,
|
||||
solana_sdk::pubkey::Pubkey::default(),
|
||||
"".to_string(),
|
||||
chrono::Utc::now().timestamp_millis(),
|
||||
0,
|
||||
None,
|
||||
program_received_time_us,
|
||||
None,
|
||||
);
|
||||
Self { metadata, slot, block_hash }
|
||||
}
|
||||
|
||||
@@ -314,8 +314,12 @@ impl_unified_event!(BonkPlatformConfigAccountEvent,);
|
||||
/// Event discriminator constants
|
||||
pub mod discriminators {
|
||||
// Event discriminators
|
||||
pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
|
||||
pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae";
|
||||
// pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
|
||||
pub const TRADE_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238];
|
||||
// pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae";
|
||||
pub const POOL_CREATE_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 151, 215, 226, 9, 118, 161, 115, 174];
|
||||
|
||||
// Instruction discriminators
|
||||
pub const BUY_EXACT_IN: &[u8] = &[250, 234, 13, 123, 213, 156, 19, 236];
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
use std::collections::HashMap;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use prost_types::Timestamp;
|
||||
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
|
||||
use solana_transaction_status::UiCompiledInstruction;
|
||||
|
||||
use crate::streaming::event_parser::{
|
||||
common::{utils::*, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::bonk::{
|
||||
bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection, VestingParams
|
||||
use crate::{
|
||||
impl_event_parser_delegate,
|
||||
streaming::event_parser::{
|
||||
common::{utils::*, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::bonk::{
|
||||
bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators,
|
||||
AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent,
|
||||
BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams,
|
||||
TradeDirection, VestingParams,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -88,7 +90,7 @@ impl BonkEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: BONK_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::Bonk,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::MIGRATE_TO_AMM,
|
||||
event_type: EventType::BonkMigrateToAmm,
|
||||
inner_instruction_parser: None,
|
||||
@@ -97,7 +99,7 @@ impl BonkEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: BONK_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::Bonk,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::MIGRATE_TO_CP_SWAP,
|
||||
event_type: EventType::BonkMigrateToCpswap,
|
||||
inner_instruction_parser: None,
|
||||
@@ -116,8 +118,6 @@ impl BonkEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = bonk_pool_create_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(metadata.signature.to_string());
|
||||
Some(Box::new(BonkPoolCreateEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -130,8 +130,6 @@ impl BonkEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = bonk_trade_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}", metadata.signature, event.pool_state));
|
||||
if metadata.event_type == EventType::BonkBuyExactIn
|
||||
|| metadata.event_type == EventType::BonkBuyExactOut
|
||||
{
|
||||
@@ -164,9 +162,6 @@ impl BonkEventParser {
|
||||
let minimum_amount_out = read_u64_le(data, 8)?;
|
||||
let share_fee_rate = read_u64_le(data, 16)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
|
||||
|
||||
Some(Box::new(BonkTradeEvent {
|
||||
metadata,
|
||||
amount_in,
|
||||
@@ -205,9 +200,6 @@ impl BonkEventParser {
|
||||
let maximum_amount_in = read_u64_le(data, 8)?;
|
||||
let share_fee_rate = read_u64_le(data, 16)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
|
||||
|
||||
Some(Box::new(BonkTradeEvent {
|
||||
metadata,
|
||||
amount_out,
|
||||
@@ -246,9 +238,6 @@ impl BonkEventParser {
|
||||
let minimum_amount_out = read_u64_le(data, 8)?;
|
||||
let share_fee_rate = read_u64_le(data, 16)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
|
||||
|
||||
Some(Box::new(BonkTradeEvent {
|
||||
metadata,
|
||||
amount_in,
|
||||
@@ -287,9 +276,6 @@ impl BonkEventParser {
|
||||
let maximum_amount_in = read_u64_le(data, 8)?;
|
||||
let share_fee_rate = read_u64_le(data, 16)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
|
||||
|
||||
Some(Box::new(BonkTradeEvent {
|
||||
metadata,
|
||||
amount_out,
|
||||
@@ -330,9 +316,6 @@ impl BonkEventParser {
|
||||
let curve_param = Self::parse_curve_params(data, &mut offset)?;
|
||||
let vesting_param = Self::parse_vesting_params(data, &mut offset)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(metadata.signature.to_string());
|
||||
|
||||
Some(Box::new(BonkPoolCreateEvent {
|
||||
metadata,
|
||||
payer: accounts[0],
|
||||
@@ -367,9 +350,6 @@ impl BonkEventParser {
|
||||
let vesting_param = Self::parse_vesting_params(data, &mut offset)?;
|
||||
let amm_fee_on = data[offset];
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(metadata.signature.to_string());
|
||||
|
||||
Some(Box::new(BonkPoolCreateEvent {
|
||||
metadata,
|
||||
payer: accounts[0],
|
||||
@@ -512,9 +492,6 @@ impl BonkEventParser {
|
||||
let quote_lot_size = u64::from_le_bytes(data[8..16].try_into().unwrap());
|
||||
let market_vault_signer_nonce = data[16];
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(metadata.signature.to_string());
|
||||
|
||||
Some(Box::new(BonkMigrateToAmmEvent {
|
||||
metadata,
|
||||
base_lot_size,
|
||||
@@ -562,9 +539,6 @@ impl BonkEventParser {
|
||||
accounts: &[Pubkey],
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(metadata.signature.to_string());
|
||||
|
||||
Some(Box::new(BonkMigrateToCpswapEvent {
|
||||
metadata,
|
||||
payer: accounts[0],
|
||||
@@ -601,59 +575,4 @@ impl BonkEventParser {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventParser for BonkEventParser {
|
||||
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
|
||||
self.inner.inner_instruction_configs()
|
||||
}
|
||||
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &UiCompiledInstruction,
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &CompiledInstruction,
|
||||
accounts: &[Pubkey],
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
impl_event_parser_delegate!(BonkEventParser);
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use prost_types::Timestamp;
|
||||
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
|
||||
use solana_transaction_status::UiCompiledInstruction;
|
||||
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use crate::streaming::event_parser::{
|
||||
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
EventParserFactory, Protocol,
|
||||
use crate::{
|
||||
impl_event_parser_delegate,
|
||||
streaming::event_parser::{
|
||||
common::filter::EventTypeFilter,
|
||||
core::traits::{GenericEventParseConfig, GenericEventParser},
|
||||
EventParserFactory, Protocol,
|
||||
},
|
||||
};
|
||||
|
||||
pub struct MutilEventParser {
|
||||
@@ -21,20 +18,22 @@ impl MutilEventParser {
|
||||
for protocol in protocols {
|
||||
let parse = EventParserFactory::create_parser(protocol);
|
||||
|
||||
// Merge inner_instruction_configs, append configurations to existing Vec
|
||||
for (key, configs) in parse.inner_instruction_configs() {
|
||||
let filtered_configs: Vec<GenericEventParseConfig> = configs.into_iter().filter(|config| {
|
||||
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
|
||||
}).collect();
|
||||
inner.inner_instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs);
|
||||
}
|
||||
|
||||
// Merge instruction_configs, append configurations to existing Vec
|
||||
for (key, configs) in parse.instruction_configs() {
|
||||
let filtered_configs: Vec<GenericEventParseConfig> = configs.into_iter().filter(|config| {
|
||||
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
|
||||
}).collect();
|
||||
inner.instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs);
|
||||
let filtered_configs: Vec<GenericEventParseConfig> = configs
|
||||
.into_iter()
|
||||
.filter(|config| {
|
||||
event_type_filter
|
||||
.as_ref()
|
||||
.map(|filter| filter.include.contains(&config.event_type))
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.collect();
|
||||
inner
|
||||
.instruction_configs
|
||||
.entry(key)
|
||||
.or_insert_with(Vec::new)
|
||||
.extend(filtered_configs);
|
||||
}
|
||||
|
||||
// Append program_ids (this is already appending)
|
||||
@@ -44,59 +43,4 @@ impl MutilEventParser {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventParser for MutilEventParser {
|
||||
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
|
||||
self.inner.inner_instruction_configs()
|
||||
}
|
||||
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &UiCompiledInstruction,
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &CompiledInstruction,
|
||||
accounts: &[Pubkey],
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
impl_event_parser_delegate!(MutilEventParser);
|
||||
|
||||
@@ -255,9 +255,15 @@ impl_unified_event!(PumpFunGlobalAccountEvent,);
|
||||
/// 事件鉴别器常量
|
||||
pub mod discriminators {
|
||||
// 事件鉴别器
|
||||
pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376";
|
||||
pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
|
||||
pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94";
|
||||
// pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376";
|
||||
pub const CREATE_TOKEN_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 27, 114, 169, 77, 222, 235, 99, 118];
|
||||
// pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee";
|
||||
pub const TRADE_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238];
|
||||
// pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94";
|
||||
pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 189, 233, 93, 185, 92, 148, 234, 148];
|
||||
|
||||
// 指令鉴别器
|
||||
pub const CREATE_TOKEN_IX: &[u8] = &[24, 30, 200, 40, 5, 28, 7, 119];
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
use std::collections::HashMap;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use prost_types::Timestamp;
|
||||
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
|
||||
use solana_transaction_status::UiCompiledInstruction;
|
||||
|
||||
use crate::streaming::event_parser::{
|
||||
common::{EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::pumpfun::{
|
||||
discriminators, pumpfun_create_token_event_log_decode, pumpfun_migrate_event_log_decode,
|
||||
pumpfun_trade_event_log_decode, PumpFunCreateTokenEvent, PumpFunMigrateEvent,
|
||||
PumpFunTradeEvent,
|
||||
use crate::{
|
||||
impl_event_parser_delegate,
|
||||
streaming::event_parser::{
|
||||
common::{EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::pumpfun::{
|
||||
discriminators, pumpfun_create_token_event_log_decode,
|
||||
pumpfun_migrate_event_log_decode, pumpfun_trade_event_log_decode,
|
||||
PumpFunCreateTokenEvent, PumpFunMigrateEvent, PumpFunTradeEvent,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -82,8 +81,6 @@ impl PumpFunEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pumpfun_migrate_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, event.user, event.mint));
|
||||
Some(Box::new(PumpFunMigrateEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -96,11 +93,6 @@ impl PumpFunEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pumpfun_create_token_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, event.name, event.symbol, event.mint
|
||||
));
|
||||
Some(Box::new(PumpFunCreateTokenEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -113,11 +105,6 @@ impl PumpFunEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pumpfun_trade_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, event.mint, event.user, event.is_buy
|
||||
));
|
||||
Some(Box::new(PumpFunTradeEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -152,9 +139,6 @@ impl PumpFunEventParser {
|
||||
Pubkey::default()
|
||||
};
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, name, symbol, accounts[0]));
|
||||
|
||||
Some(Box::new(PumpFunCreateTokenEvent {
|
||||
metadata,
|
||||
name: name.to_string(),
|
||||
@@ -181,8 +165,6 @@ impl PumpFunEventParser {
|
||||
}
|
||||
let amount = u64::from_le_bytes(data[0..8].try_into().unwrap());
|
||||
let max_sol_cost = u64::from_le_bytes(data[8..16].try_into().unwrap());
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], true));
|
||||
Some(Box::new(PumpFunTradeEvent {
|
||||
metadata,
|
||||
global: accounts[0],
|
||||
@@ -217,9 +199,6 @@ impl PumpFunEventParser {
|
||||
}
|
||||
let amount = u64::from_le_bytes(data[0..8].try_into().unwrap());
|
||||
let min_sol_output = u64::from_le_bytes(data[8..16].try_into().unwrap());
|
||||
let mut metadata = metadata;
|
||||
metadata
|
||||
.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], false));
|
||||
Some(Box::new(PumpFunTradeEvent {
|
||||
metadata,
|
||||
global: accounts[0],
|
||||
@@ -252,8 +231,6 @@ impl PumpFunEventParser {
|
||||
if accounts.len() < 24 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[5], accounts[2]));
|
||||
Some(Box::new(PumpFunMigrateEvent {
|
||||
metadata,
|
||||
global: accounts[0],
|
||||
@@ -285,59 +262,4 @@ impl PumpFunEventParser {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventParser for PumpFunEventParser {
|
||||
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
|
||||
self.inner.inner_instruction_configs()
|
||||
}
|
||||
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &UiCompiledInstruction,
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &CompiledInstruction,
|
||||
accounts: &[Pubkey],
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
impl_event_parser_delegate!(PumpFunEventParser);
|
||||
|
||||
@@ -392,11 +392,21 @@ impl_unified_event!(PumpSwapPoolAccountEvent,);
|
||||
/// 事件鉴别器常量
|
||||
pub mod discriminators {
|
||||
// 事件鉴别器
|
||||
pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777";
|
||||
pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a";
|
||||
pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774";
|
||||
pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90";
|
||||
pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0";
|
||||
// pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777";
|
||||
pub const BUY_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 103, 244, 82, 31, 44, 245, 119, 119];
|
||||
// pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a";
|
||||
pub const SELL_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 62, 47, 55, 10, 165, 3, 220, 42];
|
||||
// pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774";
|
||||
pub const CREATE_POOL_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 177, 49, 12, 210, 160, 118, 167, 116];
|
||||
// pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90";
|
||||
pub const DEPOSIT_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 120, 248, 61, 83, 31, 142, 107, 144];
|
||||
// pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0";
|
||||
pub const WITHDRAW_EVENT: &[u8] =
|
||||
&[228, 69, 165, 46, 81, 203, 154, 29, 22, 9, 133, 26, 160, 44, 71, 192];
|
||||
|
||||
// 指令鉴别器
|
||||
pub const BUY_IX: &[u8] = &[102, 6, 61, 18, 1, 218, 235, 234];
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
use std::collections::HashMap;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use prost_types::Timestamp;
|
||||
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
|
||||
use solana_transaction_status::UiCompiledInstruction;
|
||||
|
||||
use crate::streaming::event_parser::{
|
||||
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::pumpswap::{
|
||||
discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode,
|
||||
pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode,
|
||||
pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent,
|
||||
PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent,
|
||||
use crate::{
|
||||
impl_event_parser_delegate,
|
||||
streaming::event_parser::{
|
||||
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::pumpswap::{
|
||||
discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode,
|
||||
pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode,
|
||||
pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent,
|
||||
PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -92,11 +91,6 @@ impl PumpSwapEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pump_swap_buy_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, event.user, event.pool, event.base_amount_out
|
||||
));
|
||||
Some(Box::new(PumpSwapBuyEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -109,11 +103,6 @@ impl PumpSwapEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pump_swap_sell_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, event.user, event.pool, event.base_amount_in
|
||||
));
|
||||
Some(Box::new(PumpSwapSellEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -126,11 +115,6 @@ impl PumpSwapEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pump_swap_create_pool_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, event.pool, event.creator, event.base_amount_in
|
||||
));
|
||||
Some(Box::new(PumpSwapCreatePoolEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -143,11 +127,6 @@ impl PumpSwapEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pump_swap_deposit_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, event.pool, event.user, event.lp_token_amount_out
|
||||
));
|
||||
Some(Box::new(PumpSwapDepositEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -160,11 +139,6 @@ impl PumpSwapEventParser {
|
||||
metadata: EventMetadata,
|
||||
) -> Option<Box<dyn UnifiedEvent>> {
|
||||
if let Some(event) = pump_swap_withdraw_event_log_decode(data) {
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, event.pool, event.user, event.lp_token_amount_in
|
||||
));
|
||||
Some(Box::new(PumpSwapWithdrawEvent { metadata, ..event }))
|
||||
} else {
|
||||
None
|
||||
@@ -184,12 +158,6 @@ impl PumpSwapEventParser {
|
||||
let base_amount_out = read_u64_le(data, 0)?;
|
||||
let max_quote_amount_in = read_u64_le(data, 8)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[1], accounts[0], base_amount_out
|
||||
));
|
||||
|
||||
Some(Box::new(PumpSwapBuyEvent {
|
||||
metadata,
|
||||
base_amount_out,
|
||||
@@ -225,12 +193,6 @@ impl PumpSwapEventParser {
|
||||
let base_amount_in = read_u64_le(data, 0)?;
|
||||
let min_quote_amount_out = read_u64_le(data, 8)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[1], accounts[0], base_amount_in
|
||||
));
|
||||
|
||||
Some(Box::new(PumpSwapSellEvent {
|
||||
metadata,
|
||||
base_amount_in,
|
||||
@@ -272,12 +234,6 @@ impl PumpSwapEventParser {
|
||||
Pubkey::default()
|
||||
};
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[0], accounts[2], base_amount_in
|
||||
));
|
||||
|
||||
Some(Box::new(PumpSwapCreatePoolEvent {
|
||||
metadata,
|
||||
index,
|
||||
@@ -312,12 +268,6 @@ impl PumpSwapEventParser {
|
||||
let max_base_amount_in = u64::from_le_bytes(data[8..16].try_into().ok()?);
|
||||
let max_quote_amount_in = u64::from_le_bytes(data[16..24].try_into().ok()?);
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[0], accounts[2], lp_token_amount_out
|
||||
));
|
||||
|
||||
Some(Box::new(PumpSwapDepositEvent {
|
||||
metadata,
|
||||
lp_token_amount_out,
|
||||
@@ -350,12 +300,6 @@ impl PumpSwapEventParser {
|
||||
let min_base_amount_out = u64::from_le_bytes(data[8..16].try_into().ok()?);
|
||||
let min_quote_amount_out = u64::from_le_bytes(data[16..24].try_into().ok()?);
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[0], accounts[2], lp_token_amount_in
|
||||
));
|
||||
|
||||
Some(Box::new(PumpSwapWithdrawEvent {
|
||||
metadata,
|
||||
lp_token_amount_in,
|
||||
@@ -375,59 +319,4 @@ impl PumpSwapEventParser {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventParser for PumpSwapEventParser {
|
||||
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
|
||||
self.inner.inner_instruction_configs()
|
||||
}
|
||||
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &UiCompiledInstruction,
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &CompiledInstruction,
|
||||
accounts: &[Pubkey],
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
impl_event_parser_delegate!(PumpSwapEventParser);
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
use std::collections::HashMap;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use prost_types::Timestamp;
|
||||
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
|
||||
use solana_transaction_status::UiCompiledInstruction;
|
||||
|
||||
use crate::streaming::event_parser::{
|
||||
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::raydium_amm_v4::{
|
||||
discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event,
|
||||
RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent,
|
||||
use crate::{
|
||||
impl_event_parser_delegate,
|
||||
streaming::event_parser::{
|
||||
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::raydium_amm_v4::{
|
||||
discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event,
|
||||
RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -35,7 +34,7 @@ impl RaydiumAmmV4EventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumAmmV4,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::SWAP_BASE_IN,
|
||||
event_type: EventType::RaydiumAmmV4SwapBaseIn,
|
||||
inner_instruction_parser: None,
|
||||
@@ -44,7 +43,7 @@ impl RaydiumAmmV4EventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumAmmV4,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::SWAP_BASE_OUT,
|
||||
event_type: EventType::RaydiumAmmV4SwapBaseOut,
|
||||
inner_instruction_parser: None,
|
||||
@@ -53,7 +52,7 @@ impl RaydiumAmmV4EventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumAmmV4,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::DEPOSIT,
|
||||
event_type: EventType::RaydiumAmmV4Deposit,
|
||||
inner_instruction_parser: None,
|
||||
@@ -62,7 +61,7 @@ impl RaydiumAmmV4EventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumAmmV4,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::INITIALIZE2,
|
||||
event_type: EventType::RaydiumAmmV4Initialize2,
|
||||
inner_instruction_parser: None,
|
||||
@@ -71,7 +70,7 @@ impl RaydiumAmmV4EventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumAmmV4,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::WITHDRAW,
|
||||
event_type: EventType::RaydiumAmmV4Withdraw,
|
||||
inner_instruction_parser: None,
|
||||
@@ -80,7 +79,7 @@ impl RaydiumAmmV4EventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_AMM_V4_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumAmmV4,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::WITHDRAW_PNL,
|
||||
event_type: EventType::RaydiumAmmV4WithdrawPnl,
|
||||
inner_instruction_parser: None,
|
||||
@@ -103,12 +102,6 @@ impl RaydiumAmmV4EventParser {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumAmmV4WithdrawPnlEvent {
|
||||
metadata,
|
||||
token_program: accounts[0],
|
||||
@@ -142,12 +135,6 @@ impl RaydiumAmmV4EventParser {
|
||||
}
|
||||
let amount = read_u64_le(data, 0)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumAmmV4WithdrawEvent {
|
||||
metadata,
|
||||
amount,
|
||||
@@ -191,12 +178,6 @@ impl RaydiumAmmV4EventParser {
|
||||
let init_pc_amount = read_u64_le(data, 9)?;
|
||||
let init_coin_amount = read_u64_le(data, 17)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumAmmV4Initialize2Event {
|
||||
metadata,
|
||||
nonce,
|
||||
@@ -241,12 +222,6 @@ impl RaydiumAmmV4EventParser {
|
||||
let max_pc_amount = read_u64_le(data, 8)?;
|
||||
let base_side = read_u64_le(data, 16)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumAmmV4DepositEvent {
|
||||
metadata,
|
||||
max_coin_amount,
|
||||
@@ -282,12 +257,6 @@ impl RaydiumAmmV4EventParser {
|
||||
let max_amount_in = read_u64_le(data, 0)?;
|
||||
let amount_out = read_u64_le(data, 8)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
let mut accounts = accounts.to_vec();
|
||||
if accounts.len() == 17 {
|
||||
// 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符
|
||||
@@ -335,12 +304,6 @@ impl RaydiumAmmV4EventParser {
|
||||
let amount_in = read_u64_le(data, 0)?;
|
||||
let minimum_amount_out = read_u64_le(data, 8)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
let mut accounts = accounts.to_vec();
|
||||
if accounts.len() == 17 {
|
||||
// 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符
|
||||
@@ -377,59 +340,4 @@ impl RaydiumAmmV4EventParser {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventParser for RaydiumAmmV4EventParser {
|
||||
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
|
||||
self.inner.inner_instruction_configs()
|
||||
}
|
||||
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &UiCompiledInstruction,
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &CompiledInstruction,
|
||||
accounts: &[Pubkey],
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
impl_event_parser_delegate!(RaydiumAmmV4EventParser);
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
use std::collections::HashMap;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use prost_types::Timestamp;
|
||||
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
|
||||
use solana_transaction_status::UiCompiledInstruction;
|
||||
|
||||
use crate::streaming::event_parser::{
|
||||
common::{
|
||||
read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata,
|
||||
EventType, ProtocolType,
|
||||
},
|
||||
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::raydium_clmm::{
|
||||
discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent,
|
||||
RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event,
|
||||
RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent,
|
||||
RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event,
|
||||
use crate::{
|
||||
impl_event_parser_delegate,
|
||||
streaming::event_parser::{
|
||||
common::{
|
||||
read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata,
|
||||
EventType, ProtocolType,
|
||||
},
|
||||
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::raydium_clmm::{
|
||||
discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent,
|
||||
RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event,
|
||||
RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent,
|
||||
RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -40,7 +39,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::SWAP,
|
||||
event_type: EventType::RaydiumClmmSwap,
|
||||
inner_instruction_parser: None,
|
||||
@@ -49,7 +48,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::SWAP_V2,
|
||||
event_type: EventType::RaydiumClmmSwapV2,
|
||||
inner_instruction_parser: None,
|
||||
@@ -58,7 +57,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::CLOSE_POSITION,
|
||||
event_type: EventType::RaydiumClmmClosePosition,
|
||||
inner_instruction_parser: None,
|
||||
@@ -67,7 +66,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::DECREASE_LIQUIDITY_V2,
|
||||
event_type: EventType::RaydiumClmmDecreaseLiquidityV2,
|
||||
inner_instruction_parser: None,
|
||||
@@ -76,7 +75,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::CREATE_POOL,
|
||||
event_type: EventType::RaydiumClmmCreatePool,
|
||||
inner_instruction_parser: None,
|
||||
@@ -85,7 +84,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::INCREASE_LIQUIDITY_V2,
|
||||
event_type: EventType::RaydiumClmmIncreaseLiquidityV2,
|
||||
inner_instruction_parser: None,
|
||||
@@ -94,7 +93,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::OPEN_POSITION_WITH_TOKEN_22_NFT,
|
||||
event_type: EventType::RaydiumClmmOpenPositionWithToken22Nft,
|
||||
inner_instruction_parser: None,
|
||||
@@ -103,7 +102,7 @@ impl RaydiumClmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CLMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumClmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::OPEN_POSITION_V2,
|
||||
event_type: EventType::RaydiumClmmOpenPositionV2,
|
||||
inner_instruction_parser: None,
|
||||
@@ -125,8 +124,6 @@ impl RaydiumClmmEventParser {
|
||||
if data.len() < 51 || accounts.len() < 22 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumClmmOpenPositionV2Event {
|
||||
metadata,
|
||||
tick_lower_index: read_i32_le(data, 0)?,
|
||||
@@ -173,8 +170,6 @@ impl RaydiumClmmEventParser {
|
||||
if data.len() < 51 || accounts.len() < 20 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumClmmOpenPositionWithToken22NftEvent {
|
||||
metadata,
|
||||
tick_lower_index: read_i32_le(data, 0)?,
|
||||
@@ -218,8 +213,6 @@ impl RaydiumClmmEventParser {
|
||||
if data.len() < 34 || accounts.len() < 15 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumClmmIncreaseLiquidityV2Event {
|
||||
metadata,
|
||||
liquidity: read_u128_le(data, 0)?,
|
||||
@@ -253,8 +246,6 @@ impl RaydiumClmmEventParser {
|
||||
if data.len() < 24 || accounts.len() < 13 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumClmmCreatePoolEvent {
|
||||
metadata,
|
||||
sqrt_price_x64: read_u128_le(data, 0)?,
|
||||
@@ -284,8 +275,6 @@ impl RaydiumClmmEventParser {
|
||||
if data.len() < 32 || accounts.len() < 16 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumClmmDecreaseLiquidityV2Event {
|
||||
metadata,
|
||||
liquidity: read_u128_le(data, 0)?,
|
||||
@@ -320,8 +309,6 @@ impl RaydiumClmmEventParser {
|
||||
if accounts.len() < 6 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumClmmClosePositionEvent {
|
||||
metadata,
|
||||
nft_owner: accounts[0],
|
||||
@@ -348,12 +335,6 @@ impl RaydiumClmmEventParser {
|
||||
let sqrt_price_limit_x64 = read_u128_le(data, 16)?;
|
||||
let is_base_input = read_u8_le(data, 32)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[2], accounts[3], accounts[4]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumClmmSwapEvent {
|
||||
metadata,
|
||||
amount,
|
||||
@@ -388,12 +369,6 @@ impl RaydiumClmmEventParser {
|
||||
let sqrt_price_limit_x64 = read_u128_le(data, 16)?;
|
||||
let is_base_input = read_u8_le(data, 32)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[2], accounts[3], accounts[4]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumClmmSwapV2Event {
|
||||
metadata,
|
||||
amount,
|
||||
@@ -418,59 +393,4 @@ impl RaydiumClmmEventParser {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventParser for RaydiumClmmEventParser {
|
||||
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
|
||||
self.inner.inner_instruction_configs()
|
||||
}
|
||||
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &UiCompiledInstruction,
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &CompiledInstruction,
|
||||
accounts: &[Pubkey],
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
impl_event_parser_delegate!(RaydiumClmmEventParser);
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
use std::collections::HashMap;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use prost_types::Timestamp;
|
||||
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
|
||||
use solana_transaction_status::UiCompiledInstruction;
|
||||
|
||||
use crate::streaming::event_parser::{
|
||||
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::raydium_cpmm::{
|
||||
discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent, RaydiumCpmmSwapEvent,
|
||||
RaydiumCpmmWithdrawEvent,
|
||||
use crate::{
|
||||
impl_event_parser_delegate,
|
||||
streaming::event_parser::{
|
||||
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
|
||||
core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent},
|
||||
protocols::raydium_cpmm::{
|
||||
discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent,
|
||||
RaydiumCpmmSwapEvent, RaydiumCpmmWithdrawEvent,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -35,7 +34,7 @@ impl RaydiumCpmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CPMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumCpmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::SWAP_BASE_IN,
|
||||
event_type: EventType::RaydiumCpmmSwapBaseInput,
|
||||
inner_instruction_parser: None,
|
||||
@@ -44,7 +43,7 @@ impl RaydiumCpmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CPMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumCpmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::SWAP_BASE_OUT,
|
||||
event_type: EventType::RaydiumCpmmSwapBaseOutput,
|
||||
inner_instruction_parser: None,
|
||||
@@ -53,7 +52,7 @@ impl RaydiumCpmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CPMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumCpmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::DEPOSIT,
|
||||
event_type: EventType::RaydiumCpmmDeposit,
|
||||
inner_instruction_parser: None,
|
||||
@@ -62,7 +61,7 @@ impl RaydiumCpmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CPMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumCpmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::INITIALIZE,
|
||||
event_type: EventType::RaydiumCpmmInitialize,
|
||||
inner_instruction_parser: None,
|
||||
@@ -71,7 +70,7 @@ impl RaydiumCpmmEventParser {
|
||||
GenericEventParseConfig {
|
||||
program_id: RAYDIUM_CPMM_PROGRAM_ID,
|
||||
protocol_type: ProtocolType::RaydiumCpmm,
|
||||
inner_instruction_discriminator: "",
|
||||
inner_instruction_discriminator: &[],
|
||||
instruction_discriminator: discriminators::WITHDRAW,
|
||||
event_type: EventType::RaydiumCpmmWithdraw,
|
||||
inner_instruction_parser: None,
|
||||
@@ -93,8 +92,6 @@ impl RaydiumCpmmEventParser {
|
||||
if data.len() < 24 || accounts.len() < 14 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumCpmmWithdrawEvent {
|
||||
metadata,
|
||||
lp_token_amount: read_u64_le(data, 0)?,
|
||||
@@ -126,8 +123,6 @@ impl RaydiumCpmmEventParser {
|
||||
if data.len() < 24 || accounts.len() < 20 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumCpmmInitializeEvent {
|
||||
metadata,
|
||||
init_amount0: read_u64_le(data, 0)?,
|
||||
@@ -165,8 +160,6 @@ impl RaydiumCpmmEventParser {
|
||||
if data.len() < 24 || accounts.len() < 13 {
|
||||
return None;
|
||||
}
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
|
||||
Some(Box::new(RaydiumCpmmDepositEvent {
|
||||
metadata,
|
||||
lp_token_amount: read_u64_le(data, 0)?,
|
||||
@@ -201,12 +194,6 @@ impl RaydiumCpmmEventParser {
|
||||
let amount_in = read_u64_le(data, 0)?;
|
||||
let minimum_amount_out = read_u64_le(data, 8)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumCpmmSwapEvent {
|
||||
metadata,
|
||||
amount_in,
|
||||
@@ -240,12 +227,6 @@ impl RaydiumCpmmEventParser {
|
||||
let max_amount_in = read_u64_le(data, 0)?;
|
||||
let amount_out = read_u64_le(data, 8)?;
|
||||
|
||||
let mut metadata = metadata;
|
||||
metadata.set_id(format!(
|
||||
"{}-{}-{}-{}",
|
||||
metadata.signature, accounts[3], accounts[10], accounts[11]
|
||||
));
|
||||
|
||||
Some(Box::new(RaydiumCpmmSwapEvent {
|
||||
metadata,
|
||||
max_amount_in,
|
||||
@@ -268,59 +249,4 @@ impl RaydiumCpmmEventParser {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventParser for RaydiumCpmmEventParser {
|
||||
fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec<GenericEventParseConfig>> {
|
||||
self.inner.inner_instruction_configs()
|
||||
}
|
||||
fn instruction_configs(&self) -> HashMap<Vec<u8>, Vec<GenericEventParseConfig>> {
|
||||
self.inner.instruction_configs()
|
||||
}
|
||||
fn parse_events_from_inner_instruction(
|
||||
&self,
|
||||
inner_instruction: &UiCompiledInstruction,
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_inner_instruction(
|
||||
inner_instruction,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_events_from_instruction(
|
||||
&self,
|
||||
instruction: &CompiledInstruction,
|
||||
accounts: &[Pubkey],
|
||||
signature: &str,
|
||||
slot: u64,
|
||||
block_time: Option<Timestamp>,
|
||||
program_received_time_ms: i64,
|
||||
index: String,
|
||||
) -> Vec<Box<dyn UnifiedEvent>> {
|
||||
self.inner.parse_events_from_instruction(
|
||||
instruction,
|
||||
accounts,
|
||||
signature,
|
||||
slot,
|
||||
block_time,
|
||||
program_received_time_ms,
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_handle(&self, program_id: &Pubkey) -> bool {
|
||||
self.inner.should_handle(program_id)
|
||||
}
|
||||
|
||||
fn supported_program_ids(&self) -> Vec<Pubkey> {
|
||||
self.inner.supported_program_ids()
|
||||
}
|
||||
}
|
||||
impl_event_parser_delegate!(RaydiumCpmmEventParser);
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use super::types::EventPretty;
|
||||
use crate::common::AnyResult;
|
||||
use crate::streaming::common::{
|
||||
EventBatchProcessor as EventBatchCollector, MetricsEventType, MetricsManager,
|
||||
StreamClientConfig as ClientConfig,
|
||||
};
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
|
||||
use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
|
||||
use crate::streaming::event_parser::EventParser;
|
||||
use crate::streaming::event_parser::{
|
||||
core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
|
||||
};
|
||||
|
||||
/// 事件处理器
|
||||
pub struct EventProcessor {
|
||||
pub(crate) metrics_manager: MetricsManager,
|
||||
pub(crate) config: ClientConfig,
|
||||
pub(crate) parser_cache: Arc<Mutex<Option<Arc<dyn EventParser>>>>,
|
||||
}
|
||||
|
||||
impl EventProcessor {
|
||||
/// 创建新的事件处理器
|
||||
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
|
||||
Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) }
|
||||
}
|
||||
|
||||
/// 获取或创建解析器,使用缓存机制避免重复创建
|
||||
fn get_or_create_parser(
|
||||
&self,
|
||||
protocols: Vec<Protocol>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
) -> Arc<dyn EventParser> {
|
||||
let mut cache = self.parser_cache.lock().unwrap();
|
||||
if let Some(cached_parser) = cache.clone() {
|
||||
return cached_parser.clone();
|
||||
}
|
||||
let parser: Arc<dyn EventParser> =
|
||||
Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone()));
|
||||
*cache = Some(parser.clone());
|
||||
parser
|
||||
}
|
||||
|
||||
/// 使用性能监控处理事件交易
|
||||
pub async fn process_event_transaction_with_metrics<F>(
|
||||
&self,
|
||||
event_pretty: EventPretty,
|
||||
callback: &F,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
protocols: Vec<Protocol>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
) -> AnyResult<()>
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
|
||||
{
|
||||
match event_pretty {
|
||||
EventPretty::Account(account_pretty) => {
|
||||
self.metrics_manager.add_account_process_count().await;
|
||||
let start_time = std::time::Instant::now();
|
||||
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
|
||||
let account_event = AccountEventParser::parse_account_event(
|
||||
protocols.clone(),
|
||||
account_pretty,
|
||||
program_received_time_ms,
|
||||
event_type_filter,
|
||||
);
|
||||
if let Some(event) = account_event {
|
||||
callback(event);
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
// 更新性能指标(如果启用)
|
||||
self.metrics_manager
|
||||
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
|
||||
.await;
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
|
||||
}
|
||||
}
|
||||
EventPretty::Transaction(transaction_pretty) => {
|
||||
self.metrics_manager.add_tx_process_count().await;
|
||||
let start_time = std::time::Instant::now();
|
||||
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
|
||||
let slot = transaction_pretty.slot;
|
||||
let signature = transaction_pretty.signature.to_string();
|
||||
|
||||
// 使用缓存获取解析器
|
||||
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
|
||||
let all_events = parser
|
||||
.parse_transaction(
|
||||
transaction_pretty.tx.clone(),
|
||||
&signature,
|
||||
Some(slot),
|
||||
transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
|
||||
seconds: ts.seconds,
|
||||
nanos: ts.nanos,
|
||||
}),
|
||||
program_received_time_ms,
|
||||
bot_wallet,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_e| vec![]);
|
||||
|
||||
// 保存事件数量用于日志记录
|
||||
let event_count = all_events.len();
|
||||
|
||||
// 批量处理事件
|
||||
if !all_events.is_empty() {
|
||||
for event in all_events {
|
||||
callback(event);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
|
||||
// 更新性能指标(如果启用)
|
||||
self.metrics_manager
|
||||
.update_metrics(MetricsEventType::Tx, event_count as u64, processing_time_ms)
|
||||
.await;
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
|
||||
}
|
||||
EventPretty::BlockMeta(block_meta_pretty) => {
|
||||
let start_time = std::time::Instant::now();
|
||||
self.metrics_manager.add_block_meta_process_count().await;
|
||||
let block_time_ms = block_meta_pretty
|
||||
.block_time
|
||||
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
|
||||
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
|
||||
let block_meta_event = CommonEventParser::generate_block_meta_event(
|
||||
block_meta_pretty.slot,
|
||||
&block_meta_pretty.block_hash,
|
||||
block_time_ms,
|
||||
);
|
||||
callback(block_meta_event);
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
// 更新性能指标(如果启用)
|
||||
self.metrics_manager
|
||||
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
|
||||
.await;
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 使用批处理处理事件交易
|
||||
pub async fn process_event_transaction_with_batch<F>(
|
||||
&self,
|
||||
event_pretty: EventPretty,
|
||||
batch_processor: &mut EventBatchCollector<F>,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
protocols: Vec<Protocol>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
) -> AnyResult<()>
|
||||
where
|
||||
F: Fn(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
|
||||
{
|
||||
match event_pretty {
|
||||
EventPretty::Account(account_pretty) => {
|
||||
self.metrics_manager.add_account_process_count().await;
|
||||
let start_time = std::time::Instant::now();
|
||||
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
|
||||
let account_event = AccountEventParser::parse_account_event(
|
||||
protocols.clone(),
|
||||
account_pretty,
|
||||
program_received_time_ms,
|
||||
event_type_filter,
|
||||
);
|
||||
if let Some(event) = account_event {
|
||||
(batch_processor.callback)(vec![event]);
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
// 实际调用性能指标更新
|
||||
self.metrics_manager
|
||||
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
|
||||
.await;
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
|
||||
}
|
||||
}
|
||||
EventPretty::Transaction(transaction_pretty) => {
|
||||
self.metrics_manager.add_tx_process_count().await;
|
||||
let start_time = std::time::Instant::now();
|
||||
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
|
||||
let slot = transaction_pretty.slot;
|
||||
let signature = transaction_pretty.signature.to_string();
|
||||
|
||||
// 使用缓存获取解析器
|
||||
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
|
||||
let result = parser
|
||||
.parse_transaction(
|
||||
transaction_pretty.tx.clone(),
|
||||
&signature,
|
||||
Some(slot),
|
||||
transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
|
||||
seconds: ts.seconds,
|
||||
nanos: ts.nanos,
|
||||
}),
|
||||
program_received_time_ms,
|
||||
bot_wallet,
|
||||
)
|
||||
.await;
|
||||
|
||||
// 处理解析结果并使用批处理器
|
||||
let total_events = match result {
|
||||
Ok(events) => {
|
||||
let event_count = events.len();
|
||||
if !events.is_empty() {
|
||||
log::debug!("Parsed {} events", event_count);
|
||||
log::debug!("Adding {} events to batch processor", event_count);
|
||||
for event in events {
|
||||
if self.config.batch.enabled {
|
||||
batch_processor.add_event(event);
|
||||
} else {
|
||||
// 如果批处理被禁用,直接调用回调
|
||||
// 这里需要将单个事件包装成Vec来调用批处理回调
|
||||
let single_event_batch = vec![event];
|
||||
(batch_processor.callback)(single_event_batch);
|
||||
}
|
||||
}
|
||||
}
|
||||
event_count
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("Failed to parse transaction: {:?}", e);
|
||||
0
|
||||
}
|
||||
};
|
||||
|
||||
// 添加调试信息
|
||||
if total_events > 0 {
|
||||
log::debug!(
|
||||
"Total events parsed: {} for transaction {}",
|
||||
total_events,
|
||||
signature
|
||||
);
|
||||
}
|
||||
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
|
||||
// 实际调用性能指标更新
|
||||
self.metrics_manager
|
||||
.update_metrics(MetricsEventType::Tx, total_events as u64, processing_time_ms)
|
||||
.await;
|
||||
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, total_events);
|
||||
}
|
||||
EventPretty::BlockMeta(block_meta_pretty) => {
|
||||
let start_time = std::time::Instant::now();
|
||||
self.metrics_manager.add_block_meta_process_count().await;
|
||||
let block_time_ms = block_meta_pretty
|
||||
.block_time
|
||||
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
|
||||
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
|
||||
let block_meta_event = CommonEventParser::generate_block_meta_event(
|
||||
block_meta_pretty.slot,
|
||||
&block_meta_pretty.block_hash,
|
||||
block_time_ms,
|
||||
);
|
||||
(batch_processor.callback)(vec![block_meta_event]);
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
// 更新性能指标(如果启用)
|
||||
self.metrics_manager
|
||||
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
|
||||
.await;
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,15 @@
|
||||
// gRPC 相关模块
|
||||
pub mod connection;
|
||||
pub mod types;
|
||||
pub mod subscription;
|
||||
pub mod stream_handler;
|
||||
pub mod event_processor;
|
||||
pub mod types;
|
||||
|
||||
// 重新导出主要类型
|
||||
pub use connection::*;
|
||||
pub use types::*;
|
||||
pub use subscription::*;
|
||||
pub use stream_handler::*;
|
||||
pub use event_processor::*;
|
||||
pub use types::*;
|
||||
|
||||
// 从公用模块重新导出
|
||||
pub use crate::streaming::common::{
|
||||
BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsManager, PerformanceMetrics,
|
||||
StreamClientConfig as ClientConfig,
|
||||
PerformanceMetrics,
|
||||
MetricsManager,
|
||||
EventBatchProcessor as EventBatchCollector,
|
||||
BackpressureStrategy,
|
||||
BatchConfig,
|
||||
BackpressureConfig,
|
||||
ConnectionConfig,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
use chrono::Local;
|
||||
use futures::{channel::mpsc, sink::Sink, SinkExt};
|
||||
use yellowstone_grpc_proto::geyser::{
|
||||
subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, SubscribeUpdate,
|
||||
};
|
||||
|
||||
use super::types::{BlockMetaPretty, EventPretty, TransactionPretty};
|
||||
use crate::common::AnyResult;
|
||||
use crate::streaming::common::BackpressureStrategy;
|
||||
use crate::streaming::grpc::AccountPretty;
|
||||
|
||||
/// 流消息处理器
|
||||
pub struct StreamHandler;
|
||||
|
||||
impl StreamHandler {
|
||||
/// 处理单个流消息
|
||||
pub async fn handle_stream_message(
|
||||
msg: SubscribeUpdate,
|
||||
tx: &mut mpsc::Sender<EventPretty>,
|
||||
subscribe_tx: &mut (impl Sink<SubscribeRequest, Error = mpsc::SendError> + Unpin),
|
||||
backpressure_strategy: BackpressureStrategy,
|
||||
) -> AnyResult<()> {
|
||||
let created_at = msg.created_at;
|
||||
match msg.update_oneof {
|
||||
Some(UpdateOneof::Account(account)) => {
|
||||
let account_pretty = AccountPretty::from(account);
|
||||
log::debug!("Received account: {:?}", account_pretty);
|
||||
Self::handle_backpressure(
|
||||
tx,
|
||||
EventPretty::Account(account_pretty),
|
||||
backpressure_strategy,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Some(UpdateOneof::BlockMeta(sut)) => {
|
||||
let block_meta_pretty = BlockMetaPretty::from((sut, created_at));
|
||||
log::debug!("Received block meta: {:?}", block_meta_pretty);
|
||||
Self::handle_backpressure(
|
||||
tx,
|
||||
EventPretty::BlockMeta(block_meta_pretty),
|
||||
backpressure_strategy,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Some(UpdateOneof::Transaction(sut)) => {
|
||||
let transaction_pretty = TransactionPretty::from((sut, created_at));
|
||||
log::debug!(
|
||||
"Received transaction: {} at slot {}",
|
||||
transaction_pretty.signature,
|
||||
transaction_pretty.slot
|
||||
);
|
||||
|
||||
// 根据背压策略处理发送
|
||||
Self::handle_backpressure(
|
||||
tx,
|
||||
EventPretty::Transaction(transaction_pretty),
|
||||
backpressure_strategy,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Some(UpdateOneof::Ping(_)) => {
|
||||
subscribe_tx
|
||||
.send(SubscribeRequest {
|
||||
ping: Some(SubscribeRequestPing { id: 1 }),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
log::debug!("service is ping: {}", Local::now());
|
||||
}
|
||||
Some(UpdateOneof::Pong(_)) => {
|
||||
log::debug!("service is pong: {}", Local::now());
|
||||
}
|
||||
_ => {
|
||||
log::debug!("Received other message type");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 处理背压策略
|
||||
async fn handle_backpressure(
|
||||
tx: &mut mpsc::Sender<EventPretty>,
|
||||
event_pretty: EventPretty,
|
||||
backpressure_strategy: BackpressureStrategy,
|
||||
) -> AnyResult<()> {
|
||||
match backpressure_strategy {
|
||||
BackpressureStrategy::Block => {
|
||||
// 阻塞等待,直到有空间
|
||||
if let Err(e) = tx.send(event_pretty).await {
|
||||
log::error!("Failed to send transaction to channel: {:?}", e);
|
||||
return Err(anyhow::anyhow!("Channel send failed: {:?}", e));
|
||||
}
|
||||
}
|
||||
BackpressureStrategy::Drop => {
|
||||
// 尝试发送,如果失败则丢弃
|
||||
if let Err(e) = tx.try_send(event_pretty) {
|
||||
if e.is_full() {
|
||||
log::warn!("Channel is full, dropping transaction");
|
||||
} else {
|
||||
log::error!("Channel is closed: {:?}", e);
|
||||
return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
BackpressureStrategy::Retry { max_attempts, wait_ms } => {
|
||||
// 重试有限次数
|
||||
let mut retry_count = 0;
|
||||
loop {
|
||||
match tx.try_send(event_pretty.clone()) {
|
||||
Ok(_) => break,
|
||||
Err(e) => {
|
||||
if e.is_full() {
|
||||
retry_count += 1;
|
||||
if retry_count >= max_attempts {
|
||||
log::warn!(
|
||||
"Channel is full after {} attempts, dropping transaction",
|
||||
retry_count
|
||||
);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms))
|
||||
.await;
|
||||
} else {
|
||||
log::error!("Channel is closed: {:?}", e);
|
||||
return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ use crate::common::AnyResult;
|
||||
use crate::streaming::common::StreamClientConfig as ClientConfig;
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
|
||||
/// 订阅管理器
|
||||
/// Subscription manager
|
||||
#[derive(Clone)]
|
||||
pub struct SubscriptionManager {
|
||||
endpoint: String,
|
||||
@@ -23,12 +23,12 @@ pub struct SubscriptionManager {
|
||||
}
|
||||
|
||||
impl SubscriptionManager {
|
||||
/// 创建新的订阅管理器
|
||||
/// Create a new subscription manager
|
||||
pub fn new(endpoint: String, x_token: Option<String>, config: ClientConfig) -> Self {
|
||||
Self { endpoint, x_token, config }
|
||||
}
|
||||
|
||||
/// 创建 gRPC 连接
|
||||
/// Create gRPC connection
|
||||
pub async fn connect(&self) -> AnyResult<GeyserGrpcClient<impl Interceptor>> {
|
||||
let builder = GeyserGrpcClient::build_from_shared(self.endpoint.clone())?
|
||||
.x_token(self.x_token.clone())?
|
||||
@@ -39,19 +39,20 @@ impl SubscriptionManager {
|
||||
Ok(builder.connect().await?)
|
||||
}
|
||||
|
||||
/// 创建订阅请求并返回流
|
||||
/// Create subscription request and return stream
|
||||
pub async fn subscribe_with_request(
|
||||
&self,
|
||||
transactions: Option<TransactionsFilterMap>,
|
||||
accounts: Option<AccountsFilterMap>,
|
||||
commitment: Option<CommitmentLevel>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
event_type_filter: Option<&EventTypeFilter>,
|
||||
) -> AnyResult<(
|
||||
impl Sink<SubscribeRequest, Error = mpsc::SendError>,
|
||||
impl Stream<Item = Result<SubscribeUpdate, Status>>,
|
||||
SubscribeRequest,
|
||||
)> {
|
||||
let blocks_meta = if event_type_filter.is_some()
|
||||
&& event_type_filter.as_ref().unwrap().include_block_event()
|
||||
&& event_type_filter.unwrap().include_block_event()
|
||||
{
|
||||
hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} }
|
||||
} else if event_type_filter.is_none() {
|
||||
@@ -71,22 +72,22 @@ impl SubscriptionManager {
|
||||
..Default::default()
|
||||
};
|
||||
let mut client = self.connect().await?;
|
||||
let (sink, stream) = client.subscribe_with_request(Some(subscribe_request)).await?;
|
||||
Ok((sink, stream))
|
||||
let (sink, stream) = client.subscribe_with_request(Some(subscribe_request.clone())).await?;
|
||||
Ok((sink, stream, subscribe_request))
|
||||
}
|
||||
|
||||
/// 创建账户订阅请求并返回流
|
||||
/// Create account subscription request and return stream
|
||||
pub fn subscribe_with_account_request(
|
||||
&self,
|
||||
account: Vec<String>,
|
||||
owner: Vec<String>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
event_type_filter: Option<&EventTypeFilter>,
|
||||
) -> Option<AccountsFilterMap> {
|
||||
if account.len() == 0 && owner.len() == 0 {
|
||||
return None;
|
||||
}
|
||||
if event_type_filter.is_some()
|
||||
&& !event_type_filter.as_ref().unwrap().include_account_event()
|
||||
&& !event_type_filter.unwrap().include_account_event()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
@@ -103,16 +104,16 @@ impl SubscriptionManager {
|
||||
Some(accounts)
|
||||
}
|
||||
|
||||
/// 生成订阅请求过滤器
|
||||
/// Generate subscription request filter
|
||||
pub fn get_subscribe_request_filter(
|
||||
&self,
|
||||
account_include: Vec<String>,
|
||||
account_exclude: Vec<String>,
|
||||
account_required: Vec<String>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
event_type_filter: Option<&EventTypeFilter>,
|
||||
) -> Option<TransactionsFilterMap> {
|
||||
if event_type_filter.is_some()
|
||||
&& !event_type_filter.as_ref().unwrap().include_transaction_event()
|
||||
&& !event_type_filter.unwrap().include_transaction_event()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
@@ -131,7 +132,7 @@ impl SubscriptionManager {
|
||||
Some(transactions)
|
||||
}
|
||||
|
||||
/// 获取配置
|
||||
/// Get configuration
|
||||
pub fn get_config(&self) -> &ClientConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
+32
-22
@@ -1,5 +1,5 @@
|
||||
use solana_sdk::signature::Signature;
|
||||
use solana_transaction_status::{EncodedTransactionWithStatusMeta, UiTransactionEncoding};
|
||||
use solana_sdk::{pubkey::Pubkey, signature::Signature};
|
||||
use solana_transaction_status::TransactionWithStatusMeta;
|
||||
use std::{collections::HashMap, fmt};
|
||||
use yellowstone_grpc_proto::{
|
||||
geyser::{
|
||||
@@ -22,13 +22,14 @@ pub enum EventPretty {
|
||||
#[derive(Clone)]
|
||||
pub struct AccountPretty {
|
||||
pub slot: u64,
|
||||
pub signature: String,
|
||||
pub pubkey: String,
|
||||
pub signature: Signature,
|
||||
pub pubkey: Pubkey,
|
||||
pub executable: bool,
|
||||
pub lamports: u64,
|
||||
pub owner: String,
|
||||
pub owner: Pubkey,
|
||||
pub rent_epoch: u64,
|
||||
pub data: Vec<u8>,
|
||||
pub program_received_time_us: i64,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AccountPretty {
|
||||
@@ -51,6 +52,7 @@ pub struct BlockMetaPretty {
|
||||
pub slot: u64,
|
||||
pub block_hash: String,
|
||||
pub block_time: Option<Timestamp>,
|
||||
pub program_received_time_us: i64,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BlockMetaPretty {
|
||||
@@ -59,6 +61,7 @@ impl fmt::Debug for BlockMetaPretty {
|
||||
.field("slot", &self.slot)
|
||||
.field("block_hash", &self.block_hash)
|
||||
.field("block_time", &self.block_time)
|
||||
.field("program_received_time_us", &self.program_received_time_us)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -66,28 +69,23 @@ impl fmt::Debug for BlockMetaPretty {
|
||||
#[derive(Clone)]
|
||||
pub struct TransactionPretty {
|
||||
pub slot: u64,
|
||||
pub transaction_index: Option<u64>, // 新增:交易在slot中的索引
|
||||
pub block_hash: String,
|
||||
pub block_time: Option<Timestamp>,
|
||||
pub signature: Signature,
|
||||
pub is_vote: bool,
|
||||
pub tx: EncodedTransactionWithStatusMeta,
|
||||
pub tx: TransactionWithStatusMeta,
|
||||
pub program_received_time_us: i64,
|
||||
}
|
||||
|
||||
impl fmt::Debug for TransactionPretty {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
struct TxWrap<'a>(&'a EncodedTransactionWithStatusMeta);
|
||||
impl<'a> fmt::Debug for TxWrap<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let serialized = serde_json::to_string(self.0).expect("failed to serialize");
|
||||
fmt::Display::fmt(&serialized, f)
|
||||
}
|
||||
}
|
||||
|
||||
f.debug_struct("TransactionPretty")
|
||||
.field("slot", &self.slot)
|
||||
.field("transaction_index", &self.transaction_index)
|
||||
.field("signature", &self.signature)
|
||||
.field("is_vote", &self.is_vote)
|
||||
.field("tx", &TxWrap(&self.tx))
|
||||
.field("program_received_time_us", &self.program_received_time_us)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -97,13 +95,18 @@ impl From<SubscribeUpdateAccount> for AccountPretty {
|
||||
let account_info = account.account.unwrap();
|
||||
Self {
|
||||
slot: account.slot,
|
||||
signature: bs58::encode(&account_info.txn_signature.unwrap_or_default()).into_string(),
|
||||
pubkey: bs58::encode(&account_info.pubkey).into_string(),
|
||||
signature: if let Some(txn_signature) = account_info.txn_signature {
|
||||
Signature::try_from(txn_signature.as_slice()).expect("valid signature")
|
||||
} else {
|
||||
Signature::default()
|
||||
},
|
||||
pubkey: Pubkey::try_from(account_info.pubkey.as_slice()).expect("valid pubkey"),
|
||||
executable: account_info.executable,
|
||||
lamports: account_info.lamports,
|
||||
owner: bs58::encode(&account_info.owner).into_string(),
|
||||
owner: Pubkey::try_from(account_info.owner.as_slice()).expect("valid pubkey"),
|
||||
rent_epoch: account_info.rent_epoch,
|
||||
data: account_info.data,
|
||||
program_received_time_us: chrono::Utc::now().timestamp_micros(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -115,7 +118,12 @@ impl From<(SubscribeUpdateBlockMeta, Option<Timestamp>)> for BlockMetaPretty {
|
||||
Option<Timestamp>,
|
||||
),
|
||||
) -> Self {
|
||||
Self { block_hash: blockhash.to_string(), block_time, slot }
|
||||
Self {
|
||||
block_hash: blockhash.to_string(),
|
||||
block_time,
|
||||
slot,
|
||||
program_received_time_us: chrono::Utc::now().timestamp_micros(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,16 +135,18 @@ impl From<(SubscribeUpdateTransaction, Option<Timestamp>)> for TransactionPretty
|
||||
),
|
||||
) -> Self {
|
||||
let tx = transaction.expect("should be defined");
|
||||
// 根据用户说明,交易索引在 transaction.index 中
|
||||
let transaction_index = tx.index;
|
||||
Self {
|
||||
slot,
|
||||
transaction_index: Some(transaction_index), // 提取交易索引
|
||||
block_time,
|
||||
block_hash: "".to_string(),
|
||||
signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"),
|
||||
is_vote: tx.is_vote,
|
||||
tx: yellowstone_grpc_proto::convert_from::create_tx_with_meta(tx)
|
||||
.expect("valid tx with meta")
|
||||
.encode(UiTransactionEncoding::Base64, Some(u8::MAX), true)
|
||||
.expect("failed to encode"),
|
||||
.expect("valid tx with meta"),
|
||||
program_received_time_us: chrono::Utc::now().timestamp_micros(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::RwLock;
|
||||
use tokio::sync::Mutex;
|
||||
use tonic::transport::Channel;
|
||||
|
||||
@@ -13,7 +14,7 @@ use crate::streaming::common::{
|
||||
pub struct ShredStreamGrpc {
|
||||
pub shredstream_client: Arc<ShredstreamProxyClient<Channel>>,
|
||||
pub config: StreamClientConfig,
|
||||
pub metrics: Arc<Mutex<PerformanceMetrics>>,
|
||||
pub metrics: Arc<RwLock<PerformanceMetrics>>,
|
||||
pub metrics_manager: MetricsManager,
|
||||
pub subscription_handle: Arc<Mutex<Option<SubscriptionHandle>>>,
|
||||
}
|
||||
@@ -27,31 +28,38 @@ impl ShredStreamGrpc {
|
||||
/// 创建客户端,使用自定义配置
|
||||
pub async fn new_with_config(endpoint: String, config: StreamClientConfig) -> AnyResult<Self> {
|
||||
let shredstream_client = ShredstreamProxyClient::connect(endpoint.clone()).await?;
|
||||
let metrics = Arc::new(Mutex::new(PerformanceMetrics::new()));
|
||||
let config_arc = Arc::new(config.clone());
|
||||
let metrics = Arc::new(RwLock::new(PerformanceMetrics::new()));
|
||||
|
||||
let metrics_manager =
|
||||
MetricsManager::new(metrics.clone(), config_arc, "ShredStream".to_string());
|
||||
let metrics_manager = MetricsManager::new(config.enable_metrics, "ShredStream".to_string());
|
||||
|
||||
Ok(Self {
|
||||
shredstream_client: Arc::new(shredstream_client),
|
||||
config,
|
||||
metrics,
|
||||
metrics: metrics.clone(),
|
||||
metrics_manager,
|
||||
subscription_handle: Arc::new(Mutex::new(None)),
|
||||
})
|
||||
}
|
||||
|
||||
/// 创建高性能客户端(适合高并发场景)
|
||||
pub async fn new_high_performance(endpoint: String) -> AnyResult<Self> {
|
||||
Self::new_with_config(endpoint, StreamClientConfig::high_performance()).await
|
||||
/// Creates a new ShredStreamClient with high-throughput configuration.
|
||||
///
|
||||
/// This is a convenience method that creates a client optimized for high-concurrency scenarios
|
||||
/// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()`
|
||||
/// for detailed configuration information.
|
||||
pub async fn new_high_throughput(endpoint: String) -> AnyResult<Self> {
|
||||
Self::new_with_config(endpoint, StreamClientConfig::high_throughput()).await
|
||||
}
|
||||
|
||||
/// 创建低延迟客户端(适合实时场景)
|
||||
/// Creates a new ShredStreamClient with low-latency configuration.
|
||||
///
|
||||
/// This is a convenience method that creates a client optimized for real-time scenarios
|
||||
/// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()`
|
||||
/// for detailed configuration information.
|
||||
pub async fn new_low_latency(endpoint: String) -> AnyResult<Self> {
|
||||
Self::new_with_config(endpoint, StreamClientConfig::low_latency()).await
|
||||
}
|
||||
|
||||
|
||||
/// 获取当前配置
|
||||
pub fn get_config(&self) -> &StreamClientConfig {
|
||||
&self.config
|
||||
@@ -63,8 +71,8 @@ impl ShredStreamGrpc {
|
||||
}
|
||||
|
||||
/// 获取性能指标
|
||||
pub async fn get_metrics(&self) -> PerformanceMetrics {
|
||||
self.metrics_manager.get_metrics().await
|
||||
pub fn get_metrics(&self) -> PerformanceMetrics {
|
||||
self.metrics_manager.get_metrics()
|
||||
}
|
||||
|
||||
/// 启用或禁用性能监控
|
||||
@@ -73,8 +81,8 @@ impl ShredStreamGrpc {
|
||||
}
|
||||
|
||||
/// 打印性能指标
|
||||
pub async fn print_metrics(&self) {
|
||||
self.metrics_manager.print_metrics().await;
|
||||
pub fn print_metrics(&self) {
|
||||
self.metrics_manager.print_metrics();
|
||||
}
|
||||
|
||||
/// 启动自动性能监控任务
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::common::AnyResult;
|
||||
use crate::streaming::common::{
|
||||
EventBatchProcessor, MetricsEventType, MetricsManager, StreamClientConfig,
|
||||
};
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use crate::streaming::event_parser::protocols::MutilEventParser;
|
||||
use crate::streaming::event_parser::{EventParser, Protocol, UnifiedEvent};
|
||||
use crate::streaming::shred::TransactionWithSlot;
|
||||
|
||||
/// ShredStream 事件处理器
|
||||
pub struct ShredEventProcessor {
|
||||
pub(crate) metrics_manager: MetricsManager,
|
||||
pub(crate) config: StreamClientConfig,
|
||||
pub(crate) parser_cache: Arc<Mutex<Option<Arc<dyn EventParser>>>>,
|
||||
}
|
||||
|
||||
impl ShredEventProcessor {
|
||||
/// 创建新的事件处理器
|
||||
pub fn new(metrics_manager: MetricsManager, config: StreamClientConfig) -> Self {
|
||||
Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) }
|
||||
}
|
||||
|
||||
/// 获取或创建解析器,使用缓存机制避免重复创建
|
||||
fn get_or_create_parser(
|
||||
&self,
|
||||
protocols: Vec<Protocol>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
) -> Arc<dyn EventParser> {
|
||||
let mut cache = self.parser_cache.lock().unwrap();
|
||||
if let Some(cached_parser) = cache.clone() {
|
||||
return cached_parser.clone();
|
||||
}
|
||||
let parser: Arc<dyn EventParser> =
|
||||
Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone()));
|
||||
*cache = Some(parser.clone());
|
||||
parser
|
||||
}
|
||||
|
||||
/// 即时处理单个交易
|
||||
pub async fn process_transaction_immediate<F>(
|
||||
&self,
|
||||
transaction_with_slot: TransactionWithSlot,
|
||||
protocols: Vec<Protocol>,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
callback: &F,
|
||||
) -> AnyResult<()>
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
|
||||
{
|
||||
let start_time = std::time::Instant::now();
|
||||
self.metrics_manager.add_tx_process_count().await;
|
||||
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
|
||||
let slot = transaction_with_slot.slot;
|
||||
let versioned_tx = transaction_with_slot.transaction;
|
||||
let signature = versioned_tx.signatures[0];
|
||||
|
||||
// 获取缓存的解析器
|
||||
let parser = self.get_or_create_parser(protocols, event_type_filter);
|
||||
|
||||
let all_events = parser
|
||||
.parse_versioned_transaction(
|
||||
&versioned_tx,
|
||||
&signature.to_string(),
|
||||
Some(slot),
|
||||
None,
|
||||
program_received_time_ms,
|
||||
bot_wallet,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_e| vec![]);
|
||||
|
||||
// 保存事件数量用于日志记录
|
||||
let event_count = all_events.len();
|
||||
|
||||
// 即时处理事件
|
||||
for event in all_events {
|
||||
callback(event);
|
||||
}
|
||||
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
|
||||
// 实际调用性能指标更新
|
||||
self.update_metrics(event_count as u64, processing_time_ms).await;
|
||||
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 批处理模式处理单个交易
|
||||
pub async fn process_transaction_with_batch<F>(
|
||||
&self,
|
||||
transaction_with_slot: TransactionWithSlot,
|
||||
protocols: Vec<Protocol>,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
batch_processor: &mut EventBatchProcessor<F>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
) -> AnyResult<()>
|
||||
where
|
||||
F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
|
||||
{
|
||||
let start_time = std::time::Instant::now();
|
||||
self.metrics_manager.add_tx_process_count().await;
|
||||
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
|
||||
let slot = transaction_with_slot.slot;
|
||||
let versioned_tx = transaction_with_slot.transaction;
|
||||
let signature = versioned_tx.signatures[0];
|
||||
|
||||
// 获取缓存的解析器
|
||||
let parser = self.get_or_create_parser(protocols, event_type_filter);
|
||||
|
||||
let all_events = parser
|
||||
.parse_versioned_transaction(
|
||||
&versioned_tx,
|
||||
&signature.to_string(),
|
||||
Some(slot),
|
||||
None,
|
||||
program_received_time_ms,
|
||||
bot_wallet,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_e| vec![]);
|
||||
|
||||
// 保存事件数量用于日志记录
|
||||
let event_count = all_events.len();
|
||||
|
||||
// 使用批处理器处理事件
|
||||
for event in all_events {
|
||||
batch_processor.add_event(event);
|
||||
}
|
||||
|
||||
// 更新性能指标
|
||||
let processing_time = start_time.elapsed();
|
||||
let processing_time_ms = processing_time.as_millis() as f64;
|
||||
|
||||
// 实际调用性能指标更新
|
||||
self.update_metrics(event_count as u64, processing_time_ms).await;
|
||||
|
||||
// 记录慢处理操作
|
||||
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 更新性能指标
|
||||
async fn update_metrics(&self, events_processed: u64, processing_time_ms: f64) {
|
||||
// 使用统一的指标管理器,这里假设 ShredStream 主要处理交易事件
|
||||
self.metrics_manager
|
||||
.update_metrics(MetricsEventType::Tx, events_processed, processing_time_ms)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// 实现 Clone trait 以支持模块间共享
|
||||
impl Clone for ShredEventProcessor {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
metrics_manager: self.metrics_manager.clone(),
|
||||
config: self.config.clone(),
|
||||
parser_cache: self.parser_cache.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,13 @@
|
||||
// ShredStream 相关模块
|
||||
pub mod connection;
|
||||
pub mod types;
|
||||
pub mod stream_handler;
|
||||
pub mod event_processor;
|
||||
|
||||
// 重新导出主要类型
|
||||
pub use connection::*;
|
||||
pub use types::*;
|
||||
pub use stream_handler::*;
|
||||
pub use event_processor::*;
|
||||
|
||||
// 从公用模块重新导出
|
||||
pub use crate::streaming::common::{
|
||||
BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, EventBatchProcessor,
|
||||
MetricsEventType, MetricsManager, PerformanceMetrics, StreamClientConfig,
|
||||
BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsEventType, MetricsManager,
|
||||
PerformanceMetrics, StreamClientConfig,
|
||||
};
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
use futures::{channel::mpsc, StreamExt};
|
||||
use log::error;
|
||||
use solana_entry::entry::Entry;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use crate::common::AnyResult;
|
||||
use crate::protos::shredstream::{
|
||||
shredstream_proxy_client::ShredstreamProxyClient, SubscribeEntriesRequest,
|
||||
};
|
||||
use crate::streaming::shred::TransactionWithSlot;
|
||||
|
||||
/// ShredStream 流处理器
|
||||
pub struct ShredStreamHandler;
|
||||
|
||||
impl ShredStreamHandler {
|
||||
/// 启动 ShredStream 流处理任务
|
||||
///
|
||||
/// # 参数
|
||||
/// * `client` - ShredStream 客户端
|
||||
/// * `tx` - 事务发送通道
|
||||
/// * `channel_size` - 通道缓冲区大小
|
||||
///
|
||||
/// # 返回值
|
||||
/// 返回 ShredStream 流处理任务句柄和事务接收通道
|
||||
pub async fn start_stream_processing(
|
||||
mut client: ShredstreamProxyClient<tonic::transport::Channel>,
|
||||
channel_size: usize,
|
||||
) -> AnyResult<(JoinHandle<()>, mpsc::Receiver<TransactionWithSlot>)> {
|
||||
let request = tonic::Request::new(SubscribeEntriesRequest {});
|
||||
let stream = client.subscribe_entries(request).await?.into_inner();
|
||||
let (tx, rx) = mpsc::channel::<TransactionWithSlot>(channel_size);
|
||||
|
||||
let stream_task = tokio::spawn(Self::process_stream_messages(stream, tx));
|
||||
|
||||
Ok((stream_task, rx))
|
||||
}
|
||||
|
||||
/// 处理流消息
|
||||
///
|
||||
/// # 参数
|
||||
/// * `stream` - ShredStream 数据流
|
||||
/// * `tx` - 事务发送通道
|
||||
async fn process_stream_messages(
|
||||
mut stream: tonic::codec::Streaming<crate::protos::shredstream::Entry>,
|
||||
mut tx: mpsc::Sender<TransactionWithSlot>,
|
||||
) {
|
||||
while let Some(message) = stream.next().await {
|
||||
match message {
|
||||
Ok(msg) => {
|
||||
if let Err(e) = Self::handle_stream_message(msg, &mut tx).await {
|
||||
error!("Error handling stream message: {e:?}");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
error!("Stream error: {error:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 处理单个流消息
|
||||
///
|
||||
/// # 参数
|
||||
/// * `msg` - ShredStream 消息
|
||||
/// * `tx` - 事务发送通道
|
||||
async fn handle_stream_message(
|
||||
msg: crate::protos::shredstream::Entry,
|
||||
tx: &mut mpsc::Sender<TransactionWithSlot>,
|
||||
) -> AnyResult<()> {
|
||||
if let Ok(entries) = bincode::deserialize::<Vec<Entry>>(&msg.entries) {
|
||||
for entry in entries {
|
||||
for transaction in entry.transactions {
|
||||
let transaction_with_slot =
|
||||
TransactionWithSlot::new(transaction.clone(), msg.slot);
|
||||
|
||||
if let Err(e) = tx.try_send(transaction_with_slot) {
|
||||
// 如果通道满了,记录警告但不中断处理
|
||||
if e.is_full() {
|
||||
log::warn!("Transaction channel is full, dropping transaction");
|
||||
} else {
|
||||
// 通道已关闭,返回错误
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 启动事务处理任务
|
||||
///
|
||||
/// # 参数
|
||||
/// * `rx` - 事务接收通道
|
||||
/// * `processor` - 事务处理器
|
||||
pub fn start_transaction_processing<F>(
|
||||
mut rx: mpsc::Receiver<TransactionWithSlot>,
|
||||
processor: F,
|
||||
) -> JoinHandle<()>
|
||||
where
|
||||
F: Fn(TransactionWithSlot) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
while let Some(transaction_with_slot) = rx.next().await {
|
||||
if let Err(e) = processor(transaction_with_slot) {
|
||||
error!("Error processing transaction: {e:?}");
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,16 +5,16 @@ use solana_sdk::transaction::VersionedTransaction;
|
||||
pub struct TransactionWithSlot {
|
||||
pub transaction: VersionedTransaction,
|
||||
pub slot: u64,
|
||||
pub program_received_time_us: i64,
|
||||
}
|
||||
|
||||
impl TransactionWithSlot {
|
||||
/// 创建新的带槽位的交易
|
||||
pub fn new(transaction: VersionedTransaction, slot: u64) -> Self {
|
||||
Self { transaction, slot }
|
||||
}
|
||||
|
||||
/// 获取交易签名
|
||||
pub fn signature(&self) -> String {
|
||||
self.transaction.signatures[0].to_string()
|
||||
pub fn new(
|
||||
transaction: VersionedTransaction,
|
||||
slot: u64,
|
||||
program_received_time_us: i64,
|
||||
) -> Self {
|
||||
Self { transaction, slot, program_received_time_us }
|
||||
}
|
||||
}
|
||||
|
||||
+57
-110
@@ -1,10 +1,16 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::StreamExt;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
|
||||
use crate::common::AnyResult;
|
||||
use crate::streaming::common::{EventBatchProcessor, SubscriptionHandle};
|
||||
use crate::protos::shredstream::SubscribeEntriesRequest;
|
||||
use crate::streaming::common::{EventProcessor, SubscriptionHandle};
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use crate::streaming::event_parser::{Protocol, UnifiedEvent};
|
||||
use crate::streaming::shred::{ShredEventProcessor, ShredStreamHandler, TransactionWithSlot};
|
||||
use crate::streaming::shred::TransactionWithSlot;
|
||||
use log::error;
|
||||
use solana_entry::entry::Entry;
|
||||
|
||||
use super::ShredStreamGrpc;
|
||||
|
||||
@@ -29,120 +35,61 @@ impl ShredStreamGrpc {
|
||||
metrics_handle = self.metrics_manager.start_auto_monitoring().await;
|
||||
}
|
||||
|
||||
// 启动流处理
|
||||
let client = (*self.shredstream_client).clone();
|
||||
let (stream_task, rx) = ShredStreamHandler::start_stream_processing(
|
||||
client,
|
||||
self.config.backpressure.channel_size,
|
||||
)
|
||||
.await?;
|
||||
// 创建事件处理器
|
||||
let mut event_processor =
|
||||
EventProcessor::new(self.metrics_manager.clone(), self.config.clone());
|
||||
event_processor.set_protocols_and_event_type_filter(
|
||||
protocols,
|
||||
event_type_filter,
|
||||
self.config.backpressure.clone(),
|
||||
Some(Arc::new(callback)),
|
||||
);
|
||||
|
||||
// 根据配置选择处理模式并获取事件处理任务句柄
|
||||
let event_handle = if self.config.batch.enabled {
|
||||
// 批处理模式
|
||||
self.process_with_batch(rx, protocols, bot_wallet, event_type_filter, callback).await?
|
||||
} else {
|
||||
// 即时处理模式
|
||||
self.process_immediate(rx, protocols, bot_wallet, event_type_filter, callback).await?
|
||||
};
|
||||
// 启动流处理
|
||||
let mut client = (*self.shredstream_client).clone();
|
||||
let request = tonic::Request::new(SubscribeEntriesRequest {});
|
||||
let mut stream = client.subscribe_entries(request).await?.into_inner();
|
||||
let event_processor_clone = event_processor.clone();
|
||||
let stream_task = tokio::spawn(async move {
|
||||
while let Some(message) = stream.next().await {
|
||||
match message {
|
||||
Ok(msg) => {
|
||||
if let Ok(entries) = bincode::deserialize::<Vec<Entry>>(&msg.entries) {
|
||||
for entry in entries {
|
||||
for transaction in entry.transactions {
|
||||
let transaction_with_slot = TransactionWithSlot::new(
|
||||
transaction.clone(),
|
||||
msg.slot,
|
||||
chrono::Utc::now().timestamp_micros(),
|
||||
);
|
||||
// 直接处理,背压控制在 EventProcessor 内部处理
|
||||
if let Err(e) = event_processor_clone
|
||||
.process_shred_transaction_with_metrics(
|
||||
transaction_with_slot,
|
||||
bot_wallet,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error handling message: {e:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
error!("Stream error: {error:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 保存订阅句柄
|
||||
let subscription_handle = SubscriptionHandle::new(stream_task, event_handle, metrics_handle);
|
||||
let subscription_handle = SubscriptionHandle::new(stream_task, None, metrics_handle);
|
||||
let mut handle_guard = self.subscription_handle.lock().await;
|
||||
*handle_guard = Some(subscription_handle);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 批处理模式
|
||||
async fn process_with_batch<F>(
|
||||
&self,
|
||||
mut rx: futures::channel::mpsc::Receiver<TransactionWithSlot>,
|
||||
protocols: Vec<Protocol>,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
callback: F,
|
||||
) -> AnyResult<tokio::task::JoinHandle<()>>
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
|
||||
{
|
||||
use futures::StreamExt;
|
||||
|
||||
// 创建批处理器,将单个事件回调转换为批量回调
|
||||
let batch_callback = move |events: Vec<Box<dyn UnifiedEvent>>| {
|
||||
for event in events {
|
||||
callback(event);
|
||||
}
|
||||
};
|
||||
|
||||
let mut batch_processor = EventBatchProcessor::new(
|
||||
batch_callback,
|
||||
self.config.batch.batch_size,
|
||||
self.config.batch.batch_timeout_ms,
|
||||
);
|
||||
|
||||
// 创建事件处理器
|
||||
let event_processor =
|
||||
ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone());
|
||||
|
||||
let event_handle = tokio::spawn(async move {
|
||||
while let Some(transaction_with_slot) = rx.next().await {
|
||||
if let Err(e) = event_processor
|
||||
.process_transaction_with_batch(
|
||||
transaction_with_slot,
|
||||
protocols.clone(),
|
||||
bot_wallet,
|
||||
&mut batch_processor,
|
||||
event_type_filter.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
log::error!("Error processing transaction: {e:?}");
|
||||
}
|
||||
}
|
||||
|
||||
// 处理剩余的事件
|
||||
batch_processor.flush();
|
||||
});
|
||||
|
||||
Ok(event_handle)
|
||||
}
|
||||
|
||||
/// 即时处理模式
|
||||
async fn process_immediate<F>(
|
||||
&self,
|
||||
mut rx: futures::channel::mpsc::Receiver<TransactionWithSlot>,
|
||||
protocols: Vec<Protocol>,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
callback: F,
|
||||
) -> AnyResult<tokio::task::JoinHandle<()>>
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
|
||||
{
|
||||
use futures::StreamExt;
|
||||
|
||||
// 创建事件处理器
|
||||
let event_processor =
|
||||
ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone());
|
||||
|
||||
let event_handle = tokio::spawn(async move {
|
||||
while let Some(transaction_with_slot) = rx.next().await {
|
||||
if let Err(e) = event_processor
|
||||
.process_transaction_immediate(
|
||||
transaction_with_slot,
|
||||
protocols.clone(),
|
||||
bot_wallet,
|
||||
event_type_filter.clone(),
|
||||
&callback,
|
||||
)
|
||||
.await
|
||||
{
|
||||
log::error!("Error processing transaction: {e:?}");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(event_handle)
|
||||
}
|
||||
}
|
||||
|
||||
+200
-199
@@ -1,19 +1,26 @@
|
||||
use futures::{channel::mpsc, StreamExt};
|
||||
use log::error;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use yellowstone_grpc_proto::geyser::CommitmentLevel;
|
||||
|
||||
use crate::common::AnyResult;
|
||||
use crate::streaming::common::{
|
||||
EventBatchProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle,
|
||||
EventProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle,
|
||||
};
|
||||
use crate::streaming::event_parser::common::filter::EventTypeFilter;
|
||||
use crate::streaming::event_parser::{Protocol, UnifiedEvent};
|
||||
use crate::streaming::grpc::{EventPretty, EventProcessor, StreamHandler, SubscriptionManager};
|
||||
use crate::streaming::grpc::{
|
||||
AccountPretty, BlockMetaPretty, EventPretty, SubscriptionManager, TransactionPretty,
|
||||
};
|
||||
use anyhow::anyhow;
|
||||
use chrono::Local;
|
||||
use futures::channel::mpsc;
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use log::error;
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tokio::sync::Mutex;
|
||||
use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
|
||||
use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestPing};
|
||||
|
||||
/// 交易过滤器
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TransactionFilter {
|
||||
pub account_include: Vec<String>,
|
||||
pub account_exclude: Vec<String>,
|
||||
@@ -21,6 +28,7 @@ pub struct TransactionFilter {
|
||||
}
|
||||
|
||||
/// 账户过滤器
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccountFilter {
|
||||
pub account: Vec<String>,
|
||||
pub owner: Vec<String>,
|
||||
@@ -30,11 +38,15 @@ pub struct YellowstoneGrpc {
|
||||
pub endpoint: String,
|
||||
pub x_token: Option<String>,
|
||||
pub config: StreamClientConfig,
|
||||
pub metrics: Arc<Mutex<PerformanceMetrics>>,
|
||||
pub metrics: Arc<RwLock<PerformanceMetrics>>,
|
||||
pub subscription_manager: SubscriptionManager,
|
||||
pub metrics_manager: MetricsManager,
|
||||
pub event_processor: EventProcessor,
|
||||
pub subscription_handle: Arc<Mutex<Option<SubscriptionHandle>>>,
|
||||
// Dynamic subscription management fields
|
||||
pub active_subscription: Arc<AtomicBool>,
|
||||
pub control_tx: Arc<tokio::sync::Mutex<Option<mpsc::Sender<SubscribeRequest>>>>,
|
||||
pub current_request: Arc<tokio::sync::RwLock<Option<SubscribeRequest>>>,
|
||||
}
|
||||
|
||||
impl YellowstoneGrpc {
|
||||
@@ -50,43 +62,50 @@ impl YellowstoneGrpc {
|
||||
config: StreamClientConfig,
|
||||
) -> AnyResult<Self> {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default().ok();
|
||||
let metrics = Arc::new(Mutex::new(PerformanceMetrics::new()));
|
||||
let config_arc = Arc::new(config.clone());
|
||||
let metrics = Arc::new(RwLock::new(PerformanceMetrics::new()));
|
||||
|
||||
let subscription_manager =
|
||||
SubscriptionManager::new(endpoint.clone(), x_token.clone(), config.clone());
|
||||
let metrics_manager =
|
||||
MetricsManager::new(metrics.clone(), config_arc.clone(), "YellowstoneGrpc".to_string());
|
||||
let metrics_manager = MetricsManager::new_with_metrics(
|
||||
metrics.clone(),
|
||||
config.enable_metrics,
|
||||
"YellowstoneGrpc".to_string(),
|
||||
);
|
||||
let event_processor = EventProcessor::new(metrics_manager.clone(), config.clone());
|
||||
|
||||
Ok(Self {
|
||||
endpoint,
|
||||
x_token,
|
||||
config,
|
||||
metrics,
|
||||
metrics: metrics.clone(),
|
||||
subscription_manager,
|
||||
metrics_manager,
|
||||
event_processor,
|
||||
subscription_handle: Arc::new(Mutex::new(None)),
|
||||
active_subscription: Arc::new(AtomicBool::new(false)),
|
||||
control_tx: Arc::new(tokio::sync::Mutex::new(None)),
|
||||
current_request: Arc::new(tokio::sync::RwLock::new(None)),
|
||||
})
|
||||
}
|
||||
|
||||
/// 创建高性能客户端
|
||||
pub fn new_high_performance(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
|
||||
Self::new_with_config(endpoint, x_token, StreamClientConfig::high_performance())
|
||||
/// Creates a new YellowstoneGrpcClient with high-throughput configuration.
|
||||
///
|
||||
/// This is a convenience method that creates a client optimized for high-concurrency scenarios
|
||||
/// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()`
|
||||
/// for detailed configuration information.
|
||||
pub fn new_high_throughput(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
|
||||
Self::new_with_config(endpoint, x_token, StreamClientConfig::high_throughput())
|
||||
}
|
||||
|
||||
/// 创建低延迟客户端
|
||||
/// Creates a new YellowstoneGrpcClient with low-latency configuration.
|
||||
///
|
||||
/// This is a convenience method that creates a client optimized for real-time scenarios
|
||||
/// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()`
|
||||
/// for detailed configuration information.
|
||||
pub fn new_low_latency(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
|
||||
Self::new_with_config(endpoint, x_token, StreamClientConfig::low_latency())
|
||||
}
|
||||
|
||||
/// 创建即时处理客户端
|
||||
pub fn new_immediate(endpoint: String, x_token: Option<String>) -> AnyResult<Self> {
|
||||
let mut config = StreamClientConfig::low_latency();
|
||||
config.enable_metrics = false;
|
||||
Self::new_with_config(endpoint, x_token, config)
|
||||
}
|
||||
|
||||
/// 获取配置
|
||||
pub fn get_config(&self) -> &StreamClientConfig {
|
||||
@@ -99,13 +118,13 @@ impl YellowstoneGrpc {
|
||||
}
|
||||
|
||||
/// 获取性能指标
|
||||
pub async fn get_metrics(&self) -> PerformanceMetrics {
|
||||
self.metrics_manager.get_metrics().await
|
||||
pub fn get_metrics(&self) -> PerformanceMetrics {
|
||||
self.metrics_manager.get_metrics()
|
||||
}
|
||||
|
||||
/// 打印性能指标
|
||||
pub async fn print_metrics(&self) {
|
||||
self.metrics_manager.print_metrics().await;
|
||||
pub fn print_metrics(&self) {
|
||||
self.metrics_manager.print_metrics();
|
||||
}
|
||||
|
||||
/// 启用或禁用性能监控
|
||||
@@ -119,6 +138,9 @@ impl YellowstoneGrpc {
|
||||
if let Some(handle) = handle_guard.take() {
|
||||
handle.stop();
|
||||
}
|
||||
*self.control_tx.lock().await = None;
|
||||
*self.current_request.write().await = None;
|
||||
self.active_subscription.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Simplified immediate event subscription (recommended for simple scenarios)
|
||||
@@ -147,8 +169,13 @@ impl YellowstoneGrpc {
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
|
||||
{
|
||||
// 如果已有活跃订阅,先停止它
|
||||
self.stop().await;
|
||||
if self
|
||||
.active_subscription
|
||||
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
|
||||
.is_err()
|
||||
{
|
||||
return Err(anyhow!("Already subscribed. Use update_subscription() to modify filters"));
|
||||
}
|
||||
|
||||
let mut metrics_handle = None;
|
||||
// 启动自动性能监控(如果启用)
|
||||
@@ -160,205 +187,187 @@ impl YellowstoneGrpc {
|
||||
transaction_filter.account_include,
|
||||
transaction_filter.account_exclude,
|
||||
transaction_filter.account_required,
|
||||
event_type_filter.clone(),
|
||||
event_type_filter.as_ref(),
|
||||
);
|
||||
let accounts = self.subscription_manager.subscribe_with_account_request(
|
||||
account_filter.account,
|
||||
account_filter.owner,
|
||||
event_type_filter.clone(),
|
||||
event_type_filter.as_ref(),
|
||||
);
|
||||
|
||||
// 订阅事件
|
||||
let (mut subscribe_tx, mut stream) = self
|
||||
let (mut subscribe_tx, mut stream, subscribe_request) = self
|
||||
.subscription_manager
|
||||
.subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone())
|
||||
.subscribe_with_request(transactions, accounts, commitment, event_type_filter.as_ref())
|
||||
.await?;
|
||||
|
||||
// 创建通道,使用配置中的通道大小
|
||||
let (mut tx, mut rx) = mpsc::channel::<EventPretty>(self.config.backpressure.channel_size);
|
||||
// 用 Arc<Mutex<>> 包装 subscribe_tx 以支持多线程共享
|
||||
let subscribe_tx = Arc::new(Mutex::new(subscribe_tx));
|
||||
*self.current_request.write().await = Some(subscribe_request);
|
||||
let (control_tx, mut control_rx) = mpsc::channel(100);
|
||||
*self.control_tx.lock().await = Some(control_tx);
|
||||
|
||||
// 启动流处理任务
|
||||
let backpressure_strategy = self.config.backpressure.strategy;
|
||||
let mut event_processor = self.event_processor.clone();
|
||||
event_processor.set_protocols_and_event_type_filter(
|
||||
protocols,
|
||||
event_type_filter,
|
||||
self.config.backpressure.clone(),
|
||||
Some(Arc::new(callback)),
|
||||
);
|
||||
let stream_handle = tokio::spawn(async move {
|
||||
while let Some(message) = stream.next().await {
|
||||
match message {
|
||||
Ok(msg) => {
|
||||
if let Err(e) = StreamHandler::handle_stream_message(
|
||||
msg,
|
||||
&mut tx,
|
||||
&mut subscribe_tx,
|
||||
backpressure_strategy,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error handling message: {e:?}");
|
||||
loop {
|
||||
tokio::select! {
|
||||
message = stream.next() => {
|
||||
match message {
|
||||
Some(Ok(msg)) => {
|
||||
let created_at = msg.created_at;
|
||||
match msg.update_oneof {
|
||||
Some(UpdateOneof::Account(account)) => {
|
||||
let account_pretty = AccountPretty::from(account);
|
||||
log::debug!("Received account: {:?}", account_pretty);
|
||||
if let Err(e) = event_processor
|
||||
.process_grpc_event_transaction_with_metrics(
|
||||
EventPretty::Account(account_pretty),
|
||||
bot_wallet,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing account event: {e:?}");
|
||||
}
|
||||
}
|
||||
Some(UpdateOneof::BlockMeta(sut)) => {
|
||||
let block_meta_pretty =
|
||||
BlockMetaPretty::from((sut, created_at));
|
||||
log::debug!("Received block meta: {:?}", block_meta_pretty);
|
||||
if let Err(e) = event_processor
|
||||
.process_grpc_event_transaction_with_metrics(
|
||||
EventPretty::BlockMeta(block_meta_pretty),
|
||||
bot_wallet,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing block meta event: {e:?}");
|
||||
}
|
||||
}
|
||||
Some(UpdateOneof::Transaction(sut)) => {
|
||||
let transaction_pretty =
|
||||
TransactionPretty::from((sut, created_at));
|
||||
log::debug!(
|
||||
"Received transaction: {} at slot {}",
|
||||
transaction_pretty.signature,
|
||||
transaction_pretty.slot
|
||||
);
|
||||
if let Err(e) = event_processor
|
||||
.process_grpc_event_transaction_with_metrics(
|
||||
EventPretty::Transaction(transaction_pretty),
|
||||
bot_wallet,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing transaction event: {e:?}");
|
||||
}
|
||||
}
|
||||
Some(UpdateOneof::Ping(_)) => {
|
||||
// 只在需要时获取锁,并立即释放
|
||||
if let Ok(mut tx_guard) = subscribe_tx.try_lock() {
|
||||
let _ = tx_guard
|
||||
.send(SubscribeRequest {
|
||||
ping: Some(SubscribeRequestPing { id: 1 }),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
log::debug!("service is ping: {}", Local::now());
|
||||
}
|
||||
Some(UpdateOneof::Pong(_)) => {
|
||||
log::debug!("service is pong: {}", Local::now());
|
||||
}
|
||||
_ => {
|
||||
log::debug!("Received other message type");
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
error!("Stream error: {error:?}");
|
||||
break;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
Some(update) = control_rx.next() => {
|
||||
if let Err(e) = subscribe_tx.lock().await.send(update).await {
|
||||
error!("Failed to send subscription update: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
error!("Stream error: {error:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 即时处理交易,无批处理
|
||||
let event_processor = self.event_processor.clone();
|
||||
let event_handle = tokio::spawn(async move {
|
||||
while let Some(event_pretty) = rx.next().await {
|
||||
if let Err(e) = event_processor
|
||||
.process_event_transaction_with_metrics(
|
||||
event_pretty,
|
||||
&callback,
|
||||
bot_wallet,
|
||||
protocols.clone(),
|
||||
event_type_filter.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing transaction: {e:?}");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 保存订阅句柄
|
||||
let subscription_handle =
|
||||
SubscriptionHandle::new(stream_handle, event_handle, metrics_handle);
|
||||
let subscription_handle = SubscriptionHandle::new(stream_handle, None, metrics_handle);
|
||||
let mut handle_guard = self.subscription_handle.lock().await;
|
||||
*handle_guard = Some(subscription_handle);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Advanced event subscription with batch processing and backpressure handling
|
||||
/// Update subscription filters at runtime without reconnection
|
||||
///
|
||||
/// # Parameters
|
||||
/// * `protocols` - List of protocols to monitor
|
||||
/// * `bot_wallet` - Optional bot wallet address for filtering related transactions
|
||||
/// * `transaction_filter` - Transaction filter specifying accounts to include/exclude
|
||||
/// * `account_filter` - Account filter specifying accounts and owners to monitor
|
||||
/// * `event_filter` - Optional event filter for further event filtering, no filtering if None
|
||||
/// * `commitment` - Optional commitment level, defaults to Confirmed
|
||||
/// * `callback` - Event callback function that receives parsed unified events
|
||||
///
|
||||
/// # Features
|
||||
/// * Batch processing for improved throughput
|
||||
/// * Backpressure handling to prevent memory overflow
|
||||
/// * Automatic performance monitoring (if enabled)
|
||||
/// * Configurable batch size and timeout
|
||||
/// * `transaction_filter` - New transaction filter to apply
|
||||
/// * `account_filter` - New account filter to apply
|
||||
///
|
||||
/// # Returns
|
||||
/// Returns `AnyResult<()>`, `Ok(())` on success, error information on failure
|
||||
pub async fn subscribe_events_advanced<F>(
|
||||
/// Returns `AnyResult<()>` on success, error on failure
|
||||
pub async fn update_subscription(
|
||||
&self,
|
||||
protocols: Vec<Protocol>,
|
||||
bot_wallet: Option<Pubkey>,
|
||||
transaction_filter: TransactionFilter,
|
||||
account_filter: AccountFilter,
|
||||
event_type_filter: Option<EventTypeFilter>,
|
||||
commitment: Option<CommitmentLevel>,
|
||||
callback: F,
|
||||
) -> AnyResult<()>
|
||||
where
|
||||
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
|
||||
{
|
||||
// 如果已有活跃订阅,先停止它
|
||||
self.stop().await;
|
||||
) -> AnyResult<()> {
|
||||
let mut control_sender = {
|
||||
let control_guard = self.control_tx.lock().await;
|
||||
|
||||
let mut metrics_handle = None;
|
||||
// 启动自动性能监控(如果启用)
|
||||
if self.config.enable_metrics {
|
||||
metrics_handle = self.metrics_manager.start_auto_monitoring().await;
|
||||
}
|
||||
|
||||
let transactions = self.subscription_manager.get_subscribe_request_filter(
|
||||
transaction_filter.account_include,
|
||||
transaction_filter.account_exclude,
|
||||
transaction_filter.account_required,
|
||||
event_type_filter.clone(),
|
||||
);
|
||||
let accounts = self.subscription_manager.subscribe_with_account_request(
|
||||
account_filter.account,
|
||||
account_filter.owner,
|
||||
event_type_filter.clone(),
|
||||
);
|
||||
|
||||
// Subscribe to events
|
||||
let (mut subscribe_tx, mut stream) = self
|
||||
.subscription_manager
|
||||
.subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone())
|
||||
.await?;
|
||||
|
||||
// Create channel
|
||||
let (mut tx, mut rx) = mpsc::channel::<EventPretty>(self.config.backpressure.channel_size);
|
||||
|
||||
// 创建批处理器,将单个事件回调转换为批量回调
|
||||
let batch_callback = move |events: Vec<Box<dyn UnifiedEvent>>| {
|
||||
for event in events {
|
||||
callback(event);
|
||||
if !self.active_subscription.load(Ordering::Acquire) {
|
||||
return Err(anyhow!("No active subscription to update"));
|
||||
}
|
||||
|
||||
control_guard
|
||||
.as_ref()
|
||||
.ok_or_else(|| anyhow!("No active subscription to update"))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let mut batch_processor = EventBatchProcessor::new(
|
||||
batch_callback,
|
||||
self.config.batch.batch_size,
|
||||
self.config.batch.batch_timeout_ms,
|
||||
);
|
||||
let mut request = self
|
||||
.current_request
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.ok_or_else(|| anyhow!("No active subscription"))?
|
||||
.clone();
|
||||
|
||||
// Start task to process the stream
|
||||
let backpressure_strategy = self.config.backpressure.strategy;
|
||||
let stream_handle = tokio::spawn(async move {
|
||||
while let Some(message) = stream.next().await {
|
||||
match message {
|
||||
Ok(msg) => {
|
||||
if let Err(e) = StreamHandler::handle_stream_message(
|
||||
msg,
|
||||
&mut tx,
|
||||
&mut subscribe_tx,
|
||||
backpressure_strategy,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error handling message: {e:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
error!("Stream error: {error:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
request.transactions = self
|
||||
.subscription_manager
|
||||
.get_subscribe_request_filter(
|
||||
transaction_filter.account_include,
|
||||
transaction_filter.account_exclude,
|
||||
transaction_filter.account_required,
|
||||
None,
|
||||
)
|
||||
.unwrap_or_default();
|
||||
|
||||
// Process transactions with batch processing
|
||||
let event_processor = self.event_processor.clone();
|
||||
let event_handle = tokio::spawn(async move {
|
||||
while let Some(event_pretty) = rx.next().await {
|
||||
if let Err(e) = event_processor
|
||||
.process_event_transaction_with_batch(
|
||||
event_pretty,
|
||||
&mut batch_processor,
|
||||
bot_wallet,
|
||||
protocols.clone(),
|
||||
event_type_filter.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing transaction: {e:?}");
|
||||
}
|
||||
}
|
||||
request.accounts = self
|
||||
.subscription_manager
|
||||
.subscribe_with_account_request(account_filter.account, account_filter.owner, None)
|
||||
.unwrap_or_default();
|
||||
|
||||
// 处理剩余的事件
|
||||
batch_processor.flush();
|
||||
});
|
||||
control_sender
|
||||
.send(request.clone())
|
||||
.await
|
||||
.map_err(|e| anyhow!("Failed to send update: {}", e))?;
|
||||
|
||||
// 保存订阅句柄
|
||||
let subscription_handle =
|
||||
SubscriptionHandle::new(stream_handle, event_handle, metrics_handle);
|
||||
let mut handle_guard = self.subscription_handle.lock().await;
|
||||
*handle_guard = Some(subscription_handle);
|
||||
*self.current_request.write().await = Some(request);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -376,17 +385,9 @@ impl Clone for YellowstoneGrpc {
|
||||
metrics_manager: self.metrics_manager.clone(),
|
||||
event_processor: self.event_processor.clone(),
|
||||
subscription_handle: self.subscription_handle.clone(), // 共享同一个 Arc<Mutex<>>
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 实现 Clone trait 以支持模块间共享
|
||||
impl Clone for EventProcessor {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
metrics_manager: self.metrics_manager.clone(),
|
||||
config: self.config.clone(),
|
||||
parser_cache: self.parser_cache.clone(),
|
||||
active_subscription: self.active_subscription.clone(),
|
||||
control_tx: self.control_tx.clone(),
|
||||
current_request: self.current_request.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
use crate::{
|
||||
common::AnyResult,
|
||||
streaming::{
|
||||
grpc::{BackpressureStrategy, EventPretty, StreamHandler},
|
||||
grpc::{EventPretty, TransactionPretty},
|
||||
yellowstone_grpc::YellowstoneGrpc,
|
||||
},
|
||||
};
|
||||
use futures::{channel::mpsc, StreamExt};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use log::error;
|
||||
use solana_program::pubkey;
|
||||
use solana_sdk::{pubkey::Pubkey, transaction::VersionedTransaction};
|
||||
use solana_transaction_status::EncodedTransactionWithStatusMeta;
|
||||
use solana_transaction_status::TransactionWithStatusMeta;
|
||||
use yellowstone_grpc_proto::geyser::{
|
||||
subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing,
|
||||
};
|
||||
|
||||
const SYSTEM_PROGRAM_ID: Pubkey = pubkey!("11111111111111111111111111111111");
|
||||
// 根据实际并发量调整通道大小,避免背压
|
||||
const CHANNEL_SIZE: usize = 50000; // 增加到 50000
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum SystemEvent {
|
||||
@@ -36,7 +37,7 @@ impl YellowstoneGrpc {
|
||||
account_exclude: Option<Vec<String>>,
|
||||
) -> AnyResult<()>
|
||||
where
|
||||
F: Fn(SystemEvent) + Send + Sync + 'static,
|
||||
F: Fn(SystemEvent) + Send + Sync + Clone + 'static,
|
||||
{
|
||||
let addrs = vec![SYSTEM_PROGRAM_ID.to_string()];
|
||||
let account_include = account_include.unwrap_or_default();
|
||||
@@ -47,11 +48,10 @@ impl YellowstoneGrpc {
|
||||
addrs,
|
||||
None,
|
||||
);
|
||||
let (mut subscribe_tx, mut stream) = self
|
||||
let (mut subscribe_tx, mut stream, _) = self
|
||||
.subscription_manager
|
||||
.subscribe_with_request(transactions, None, None, None)
|
||||
.await?;
|
||||
let (mut tx, mut rx) = mpsc::channel::<EventPretty>(CHANNEL_SIZE);
|
||||
|
||||
let callback = Box::new(callback);
|
||||
|
||||
@@ -59,16 +59,34 @@ impl YellowstoneGrpc {
|
||||
while let Some(message) = stream.next().await {
|
||||
match message {
|
||||
Ok(msg) => {
|
||||
if let Err(e) = StreamHandler::handle_stream_message(
|
||||
msg,
|
||||
&mut tx,
|
||||
&mut subscribe_tx,
|
||||
BackpressureStrategy::Block,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error handling message: {e:?}");
|
||||
break;
|
||||
let created_at = msg.created_at;
|
||||
match msg.update_oneof {
|
||||
Some(UpdateOneof::Transaction(sut)) => {
|
||||
let transaction_pretty = TransactionPretty::from((sut, created_at));
|
||||
let event_pretty = EventPretty::Transaction(transaction_pretty);
|
||||
if let Err(e) = Self::process_system_transaction(
|
||||
event_pretty,
|
||||
&*callback,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error processing transaction: {e:?}");
|
||||
}
|
||||
}
|
||||
Some(UpdateOneof::Ping(_)) => {
|
||||
let _ = subscribe_tx
|
||||
.send(SubscribeRequest {
|
||||
ping: Some(SubscribeRequestPing { id: 1 }),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
Some(UpdateOneof::Pong(_)) => {
|
||||
// Pong response, no action needed
|
||||
}
|
||||
_ => {
|
||||
// Other message types, ignore for system subscription
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -78,12 +96,6 @@ impl YellowstoneGrpc {
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
while let Some(event_pretty) = rx.next().await {
|
||||
if let Err(e) = Self::process_system_transaction(event_pretty, &*callback).await {
|
||||
error!("Error processing transaction: {e:?}");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -93,20 +105,19 @@ impl YellowstoneGrpc {
|
||||
{
|
||||
match event_pretty {
|
||||
EventPretty::Transaction(transaction_pretty) => {
|
||||
let trade_raw: EncodedTransactionWithStatusMeta = transaction_pretty.tx;
|
||||
let meta = trade_raw
|
||||
.meta
|
||||
.as_ref()
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing transaction metadata"))?;
|
||||
let trade_raw: TransactionWithStatusMeta = transaction_pretty.tx;
|
||||
let meta = trade_raw.get_status_meta();
|
||||
|
||||
if meta.err.is_some() {
|
||||
if meta.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let transaction = trade_raw.get_transaction();
|
||||
|
||||
callback(SystemEvent::NewTransfer(TransferInfo {
|
||||
slot: transaction_pretty.slot,
|
||||
signature: transaction_pretty.signature.to_string(),
|
||||
tx: trade_raw.transaction.decode(),
|
||||
tx: Some(transaction),
|
||||
}));
|
||||
}
|
||||
_ => {}
|
||||
|
||||
Reference in New Issue
Block a user