gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-22 21:38:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Major refactor with batch processing and performance optimization

Browse Source 
- Refactor streaming architecture with modular grpc components
- Add batch processing system with backpressure strategies
- Implement performance monitoring and metrics collection
- Add multi-protocol parser with block metadata support
- Enhance configuration system with preset profiles
- Add parse_tx_events example and update documentation
- Optimize yellowstone_grpc client complexity
This commit is contained in:
ysq
2025-08-11 01:04:16 +08:00
parent 87465cbfdb
commit 97faba9406
35 changed files with 2067 additions and 1434 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README_CN.md
+58 -27
View File
@@ -18,7 +18,7 @@
6. **事件解析系统**: 自动解析和分类协议特定事件
7. **高性能**: 针对低延迟事件处理进行优化
8. **批处理优化**: 批量处理事件以减少回调开销
9. **性能监控**: 内置性能指标监控，包括事件处理速度、内存使用等
9. **性能监控**: 内置性能指标监控，包括事件处理速度等
10. **内存优化**: 对象池和缓存机制减少内存分配
11. **灵活配置系统**: 支持自定义批处理大小、背压策略、通道大小等参数
12. **预设配置**: 提供高性能、低延迟、有序处理等预设配置
@@ -41,25 +41,45 @@ git clone https://github.com/0xfnzero/solana-streamer
```toml
# 添加到您的 Cargo.toml
solana-streamer-sdk = { path = "./solana-streamer", version = "0.1.11" }
solana-streamer-sdk = { path = "./solana-streamer", version = "0.2.0" }
```
### 使用 crates.io
```toml
# 添加到您的 Cargo.toml
solana-streamer-sdk = "0.1.11"
solana-streamer-sdk = "0.2.0"
```
## 使用示例
### 快速开始 - 解析交易事件
您可以通过运行内置示例来快速测试库的交易事件解析功能：
```bash
cargo run --example parse_tx_events
```
该示例演示了：
- 如何使用 RPC 从 Solana 主网解析交易数据
- 多协议事件解析（PumpFun、PumpSwap、Bonk、Raydium CPMM/CLMM）
- 交易详情提取，包括费用、日志和计算单元
该示例使用预定义的交易签名，展示如何从交易数据中提取协议特定的事件。
### 高级用法示例
```rust
use solana_streamer_sdk::{
match_event,
streaming::{
event_parser::{
protocols::{
bonk::{parser::BONK_PROGRAM_ID, BonkPoolCreateEvent, BonkTradeEvent},
bonk::{
parser::BONK_PROGRAM_ID, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent,
BonkPoolCreateEvent, BonkTradeEvent,
},
pumpfun::{parser::PUMPFUN_PROGRAM_ID, PumpFunCreateTokenEvent, PumpFunTradeEvent},
pumpswap::{
parser::PUMPSWAP_PROGRAM_ID, PumpSwapBuyEvent, PumpSwapCreatePoolEvent,
@@ -68,23 +88,23 @@ use solana_streamer_sdk::{
raydium_clmm::{
parser::RAYDIUM_CLMM_PROGRAM_ID, RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event,
},
raydium_cpmm::{parser::RAYDIUM_CPMM_PROGRAM_ID, RaydiumCpmmSwapEvent},
raydium_cpmm::{parser::RAYDIUM_CPMM_PROGRAM_ID, RaydiumCpmmSwapEvent}, BlockMetaEvent,
},
Protocol, UnifiedEvent,
},
ShredStreamGrpc, YellowstoneGrpc,
}, grpc::ClientConfig, shred_stream::ShredClientConfig, ShredStreamGrpc, YellowstoneGrpc
},
};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("Starting Solana Streamer...");
test_grpc().await?;
test_shreds().await?;
Ok(())
}
async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
println!("正在订阅 Yellowstone gRPC 事件...");
println!("Subscribing to Yellowstone gRPC events...");
// 创建低延迟配置
let mut config = ClientConfig::low_latency();
@@ -96,6 +116,8 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
config,
)?;
println!("GRPC client created successfully");
let callback = create_event_callback();
// 将会从交易中尝试解析对应的协议事件
@@ -107,6 +129,8 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
Protocol::RaydiumClmm,
];
println!("Protocols to monitor: {:?}", protocols);
// 过滤账号
let account_include = vec![
PUMPFUN_PROGRAM_ID.to_string(), // 监听 pumpfun 程序ID
@@ -119,8 +143,12 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
let account_exclude = vec![];
let account_required = vec![];
println!("开始监听事件，按 Ctrl+C 停止...");
grpc.subscribe_events_v2(
println!("Starting to listen for events, press Ctrl+C to stop...");
println!("Monitoring programs: {:?}", account_include);
println!("Starting subscription...");
grpc.subscribe_events_immediate(
protocols,
None,
account_include,
@@ -135,13 +163,15 @@ async fn test_grpc() -> Result<(), Box<dyn std::error::Error>> {
}
async fn test_shreds() -> Result<(), Box<dyn std::error::Error>> {
println!("正在订阅 ShredStream 事件...");
println!("Subscribing to ShredStream events...");
// 创建低延迟配置
let mut config = ShredClientConfig::low_latency();
// 启用性能监控, 有性能损耗, 默认关闭
config.enable_metrics = true;
let shred_stream =
ShredStreamGrpc::new_with_config("http://127.0.0.1:10800".to_string(), config).await?;
let callback = create_event_callback();
let protocols = vec![
Protocol::PumpFun,
@@ -151,24 +181,26 @@ async fn test_shreds() -> Result<(), Box<dyn std::error::Error>> {
Protocol::RaydiumClmm,
];
println!("开始监听事件，按 Ctrl+C 停止...");
shred_stream
.shredstream_subscribe(protocols, None, callback)
.await?;
println!("Listening for events, press Ctrl+C to stop...");
shred_stream.shredstream_subscribe(protocols, None, callback).await?;
Ok(())
}
fn create_event_callback() -> impl Fn(Box<dyn UnifiedEvent>) {
|event: Box<dyn UnifiedEvent>| {
println!("🎉 Event received! Type: {:?}, ID: {}", event.event_type(), event.id());
match_event!(event, {
BlockMetaEvent => |e: BlockMetaEvent| {
println!("BlockMetaEvent: {e:?}");
},
BonkPoolCreateEvent => |e: BonkPoolCreateEvent| {
// 使用grpc的时候，可以从每个事件中获取到block_time
println!("block_time: {:?}, block_time_ms: {:?}", e.metadata.block_time, e.metadata.block_time_ms);
println!("BonkPoolCreateEvent: {:?}", e.base_mint_param.symbol);
},
BonkTradeEvent => |e: BonkTradeEvent| {
println!("BonkTradeEvent: {:?}", e);
println!("BonkTradeEvent: {e:?}");
},
BonkMigrateToAmmEvent => |e: BonkMigrateToAmmEvent| {
println!("BonkMigrateToAmmEvent: {e:?}");
@@ -177,34 +209,34 @@ fn create_event_callback() -> impl Fn(Box<dyn UnifiedEvent>) {
println!("BonkMigrateToCpswapEvent: {e:?}");
},
PumpFunTradeEvent => |e: PumpFunTradeEvent| {
println!("PumpFunTradeEvent: {:?}", e);
println!("PumpFunTradeEvent: {e:?}");
},
PumpFunCreateTokenEvent => |e: PumpFunCreateTokenEvent| {
println!("PumpFunCreateTokenEvent: {:?}", e);
println!("PumpFunCreateTokenEvent: {e:?}");
},
PumpSwapBuyEvent => |e: PumpSwapBuyEvent| {
println!("Buy event: {:?}", e);
println!("Buy event: {e:?}");
},
PumpSwapSellEvent => |e: PumpSwapSellEvent| {
println!("Sell event: {:?}", e);
println!("Sell event: {e:?}");
},
PumpSwapCreatePoolEvent => |e: PumpSwapCreatePoolEvent| {
println!("CreatePool event: {:?}", e);
println!("CreatePool event: {e:?}");
},
PumpSwapDepositEvent => |e: PumpSwapDepositEvent| {
println!("Deposit event: {:?}", e);
println!("Deposit event: {e:?}");
},
PumpSwapWithdrawEvent => |e: PumpSwapWithdrawEvent| {
println!("Withdraw event: {:?}", e);
println!("Withdraw event: {e:?}");
},
RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| {
println!("RaydiumCpmmSwapEvent: {:?}", e);
println!("RaydiumCpmmSwapEvent: {e:?}");
},
RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| {
println!("RaydiumClmmSwapEvent: {:?}", e);
println!("RaydiumClmmSwapEvent: {e:?}");
},
RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| {
println!("RaydiumClmmSwapV2Event: {:?}", e);
println!("RaydiumClmmSwapV2Event: {e:?}");
}
});
}
@@ -291,7 +323,6 @@ MIT 许可证
1. **网络稳定性**: 确保稳定的网络连接以进行连续的事件流传输
2. **速率限制**: 注意公共 gRPC 端点的速率限制
3. **错误恢复**: 实现适当的错误处理和重连逻辑
4. **资源管理**: 监控长时间运行流的内存和 CPU 使用情况
5. **合规性**: 确保遵守相关法律法规
## 语言版本