Release solana-streamer-sdk v1.4.7

This commit is contained in:
0xfnzero
2026-05-19 00:27:06 +08:00
parent ea0f9b9145
commit 5494fd5fb3
9 changed files with 513 additions and 62 deletions
+2 -3
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "solana-streamer-sdk" name = "solana-streamer-sdk"
version = "1.4.6" version = "1.4.7"
edition = "2021" edition = "2021"
authors = ["William <byteblock6@gmail.com>", "sgxiang <sgxiang@gmail.com>", "wei <1415121722@qq.com>"] authors = ["William <byteblock6@gmail.com>", "sgxiang <sgxiang@gmail.com>", "wei <1415121722@qq.com>"]
repository = "https://github.com/0xfnzero/solana-streamer" repository = "https://github.com/0xfnzero/solana-streamer"
@@ -21,8 +21,7 @@ sdk-perf-stats = ["sol-parser-sdk/perf-stats"]
sdk-ultra-perf = ["sol-parser-sdk/ultra-perf"] sdk-ultra-perf = ["sol-parser-sdk/ultra-perf"]
[dependencies] [dependencies]
# Keep the streamer facade in lockstep with the sibling SDK while both crates evolve together. sol-parser-sdk = { version = "0.4.11", default-features = false }
sol-parser-sdk = { path = "../sol-parser-sdk", version = "0.4.11", default-features = false }
solana-sdk = "3.0.0" solana-sdk = "3.0.0"
solana-client = "3.1.12" solana-client = "3.1.12"
solana-transaction-status = "3.1.12" solana-transaction-status = "3.1.12"
+17 -7
View File
@@ -123,39 +123,40 @@ Add the dependency to your `Cargo.toml`:
```toml ```toml
# Add to your Cargo.toml # Add to your Cargo.toml
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.6" } solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.7" }
``` ```
### Use crates.io ### Use crates.io
```toml ```toml
# Add to your Cargo.toml # Add to your Cargo.toml
solana-streamer-sdk = "1.4.6" solana-streamer-sdk = "1.4.7"
``` ```
Parser backend features: Parser backend features:
```toml ```toml
# Default: sol-parser-sdk parse-borsh backend # Default: sol-parser-sdk parse-borsh backend
solana-streamer-sdk = "1.4.6" solana-streamer-sdk = "1.4.7"
# Zero-copy parser backend for latency-sensitive bots # Zero-copy parser backend for latency-sensitive bots
solana-streamer-sdk = { version = "1.4.6", default-features = false, features = ["sdk-parse-zero-copy"] } solana-streamer-sdk = { version = "1.4.7", default-features = false, features = ["sdk-parse-zero-copy"] }
``` ```
If both `sdk-parse-borsh` and `sdk-parse-zero-copy` are enabled, `sol-parser-sdk 0.4.11+` uses the zero-copy backend. If both `sdk-parse-borsh` and `sdk-parse-zero-copy` are enabled, `sol-parser-sdk 0.4.11+` uses the zero-copy backend.
## 🔄 Migration Guide ## 🔄 Migration Guide
### Upgrading to v1.4.6 ### Upgrading to v1.4.7
Version 1.4.6 uses `sol-parser-sdk 0.4.11` from crates.io and supports the Pump.fun / PumpSwap fee-recipient upgrade accounts and Pump.fun v2 trade instructions across Yellowstone gRPC, ShredStream, RPC transaction parsing, and account parsing while preserving the existing subscription and callback API. Existing bots can usually upgrade by changing only the crate version. Version 1.4.7 uses `sol-parser-sdk 0.4.11` from crates.io and adds the SDK-compatible Yellowstone gRPC ordering modes to the streamer facade while preserving the existing subscription and callback API. Existing bots can keep the default ultra-low-latency `Unordered` mode or opt into `Ordered`, `StreamingOrdered`, or `MicroBatch` through `ClientConfig`.
New optional capabilities: New optional capabilities:
- `solana_streamer_sdk::parser_sdk` re-exports the raw `sol-parser-sdk` crate. - `solana_streamer_sdk::parser_sdk` re-exports the raw `sol-parser-sdk` crate.
- `solana_streamer_sdk::sdk_bridge` adapts raw SDK events back into streamer `DexEvent`. - `solana_streamer_sdk::sdk_bridge` adapts raw SDK events back into streamer `DexEvent`.
- `fetch_rpc_transaction_as_streamer_events` and `parse_encoded_rpc_transaction_as_streamer_events` parse RPC transactions into streamer events. - `fetch_rpc_transaction_as_streamer_events` and `parse_encoded_rpc_transaction_as_streamer_events` parse RPC transactions into streamer events.
- `grpc::ClientConfig::order_mode` supports `Unordered`, `Ordered`, `StreamingOrdered`, and `MicroBatch`.
- `sdk-parse-zero-copy` enables the SDK zero-copy parser backend. - `sdk-parse-zero-copy` enables the SDK zero-copy parser backend.
### Migrating from v0.5.x to v1.x.x ### Migrating from v0.5.x to v1.x.x
@@ -190,7 +191,10 @@ let callback = |event: DexEvent| {
You can customize client configuration: You can customize client configuration:
```rust ```rust
use solana_streamer_sdk::streaming::{grpc::ClientConfig, YellowstoneGrpc}; use solana_streamer_sdk::streaming::{
grpc::{ClientConfig, OrderMode},
YellowstoneGrpc,
};
// Use default configuration // Use default configuration
let grpc = YellowstoneGrpc::new(endpoint, token)?; let grpc = YellowstoneGrpc::new(endpoint, token)?;
@@ -200,6 +204,9 @@ let mut config = ClientConfig::default();
config.enable_metrics = true; // Enable performance monitoring config.enable_metrics = true; // Enable performance monitoring
config.connection.connect_timeout = 30; // 30 seconds config.connection.connect_timeout = 30; // 30 seconds
config.connection.request_timeout = 120; // 120 seconds config.connection.request_timeout = 120; // 120 seconds
config.order_mode = OrderMode::MicroBatch; // Unordered / Ordered / StreamingOrdered / MicroBatch
config.order_timeout_ms = 100;
config.micro_batch_us = 100;
let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?; let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
``` ```
@@ -209,6 +216,9 @@ let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
- `connection.connect_timeout`: Connection timeout in seconds (default: 10) - `connection.connect_timeout`: Connection timeout in seconds (default: 10)
- `connection.request_timeout`: Request timeout in seconds (default: 60) - `connection.request_timeout`: Request timeout in seconds (default: 60)
- `connection.max_decoding_message_size`: Maximum message size in bytes (default: 10MB) - `connection.max_decoding_message_size`: Maximum message size in bytes (default: 10MB)
- `order_mode`: Transaction event output ordering mode (default: `Unordered`)
- `order_timeout_ms`: Flush timeout for `Ordered` and `StreamingOrdered` modes (default: 100)
- `micro_batch_us`: Micro-batch window for `MicroBatch` mode (default: 100)
### Minimal gRPC Subscription ### Minimal gRPC Subscription
+17 -7
View File
@@ -122,39 +122,40 @@ git clone https://github.com/0xfnzero/solana-streamer
```toml ```toml
# 添加到您的 Cargo.toml # 添加到您的 Cargo.toml
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.6" } solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.7" }
``` ```
### 使用 crates.io ### 使用 crates.io
```toml ```toml
# 添加到您的 Cargo.toml # 添加到您的 Cargo.toml
solana-streamer-sdk = "1.4.6" solana-streamer-sdk = "1.4.7"
``` ```
解析后端 feature 解析后端 feature
```toml ```toml
# 默认:sol-parser-sdk parse-borsh 后端 # 默认:sol-parser-sdk parse-borsh 后端
solana-streamer-sdk = "1.4.6" solana-streamer-sdk = "1.4.7"
# 面向低延迟 Bot 的 zero-copy 解析后端 # 面向低延迟 Bot 的 zero-copy 解析后端
solana-streamer-sdk = { version = "1.4.6", default-features = false, features = ["sdk-parse-zero-copy"] } solana-streamer-sdk = { version = "1.4.7", default-features = false, features = ["sdk-parse-zero-copy"] }
``` ```
如果同时启用 `sdk-parse-borsh``sdk-parse-zero-copy``sol-parser-sdk 0.4.11+` 会优先使用 zero-copy 后端。 如果同时启用 `sdk-parse-borsh``sdk-parse-zero-copy``sol-parser-sdk 0.4.11+` 会优先使用 zero-copy 后端。
## 🔄 迁移指南 ## 🔄 迁移指南
### 升级到 v1.4.6 ### 升级到 v1.4.7
v1.4.6 使用 crates.io 上的 `sol-parser-sdk 0.4.11`,并在 Yellowstone gRPC、ShredStream、RPC 交易解析和账户解析中支持 Pump.fun / PumpSwap fee-recipient 升级账户以及 Pump.fun v2 交易指令,同时保留已有订阅和回调 API。大多数 Bot 只需要修改 crate 版本即可升级 v1.4.7 使用 crates.io 上的 `sol-parser-sdk 0.4.11`,并在 streamer facade 中补齐与 SDK 兼容的 Yellowstone gRPC 输出顺序模式,同时保留已有订阅和回调 API。现有 Bot 可以继续使用默认的超低延迟 `Unordered` 模式,也可以通过 `ClientConfig` 选择 `Ordered``StreamingOrdered``MicroBatch`
新增可选能力: 新增可选能力:
- `solana_streamer_sdk::parser_sdk` 重新导出原始 `sol-parser-sdk` crate。 - `solana_streamer_sdk::parser_sdk` 重新导出原始 `sol-parser-sdk` crate。
- `solana_streamer_sdk::sdk_bridge` 可将原始 SDK 事件适配回 streamer `DexEvent` - `solana_streamer_sdk::sdk_bridge` 可将原始 SDK 事件适配回 streamer `DexEvent`
- `fetch_rpc_transaction_as_streamer_events``parse_encoded_rpc_transaction_as_streamer_events` 可将 RPC 交易解析为 streamer 事件。 - `fetch_rpc_transaction_as_streamer_events``parse_encoded_rpc_transaction_as_streamer_events` 可将 RPC 交易解析为 streamer 事件。
- `grpc::ClientConfig::order_mode` 支持 `Unordered``Ordered``StreamingOrdered``MicroBatch`
- `sdk-parse-zero-copy` 可启用 SDK zero-copy 解析后端。 - `sdk-parse-zero-copy` 可启用 SDK zero-copy 解析后端。
### 从 v0.5.x 迁移到 v1.x.x ### 从 v0.5.x 迁移到 v1.x.x
@@ -189,7 +190,10 @@ let callback = |event: DexEvent| {
您可以自定义客户端配置: 您可以自定义客户端配置:
```rust ```rust
use solana_streamer_sdk::streaming::{grpc::ClientConfig, YellowstoneGrpc}; use solana_streamer_sdk::streaming::{
grpc::{ClientConfig, OrderMode},
YellowstoneGrpc,
};
// 使用默认配置 // 使用默认配置
let grpc = YellowstoneGrpc::new(endpoint, token)?; let grpc = YellowstoneGrpc::new(endpoint, token)?;
@@ -199,6 +203,9 @@ let mut config = ClientConfig::default();
config.enable_metrics = true; // 启用性能监控 config.enable_metrics = true; // 启用性能监控
config.connection.connect_timeout = 30; // 30 秒 config.connection.connect_timeout = 30; // 30 秒
config.connection.request_timeout = 120; // 120 秒 config.connection.request_timeout = 120; // 120 秒
config.order_mode = OrderMode::MicroBatch; // Unordered / Ordered / StreamingOrdered / MicroBatch
config.order_timeout_ms = 100;
config.micro_batch_us = 100;
let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?; let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
``` ```
@@ -208,6 +215,9 @@ let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
- `connection.connect_timeout`: 连接超时(秒)(默认:10 - `connection.connect_timeout`: 连接超时(秒)(默认:10
- `connection.request_timeout`: 请求超时(秒)(默认:60 - `connection.request_timeout`: 请求超时(秒)(默认:60
- `connection.max_decoding_message_size`: 最大消息大小(字节)(默认:10MB) - `connection.max_decoding_message_size`: 最大消息大小(字节)(默认:10MB)
- `order_mode`: 交易事件输出顺序模式(默认:`Unordered`
- `order_timeout_ms`: `Ordered``StreamingOrdered` 模式的刷新超时(默认:100
- `micro_batch_us`: `MicroBatch` 模式的微批窗口(默认:100
### 最小 gRPC 订阅 ### 最小 gRPC 订阅
+14 -1
View File
@@ -1,4 +1,5 @@
use super::constants::*; use super::constants::*;
pub use sol_parser_sdk::grpc::types::OrderMode;
/// Connection configuration /// Connection configuration
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -28,10 +29,22 @@ pub struct StreamClientConfig {
pub connection: ConnectionConfig, pub connection: ConnectionConfig,
/// Whether performance monitoring is enabled (default: false) /// Whether performance monitoring is enabled (default: false)
pub enable_metrics: bool, pub enable_metrics: bool,
/// Event output ordering mode for gRPC transaction events.
pub order_mode: OrderMode,
/// Slot timeout in milliseconds for ordered modes.
pub order_timeout_ms: u64,
/// MicroBatch window size in microseconds.
pub micro_batch_us: u64,
} }
impl Default for StreamClientConfig { impl Default for StreamClientConfig {
fn default() -> Self { fn default() -> Self {
Self { connection: ConnectionConfig::default(), enable_metrics: false } Self {
connection: ConnectionConfig::default(),
enable_metrics: false,
order_mode: OrderMode::Unordered,
order_timeout_ms: 100,
micro_batch_us: 100,
}
} }
} }
+57 -32
View File
@@ -41,6 +41,57 @@ fn create_metrics_callback(
}) })
} }
#[inline]
pub fn transaction_metrics_callback(
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
) -> Arc<dyn Fn(DexEvent) + Send + Sync> {
create_metrics_callback(callback)
}
pub fn parse_grpc_transaction_events(
transaction_pretty: crate::streaming::grpc::TransactionPretty,
protocols: &[Protocol],
event_type_filter: Option<&EventTypeFilter>,
bot_wallet: Option<Pubkey>,
) -> Vec<DexEvent> {
MetricsManager::global().add_tx_process_count();
let slot = transaction_pretty.slot;
let block_time = transaction_pretty.block_time;
let recv_us = transaction_pretty.recv_us;
let grpc_tx = transaction_pretty.grpc_tx;
let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
let update =
SubscribeUpdateTransaction { slot, transaction: Some(grpc_tx), ..Default::default() };
let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
let sdk_events = parse_subscribe_update_transaction_low_latency(
&update,
recv_us,
block_time_us,
sdk_parse_filter.as_ref(),
);
let mut events = adapt_parser_events_list(
sdk_events,
block_time.as_ref(),
recv_us,
protocols,
event_type_filter,
);
for event in events.iter_mut() {
event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
}
events
.into_iter()
.map(|event| {
crate::streaming::event_parser::core::event_parser::helpers::process_event(
event, bot_wallet,
)
})
.collect()
}
/// Process GRPC transaction events /// Process GRPC transaction events
pub async fn process_grpc_transaction( pub async fn process_grpc_transaction(
event_pretty: EventPretty, event_pretty: EventPretty,
@@ -66,40 +117,14 @@ pub async fn process_grpc_transaction(
} }
} }
EventPretty::Transaction(transaction_pretty) => { EventPretty::Transaction(transaction_pretty) => {
MetricsManager::global().add_tx_process_count(); let callback = transaction_metrics_callback(callback);
for event in parse_grpc_transaction_events(
let slot = transaction_pretty.slot; transaction_pretty,
let block_time = transaction_pretty.block_time;
let recv_us = transaction_pretty.recv_us;
let grpc_tx = transaction_pretty.grpc_tx;
let adapter_callback = create_metrics_callback(callback.clone());
let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
let update = SubscribeUpdateTransaction {
slot,
transaction: Some(grpc_tx),
..Default::default()
};
let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
let sdk_events = parse_subscribe_update_transaction_low_latency(
&update,
recv_us,
block_time_us,
sdk_parse_filter.as_ref(),
);
let events = adapt_parser_events_list(
sdk_events,
block_time.as_ref(),
recv_us,
protocols, protocols,
event_type_filter, event_type_filter,
); bot_wallet,
) {
for mut event in events { callback(event);
event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
event = crate::streaming::event_parser::core::event_parser::helpers::process_event(
event, bot_wallet,
);
adapter_callback(event);
} }
} }
EventPretty::BlockMeta(block_meta_pretty) => { EventPretty::BlockMeta(block_meta_pretty) => {
+2
View File
@@ -3,6 +3,7 @@ pub mod config;
pub mod constants; pub mod constants;
pub mod event_processor; pub mod event_processor;
pub mod metrics; pub mod metrics;
pub mod order_buffer;
pub mod subscription; pub mod subscription;
// 重新导出主要类型 // 重新导出主要类型
@@ -10,4 +11,5 @@ pub use config::*;
pub use constants::*; pub use constants::*;
pub use event_processor::*; pub use event_processor::*;
pub use metrics::*; pub use metrics::*;
pub use order_buffer::*;
pub use subscription::*; pub use subscription::*;
+196
View File
@@ -0,0 +1,196 @@
use crate::streaming::event_parser::DexEvent;
use std::collections::{BTreeMap, HashMap};
use tokio::time::Instant;
#[derive(Default)]
pub struct SlotBuffer {
slots: BTreeMap<u64, Vec<(u64, DexEvent)>>,
current_slot: u64,
last_flush_time: Option<Instant>,
streaming_watermarks: HashMap<u64, u64>,
}
impl SlotBuffer {
#[inline]
pub fn new() -> Self {
Self {
slots: BTreeMap::new(),
current_slot: 0,
last_flush_time: Some(Instant::now()),
streaming_watermarks: HashMap::new(),
}
}
#[inline]
pub fn push(&mut self, slot: u64, tx_index: u64, event: DexEvent) {
if self.slots.is_empty() {
self.last_flush_time = Some(Instant::now());
}
self.slots.entry(slot).or_default().push((tx_index, event));
if slot > self.current_slot {
self.current_slot = slot;
}
}
#[inline]
pub fn is_empty(&self) -> bool {
self.slots.is_empty()
}
pub fn flush_before(&mut self, current_slot: u64) -> Vec<DexEvent> {
let slots_to_flush: Vec<u64> =
self.slots.keys().filter(|&&s| s < current_slot).copied().collect();
let mut result = Vec::with_capacity(slots_to_flush.len() * 4);
for slot in slots_to_flush {
if let Some(mut events) = self.slots.remove(&slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
pub fn flush_all(&mut self) -> Vec<DexEvent> {
let all_slots: Vec<u64> = self.slots.keys().copied().collect();
let mut result = Vec::with_capacity(all_slots.len() * 4);
for slot in all_slots {
if let Some(mut events) = self.slots.remove(&slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
#[inline]
pub fn should_timeout(&self, timeout_ms: u64) -> bool {
self.last_flush_time
.map(|t| !self.slots.is_empty() && t.elapsed().as_millis() as u64 > timeout_ms)
.unwrap_or(false)
}
pub fn push_streaming(&mut self, slot: u64, tx_index: u64, event: DexEvent) -> Vec<DexEvent> {
let mut result = Vec::new();
if slot > self.current_slot && self.current_slot > 0 {
let old_slots: Vec<u64> = self.slots.keys().filter(|&&s| s < slot).copied().collect();
for old_slot in old_slots {
if let Some(mut events) = self.slots.remove(&old_slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
self.streaming_watermarks.remove(&old_slot);
}
}
if slot > self.current_slot {
self.current_slot = slot;
}
let next_expected = *self.streaming_watermarks.get(&slot).unwrap_or(&0);
if tx_index == next_expected {
result.push(event);
let mut watermark = next_expected + 1;
if let Some(buffered) = self.slots.get_mut(&slot) {
buffered.sort_unstable_by_key(|(idx, _)| *idx);
while let Some(pos) = buffered.iter().position(|(idx, _)| *idx == watermark) {
result.push(buffered.remove(pos).1);
watermark += 1;
}
}
self.streaming_watermarks.insert(slot, watermark);
} else if tx_index > next_expected {
if self.slots.is_empty() {
self.last_flush_time = Some(Instant::now());
}
self.slots.entry(slot).or_default().push((tx_index, event));
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
pub fn flush_streaming_timeout(&mut self) -> Vec<DexEvent> {
let mut result = Vec::new();
for (slot, mut events) in std::mem::take(&mut self.slots) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
self.streaming_watermarks.remove(&slot);
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
}
pub struct MicroBatchBuffer {
events: Vec<(u64, u64, DexEvent)>,
window_start_us: i64,
}
impl MicroBatchBuffer {
#[inline]
pub fn new() -> Self {
Self { events: Vec::with_capacity(64), window_start_us: 0 }
}
#[inline]
pub fn push(
&mut self,
slot: u64,
tx_index: u64,
event: DexEvent,
now_us: i64,
window_us: u64,
) -> bool {
if self.events.is_empty() {
self.window_start_us = now_us;
}
self.events.push((slot, tx_index, event));
(now_us - self.window_start_us) as u64 >= window_us
}
#[inline]
pub fn flush(&mut self) -> Vec<DexEvent> {
if self.events.is_empty() {
return Vec::new();
}
self.events.sort_unstable_by_key(|(slot, tx_index, _)| (*slot, *tx_index));
let result =
std::mem::take(&mut self.events).into_iter().map(|(_, _, event)| event).collect();
self.window_start_us = 0;
result
}
#[inline]
pub fn should_flush(&self, now_us: i64, window_us: u64) -> bool {
!self.events.is_empty() && (now_us - self.window_start_us) as u64 >= window_us
}
#[inline]
pub fn is_empty(&self) -> bool {
self.events.is_empty()
}
}
impl Default for MicroBatchBuffer {
fn default() -> Self {
Self::new()
}
}
+2 -1
View File
@@ -12,5 +12,6 @@ pub use types::*;
// 从公用模块重新导出 // 从公用模块重新导出
pub use crate::streaming::common::{ pub use crate::streaming::common::{
ConnectionConfig, MetricsManager, PerformanceMetrics, StreamClientConfig as ClientConfig, ConnectionConfig, MetricsManager, OrderMode, PerformanceMetrics,
StreamClientConfig as ClientConfig,
}; };
+206 -11
View File
@@ -1,21 +1,24 @@
use crate::common::AnyResult; use crate::common::AnyResult;
use crate::streaming::common::{ use crate::streaming::common::{
process_grpc_transaction, MetricsManager, PerformanceMetrics, StreamClientConfig, parse_grpc_transaction_events, process_grpc_transaction, transaction_metrics_callback,
MetricsManager, MicroBatchBuffer, PerformanceMetrics, SlotBuffer, StreamClientConfig,
SubscriptionHandle, SubscriptionHandle,
}; };
use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::{DexEvent, Protocol}; use crate::streaming::event_parser::{DexEvent, Protocol};
use crate::streaming::grpc::pool::factory; use crate::streaming::grpc::pool::factory;
use crate::streaming::grpc::{EventPretty, SubscriptionManager}; use crate::streaming::grpc::{EventPretty, SubscriptionManager, TransactionPretty};
use anyhow::anyhow; use anyhow::anyhow;
use futures::channel::mpsc; use futures::channel::mpsc;
use futures::{SinkExt, StreamExt}; use futures::{SinkExt, StreamExt};
use log::error; use log::error;
use sol_parser_sdk::grpc::OrderMode;
use solana_sdk::pubkey::Pubkey; use solana_sdk::pubkey::Pubkey;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tokio::time::{Duration, Instant};
use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
use yellowstone_grpc_proto::geyser::{ use yellowstone_grpc_proto::geyser::{
CommitmentLevel, SubscribeRequest, SubscribeRequestFilterAccountsFilter, SubscribeRequestPing, CommitmentLevel, SubscribeRequest, SubscribeRequestFilterAccountsFilter, SubscribeRequestPing,
@@ -179,9 +182,35 @@ impl YellowstoneGrpc {
// Wrap callback once before the async block // Wrap callback once before the async block
let callback = Arc::new(callback); let callback = Arc::new(callback);
let transaction_callback = transaction_metrics_callback(callback.clone());
let order_mode = self.config.order_mode;
let order_timeout_ms = self.config.order_timeout_ms;
let micro_batch_us = self.config.micro_batch_us;
let stream_handle = tokio::spawn(async move { let stream_handle = tokio::spawn(async move {
let mut slot_buffer = SlotBuffer::new();
let mut micro_batch = MicroBatchBuffer::new();
let mut last_slot = 0u64;
let check_interval = match order_mode {
OrderMode::MicroBatch => Duration::from_micros(micro_batch_us.max(1)),
_ => Duration::from_millis((order_timeout_ms / 2).max(1)),
};
let mut next_check = Instant::now() + check_interval;
loop { loop {
if has_buffered_events(order_mode, &slot_buffer, &micro_batch) {
flush_ordered_timeouts(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
order_timeout_ms,
micro_batch_us,
&mut next_check,
check_interval,
);
}
tokio::select! { tokio::select! {
message = stream.next() => { message = stream.next() => {
match message { match message {
@@ -225,17 +254,18 @@ impl YellowstoneGrpc {
transaction_pretty.signature, transaction_pretty.signature,
transaction_pretty.slot transaction_pretty.slot
); );
if let Err(e) = process_grpc_transaction( handle_ordered_transaction(
EventPretty::Transaction(transaction_pretty), transaction_pretty,
&protocols, &protocols,
event_type_filter.as_ref(), event_type_filter.as_ref(),
callback.clone(), transaction_callback.clone(),
bot_wallet, bot_wallet,
) order_mode,
.await &mut slot_buffer,
{ &mut micro_batch,
error!("Error processing transaction event: {e:?}"); &mut last_slot,
} micro_batch_us,
);
} }
Some(UpdateOneof::Ping(_)) => { Some(UpdateOneof::Ping(_)) => {
// 只在需要时获取锁,并立即释放 // 只在需要时获取锁,并立即释放
@@ -261,17 +291,49 @@ impl YellowstoneGrpc {
} }
Some(Err(error)) => { Some(Err(error)) => {
error!("Stream error: {error:?}"); error!("Stream error: {error:?}");
flush_ordered_on_disconnect(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
);
break;
}
None => {
flush_ordered_on_disconnect(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
);
break; break;
} }
None => break,
} }
} }
Some(update) = control_rx.next() => { Some(update) = control_rx.next() => {
if let Err(e) = subscribe_tx.lock().await.send(update).await { if let Err(e) = subscribe_tx.lock().await.send(update).await {
error!("Failed to send subscription update: {}", e); error!("Failed to send subscription update: {}", e);
flush_ordered_on_disconnect(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
);
break; break;
} }
} }
_ = tokio::time::sleep_until(next_check), if has_buffered_events(order_mode, &slot_buffer, &micro_batch) => {
flush_ordered_timeouts(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
order_timeout_ms,
micro_batch_us,
&mut next_check,
check_interval,
);
}
} }
} }
}); });
@@ -345,6 +407,139 @@ impl YellowstoneGrpc {
} }
} }
#[allow(clippy::too_many_arguments)]
fn handle_ordered_transaction(
transaction_pretty: TransactionPretty,
protocols: &[Protocol],
event_type_filter: Option<&EventTypeFilter>,
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
bot_wallet: Option<Pubkey>,
order_mode: OrderMode,
slot_buffer: &mut SlotBuffer,
micro_batch: &mut MicroBatchBuffer,
last_slot: &mut u64,
micro_batch_us: u64,
) {
let fallback_slot = transaction_pretty.slot;
let fallback_tx_index = transaction_pretty.tx_index.unwrap_or(0);
let recv_us = transaction_pretty.recv_us;
let events =
parse_grpc_transaction_events(transaction_pretty, protocols, event_type_filter, bot_wallet);
match order_mode {
OrderMode::Unordered => {
for event in events {
callback(event);
}
}
OrderMode::Ordered => {
if fallback_slot > *last_slot && *last_slot > 0 {
deliver_events(callback.clone(), slot_buffer.flush_before(fallback_slot));
}
*last_slot = fallback_slot;
for event in events {
let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
slot_buffer.push(slot, tx_index, event);
}
}
OrderMode::StreamingOrdered => {
for event in events {
let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
deliver_events(callback.clone(), slot_buffer.push_streaming(slot, tx_index, event));
}
}
OrderMode::MicroBatch => {
for event in events {
let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
if micro_batch.push(slot, tx_index, event, recv_us, micro_batch_us) {
deliver_events(callback.clone(), micro_batch.flush());
}
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn flush_ordered_timeouts(
order_mode: OrderMode,
slot_buffer: &mut SlotBuffer,
micro_batch: &mut MicroBatchBuffer,
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
order_timeout_ms: u64,
micro_batch_us: u64,
next_check: &mut Instant,
check_interval: Duration,
) {
if Instant::now() < *next_check {
return;
}
*next_check = Instant::now() + check_interval;
match order_mode {
OrderMode::Ordered => {
if slot_buffer.should_timeout(order_timeout_ms) {
deliver_events(callback, slot_buffer.flush_all());
}
}
OrderMode::StreamingOrdered => {
if slot_buffer.should_timeout(order_timeout_ms) {
deliver_events(callback, slot_buffer.flush_streaming_timeout());
}
}
OrderMode::MicroBatch => {
let now_us =
crate::streaming::event_parser::common::high_performance_clock::get_high_perf_clock(
);
if micro_batch.should_flush(now_us, micro_batch_us) {
deliver_events(callback, micro_batch.flush());
}
}
OrderMode::Unordered => {}
}
}
fn flush_ordered_on_disconnect(
order_mode: OrderMode,
slot_buffer: &mut SlotBuffer,
micro_batch: &mut MicroBatchBuffer,
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
) {
match order_mode {
OrderMode::Ordered => deliver_events(callback, slot_buffer.flush_all()),
OrderMode::StreamingOrdered => {
deliver_events(callback, slot_buffer.flush_streaming_timeout())
}
OrderMode::MicroBatch => deliver_events(callback, micro_batch.flush()),
OrderMode::Unordered => {}
}
}
#[inline]
fn event_order_key(event: &DexEvent, fallback_slot: u64, fallback_tx_index: u64) -> (u64, u64) {
let metadata = event.metadata();
(metadata.slot.max(fallback_slot), metadata.tx_index.unwrap_or(fallback_tx_index))
}
#[inline]
fn deliver_events(callback: Arc<dyn Fn(DexEvent) + Send + Sync>, events: Vec<DexEvent>) {
for event in events {
callback(event);
}
}
#[inline]
fn has_buffered_events(
order_mode: OrderMode,
slot_buffer: &SlotBuffer,
micro_batch: &MicroBatchBuffer,
) -> bool {
match order_mode {
OrderMode::Ordered | OrderMode::StreamingOrdered => !slot_buffer.is_empty(),
OrderMode::MicroBatch => !micro_batch.is_empty(),
OrderMode::Unordered => false,
}
}
// 实现 Clone trait 以支持模块间共享 // 实现 Clone trait 以支持模块间共享
impl Clone for YellowstoneGrpc { impl Clone for YellowstoneGrpc {
fn clone(&self) -> Self { fn clone(&self) -> Self {