gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-05 21:27:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Release solana-streamer-sdk v1.4.7

Browse Source
This commit is contained in:
0xfnzero
2026-05-19 00:27:06 +08:00
parent ea0f9b9145
commit 5494fd5fb3
9 changed files with 513 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+2 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "solana-streamer-sdk"
version = "1.4.6"
version = "1.4.7"
edition = "2021"
authors = ["William <byteblock6@gmail.com>", "sgxiang <sgxiang@gmail.com>", "wei <1415121722@qq.com>"]
repository = "https://github.com/0xfnzero/solana-streamer"
@@ -21,8 +21,7 @@ sdk-perf-stats = ["sol-parser-sdk/perf-stats"]
sdk-ultra-perf = ["sol-parser-sdk/ultra-perf"]
[dependencies]
# Keep the streamer facade in lockstep with the sibling SDK while both crates evolve together.
sol-parser-sdk = { path = "../sol-parser-sdk", version = "0.4.11", default-features = false }
sol-parser-sdk = { version = "0.4.11", default-features = false }
solana-sdk = "3.0.0"
solana-client = "3.1.12"
solana-transaction-status = "3.1.12"
README.md
+17 -7
View File
@@ -123,39 +123,40 @@ Add the dependency to your `Cargo.toml`:
```toml
# Add to your Cargo.toml
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.6" }
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.7" }
```
### Use crates.io
```toml
# Add to your Cargo.toml
solana-streamer-sdk = "1.4.6"
solana-streamer-sdk = "1.4.7"
```
Parser backend features:
```toml
# Default: sol-parser-sdk parse-borsh backend
solana-streamer-sdk = "1.4.6"
solana-streamer-sdk = "1.4.7"
# Zero-copy parser backend for latency-sensitive bots
solana-streamer-sdk = { version = "1.4.6", default-features = false, features = ["sdk-parse-zero-copy"] }
solana-streamer-sdk = { version = "1.4.7", default-features = false, features = ["sdk-parse-zero-copy"] }
```
If both `sdk-parse-borsh` and `sdk-parse-zero-copy` are enabled, `sol-parser-sdk 0.4.11+` uses the zero-copy backend.
## 🔄 Migration Guide
### Upgrading to v1.4.6
### Upgrading to v1.4.7
Version 1.4.6 uses `sol-parser-sdk 0.4.11` from crates.io and supports the Pump.fun / PumpSwap fee-recipient upgrade accounts and Pump.fun v2 trade instructions across Yellowstone gRPC, ShredStream, RPC transaction parsing, and account parsing while preserving the existing subscription and callback API. Existing bots can usually upgrade by changing only the crate version.
Version 1.4.7 uses `sol-parser-sdk 0.4.11` from crates.io and adds the SDK-compatible Yellowstone gRPC ordering modes to the streamer facade while preserving the existing subscription and callback API. Existing bots can keep the default ultra-low-latency `Unordered` mode or opt into `Ordered`, `StreamingOrdered`, or `MicroBatch` through `ClientConfig`.
New optional capabilities:
- `solana_streamer_sdk::parser_sdk` re-exports the raw `sol-parser-sdk` crate.
- `solana_streamer_sdk::sdk_bridge` adapts raw SDK events back into streamer `DexEvent`.
- `fetch_rpc_transaction_as_streamer_events` and `parse_encoded_rpc_transaction_as_streamer_events` parse RPC transactions into streamer events.
- `grpc::ClientConfig::order_mode` supports `Unordered`, `Ordered`, `StreamingOrdered`, and `MicroBatch`.
- `sdk-parse-zero-copy` enables the SDK zero-copy parser backend.
### Migrating from v0.5.x to v1.x.x
@@ -190,7 +191,10 @@ let callback = |event: DexEvent| {
You can customize client configuration:
```rust
use solana_streamer_sdk::streaming::{grpc::ClientConfig, YellowstoneGrpc};
use solana_streamer_sdk::streaming::{
grpc::{ClientConfig, OrderMode},
YellowstoneGrpc,
};
// Use default configuration
let grpc = YellowstoneGrpc::new(endpoint, token)?;
@@ -200,6 +204,9 @@ let mut config = ClientConfig::default();
config.enable_metrics = true; // Enable performance monitoring
config.connection.connect_timeout = 30; // 30 seconds
config.connection.request_timeout = 120; // 120 seconds
config.order_mode = OrderMode::MicroBatch; // Unordered / Ordered / StreamingOrdered / MicroBatch
config.order_timeout_ms = 100;
config.micro_batch_us = 100;
let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
```
@@ -209,6 +216,9 @@ let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
- `connection.connect_timeout`: Connection timeout in seconds (default: 10)
- `connection.request_timeout`: Request timeout in seconds (default: 60)
- `connection.max_decoding_message_size`: Maximum message size in bytes (default: 10MB)
- `order_mode`: Transaction event output ordering mode (default: `Unordered`)
- `order_timeout_ms`: Flush timeout for `Ordered` and `StreamingOrdered` modes (default: 100)
- `micro_batch_us`: Micro-batch window for `MicroBatch` mode (default: 100)
### Minimal gRPC Subscription
README_CN.md
+17 -7
View File
@@ -122,39 +122,40 @@ git clone https://github.com/0xfnzero/solana-streamer
```toml
# 添加到您的 Cargo.toml
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.6" }
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.7" }
```
### 使用 crates.io
```toml
# 添加到您的 Cargo.toml
solana-streamer-sdk = "1.4.6"
solana-streamer-sdk = "1.4.7"
```
解析后端 feature：
```toml
# 默认：sol-parser-sdk parse-borsh 后端
solana-streamer-sdk = "1.4.6"
solana-streamer-sdk = "1.4.7"
# 面向低延迟 Bot 的 zero-copy 解析后端
solana-streamer-sdk = { version = "1.4.6", default-features = false, features = ["sdk-parse-zero-copy"] }
solana-streamer-sdk = { version = "1.4.7", default-features = false, features = ["sdk-parse-zero-copy"] }
```
如果同时启用 `sdk-parse-borsh` 和 `sdk-parse-zero-copy`，`sol-parser-sdk 0.4.11+` 会优先使用 zero-copy 后端。
## 🔄 迁移指南
### 升级到 v1.4.6
### 升级到 v1.4.7
v1.4.6 使用 crates.io 上的 `sol-parser-sdk 0.4.11`，并在 Yellowstone gRPC、ShredStream、RPC 交易解析和账户解析中支持 Pump.fun / PumpSwap fee-recipient 升级账户以及 Pump.fun v2 交易指令，同时保留已有订阅和回调 API。大多数 Bot 只需要修改 crate 版本即可升级。
v1.4.7 使用 crates.io 上的 `sol-parser-sdk 0.4.11`，并在 streamer facade 中补齐与 SDK 兼容的 Yellowstone gRPC 输出顺序模式，同时保留已有订阅和回调 API。现有 Bot 可以继续使用默认的超低延迟 `Unordered` 模式，也可以通过 `ClientConfig` 选择 `Ordered`、`StreamingOrdered` 或 `MicroBatch`。
新增可选能力：
- `solana_streamer_sdk::parser_sdk` 重新导出原始 `sol-parser-sdk` crate。
- `solana_streamer_sdk::sdk_bridge` 可将原始 SDK 事件适配回 streamer `DexEvent`。
- `fetch_rpc_transaction_as_streamer_events` 和 `parse_encoded_rpc_transaction_as_streamer_events` 可将 RPC 交易解析为 streamer 事件。
- `grpc::ClientConfig::order_mode` 支持 `Unordered`、`Ordered`、`StreamingOrdered` 和 `MicroBatch`。
- `sdk-parse-zero-copy` 可启用 SDK zero-copy 解析后端。
### 从 v0.5.x 迁移到 v1.x.x
@@ -189,7 +190,10 @@ let callback = |event: DexEvent| {
您可以自定义客户端配置：
```rust
use solana_streamer_sdk::streaming::{grpc::ClientConfig, YellowstoneGrpc};
use solana_streamer_sdk::streaming::{
grpc::{ClientConfig, OrderMode},
YellowstoneGrpc,
};
// 使用默认配置
let grpc = YellowstoneGrpc::new(endpoint, token)?;
@@ -199,6 +203,9 @@ let mut config = ClientConfig::default();
config.enable_metrics = true; // 启用性能监控
config.connection.connect_timeout = 30; // 30 秒
config.connection.request_timeout = 120; // 120 秒
config.order_mode = OrderMode::MicroBatch; // Unordered / Ordered / StreamingOrdered / MicroBatch
config.order_timeout_ms = 100;
config.micro_batch_us = 100;
let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
```
@@ -208,6 +215,9 @@ let grpc = YellowstoneGrpc::new_with_config(endpoint, token, config)?;
- `connection.connect_timeout`: 连接超时（秒）（默认：10）
- `connection.request_timeout`: 请求超时（秒）（默认：60）
- `connection.max_decoding_message_size`: 最大消息大小（字节）（默认：10MB）
- `order_mode`: 交易事件输出顺序模式（默认：`Unordered`）
- `order_timeout_ms`: `Ordered` 和 `StreamingOrdered` 模式的刷新超时（默认：100）
- `micro_batch_us`: `MicroBatch` 模式的微批窗口（默认：100）
### 最小 gRPC 订阅
src/streaming/common/config.rs
+14 -1
View File
@@ -1,4 +1,5 @@
use super::constants::*;
pub use sol_parser_sdk::grpc::types::OrderMode;
/// Connection configuration
#[derive(Debug, Clone)]
@@ -28,10 +29,22 @@ pub struct StreamClientConfig {
pub connection: ConnectionConfig,
/// Whether performance monitoring is enabled (default: false)
pub enable_metrics: bool,
/// Event output ordering mode for gRPC transaction events.
pub order_mode: OrderMode,
/// Slot timeout in milliseconds for ordered modes.
pub order_timeout_ms: u64,
/// MicroBatch window size in microseconds.
pub micro_batch_us: u64,
}
impl Default for StreamClientConfig {
fn default() -> Self {
Self { connection: ConnectionConfig::default(), enable_metrics: false }
Self {
connection: ConnectionConfig::default(),
enable_metrics: false,
order_mode: OrderMode::Unordered,
order_timeout_ms: 100,
micro_batch_us: 100,
}
}
}
src/streaming/common/event_processor.rs
+57 -32
View File
@@ -41,6 +41,57 @@ fn create_metrics_callback(
})
}
#[inline]
pub fn transaction_metrics_callback(
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
) -> Arc<dyn Fn(DexEvent) + Send + Sync> {
create_metrics_callback(callback)
}
pub fn parse_grpc_transaction_events(
transaction_pretty: crate::streaming::grpc::TransactionPretty,
protocols: &[Protocol],
event_type_filter: Option<&EventTypeFilter>,
bot_wallet: Option<Pubkey>,
) -> Vec<DexEvent> {
MetricsManager::global().add_tx_process_count();
let slot = transaction_pretty.slot;
let block_time = transaction_pretty.block_time;
let recv_us = transaction_pretty.recv_us;
let grpc_tx = transaction_pretty.grpc_tx;
let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
let update =
SubscribeUpdateTransaction { slot, transaction: Some(grpc_tx), ..Default::default() };
let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
let sdk_events = parse_subscribe_update_transaction_low_latency(
&update,
recv_us,
block_time_us,
sdk_parse_filter.as_ref(),
);
let mut events = adapt_parser_events_list(
sdk_events,
block_time.as_ref(),
recv_us,
protocols,
event_type_filter,
);
for event in events.iter_mut() {
event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
}
events
.into_iter()
.map(|event| {
crate::streaming::event_parser::core::event_parser::helpers::process_event(
event, bot_wallet,
)
})
.collect()
}
/// Process GRPC transaction events
pub async fn process_grpc_transaction(
event_pretty: EventPretty,
@@ -66,40 +117,14 @@ pub async fn process_grpc_transaction(
}
}
EventPretty::Transaction(transaction_pretty) => {
MetricsManager::global().add_tx_process_count();
let slot = transaction_pretty.slot;
let block_time = transaction_pretty.block_time;
let recv_us = transaction_pretty.recv_us;
let grpc_tx = transaction_pretty.grpc_tx;
let adapter_callback = create_metrics_callback(callback.clone());
let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
let update = SubscribeUpdateTransaction {
slot,
transaction: Some(grpc_tx),
..Default::default()
};
let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
let sdk_events = parse_subscribe_update_transaction_low_latency(
&update,
recv_us,
block_time_us,
sdk_parse_filter.as_ref(),
);
let events = adapt_parser_events_list(
sdk_events,
block_time.as_ref(),
recv_us,
let callback = transaction_metrics_callback(callback);
for event in parse_grpc_transaction_events(
transaction_pretty,
protocols,
event_type_filter,
);
for mut event in events {
event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
event = crate::streaming::event_parser::core::event_parser::helpers::process_event(
event, bot_wallet,
);
adapter_callback(event);
bot_wallet,
) {
callback(event);
}
}
EventPretty::BlockMeta(block_meta_pretty) => {
src/streaming/common/mod.rs
+2
View File
@@ -3,6 +3,7 @@ pub mod config;
pub mod constants;
pub mod event_processor;
pub mod metrics;
pub mod order_buffer;
pub mod subscription;
// 重新导出主要类型
@@ -10,4 +11,5 @@ pub use config::*;
pub use constants::*;
pub use event_processor::*;
pub use metrics::*;
pub use order_buffer::*;
pub use subscription::*;
src/streaming/common/order_buffer.rs
+196
View File
@@ -0,0 +1,196 @@
use crate::streaming::event_parser::DexEvent;
use std::collections::{BTreeMap, HashMap};
use tokio::time::Instant;
#[derive(Default)]
pub struct SlotBuffer {
slots: BTreeMap<u64, Vec<(u64, DexEvent)>>,
current_slot: u64,
last_flush_time: Option<Instant>,
streaming_watermarks: HashMap<u64, u64>,
}
impl SlotBuffer {
#[inline]
pub fn new() -> Self {
Self {
slots: BTreeMap::new(),
current_slot: 0,
last_flush_time: Some(Instant::now()),
streaming_watermarks: HashMap::new(),
}
}
#[inline]
pub fn push(&mut self, slot: u64, tx_index: u64, event: DexEvent) {
if self.slots.is_empty() {
self.last_flush_time = Some(Instant::now());
}
self.slots.entry(slot).or_default().push((tx_index, event));
if slot > self.current_slot {
self.current_slot = slot;
}
}
#[inline]
pub fn is_empty(&self) -> bool {
self.slots.is_empty()
}
pub fn flush_before(&mut self, current_slot: u64) -> Vec<DexEvent> {
let slots_to_flush: Vec<u64> =
self.slots.keys().filter(|&&s| s < current_slot).copied().collect();
let mut result = Vec::with_capacity(slots_to_flush.len() * 4);
for slot in slots_to_flush {
if let Some(mut events) = self.slots.remove(&slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
pub fn flush_all(&mut self) -> Vec<DexEvent> {
let all_slots: Vec<u64> = self.slots.keys().copied().collect();
let mut result = Vec::with_capacity(all_slots.len() * 4);
for slot in all_slots {
if let Some(mut events) = self.slots.remove(&slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
#[inline]
pub fn should_timeout(&self, timeout_ms: u64) -> bool {
self.last_flush_time
.map(|t| !self.slots.is_empty() && t.elapsed().as_millis() as u64 > timeout_ms)
.unwrap_or(false)
}
pub fn push_streaming(&mut self, slot: u64, tx_index: u64, event: DexEvent) -> Vec<DexEvent> {
let mut result = Vec::new();
if slot > self.current_slot && self.current_slot > 0 {
let old_slots: Vec<u64> = self.slots.keys().filter(|&&s| s < slot).copied().collect();
for old_slot in old_slots {
if let Some(mut events) = self.slots.remove(&old_slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
self.streaming_watermarks.remove(&old_slot);
}
}
if slot > self.current_slot {
self.current_slot = slot;
}
let next_expected = *self.streaming_watermarks.get(&slot).unwrap_or(&0);
if tx_index == next_expected {
result.push(event);
let mut watermark = next_expected + 1;
if let Some(buffered) = self.slots.get_mut(&slot) {
buffered.sort_unstable_by_key(|(idx, _)| *idx);
while let Some(pos) = buffered.iter().position(|(idx, _)| *idx == watermark) {
result.push(buffered.remove(pos).1);
watermark += 1;
}
}
self.streaming_watermarks.insert(slot, watermark);
} else if tx_index > next_expected {
if self.slots.is_empty() {
self.last_flush_time = Some(Instant::now());
}
self.slots.entry(slot).or_default().push((tx_index, event));
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
pub fn flush_streaming_timeout(&mut self) -> Vec<DexEvent> {
let mut result = Vec::new();
for (slot, mut events) in std::mem::take(&mut self.slots) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
self.streaming_watermarks.remove(&slot);
}
if !result.is_empty() {
self.last_flush_time = Some(Instant::now());
}
result
}
}
pub struct MicroBatchBuffer {
events: Vec<(u64, u64, DexEvent)>,
window_start_us: i64,
}
impl MicroBatchBuffer {
#[inline]
pub fn new() -> Self {
Self { events: Vec::with_capacity(64), window_start_us: 0 }
}
#[inline]
pub fn push(
&mut self,
slot: u64,
tx_index: u64,
event: DexEvent,
now_us: i64,
window_us: u64,
) -> bool {
if self.events.is_empty() {
self.window_start_us = now_us;
}
self.events.push((slot, tx_index, event));
(now_us - self.window_start_us) as u64 >= window_us
}
#[inline]
pub fn flush(&mut self) -> Vec<DexEvent> {
if self.events.is_empty() {
return Vec::new();
}
self.events.sort_unstable_by_key(|(slot, tx_index, _)| (*slot, *tx_index));
let result =
std::mem::take(&mut self.events).into_iter().map(|(_, _, event)| event).collect();
self.window_start_us = 0;
result
}
#[inline]
pub fn should_flush(&self, now_us: i64, window_us: u64) -> bool {
!self.events.is_empty() && (now_us - self.window_start_us) as u64 >= window_us
}
#[inline]
pub fn is_empty(&self) -> bool {
self.events.is_empty()
}
}
impl Default for MicroBatchBuffer {
fn default() -> Self {
Self::new()
}
}
src/streaming/grpc/mod.rs
+2 -1
View File
@@ -12,5 +12,6 @@ pub use types::*;
// 从公用模块重新导出
pub use crate::streaming::common::{
ConnectionConfig, MetricsManager, PerformanceMetrics, StreamClientConfig as ClientConfig,
ConnectionConfig, MetricsManager, OrderMode, PerformanceMetrics,
StreamClientConfig as ClientConfig,
};
src/streaming/yellowstone_grpc.rs
+206 -11
View File
@@ -1,21 +1,24 @@
use crate::common::AnyResult;
use crate::streaming::common::{
process_grpc_transaction, MetricsManager, PerformanceMetrics, StreamClientConfig,
parse_grpc_transaction_events, process_grpc_transaction, transaction_metrics_callback,
MetricsManager, MicroBatchBuffer, PerformanceMetrics, SlotBuffer, StreamClientConfig,
SubscriptionHandle,
};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::{DexEvent, Protocol};
use crate::streaming::grpc::pool::factory;
use crate::streaming::grpc::{EventPretty, SubscriptionManager};
use crate::streaming::grpc::{EventPretty, SubscriptionManager, TransactionPretty};
use anyhow::anyhow;
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use log::error;
use sol_parser_sdk::grpc::OrderMode;
use solana_sdk::pubkey::Pubkey;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::Mutex;
use tokio::time::{Duration, Instant};
use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
use yellowstone_grpc_proto::geyser::{
CommitmentLevel, SubscribeRequest, SubscribeRequestFilterAccountsFilter, SubscribeRequestPing,
@@ -179,9 +182,35 @@ impl YellowstoneGrpc {
// Wrap callback once before the async block
let callback = Arc::new(callback);
let transaction_callback = transaction_metrics_callback(callback.clone());
let order_mode = self.config.order_mode;
let order_timeout_ms = self.config.order_timeout_ms;
let micro_batch_us = self.config.micro_batch_us;
let stream_handle = tokio::spawn(async move {
let mut slot_buffer = SlotBuffer::new();
let mut micro_batch = MicroBatchBuffer::new();
let mut last_slot = 0u64;
let check_interval = match order_mode {
OrderMode::MicroBatch => Duration::from_micros(micro_batch_us.max(1)),
_ => Duration::from_millis((order_timeout_ms / 2).max(1)),
};
let mut next_check = Instant::now() + check_interval;
loop {
if has_buffered_events(order_mode, &slot_buffer, &micro_batch) {
flush_ordered_timeouts(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
order_timeout_ms,
micro_batch_us,
&mut next_check,
check_interval,
);
}
tokio::select! {
message = stream.next() => {
match message {
@@ -225,17 +254,18 @@ impl YellowstoneGrpc {
transaction_pretty.signature,
transaction_pretty.slot
);
if let Err(e) = process_grpc_transaction(
EventPretty::Transaction(transaction_pretty),
handle_ordered_transaction(
transaction_pretty,
&protocols,
event_type_filter.as_ref(),
callback.clone(),
transaction_callback.clone(),
bot_wallet,
)
.await
{
error!("Error processing transaction event: {e:?}");
}
order_mode,
&mut slot_buffer,
&mut micro_batch,
&mut last_slot,
micro_batch_us,
);
}
Some(UpdateOneof::Ping(_)) => {
// 只在需要时获取锁，并立即释放
@@ -261,17 +291,49 @@ impl YellowstoneGrpc {
}
Some(Err(error)) => {
error!("Stream error: {error:?}");
flush_ordered_on_disconnect(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
);
break;
}
None => {
flush_ordered_on_disconnect(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
);
break;
}
None => break,
}
}
Some(update) = control_rx.next() => {
if let Err(e) = subscribe_tx.lock().await.send(update).await {
error!("Failed to send subscription update: {}", e);
flush_ordered_on_disconnect(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
);
break;
}
}
_ = tokio::time::sleep_until(next_check), if has_buffered_events(order_mode, &slot_buffer, &micro_batch) => {
flush_ordered_timeouts(
order_mode,
&mut slot_buffer,
&mut micro_batch,
transaction_callback.clone(),
order_timeout_ms,
micro_batch_us,
&mut next_check,
check_interval,
);
}
}
}
});
@@ -345,6 +407,139 @@ impl YellowstoneGrpc {
}
}
#[allow(clippy::too_many_arguments)]
fn handle_ordered_transaction(
transaction_pretty: TransactionPretty,
protocols: &[Protocol],
event_type_filter: Option<&EventTypeFilter>,
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
bot_wallet: Option<Pubkey>,
order_mode: OrderMode,
slot_buffer: &mut SlotBuffer,
micro_batch: &mut MicroBatchBuffer,
last_slot: &mut u64,
micro_batch_us: u64,
) {
let fallback_slot = transaction_pretty.slot;
let fallback_tx_index = transaction_pretty.tx_index.unwrap_or(0);
let recv_us = transaction_pretty.recv_us;
let events =
parse_grpc_transaction_events(transaction_pretty, protocols, event_type_filter, bot_wallet);
match order_mode {
OrderMode::Unordered => {
for event in events {
callback(event);
}
}
OrderMode::Ordered => {
if fallback_slot > *last_slot && *last_slot > 0 {
deliver_events(callback.clone(), slot_buffer.flush_before(fallback_slot));
}
*last_slot = fallback_slot;
for event in events {
let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
slot_buffer.push(slot, tx_index, event);
}
}
OrderMode::StreamingOrdered => {
for event in events {
let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
deliver_events(callback.clone(), slot_buffer.push_streaming(slot, tx_index, event));
}
}
OrderMode::MicroBatch => {
for event in events {
let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
if micro_batch.push(slot, tx_index, event, recv_us, micro_batch_us) {
deliver_events(callback.clone(), micro_batch.flush());
}
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn flush_ordered_timeouts(
order_mode: OrderMode,
slot_buffer: &mut SlotBuffer,
micro_batch: &mut MicroBatchBuffer,
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
order_timeout_ms: u64,
micro_batch_us: u64,
next_check: &mut Instant,
check_interval: Duration,
) {
if Instant::now() < *next_check {
return;
}
*next_check = Instant::now() + check_interval;
match order_mode {
OrderMode::Ordered => {
if slot_buffer.should_timeout(order_timeout_ms) {
deliver_events(callback, slot_buffer.flush_all());
}
}
OrderMode::StreamingOrdered => {
if slot_buffer.should_timeout(order_timeout_ms) {
deliver_events(callback, slot_buffer.flush_streaming_timeout());
}
}
OrderMode::MicroBatch => {
let now_us =
crate::streaming::event_parser::common::high_performance_clock::get_high_perf_clock(
);
if micro_batch.should_flush(now_us, micro_batch_us) {
deliver_events(callback, micro_batch.flush());
}
}
OrderMode::Unordered => {}
}
}
fn flush_ordered_on_disconnect(
order_mode: OrderMode,
slot_buffer: &mut SlotBuffer,
micro_batch: &mut MicroBatchBuffer,
callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
) {
match order_mode {
OrderMode::Ordered => deliver_events(callback, slot_buffer.flush_all()),
OrderMode::StreamingOrdered => {
deliver_events(callback, slot_buffer.flush_streaming_timeout())
}
OrderMode::MicroBatch => deliver_events(callback, micro_batch.flush()),
OrderMode::Unordered => {}
}
}
#[inline]
fn event_order_key(event: &DexEvent, fallback_slot: u64, fallback_tx_index: u64) -> (u64, u64) {
let metadata = event.metadata();
(metadata.slot.max(fallback_slot), metadata.tx_index.unwrap_or(fallback_tx_index))
}
#[inline]
fn deliver_events(callback: Arc<dyn Fn(DexEvent) + Send + Sync>, events: Vec<DexEvent>) {
for event in events {
callback(event);
}
}
#[inline]
fn has_buffered_events(
order_mode: OrderMode,
slot_buffer: &SlotBuffer,
micro_batch: &MicroBatchBuffer,
) -> bool {
match order_mode {
OrderMode::Ordered | OrderMode::StreamingOrdered => !slot_buffer.is_empty(),
OrderMode::MicroBatch => !micro_batch.is_empty(),
OrderMode::Unordered => false,
}
}
// 实现 Clone trait 以支持模块间共享
impl Clone for YellowstoneGrpc {
fn clone(&self) -> Self {