Release solana-streamer-sdk v1.4.7

src/streaming/common/config.rs

@@ -1,4 +1,5 @@
 use super::constants::*;
+pub use sol_parser_sdk::grpc::types::OrderMode;
 
 /// Connection configuration
 #[derive(Debug, Clone)]
@@ -28,10 +29,22 @@ pub struct StreamClientConfig {
     pub connection: ConnectionConfig,
     /// Whether performance monitoring is enabled (default: false)
     pub enable_metrics: bool,
+    /// Event output ordering mode for gRPC transaction events.
+    pub order_mode: OrderMode,
+    /// Slot timeout in milliseconds for ordered modes.
+    pub order_timeout_ms: u64,
+    /// MicroBatch window size in microseconds.
+    pub micro_batch_us: u64,
 }
 
 impl Default for StreamClientConfig {
     fn default() -> Self {
-        Self { connection: ConnectionConfig::default(), enable_metrics: false }
+        Self {
+            connection: ConnectionConfig::default(),
+            enable_metrics: false,
+            order_mode: OrderMode::Unordered,
+            order_timeout_ms: 100,
+            micro_batch_us: 100,
+        }
     }
 }

src/streaming/common/event_processor.rs

@@ -41,6 +41,57 @@ fn create_metrics_callback(
     })
 }
 
+#[inline]
+pub fn transaction_metrics_callback(
+    callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
+) -> Arc<dyn Fn(DexEvent) + Send + Sync> {
+    create_metrics_callback(callback)
+}
+
+pub fn parse_grpc_transaction_events(
+    transaction_pretty: crate::streaming::grpc::TransactionPretty,
+    protocols: &[Protocol],
+    event_type_filter: Option<&EventTypeFilter>,
+    bot_wallet: Option<Pubkey>,
+) -> Vec<DexEvent> {
+    MetricsManager::global().add_tx_process_count();
+
+    let slot = transaction_pretty.slot;
+    let block_time = transaction_pretty.block_time;
+    let recv_us = transaction_pretty.recv_us;
+    let grpc_tx = transaction_pretty.grpc_tx;
+    let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
+    let update =
+        SubscribeUpdateTransaction { slot, transaction: Some(grpc_tx), ..Default::default() };
+    let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
+    let sdk_events = parse_subscribe_update_transaction_low_latency(
+        &update,
+        recv_us,
+        block_time_us,
+        sdk_parse_filter.as_ref(),
+    );
+    let mut events = adapt_parser_events_list(
+        sdk_events,
+        block_time.as_ref(),
+        recv_us,
+        protocols,
+        event_type_filter,
+    );
+
+    for event in events.iter_mut() {
+        event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
+    }
+
+    events
+        .into_iter()
+        .map(|event| {
+            crate::streaming::event_parser::core::event_parser::helpers::process_event(
+                event, bot_wallet,
+            )
+        })
+        .collect()
+}
+
 /// Process GRPC transaction events
 pub async fn process_grpc_transaction(
     event_pretty: EventPretty,
@@ -66,40 +117,14 @@ pub async fn process_grpc_transaction(
             }
         }
         EventPretty::Transaction(transaction_pretty) => {
-            MetricsManager::global().add_tx_process_count();
-
-            let slot = transaction_pretty.slot;
-            let block_time = transaction_pretty.block_time;
-            let recv_us = transaction_pretty.recv_us;
-            let grpc_tx = transaction_pretty.grpc_tx;
-            let adapter_callback = create_metrics_callback(callback.clone());
-            let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
-            let update = SubscribeUpdateTransaction {
-                slot,
-                transaction: Some(grpc_tx),
-                ..Default::default()
-            };
-            let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
-            let sdk_events = parse_subscribe_update_transaction_low_latency(
-                &update,
-                recv_us,
-                block_time_us,
-                sdk_parse_filter.as_ref(),
-            );
-            let events = adapt_parser_events_list(
-                sdk_events,
-                block_time.as_ref(),
-                recv_us,
+            let callback = transaction_metrics_callback(callback);
+            for event in parse_grpc_transaction_events(
+                transaction_pretty,
                 protocols,
                 event_type_filter,
-            );
-
-            for mut event in events {
-                event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
-                event = crate::streaming::event_parser::core::event_parser::helpers::process_event(
-                    event, bot_wallet,
-                );
-                adapter_callback(event);
+                bot_wallet,
+            ) {
+                callback(event);
             }
         }
         EventPretty::BlockMeta(block_meta_pretty) => {

src/streaming/common/mod.rs

@@ -3,6 +3,7 @@ pub mod config;
 pub mod constants;
 pub mod event_processor;
 pub mod metrics;
+pub mod order_buffer;
 pub mod subscription;
 
 // 重新导出主要类型
@@ -10,4 +11,5 @@ pub use config::*;
 pub use constants::*;
 pub use event_processor::*;
 pub use metrics::*;
+pub use order_buffer::*;
 pub use subscription::*;

src/streaming/common/order_buffer.rs

@@ -0,0 +1,196 @@
+use crate::streaming::event_parser::DexEvent;
+use std::collections::{BTreeMap, HashMap};
+use tokio::time::Instant;
+
+#[derive(Default)]
+pub struct SlotBuffer {
+    slots: BTreeMap<u64, Vec<(u64, DexEvent)>>,
+    current_slot: u64,
+    last_flush_time: Option<Instant>,
+    streaming_watermarks: HashMap<u64, u64>,
+}
+
+impl SlotBuffer {
+    #[inline]
+    pub fn new() -> Self {
+        Self {
+            slots: BTreeMap::new(),
+            current_slot: 0,
+            last_flush_time: Some(Instant::now()),
+            streaming_watermarks: HashMap::new(),
+        }
+    }
+
+    #[inline]
+    pub fn push(&mut self, slot: u64, tx_index: u64, event: DexEvent) {
+        if self.slots.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        self.slots.entry(slot).or_default().push((tx_index, event));
+        if slot > self.current_slot {
+            self.current_slot = slot;
+        }
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.slots.is_empty()
+    }
+
+    pub fn flush_before(&mut self, current_slot: u64) -> Vec<DexEvent> {
+        let slots_to_flush: Vec<u64> =
+            self.slots.keys().filter(|&&s| s < current_slot).copied().collect();
+
+        let mut result = Vec::with_capacity(slots_to_flush.len() * 4);
+        for slot in slots_to_flush {
+            if let Some(mut events) = self.slots.remove(&slot) {
+                events.sort_unstable_by_key(|(idx, _)| *idx);
+                result.extend(events.into_iter().map(|(_, event)| event));
+            }
+        }
+
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+
+    pub fn flush_all(&mut self) -> Vec<DexEvent> {
+        let all_slots: Vec<u64> = self.slots.keys().copied().collect();
+        let mut result = Vec::with_capacity(all_slots.len() * 4);
+
+        for slot in all_slots {
+            if let Some(mut events) = self.slots.remove(&slot) {
+                events.sort_unstable_by_key(|(idx, _)| *idx);
+                result.extend(events.into_iter().map(|(_, event)| event));
+            }
+        }
+
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+
+    #[inline]
+    pub fn should_timeout(&self, timeout_ms: u64) -> bool {
+        self.last_flush_time
+            .map(|t| !self.slots.is_empty() && t.elapsed().as_millis() as u64 > timeout_ms)
+            .unwrap_or(false)
+    }
+
+    pub fn push_streaming(&mut self, slot: u64, tx_index: u64, event: DexEvent) -> Vec<DexEvent> {
+        let mut result = Vec::new();
+
+        if slot > self.current_slot && self.current_slot > 0 {
+            let old_slots: Vec<u64> = self.slots.keys().filter(|&&s| s < slot).copied().collect();
+            for old_slot in old_slots {
+                if let Some(mut events) = self.slots.remove(&old_slot) {
+                    events.sort_unstable_by_key(|(idx, _)| *idx);
+                    result.extend(events.into_iter().map(|(_, event)| event));
+                }
+                self.streaming_watermarks.remove(&old_slot);
+            }
+        }
+
+        if slot > self.current_slot {
+            self.current_slot = slot;
+        }
+
+        let next_expected = *self.streaming_watermarks.get(&slot).unwrap_or(&0);
+
+        if tx_index == next_expected {
+            result.push(event);
+            let mut watermark = next_expected + 1;
+
+            if let Some(buffered) = self.slots.get_mut(&slot) {
+                buffered.sort_unstable_by_key(|(idx, _)| *idx);
+                while let Some(pos) = buffered.iter().position(|(idx, _)| *idx == watermark) {
+                    result.push(buffered.remove(pos).1);
+                    watermark += 1;
+                }
+            }
+            self.streaming_watermarks.insert(slot, watermark);
+        } else if tx_index > next_expected {
+            if self.slots.is_empty() {
+                self.last_flush_time = Some(Instant::now());
+            }
+            self.slots.entry(slot).or_default().push((tx_index, event));
+        }
+
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+
+    pub fn flush_streaming_timeout(&mut self) -> Vec<DexEvent> {
+        let mut result = Vec::new();
+        for (slot, mut events) in std::mem::take(&mut self.slots) {
+            events.sort_unstable_by_key(|(idx, _)| *idx);
+            result.extend(events.into_iter().map(|(_, event)| event));
+            self.streaming_watermarks.remove(&slot);
+        }
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+}
+
+pub struct MicroBatchBuffer {
+    events: Vec<(u64, u64, DexEvent)>,
+    window_start_us: i64,
+}
+
+impl MicroBatchBuffer {
+    #[inline]
+    pub fn new() -> Self {
+        Self { events: Vec::with_capacity(64), window_start_us: 0 }
+    }
+
+    #[inline]
+    pub fn push(
+        &mut self,
+        slot: u64,
+        tx_index: u64,
+        event: DexEvent,
+        now_us: i64,
+        window_us: u64,
+    ) -> bool {
+        if self.events.is_empty() {
+            self.window_start_us = now_us;
+        }
+        self.events.push((slot, tx_index, event));
+        (now_us - self.window_start_us) as u64 >= window_us
+    }
+
+    #[inline]
+    pub fn flush(&mut self) -> Vec<DexEvent> {
+        if self.events.is_empty() {
+            return Vec::new();
+        }
+
+        self.events.sort_unstable_by_key(|(slot, tx_index, _)| (*slot, *tx_index));
+        let result =
+            std::mem::take(&mut self.events).into_iter().map(|(_, _, event)| event).collect();
+        self.window_start_us = 0;
+        result
+    }
+
+    #[inline]
+    pub fn should_flush(&self, now_us: i64, window_us: u64) -> bool {
+        !self.events.is_empty() && (now_us - self.window_start_us) as u64 >= window_us
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.events.is_empty()
+    }
+}
+
+impl Default for MicroBatchBuffer {
+    fn default() -> Self {
+        Self::new()
+    }
+}

src/streaming/grpc/mod.rs

@@ -12,5 +12,6 @@ pub use types::*;
 
 // 从公用模块重新导出
 pub use crate::streaming::common::{
-    ConnectionConfig, MetricsManager, PerformanceMetrics, StreamClientConfig as ClientConfig,
+    ConnectionConfig, MetricsManager, OrderMode, PerformanceMetrics,
+    StreamClientConfig as ClientConfig,
 };

src/streaming/yellowstone_grpc.rs

@@ -1,21 +1,24 @@
 use crate::common::AnyResult;
 use crate::streaming::common::{
-    process_grpc_transaction, MetricsManager, PerformanceMetrics, StreamClientConfig,
+    parse_grpc_transaction_events, process_grpc_transaction, transaction_metrics_callback,
+    MetricsManager, MicroBatchBuffer, PerformanceMetrics, SlotBuffer, StreamClientConfig,
     SubscriptionHandle,
 };
 use crate::streaming::event_parser::common::filter::EventTypeFilter;
 use crate::streaming::event_parser::{DexEvent, Protocol};
 use crate::streaming::grpc::pool::factory;
-use crate::streaming::grpc::{EventPretty, SubscriptionManager};
+use crate::streaming::grpc::{EventPretty, SubscriptionManager, TransactionPretty};
 use anyhow::anyhow;
 use futures::channel::mpsc;
 use futures::{SinkExt, StreamExt};
 use log::error;
+use sol_parser_sdk::grpc::OrderMode;
 use solana_sdk::pubkey::Pubkey;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::sync::Mutex;
+use tokio::time::{Duration, Instant};
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::{
     CommitmentLevel, SubscribeRequest, SubscribeRequestFilterAccountsFilter, SubscribeRequestPing,
@@ -179,9 +182,35 @@ impl YellowstoneGrpc {
 
         // Wrap callback once before the async block
         let callback = Arc::new(callback);
+        let transaction_callback = transaction_metrics_callback(callback.clone());
+        let order_mode = self.config.order_mode;
+        let order_timeout_ms = self.config.order_timeout_ms;
+        let micro_batch_us = self.config.micro_batch_us;
 
         let stream_handle = tokio::spawn(async move {
+            let mut slot_buffer = SlotBuffer::new();
+            let mut micro_batch = MicroBatchBuffer::new();
+            let mut last_slot = 0u64;
+            let check_interval = match order_mode {
+                OrderMode::MicroBatch => Duration::from_micros(micro_batch_us.max(1)),
+                _ => Duration::from_millis((order_timeout_ms / 2).max(1)),
+            };
+            let mut next_check = Instant::now() + check_interval;
+
             loop {
+                if has_buffered_events(order_mode, &slot_buffer, &micro_batch) {
+                    flush_ordered_timeouts(
+                        order_mode,
+                        &mut slot_buffer,
+                        &mut micro_batch,
+                        transaction_callback.clone(),
+                        order_timeout_ms,
+                        micro_batch_us,
+                        &mut next_check,
+                        check_interval,
+                    );
+                }
+
                 tokio::select! {
                     message = stream.next() => {
                         match message {
@@ -225,17 +254,18 @@ impl YellowstoneGrpc {
                                             transaction_pretty.signature,
                                             transaction_pretty.slot
                                         );
-                                        if let Err(e) = process_grpc_transaction(
-                                            EventPretty::Transaction(transaction_pretty),
+                                        handle_ordered_transaction(
+                                            transaction_pretty,
                                             &protocols,
                                             event_type_filter.as_ref(),
-                                            callback.clone(),
+                                            transaction_callback.clone(),
                                             bot_wallet,
-                                        )
-                                        .await
-                                        {
-                                            error!("Error processing transaction event: {e:?}");
-                                        }
+                                            order_mode,
+                                            &mut slot_buffer,
+                                            &mut micro_batch,
+                                            &mut last_slot,
+                                            micro_batch_us,
+                                        );
                                     }
                                     Some(UpdateOneof::Ping(_)) => {
                                         // 只在需要时获取锁，并立即释放
@@ -261,17 +291,49 @@ impl YellowstoneGrpc {
                             }
                             Some(Err(error)) => {
                                 error!("Stream error: {error:?}");
+                                flush_ordered_on_disconnect(
+                                    order_mode,
+                                    &mut slot_buffer,
+                                    &mut micro_batch,
+                                    transaction_callback.clone(),
+                                );
+                                break;
+                            }
+                            None => {
+                                flush_ordered_on_disconnect(
+                                    order_mode,
+                                    &mut slot_buffer,
+                                    &mut micro_batch,
+                                    transaction_callback.clone(),
+                                );
                                 break;
                             }
-                            None => break,
                         }
                     }
                     Some(update) = control_rx.next() => {
                         if let Err(e) = subscribe_tx.lock().await.send(update).await {
                             error!("Failed to send subscription update: {}", e);
+                            flush_ordered_on_disconnect(
+                                order_mode,
+                                &mut slot_buffer,
+                                &mut micro_batch,
+                                transaction_callback.clone(),
+                            );
                             break;
                         }
                     }
+                    _ = tokio::time::sleep_until(next_check), if has_buffered_events(order_mode, &slot_buffer, &micro_batch) => {
+                        flush_ordered_timeouts(
+                            order_mode,
+                            &mut slot_buffer,
+                            &mut micro_batch,
+                            transaction_callback.clone(),
+                            order_timeout_ms,
+                            micro_batch_us,
+                            &mut next_check,
+                            check_interval,
+                        );
+                    }
                 }
             }
         });
@@ -345,6 +407,139 @@ impl YellowstoneGrpc {
     }
 }
 
+#[allow(clippy::too_many_arguments)]
+fn handle_ordered_transaction(
+    transaction_pretty: TransactionPretty,
+    protocols: &[Protocol],
+    event_type_filter: Option<&EventTypeFilter>,
+    callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
+    bot_wallet: Option<Pubkey>,
+    order_mode: OrderMode,
+    slot_buffer: &mut SlotBuffer,
+    micro_batch: &mut MicroBatchBuffer,
+    last_slot: &mut u64,
+    micro_batch_us: u64,
+) {
+    let fallback_slot = transaction_pretty.slot;
+    let fallback_tx_index = transaction_pretty.tx_index.unwrap_or(0);
+    let recv_us = transaction_pretty.recv_us;
+    let events =
+        parse_grpc_transaction_events(transaction_pretty, protocols, event_type_filter, bot_wallet);
+
+    match order_mode {
+        OrderMode::Unordered => {
+            for event in events {
+                callback(event);
+            }
+        }
+        OrderMode::Ordered => {
+            if fallback_slot > *last_slot && *last_slot > 0 {
+                deliver_events(callback.clone(), slot_buffer.flush_before(fallback_slot));
+            }
+            *last_slot = fallback_slot;
+            for event in events {
+                let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
+                slot_buffer.push(slot, tx_index, event);
+            }
+        }
+        OrderMode::StreamingOrdered => {
+            for event in events {
+                let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
+                deliver_events(callback.clone(), slot_buffer.push_streaming(slot, tx_index, event));
+            }
+        }
+        OrderMode::MicroBatch => {
+            for event in events {
+                let (slot, tx_index) = event_order_key(&event, fallback_slot, fallback_tx_index);
+                if micro_batch.push(slot, tx_index, event, recv_us, micro_batch_us) {
+                    deliver_events(callback.clone(), micro_batch.flush());
+                }
+            }
+        }
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn flush_ordered_timeouts(
+    order_mode: OrderMode,
+    slot_buffer: &mut SlotBuffer,
+    micro_batch: &mut MicroBatchBuffer,
+    callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
+    order_timeout_ms: u64,
+    micro_batch_us: u64,
+    next_check: &mut Instant,
+    check_interval: Duration,
+) {
+    if Instant::now() < *next_check {
+        return;
+    }
+    *next_check = Instant::now() + check_interval;
+
+    match order_mode {
+        OrderMode::Ordered => {
+            if slot_buffer.should_timeout(order_timeout_ms) {
+                deliver_events(callback, slot_buffer.flush_all());
+            }
+        }
+        OrderMode::StreamingOrdered => {
+            if slot_buffer.should_timeout(order_timeout_ms) {
+                deliver_events(callback, slot_buffer.flush_streaming_timeout());
+            }
+        }
+        OrderMode::MicroBatch => {
+            let now_us =
+                crate::streaming::event_parser::common::high_performance_clock::get_high_perf_clock(
+                );
+            if micro_batch.should_flush(now_us, micro_batch_us) {
+                deliver_events(callback, micro_batch.flush());
+            }
+        }
+        OrderMode::Unordered => {}
+    }
+}
+
+fn flush_ordered_on_disconnect(
+    order_mode: OrderMode,
+    slot_buffer: &mut SlotBuffer,
+    micro_batch: &mut MicroBatchBuffer,
+    callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
+) {
+    match order_mode {
+        OrderMode::Ordered => deliver_events(callback, slot_buffer.flush_all()),
+        OrderMode::StreamingOrdered => {
+            deliver_events(callback, slot_buffer.flush_streaming_timeout())
+        }
+        OrderMode::MicroBatch => deliver_events(callback, micro_batch.flush()),
+        OrderMode::Unordered => {}
+    }
+}
+
+#[inline]
+fn event_order_key(event: &DexEvent, fallback_slot: u64, fallback_tx_index: u64) -> (u64, u64) {
+    let metadata = event.metadata();
+    (metadata.slot.max(fallback_slot), metadata.tx_index.unwrap_or(fallback_tx_index))
+}
+
+#[inline]
+fn deliver_events(callback: Arc<dyn Fn(DexEvent) + Send + Sync>, events: Vec<DexEvent>) {
+    for event in events {
+        callback(event);
+    }
+}
+
+#[inline]
+fn has_buffered_events(
+    order_mode: OrderMode,
+    slot_buffer: &SlotBuffer,
+    micro_batch: &MicroBatchBuffer,
+) -> bool {
+    match order_mode {
+        OrderMode::Ordered | OrderMode::StreamingOrdered => !slot_buffer.is_empty(),
+        OrderMode::MicroBatch => !micro_batch.is_empty(),
+        OrderMode::Unordered => false,
+    }
+}
+
 // 实现 Clone trait 以支持模块间共享
 impl Clone for YellowstoneGrpc {
     fn clone(&self) -> Self {