Release solana-streamer-sdk v1.4.7

src/streaming/common/config.rs

@@ -1,4 +1,5 @@
 use super::constants::*;
+pub use sol_parser_sdk::grpc::types::OrderMode;
 
 /// Connection configuration
 #[derive(Debug, Clone)]
@@ -28,10 +29,22 @@ pub struct StreamClientConfig {
     pub connection: ConnectionConfig,
     /// Whether performance monitoring is enabled (default: false)
     pub enable_metrics: bool,
+    /// Event output ordering mode for gRPC transaction events.
+    pub order_mode: OrderMode,
+    /// Slot timeout in milliseconds for ordered modes.
+    pub order_timeout_ms: u64,
+    /// MicroBatch window size in microseconds.
+    pub micro_batch_us: u64,
 }
 
 impl Default for StreamClientConfig {
     fn default() -> Self {
-        Self { connection: ConnectionConfig::default(), enable_metrics: false }
+        Self {
+            connection: ConnectionConfig::default(),
+            enable_metrics: false,
+            order_mode: OrderMode::Unordered,
+            order_timeout_ms: 100,
+            micro_batch_us: 100,
+        }
     }
 }

src/streaming/common/event_processor.rs

@@ -41,6 +41,57 @@ fn create_metrics_callback(
     })
 }
 
+#[inline]
+pub fn transaction_metrics_callback(
+    callback: Arc<dyn Fn(DexEvent) + Send + Sync>,
+) -> Arc<dyn Fn(DexEvent) + Send + Sync> {
+    create_metrics_callback(callback)
+}
+
+pub fn parse_grpc_transaction_events(
+    transaction_pretty: crate::streaming::grpc::TransactionPretty,
+    protocols: &[Protocol],
+    event_type_filter: Option<&EventTypeFilter>,
+    bot_wallet: Option<Pubkey>,
+) -> Vec<DexEvent> {
+    MetricsManager::global().add_tx_process_count();
+
+    let slot = transaction_pretty.slot;
+    let block_time = transaction_pretty.block_time;
+    let recv_us = transaction_pretty.recv_us;
+    let grpc_tx = transaction_pretty.grpc_tx;
+    let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
+    let update =
+        SubscribeUpdateTransaction { slot, transaction: Some(grpc_tx), ..Default::default() };
+    let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
+    let sdk_events = parse_subscribe_update_transaction_low_latency(
+        &update,
+        recv_us,
+        block_time_us,
+        sdk_parse_filter.as_ref(),
+    );
+    let mut events = adapt_parser_events_list(
+        sdk_events,
+        block_time.as_ref(),
+        recv_us,
+        protocols,
+        event_type_filter,
+    );
+
+    for event in events.iter_mut() {
+        event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
+    }
+
+    events
+        .into_iter()
+        .map(|event| {
+            crate::streaming::event_parser::core::event_parser::helpers::process_event(
+                event, bot_wallet,
+            )
+        })
+        .collect()
+}
+
 /// Process GRPC transaction events
 pub async fn process_grpc_transaction(
     event_pretty: EventPretty,
@@ -66,40 +117,14 @@ pub async fn process_grpc_transaction(
             }
         }
         EventPretty::Transaction(transaction_pretty) => {
-            MetricsManager::global().add_tx_process_count();
-
-            let slot = transaction_pretty.slot;
-            let block_time = transaction_pretty.block_time;
-            let recv_us = transaction_pretty.recv_us;
-            let grpc_tx = transaction_pretty.grpc_tx;
-            let adapter_callback = create_metrics_callback(callback.clone());
-            let block_time_us = block_time.map(|t| t.seconds * 1_000_000 + t.nanos as i64 / 1_000);
-            let update = SubscribeUpdateTransaction {
-                slot,
-                transaction: Some(grpc_tx),
-                ..Default::default()
-            };
-            let sdk_parse_filter = build_sdk_parse_event_filter(event_type_filter);
-            let sdk_events = parse_subscribe_update_transaction_low_latency(
-                &update,
-                recv_us,
-                block_time_us,
-                sdk_parse_filter.as_ref(),
-            );
-            let events = adapt_parser_events_list(
-                sdk_events,
-                block_time.as_ref(),
-                recv_us,
+            let callback = transaction_metrics_callback(callback);
+            for event in parse_grpc_transaction_events(
+                transaction_pretty,
                 protocols,
                 event_type_filter,
-            );
-
-            for mut event in events {
-                event.metadata_mut().handle_us = elapsed_micros_since(recv_us);
-                event = crate::streaming::event_parser::core::event_parser::helpers::process_event(
-                    event, bot_wallet,
-                );
-                adapter_callback(event);
+                bot_wallet,
+            ) {
+                callback(event);
             }
         }
         EventPretty::BlockMeta(block_meta_pretty) => {

src/streaming/common/mod.rs

@@ -3,6 +3,7 @@ pub mod config;
 pub mod constants;
 pub mod event_processor;
 pub mod metrics;
+pub mod order_buffer;
 pub mod subscription;
 
 // 重新导出主要类型
@@ -10,4 +11,5 @@ pub use config::*;
 pub use constants::*;
 pub use event_processor::*;
 pub use metrics::*;
+pub use order_buffer::*;
 pub use subscription::*;

src/streaming/common/order_buffer.rs

@@ -0,0 +1,196 @@
+use crate::streaming::event_parser::DexEvent;
+use std::collections::{BTreeMap, HashMap};
+use tokio::time::Instant;
+
+#[derive(Default)]
+pub struct SlotBuffer {
+    slots: BTreeMap<u64, Vec<(u64, DexEvent)>>,
+    current_slot: u64,
+    last_flush_time: Option<Instant>,
+    streaming_watermarks: HashMap<u64, u64>,
+}
+
+impl SlotBuffer {
+    #[inline]
+    pub fn new() -> Self {
+        Self {
+            slots: BTreeMap::new(),
+            current_slot: 0,
+            last_flush_time: Some(Instant::now()),
+            streaming_watermarks: HashMap::new(),
+        }
+    }
+
+    #[inline]
+    pub fn push(&mut self, slot: u64, tx_index: u64, event: DexEvent) {
+        if self.slots.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        self.slots.entry(slot).or_default().push((tx_index, event));
+        if slot > self.current_slot {
+            self.current_slot = slot;
+        }
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.slots.is_empty()
+    }
+
+    pub fn flush_before(&mut self, current_slot: u64) -> Vec<DexEvent> {
+        let slots_to_flush: Vec<u64> =
+            self.slots.keys().filter(|&&s| s < current_slot).copied().collect();
+
+        let mut result = Vec::with_capacity(slots_to_flush.len() * 4);
+        for slot in slots_to_flush {
+            if let Some(mut events) = self.slots.remove(&slot) {
+                events.sort_unstable_by_key(|(idx, _)| *idx);
+                result.extend(events.into_iter().map(|(_, event)| event));
+            }
+        }
+
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+
+    pub fn flush_all(&mut self) -> Vec<DexEvent> {
+        let all_slots: Vec<u64> = self.slots.keys().copied().collect();
+        let mut result = Vec::with_capacity(all_slots.len() * 4);
+
+        for slot in all_slots {
+            if let Some(mut events) = self.slots.remove(&slot) {
+                events.sort_unstable_by_key(|(idx, _)| *idx);
+                result.extend(events.into_iter().map(|(_, event)| event));
+            }
+        }
+
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+
+    #[inline]
+    pub fn should_timeout(&self, timeout_ms: u64) -> bool {
+        self.last_flush_time
+            .map(|t| !self.slots.is_empty() && t.elapsed().as_millis() as u64 > timeout_ms)
+            .unwrap_or(false)
+    }
+
+    pub fn push_streaming(&mut self, slot: u64, tx_index: u64, event: DexEvent) -> Vec<DexEvent> {
+        let mut result = Vec::new();
+
+        if slot > self.current_slot && self.current_slot > 0 {
+            let old_slots: Vec<u64> = self.slots.keys().filter(|&&s| s < slot).copied().collect();
+            for old_slot in old_slots {
+                if let Some(mut events) = self.slots.remove(&old_slot) {
+                    events.sort_unstable_by_key(|(idx, _)| *idx);
+                    result.extend(events.into_iter().map(|(_, event)| event));
+                }
+                self.streaming_watermarks.remove(&old_slot);
+            }
+        }
+
+        if slot > self.current_slot {
+            self.current_slot = slot;
+        }
+
+        let next_expected = *self.streaming_watermarks.get(&slot).unwrap_or(&0);
+
+        if tx_index == next_expected {
+            result.push(event);
+            let mut watermark = next_expected + 1;
+
+            if let Some(buffered) = self.slots.get_mut(&slot) {
+                buffered.sort_unstable_by_key(|(idx, _)| *idx);
+                while let Some(pos) = buffered.iter().position(|(idx, _)| *idx == watermark) {
+                    result.push(buffered.remove(pos).1);
+                    watermark += 1;
+                }
+            }
+            self.streaming_watermarks.insert(slot, watermark);
+        } else if tx_index > next_expected {
+            if self.slots.is_empty() {
+                self.last_flush_time = Some(Instant::now());
+            }
+            self.slots.entry(slot).or_default().push((tx_index, event));
+        }
+
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+
+    pub fn flush_streaming_timeout(&mut self) -> Vec<DexEvent> {
+        let mut result = Vec::new();
+        for (slot, mut events) in std::mem::take(&mut self.slots) {
+            events.sort_unstable_by_key(|(idx, _)| *idx);
+            result.extend(events.into_iter().map(|(_, event)| event));
+            self.streaming_watermarks.remove(&slot);
+        }
+        if !result.is_empty() {
+            self.last_flush_time = Some(Instant::now());
+        }
+        result
+    }
+}
+
+pub struct MicroBatchBuffer {
+    events: Vec<(u64, u64, DexEvent)>,
+    window_start_us: i64,
+}
+
+impl MicroBatchBuffer {
+    #[inline]
+    pub fn new() -> Self {
+        Self { events: Vec::with_capacity(64), window_start_us: 0 }
+    }
+
+    #[inline]
+    pub fn push(
+        &mut self,
+        slot: u64,
+        tx_index: u64,
+        event: DexEvent,
+        now_us: i64,
+        window_us: u64,
+    ) -> bool {
+        if self.events.is_empty() {
+            self.window_start_us = now_us;
+        }
+        self.events.push((slot, tx_index, event));
+        (now_us - self.window_start_us) as u64 >= window_us
+    }
+
+    #[inline]
+    pub fn flush(&mut self) -> Vec<DexEvent> {
+        if self.events.is_empty() {
+            return Vec::new();
+        }
+
+        self.events.sort_unstable_by_key(|(slot, tx_index, _)| (*slot, *tx_index));
+        let result =
+            std::mem::take(&mut self.events).into_iter().map(|(_, _, event)| event).collect();
+        self.window_start_us = 0;
+        result
+    }
+
+    #[inline]
+    pub fn should_flush(&self, now_us: i64, window_us: u64) -> bool {
+        !self.events.is_empty() && (now_us - self.window_start_us) as u64 >= window_us
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.events.is_empty()
+    }
+}
+
+impl Default for MicroBatchBuffer {
+    fn default() -> Self {
+        Self::new()
+    }
+}