feat: refactor shred streaming with modular architecture

src/streaming/shred/connection.rs

@@ -0,0 +1,85 @@
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use tonic::transport::Channel;
+
+use crate::common::AnyResult;
+use crate::streaming::common::{
+    MetricsManager, PerformanceMetrics, StreamClientConfig,
+};
+use crate::protos::shredstream::shredstream_proxy_client::ShredstreamProxyClient;
+
+/// ShredStream gRPC 客户端
+#[derive(Clone)]
+pub struct ShredStreamGrpc {
+    pub shredstream_client: Arc<ShredstreamProxyClient<Channel>>,
+    pub config: StreamClientConfig,
+    pub metrics: Arc<Mutex<PerformanceMetrics>>,
+    pub metrics_manager: MetricsManager,
+}
+
+impl ShredStreamGrpc {
+    /// 创建客户端，使用默认配置
+    pub async fn new(endpoint: String) -> AnyResult<Self> {
+        Self::new_with_config(endpoint, StreamClientConfig::default()).await
+    }
+
+    /// 创建客户端，使用自定义配置
+    pub async fn new_with_config(endpoint: String, config: StreamClientConfig) -> AnyResult<Self> {
+        let shredstream_client = ShredstreamProxyClient::connect(endpoint.clone()).await?;
+        let metrics = Arc::new(Mutex::new(PerformanceMetrics::new()));
+        let config_arc = Arc::new(config.clone());
+        
+        let metrics_manager = MetricsManager::new(
+            metrics.clone(), 
+            config_arc, 
+            "ShredStream".to_string()
+        );
+        
+        Ok(Self {
+            shredstream_client: Arc::new(shredstream_client),
+            config,
+            metrics,
+            metrics_manager,
+        })
+    }
+
+    /// 创建高性能客户端（适合高并发场景）
+    pub async fn new_high_performance(endpoint: String) -> AnyResult<Self> {
+        Self::new_with_config(endpoint, StreamClientConfig::high_performance()).await
+    }
+
+    /// 创建低延迟客户端（适合实时场景）
+    pub async fn new_low_latency(endpoint: String) -> AnyResult<Self> {
+        Self::new_with_config(endpoint, StreamClientConfig::low_latency()).await
+    }
+
+    /// 获取当前配置
+    pub fn get_config(&self) -> &StreamClientConfig {
+        &self.config
+    }
+
+    /// 更新配置
+    pub fn update_config(&mut self, config: StreamClientConfig) {
+        self.config = config;
+    }
+
+    /// 获取性能指标
+    pub async fn get_metrics(&self) -> PerformanceMetrics {
+        self.metrics_manager.get_metrics().await
+    }
+
+    /// 启用或禁用性能监控
+    pub fn set_enable_metrics(&mut self, enabled: bool) {
+        self.config.enable_metrics = enabled;
+    }
+
+    /// 打印性能指标
+    pub async fn print_metrics(&self) {
+        self.metrics_manager.print_metrics().await;
+    }
+
+    /// 启动自动性能监控任务
+    pub async fn start_auto_metrics_monitoring(&self) {
+        self.metrics_manager.start_auto_monitoring().await;
+    }
+}

src/streaming/shred/event_processor.rs

@@ -0,0 +1,170 @@
+use solana_sdk::pubkey::Pubkey;
+use std::sync::{Arc, Mutex};
+
+use crate::common::AnyResult;
+use crate::streaming::common::{
+    EventBatchProcessor, MetricsEventType, MetricsManager, StreamClientConfig,
+};
+use crate::streaming::event_parser::common::filter::EventTypeFilter;
+use crate::streaming::event_parser::protocols::MutilEventParser;
+use crate::streaming::event_parser::{EventParser, Protocol, UnifiedEvent};
+use crate::streaming::shred::TransactionWithSlot;
+
+/// ShredStream 事件处理器
+pub struct ShredEventProcessor {
+    pub(crate) metrics_manager: MetricsManager,
+    pub(crate) config: StreamClientConfig,
+    pub(crate) parser_cache: Arc<Mutex<Option<Arc<dyn EventParser>>>>,
+}
+
+impl ShredEventProcessor {
+    /// 创建新的事件处理器
+    pub fn new(metrics_manager: MetricsManager, config: StreamClientConfig) -> Self {
+        Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) }
+    }
+
+    /// 获取或创建解析器，使用缓存机制避免重复创建
+    fn get_or_create_parser(
+        &self,
+        protocols: Vec<Protocol>,
+        event_type_filter: Option<EventTypeFilter>,
+    ) -> Arc<dyn EventParser> {
+        let mut cache = self.parser_cache.lock().unwrap();
+        if let Some(cached_parser) = cache.clone() {
+            return cached_parser.clone();
+        }
+        let parser: Arc<dyn EventParser> =
+            Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone()));
+        *cache = Some(parser.clone());
+        parser
+    }
+
+    /// 即时处理单个交易
+    pub async fn process_transaction_immediate<F>(
+        &self,
+        transaction_with_slot: TransactionWithSlot,
+        protocols: Vec<Protocol>,
+        bot_wallet: Option<Pubkey>,
+        event_type_filter: Option<EventTypeFilter>,
+        callback: &F,
+    ) -> AnyResult<()>
+    where
+        F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
+    {
+        let start_time = std::time::Instant::now();
+        self.metrics_manager.add_tx_process_count().await;
+        let program_received_time_ms = chrono::Utc::now().timestamp_millis();
+        let slot = transaction_with_slot.slot;
+        let versioned_tx = transaction_with_slot.transaction;
+        let signature = versioned_tx.signatures[0];
+
+        // 获取缓存的解析器
+        let parser = self.get_or_create_parser(protocols, event_type_filter);
+
+        let all_events = parser
+            .parse_versioned_transaction(
+                &versioned_tx,
+                &signature.to_string(),
+                Some(slot),
+                None,
+                program_received_time_ms,
+                bot_wallet,
+            )
+            .await
+            .unwrap_or_else(|_e| vec![]);
+
+        // 保存事件数量用于日志记录
+        let event_count = all_events.len();
+
+        // 即时处理事件
+        for event in all_events {
+            callback(event);
+        }
+
+        // 更新性能指标
+        let processing_time = start_time.elapsed();
+        let processing_time_ms = processing_time.as_millis() as f64;
+
+        // 实际调用性能指标更新
+        self.update_metrics(event_count as u64, processing_time_ms).await;
+
+        // 记录慢处理操作
+        self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
+
+        Ok(())
+    }
+
+    /// 批处理模式处理单个交易
+    pub async fn process_transaction_with_batch<F>(
+        &self,
+        transaction_with_slot: TransactionWithSlot,
+        protocols: Vec<Protocol>,
+        bot_wallet: Option<Pubkey>,
+        batch_processor: &mut EventBatchProcessor<F>,
+        event_type_filter: Option<EventTypeFilter>,
+    ) -> AnyResult<()>
+    where
+        F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
+    {
+        let start_time = std::time::Instant::now();
+        self.metrics_manager.add_tx_process_count().await;
+        let program_received_time_ms = chrono::Utc::now().timestamp_millis();
+        let slot = transaction_with_slot.slot;
+        let versioned_tx = transaction_with_slot.transaction;
+        let signature = versioned_tx.signatures[0];
+
+        // 获取缓存的解析器
+        let parser = self.get_or_create_parser(protocols, event_type_filter);
+
+        let all_events = parser
+            .parse_versioned_transaction(
+                &versioned_tx,
+                &signature.to_string(),
+                Some(slot),
+                None,
+                program_received_time_ms,
+                bot_wallet,
+            )
+            .await
+            .unwrap_or_else(|_e| vec![]);
+
+        // 保存事件数量用于日志记录
+        let event_count = all_events.len();
+
+        // 使用批处理器处理事件
+        for event in all_events {
+            batch_processor.add_event(event);
+        }
+
+        // 更新性能指标
+        let processing_time = start_time.elapsed();
+        let processing_time_ms = processing_time.as_millis() as f64;
+
+        // 实际调用性能指标更新
+        self.update_metrics(event_count as u64, processing_time_ms).await;
+
+        // 记录慢处理操作
+        self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
+
+        Ok(())
+    }
+
+    /// 更新性能指标
+    async fn update_metrics(&self, events_processed: u64, processing_time_ms: f64) {
+        // 使用统一的指标管理器，这里假设 ShredStream 主要处理交易事件
+        self.metrics_manager
+            .update_metrics(MetricsEventType::Tx, events_processed, processing_time_ms)
+            .await;
+    }
+}
+
+// 实现 Clone trait 以支持模块间共享
+impl Clone for ShredEventProcessor {
+    fn clone(&self) -> Self {
+        Self {
+            metrics_manager: self.metrics_manager.clone(),
+            config: self.config.clone(),
+            parser_cache: self.parser_cache.clone(),
+        }
+    }
+}

src/streaming/shred/mod.rs

@@ -0,0 +1,17 @@
+// ShredStream 相关模块
+pub mod connection;
+pub mod types;
+pub mod stream_handler;
+pub mod event_processor;
+
+// 重新导出主要类型
+pub use connection::*;
+pub use types::*;
+pub use stream_handler::*;
+pub use event_processor::*;
+
+// 从公用模块重新导出
+pub use crate::streaming::common::{
+    BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, EventBatchProcessor,
+    MetricsEventType, MetricsManager, PerformanceMetrics, StreamClientConfig,
+};

src/streaming/shred/stream_handler.rs

@@ -0,0 +1,116 @@
+use futures::{channel::mpsc, StreamExt};
+use log::error;
+use solana_entry::entry::Entry;
+use tokio::task::JoinHandle;
+
+use crate::common::AnyResult;
+use crate::protos::shredstream::{
+    shredstream_proxy_client::ShredstreamProxyClient, SubscribeEntriesRequest,
+};
+use crate::streaming::shred::TransactionWithSlot;
+
+/// ShredStream 流处理器
+pub struct ShredStreamHandler;
+
+impl ShredStreamHandler {
+    /// 启动 ShredStream 流处理任务
+    ///
+    /// # 参数
+    /// * `client` - ShredStream 客户端
+    /// * `tx` - 事务发送通道
+    /// * `channel_size` - 通道缓冲区大小
+    ///
+    /// # 返回值
+    /// 返回 ShredStream 流处理任务句柄和事务接收通道
+    pub async fn start_stream_processing(
+        mut client: ShredstreamProxyClient<tonic::transport::Channel>,
+        channel_size: usize,
+    ) -> AnyResult<(JoinHandle<()>, mpsc::Receiver<TransactionWithSlot>)> {
+        let request = tonic::Request::new(SubscribeEntriesRequest {});
+        let stream = client.subscribe_entries(request).await?.into_inner();
+        let (tx, rx) = mpsc::channel::<TransactionWithSlot>(channel_size);
+
+        let stream_task = tokio::spawn(Self::process_stream_messages(stream, tx));
+
+        Ok((stream_task, rx))
+    }
+
+    /// 处理流消息
+    ///
+    /// # 参数
+    /// * `stream` - ShredStream 数据流
+    /// * `tx` - 事务发送通道
+    async fn process_stream_messages(
+        mut stream: tonic::codec::Streaming<crate::protos::shredstream::Entry>,
+        mut tx: mpsc::Sender<TransactionWithSlot>,
+    ) {
+        while let Some(message) = stream.next().await {
+            match message {
+                Ok(msg) => {
+                    if let Err(e) = Self::handle_stream_message(msg, &mut tx).await {
+                        error!("Error handling stream message: {e:?}");
+                        continue;
+                    }
+                }
+                Err(error) => {
+                    error!("Stream error: {error:?}");
+                    break;
+                }
+            }
+        }
+    }
+
+    /// 处理单个流消息
+    ///
+    /// # 参数
+    /// * `msg` - ShredStream 消息
+    /// * `tx` - 事务发送通道
+    async fn handle_stream_message(
+        msg: crate::protos::shredstream::Entry,
+        tx: &mut mpsc::Sender<TransactionWithSlot>,
+    ) -> AnyResult<()> {
+        if let Ok(entries) = bincode::deserialize::<Vec<Entry>>(&msg.entries) {
+            for entry in entries {
+                for transaction in entry.transactions {
+                    let transaction_with_slot =
+                        TransactionWithSlot::new(transaction.clone(), msg.slot);
+
+                    if let Err(e) = tx.try_send(transaction_with_slot) {
+                        // 如果通道满了，记录警告但不中断处理
+                        if e.is_full() {
+                            log::warn!("Transaction channel is full, dropping transaction");
+                        } else {
+                            // 通道已关闭，返回错误
+                            return Err(e.into());
+                        }
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// 启动事务处理任务
+    ///
+    /// # 参数
+    /// * `rx` - 事务接收通道
+    /// * `processor` - 事务处理器
+    pub fn start_transaction_processing<F>(
+        mut rx: mpsc::Receiver<TransactionWithSlot>,
+        processor: F,
+    ) -> JoinHandle<()>
+    where
+        F: Fn(TransactionWithSlot) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+            + Send
+            + Sync
+            + 'static,
+    {
+        tokio::spawn(async move {
+            while let Some(transaction_with_slot) = rx.next().await {
+                if let Err(e) = processor(transaction_with_slot) {
+                    error!("Error processing transaction: {e:?}");
+                }
+            }
+        })
+    }
+}

src/streaming/shred/types.rs

@@ -0,0 +1,20 @@
+use solana_sdk::transaction::VersionedTransaction;
+
+/// 携带槽位信息的交易
+#[derive(Debug, Clone)]
+pub struct TransactionWithSlot {
+    pub transaction: VersionedTransaction,
+    pub slot: u64,
+}
+
+impl TransactionWithSlot {
+    /// 创建新的带槽位的交易
+    pub fn new(transaction: VersionedTransaction, slot: u64) -> Self {
+        Self { transaction, slot }
+    }
+
+    /// 获取交易签名
+    pub fn signature(&self) -> String {
+        self.transaction.signatures[0].to_string()
+    }
+}