feat: Major refactor with batch processing and performance optimization

- Refactor streaming architecture with modular grpc components
- Add batch processing system with backpressure strategies
- Implement performance monitoring and metrics collection
- Add multi-protocol parser with block metadata support
- Enhance configuration system with preset profiles
- Add parse_tx_events example and update documentation
- Optimize yellowstone_grpc client complexity

src/streaming/grpc/connection.rs

@@ -0,0 +1,33 @@
+use std::time::Duration;
+use tonic::transport::channel::ClientTlsConfig;
+use yellowstone_grpc_client::{GeyserGrpcClient, Interceptor};
+use crate::common::AnyResult;
+use crate::streaming::common::constants::{
+    DEFAULT_CONNECT_TIMEOUT, DEFAULT_REQUEST_TIMEOUT, DEFAULT_MAX_DECODING_MESSAGE_SIZE
+};
+
+/// gRPC连接池 - 简化版本
+pub struct GrpcConnectionPool {
+    endpoint: String,
+    x_token: Option<String>,
+}
+
+impl GrpcConnectionPool {
+    pub fn new(endpoint: String, x_token: Option<String>) -> Self {
+        Self {
+            endpoint,
+            x_token,
+        }
+    }
+
+    pub async fn create_connection(&self) -> AnyResult<GeyserGrpcClient<impl Interceptor>> {
+        let builder = GeyserGrpcClient::build_from_shared(self.endpoint.clone())?
+            .x_token(self.x_token.clone())?
+            .tls_config(ClientTlsConfig::new().with_native_roots())?
+            .max_decoding_message_size(DEFAULT_MAX_DECODING_MESSAGE_SIZE)
+            .connect_timeout(Duration::from_secs(DEFAULT_CONNECT_TIMEOUT))
+            .timeout(Duration::from_secs(DEFAULT_REQUEST_TIMEOUT));
+
+        Ok(builder.connect().await?)
+    }
+}

src/streaming/grpc/event_processor.rs

@@ -0,0 +1,201 @@
+use std::sync::Arc;
+
+use solana_sdk::pubkey::Pubkey;
+
+use super::types::EventPretty;
+use crate::common::AnyResult;
+use crate::streaming::common::{
+    EventBatchProcessor as EventBatchCollector, MetricsManager, StreamClientConfig as ClientConfig,
+};
+use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
+use crate::streaming::event_parser::EventParser;
+use crate::streaming::event_parser::{
+    core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
+};
+
+/// 事件处理器
+pub struct EventProcessor {
+    pub(crate) metrics_manager: MetricsManager,
+    pub(crate) config: ClientConfig,
+}
+
+impl EventProcessor {
+    /// 创建新的事件处理器
+    pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
+        Self { metrics_manager, config }
+    }
+
+    /// 使用性能监控处理事件交易
+    pub async fn process_event_transaction_with_metrics<F>(
+        &self,
+        event_pretty: EventPretty,
+        callback: &F,
+        bot_wallet: Option<Pubkey>,
+        protocols: Vec<Protocol>,
+    ) -> AnyResult<()>
+    where
+        F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
+    {
+        match event_pretty {
+            EventPretty::Transaction(transaction_pretty) => {
+                let start_time = std::time::Instant::now();
+                let program_received_time_ms = chrono::Utc::now().timestamp_millis();
+                let slot = transaction_pretty.slot;
+                let signature = transaction_pretty.signature.to_string();
+
+                // 直接创建解析器并处理事务
+                let parser: Arc<dyn EventParser> =
+                    Arc::new(MutilEventParser::new(protocols.clone()));
+                let all_events = parser
+                    .parse_transaction(
+                        transaction_pretty.tx.clone(),
+                        &signature,
+                        Some(slot),
+                        transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
+                            seconds: ts.seconds,
+                            nanos: ts.nanos,
+                        }),
+                        program_received_time_ms,
+                        bot_wallet,
+                    )
+                    .await
+                    .unwrap_or_else(|_e| vec![]);
+
+                // 保存事件数量用于日志记录
+                let event_count = all_events.len();
+
+                // 批量处理事件
+                if !all_events.is_empty() {
+                    for event in all_events {
+                        callback(event);
+                    }
+                }
+
+                // 更新性能指标
+                let processing_time = start_time.elapsed();
+                let processing_time_ms = processing_time.as_millis() as f64;
+
+                // 更新性能指标（如果启用）
+                if self.config.enable_metrics {
+                    self.metrics_manager
+                        .update_metrics(event_count as u64, processing_time_ms)
+                        .await;
+                }
+
+                // 记录慢处理操作
+                self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
+            }
+            EventPretty::BlockMeta(block_meta_pretty) => {
+                let block_time_ms = block_meta_pretty
+                    .block_time
+                    .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
+                    .unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
+                let block_meta_event = CommonEventParser::generate_block_meta_event(
+                    block_meta_pretty.slot,
+                    &block_meta_pretty.block_hash,
+                    block_time_ms,
+                );
+                callback(block_meta_event);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// 使用批处理处理事件交易
+    pub async fn process_event_transaction_with_batch<F>(
+        &self,
+        event_pretty: EventPretty,
+        batch_processor: &mut EventBatchCollector<F>,
+        bot_wallet: Option<Pubkey>,
+        protocols: Vec<Protocol>,
+    ) -> AnyResult<()>
+    where
+        F: Fn(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
+    {
+        match event_pretty {
+            EventPretty::Transaction(transaction_pretty) => {
+                let start_time = std::time::Instant::now();
+                let program_received_time_ms = chrono::Utc::now().timestamp_millis();
+                let slot = transaction_pretty.slot;
+                let signature = transaction_pretty.signature.to_string();
+
+                // 直接创建解析器并处理事务
+                let parser: Arc<dyn EventParser> =
+                    Arc::new(MutilEventParser::new(protocols.clone()));
+                let result = parser
+                    .parse_transaction(
+                        transaction_pretty.tx.clone(),
+                        &signature,
+                        Some(slot),
+                        transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
+                            seconds: ts.seconds,
+                            nanos: ts.nanos,
+                        }),
+                        program_received_time_ms,
+                        bot_wallet,
+                    )
+                    .await;
+
+                // 处理解析结果并使用批处理器
+                let total_events = match result {
+                    Ok(events) => {
+                        let event_count = events.len();
+                        if !events.is_empty() {
+                            log::info!("Parsed {} events", event_count);
+                            log::info!("Adding {} events to batch processor", event_count);
+                            for event in events {
+                                if self.config.batch.enabled {
+                                    batch_processor.add_event(event);
+                                } else {
+                                    // 如果批处理被禁用，直接调用回调
+                                    // 这里需要将单个事件包装成Vec来调用批处理回调
+                                    let single_event_batch = vec![event];
+                                    (batch_processor.callback)(single_event_batch);
+                                }
+                            }
+                        }
+                        event_count
+                    }
+                    Err(e) => {
+                        log::warn!("Failed to parse transaction: {:?}", e);
+                        0
+                    }
+                };
+
+                // 添加调试信息
+                if total_events > 0 {
+                    log::info!(
+                        "Total events parsed: {} for transaction {}",
+                        total_events,
+                        signature
+                    );
+                }
+
+                // 更新性能指标
+                let processing_time = start_time.elapsed();
+                let processing_time_ms = processing_time.as_millis() as f64;
+
+                // 实际调用性能指标更新
+                self.metrics_manager.update_metrics(total_events as u64, processing_time_ms).await;
+
+                // 记录慢处理操作
+                self.metrics_manager.log_slow_processing(processing_time_ms, total_events);
+            }
+            EventPretty::BlockMeta(block_meta_pretty) => {
+                let block_time_ms = block_meta_pretty
+                    .block_time
+                    .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
+                    .unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
+                let block_meta_event = CommonEventParser::generate_block_meta_event(
+                    block_meta_pretty.slot,
+                    &block_meta_pretty.block_hash,
+                    block_time_ms,
+                );
+                (batch_processor.callback)(vec![block_meta_event]);
+            }
+        }
+
+        Ok(())
+    }
+}

src/streaming/grpc/mod.rs

@@ -0,0 +1,25 @@
+// gRPC 相关模块
+pub mod connection;
+pub mod types;
+pub mod subscription;
+pub mod stream_handler;
+pub mod event_processor;
+
+// 重新导出主要类型
+pub use connection::*;
+pub use types::*;
+pub use subscription::*;
+pub use stream_handler::*;
+pub use event_processor::*;
+
+// 从公用模块重新导出
+pub use crate::streaming::common::{
+    StreamClientConfig as ClientConfig,
+    PerformanceMetrics,
+    MetricsManager,
+    EventBatchProcessor as EventBatchCollector,
+    BackpressureStrategy,
+    BatchConfig,
+    BackpressureConfig,
+    ConnectionConfig,
+};

src/streaming/grpc/stream_handler.rs

@@ -0,0 +1,124 @@
+use chrono::Local;
+use futures::{channel::mpsc, sink::Sink, SinkExt};
+use log::info;
+use yellowstone_grpc_proto::geyser::{
+    subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, SubscribeUpdate,
+};
+
+use super::types::{BlockMetaPretty, EventPretty, TransactionPretty};
+use crate::common::AnyResult;
+use crate::streaming::common::BackpressureStrategy;
+
+/// 流消息处理器
+pub struct StreamHandler;
+
+impl StreamHandler {
+    /// 处理单个流消息
+    pub async fn handle_stream_message(
+        msg: SubscribeUpdate,
+        tx: &mut mpsc::Sender<EventPretty>,
+        subscribe_tx: &mut (impl Sink<SubscribeRequest, Error = mpsc::SendError> + Unpin),
+        backpressure_strategy: BackpressureStrategy,
+    ) -> AnyResult<()> {
+        let created_at = msg.created_at;
+        match msg.update_oneof {
+            Some(UpdateOneof::BlockMeta(sut)) => {
+                let block_meta_pretty = BlockMetaPretty::from((sut, created_at));
+                log::info!("Received block meta: {:?}", block_meta_pretty);
+                Self::handle_backpressure(
+                    tx,
+                    EventPretty::BlockMeta(block_meta_pretty),
+                    backpressure_strategy,
+                )
+                .await?;
+            }
+            Some(UpdateOneof::Transaction(sut)) => {
+                let transaction_pretty = TransactionPretty::from((sut, created_at));
+                log::info!(
+                    "Received transaction: {} at slot {}",
+                    transaction_pretty.signature,
+                    transaction_pretty.slot
+                );
+
+                // 根据背压策略处理发送
+                Self::handle_backpressure(
+                    tx,
+                    EventPretty::Transaction(transaction_pretty),
+                    backpressure_strategy,
+                )
+                .await?;
+            }
+            Some(UpdateOneof::Ping(_)) => {
+                subscribe_tx
+                    .send(SubscribeRequest {
+                        ping: Some(SubscribeRequestPing { id: 1 }),
+                        ..Default::default()
+                    })
+                    .await?;
+                info!("service is ping: {}", Local::now());
+            }
+            Some(UpdateOneof::Pong(_)) => {
+                info!("service is pong: {}", Local::now());
+            }
+            _ => {
+                log::debug!("Received other message type");
+            }
+        }
+        Ok(())
+    }
+
+    /// 处理背压策略
+    async fn handle_backpressure(
+        tx: &mut mpsc::Sender<EventPretty>,
+        event_pretty: EventPretty,
+        backpressure_strategy: BackpressureStrategy,
+    ) -> AnyResult<()> {
+        match backpressure_strategy {
+            BackpressureStrategy::Block => {
+                // 阻塞等待，直到有空间
+                if let Err(e) = tx.send(event_pretty).await {
+                    log::error!("Failed to send transaction to channel: {:?}", e);
+                    return Err(anyhow::anyhow!("Channel send failed: {:?}", e));
+                }
+            }
+            BackpressureStrategy::Drop => {
+                // 尝试发送，如果失败则丢弃
+                if let Err(e) = tx.try_send(event_pretty) {
+                    if e.is_full() {
+                        log::warn!("Channel is full, dropping transaction");
+                    } else {
+                        log::error!("Channel is closed: {:?}", e);
+                        return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
+                    }
+                }
+            }
+            BackpressureStrategy::Retry { max_attempts, wait_ms } => {
+                // 重试有限次数
+                let mut retry_count = 0;
+                loop {
+                    match tx.try_send(event_pretty.clone()) {
+                        Ok(_) => break,
+                        Err(e) => {
+                            if e.is_full() {
+                                retry_count += 1;
+                                if retry_count >= max_attempts {
+                                    log::warn!(
+                                        "Channel is full after {} attempts, dropping transaction",
+                                        retry_count
+                                    );
+                                    break;
+                                }
+                                tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms))
+                                    .await;
+                            } else {
+                                log::error!("Channel is closed: {:?}", e);
+                                return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}

src/streaming/grpc/subscription.rs

@@ -0,0 +1,106 @@
+use futures::{channel::mpsc, sink::Sink, Stream};
+use maplit::hashmap;
+use std::{collections::HashMap, time::Duration};
+use tonic::{transport::channel::ClientTlsConfig, Status};
+use yellowstone_grpc_client::{GeyserGrpcClient, Interceptor};
+use yellowstone_grpc_proto::geyser::{
+    CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocksMeta,
+    SubscribeRequestFilterTransactions, SubscribeUpdate,
+};
+
+use super::types::TransactionsFilterMap;
+use crate::common::AnyResult;
+use crate::streaming::common::StreamClientConfig as ClientConfig;
+
+/// 订阅管理器
+#[derive(Clone)]
+pub struct SubscriptionManager {
+    endpoint: String,
+    x_token: Option<String>,
+    config: ClientConfig,
+}
+
+impl SubscriptionManager {
+    /// 创建新的订阅管理器
+    pub fn new(endpoint: String, x_token: Option<String>, config: ClientConfig) -> Self {
+        Self { endpoint, x_token, config }
+    }
+
+    /// 创建 gRPC 连接
+    pub async fn connect(&self) -> AnyResult<GeyserGrpcClient<impl Interceptor>> {
+        let builder = GeyserGrpcClient::build_from_shared(self.endpoint.clone())?
+            .x_token(self.x_token.clone())?
+            .tls_config(ClientTlsConfig::new().with_native_roots())?
+            .max_decoding_message_size(self.config.connection.max_decoding_message_size)
+            .connect_timeout(Duration::from_secs(self.config.connection.connect_timeout))
+            .timeout(Duration::from_secs(self.config.connection.request_timeout));
+        Ok(builder.connect().await?)
+    }
+
+    /// 创建订阅请求并返回流
+    pub async fn subscribe_with_request(
+        &self,
+        transactions: TransactionsFilterMap,
+        commitment: Option<CommitmentLevel>,
+    ) -> AnyResult<(
+        impl Sink<SubscribeRequest, Error = mpsc::SendError>,
+        impl Stream<Item = Result<SubscribeUpdate, Status>>,
+    )> {
+        let subscribe_request = SubscribeRequest {
+            transactions,
+            blocks_meta: hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} },
+            commitment: if let Some(commitment) = commitment {
+                Some(commitment as i32)
+            } else {
+                Some(CommitmentLevel::Processed.into())
+            },
+            ..Default::default()
+        };
+
+        let mut client = self.connect().await?;
+        let (sink, stream) = client.subscribe_with_request(Some(subscribe_request)).await?;
+        Ok((sink, stream))
+    }
+
+    /// 生成订阅请求过滤器
+    pub fn get_subscribe_request_filter(
+        &self,
+        account_include: Vec<String>,
+        account_exclude: Vec<String>,
+        account_required: Vec<String>,
+    ) -> TransactionsFilterMap {
+        let mut transactions = HashMap::new();
+        transactions.insert(
+            "client".to_string(),
+            SubscribeRequestFilterTransactions {
+                vote: Some(false),
+                failed: Some(false),
+                signature: None,
+                account_include,
+                account_exclude,
+                account_required,
+            },
+        );
+        transactions
+    }
+
+    /// 验证订阅参数
+    pub fn validate_subscription_params(
+        &self,
+        account_include: &[String],
+        account_exclude: &[String],
+        account_required: &[String],
+    ) -> AnyResult<()> {
+        if account_include.is_empty() && account_exclude.is_empty() && account_required.is_empty() {
+            return Err(anyhow::anyhow!(
+                "account_include or account_exclude or account_required cannot be empty"
+            ));
+        }
+        Ok(())
+    }
+
+    /// 获取配置
+    pub fn get_config(&self) -> &ClientConfig {
+        &self.config
+    }
+}

src/streaming/grpc/types.rs

@@ -0,0 +1,96 @@
+use solana_sdk::signature::Signature;
+use solana_transaction_status::{EncodedTransactionWithStatusMeta, UiTransactionEncoding};
+use std::{collections::HashMap, fmt};
+use yellowstone_grpc_proto::{
+    geyser::{
+        SubscribeRequestFilterTransactions, SubscribeUpdateBlockMeta, SubscribeUpdateTransaction,
+    },
+    prost_types::Timestamp,
+};
+
+pub type TransactionsFilterMap = HashMap<String, SubscribeRequestFilterTransactions>;
+
+#[derive(Clone)]
+pub enum EventPretty {
+    BlockMeta(BlockMetaPretty),
+    Transaction(TransactionPretty),
+}
+
+#[derive(Clone)]
+pub struct BlockMetaPretty {
+    pub slot: u64,
+    pub block_hash: String,
+    pub block_time: Option<Timestamp>,
+}
+
+impl fmt::Debug for BlockMetaPretty {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("BlockMetaPretty")
+            .field("slot", &self.slot)
+            .field("block_hash", &self.block_hash)
+            .field("block_time", &self.block_time)
+            .finish()
+    }
+}
+
+#[derive(Clone)]
+pub struct TransactionPretty {
+    pub slot: u64,
+    pub block_hash: String,
+    pub block_time: Option<Timestamp>,
+    pub signature: Signature,
+    pub is_vote: bool,
+    pub tx: EncodedTransactionWithStatusMeta,
+}
+
+impl fmt::Debug for TransactionPretty {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        struct TxWrap<'a>(&'a EncodedTransactionWithStatusMeta);
+        impl<'a> fmt::Debug for TxWrap<'a> {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                let serialized = serde_json::to_string(self.0).expect("failed to serialize");
+                fmt::Display::fmt(&serialized, f)
+            }
+        }
+
+        f.debug_struct("TransactionPretty")
+            .field("slot", &self.slot)
+            .field("signature", &self.signature)
+            .field("is_vote", &self.is_vote)
+            .field("tx", &TxWrap(&self.tx))
+            .finish()
+    }
+}
+
+impl From<(SubscribeUpdateBlockMeta, Option<Timestamp>)> for BlockMetaPretty {
+    fn from(
+        (SubscribeUpdateBlockMeta { slot, blockhash, .. }, block_time): (
+            SubscribeUpdateBlockMeta,
+            Option<Timestamp>,
+        ),
+    ) -> Self {
+        Self { block_hash: blockhash.to_string(), block_time, slot }
+    }
+}
+
+impl From<(SubscribeUpdateTransaction, Option<Timestamp>)> for TransactionPretty {
+    fn from(
+        (SubscribeUpdateTransaction { transaction, slot }, block_time): (
+            SubscribeUpdateTransaction,
+            Option<Timestamp>,
+        ),
+    ) -> Self {
+        let tx = transaction.expect("should be defined");
+        Self {
+            slot,
+            block_time,
+            block_hash: "".to_string(),
+            signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"),
+            is_vote: tx.is_vote,
+            tx: yellowstone_grpc_proto::convert_from::create_tx_with_meta(tx)
+                .expect("valid tx with meta")
+                .encode(UiTransactionEncoding::Base64, Some(u8::MAX), true)
+                .expect("failed to encode"),
+        }
+    }
+}