perf: Refactor event processing system for better performance

src/streaming/common/constants.rs

@@ -11,4 +11,4 @@ pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5;
 // 性能监控相关常量
 pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5;
 pub const DEFAULT_METRICS_PRINT_INTERVAL_SECONDS: u64 = 10;
-pub const SLOW_PROCESSING_THRESHOLD_MS: f64 = 10.0;
+pub const SLOW_PROCESSING_THRESHOLD_US: f64 = 500.0;

src/streaming/common/event_processor.rs

@@ -0,0 +1,254 @@
+use std::sync::Arc;
+use tokio::task::JoinHandle;
+
+use solana_sdk::pubkey::Pubkey;
+
+use crate::common::AnyResult;
+use crate::streaming::common::{
+    MetricsEventType, MetricsManager, StreamClientConfig as ClientConfig,
+};
+use crate::streaming::event_parser::common::filter::EventTypeFilter;
+use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
+use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
+use crate::streaming::event_parser::EventParser;
+use crate::streaming::event_parser::{
+    core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
+};
+use crate::streaming::grpc::{BackpressureStrategy, BatchConfig, EventPretty};
+use crate::streaming::shred::TransactionWithSlot;
+use once_cell::sync::OnceCell;
+
+/// 事件处理器
+pub struct EventProcessor {
+    pub(crate) metrics_manager: MetricsManager,
+    pub(crate) config: ClientConfig,
+    pub(crate) parser_cache: OnceCell<Arc<dyn EventParser>>,
+    pub(crate) protocols: Vec<Protocol>,
+    pub(crate) event_type_filter: Option<EventTypeFilter>,
+    pub(crate) backpressure_strategy: BackpressureStrategy,
+    pub(crate) batch_config: BatchConfig,
+}
+
+impl EventProcessor {
+    /// 创建新的事件处理器
+    pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
+        Self {
+            metrics_manager,
+            config,
+            parser_cache: OnceCell::new(),
+            protocols: vec![],
+            event_type_filter: None,
+            backpressure_strategy: BackpressureStrategy::Block,
+            batch_config: BatchConfig::default(),
+        }
+    }
+
+    pub fn set_protocols_and_event_type_filter(
+        &mut self,
+        protocols: Vec<Protocol>,
+        event_type_filter: Option<EventTypeFilter>,
+        backpressure_strategy: BackpressureStrategy,
+        batch_config: BatchConfig,
+    ) {
+        self.protocols = protocols.clone();
+        self.event_type_filter = event_type_filter.clone();
+        self.backpressure_strategy = backpressure_strategy;
+        self.batch_config = batch_config;
+        self.parser_cache
+            .get_or_init(|| Arc::new(MutilEventParser::new(protocols, event_type_filter)));
+    }
+
+    pub fn get_parser(&self) -> Arc<dyn EventParser> {
+        self.parser_cache.get().unwrap().clone()
+    }
+
+    pub fn get_event_handle(&self) -> Option<JoinHandle<()>> {
+        return None;
+    }
+
+    pub async fn process_grpc_event_transaction_with_metrics<F>(
+        &self,
+        event_pretty: EventPretty,
+        callback: &F,
+        bot_wallet: Option<Pubkey>,
+    ) -> AnyResult<()>
+    where
+        F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
+    {
+        self.process_grpc_event_transaction(event_pretty, callback, bot_wallet).await?;
+        Ok(())
+    }
+
+    async fn process_grpc_event_transaction<F>(
+        &self,
+        event_pretty: EventPretty,
+        callback: &F,
+        bot_wallet: Option<Pubkey>,
+    ) -> AnyResult<()>
+    where
+        F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
+    {
+        match event_pretty {
+            EventPretty::Account(account_pretty) => {
+                self.metrics_manager.add_account_process_count();
+                let account_event = AccountEventParser::parse_account_event(
+                    self.protocols.clone(),
+                    account_pretty,
+                    self.event_type_filter.clone(),
+                );
+                if let Some(event) = account_event {
+                    let processing_time_us = event.program_handle_time_consuming_us() as f64;
+                    callback(event);
+                    // 更新性能指标（如果启用）
+                    self.metrics_manager.update_metrics(
+                        MetricsEventType::Account,
+                        1,
+                        processing_time_us,
+                    );
+                }
+            }
+            EventPretty::Transaction(transaction_pretty) => {
+                self.metrics_manager.add_tx_process_count();
+                let slot = transaction_pretty.slot;
+                let signature = transaction_pretty.signature;
+                // 使用缓存获取解析器
+                let parser = self.get_parser();
+                let all_events = parser
+                    .parse_transaction(
+                        transaction_pretty.tx.clone(),
+                        signature,
+                        Some(slot),
+                        transaction_pretty.block_time,
+                        transaction_pretty.program_received_time_us,
+                        bot_wallet,
+                    )
+                    .await
+                    .unwrap_or_else(|_e| vec![]);
+
+                let max_time_consuming_us = all_events
+                    .iter()
+                    .map(|event| event.program_handle_time_consuming_us())
+                    .max()
+                    .unwrap_or(0);
+
+                // 保存事件数量用于日志记录
+                let event_count = all_events.len();
+
+                // 批量处理事件
+                if !all_events.is_empty() {
+                    for mut event in all_events {
+                        event.set_program_handle_time_consuming_us(
+                            chrono::Utc::now().timestamp_micros()
+                                - event.program_received_time_us(),
+                        );
+                        callback(event);
+                    }
+                }
+
+                // 更新性能指标
+                // 更新性能指标（如果启用）
+                self.metrics_manager.update_metrics(
+                    MetricsEventType::Tx,
+                    event_count as u64,
+                    max_time_consuming_us as f64,
+                );
+            }
+            EventPretty::BlockMeta(block_meta_pretty) => {
+                self.metrics_manager.add_block_meta_process_count();
+                let block_time_ms = block_meta_pretty
+                    .block_time
+                    .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
+                    .unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
+                let block_meta_event = CommonEventParser::generate_block_meta_event(
+                    block_meta_pretty.slot,
+                    &block_meta_pretty.block_hash,
+                    block_time_ms,
+                    block_meta_pretty.program_received_time_us,
+                );
+                let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64;
+                callback(block_meta_event);
+                // 更新性能指标（如果启用）
+                self.metrics_manager.update_metrics(
+                    MetricsEventType::BlockMeta,
+                    1,
+                    processing_time_us,
+                );
+            }
+        }
+
+        Ok(())
+    }
+
+    /// 即时处理单个交易
+    pub async fn process_shred_transaction_immediate<F>(
+        &self,
+        transaction_with_slot: TransactionWithSlot,
+        bot_wallet: Option<Pubkey>,
+        callback: &F,
+    ) -> AnyResult<()>
+    where
+        F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
+    {
+        self.metrics_manager.add_tx_process_count();
+        let program_received_time_us = chrono::Utc::now().timestamp_micros();
+        let slot = transaction_with_slot.slot;
+        let versioned_tx = transaction_with_slot.transaction;
+        let signature = versioned_tx.signatures[0];
+
+        // 获取缓存的解析器
+        let parser = self.get_parser();
+
+        let all_events = parser
+            .parse_versioned_transaction(
+                &versioned_tx,
+                signature,
+                Some(slot),
+                None,
+                program_received_time_us,
+                bot_wallet,
+            )
+            .await
+            .unwrap_or_else(|_e| vec![]);
+
+        let max_time_consuming_us = all_events
+            .iter()
+            .map(|event| event.program_handle_time_consuming_us())
+            .max()
+            .unwrap_or(0);
+
+        // 保存事件数量用于日志记录
+        let event_count = all_events.len();
+
+        // 即时处理事件
+        for mut event in all_events {
+            event.set_program_handle_time_consuming_us(
+                chrono::Utc::now().timestamp_micros() - event.program_received_time_us(),
+            );
+            callback(event);
+        }
+
+        // 实际调用性能指标更新
+        self.metrics_manager.update_metrics(
+            MetricsEventType::Tx,
+            event_count as u64,
+            max_time_consuming_us as f64,
+        );
+
+        Ok(())
+    }
+}
+
+// 实现 Clone trait 以支持模块间共享
+impl Clone for EventProcessor {
+    fn clone(&self) -> Self {
+        Self {
+            metrics_manager: self.metrics_manager.clone(),
+            config: self.config.clone(),
+            parser_cache: self.parser_cache.clone(),
+            protocols: self.protocols.clone(),
+            event_type_filter: self.event_type_filter.clone(),
+            backpressure_strategy: self.backpressure_strategy.clone(),
+            batch_config: self.batch_config.clone(),
+        }
+    }
+}

src/streaming/common/metrics.rs

@@ -1,5 +1,7 @@
 use std::sync::Arc;
-use tokio::sync::Mutex;
+use crossbeam::utils::Backoff;
+use crossbeam::atomic::AtomicCell;
+use std::sync::RwLock;
 
 use super::config::StreamClientConfig;
 use super::constants::*;
@@ -31,9 +33,9 @@ impl EventMetrics {
 pub struct PerformanceMetrics {
     pub start_time: std::time::Instant,
     pub event_metrics: [EventMetrics; 3], // [Tx, Account, BlockMeta]
-    pub average_processing_time_ms: f64,
-    pub min_processing_time_ms: f64,
-    pub max_processing_time_ms: f64,
+    pub average_processing_time_us: f64,
+    pub min_processing_time_us: f64,
+    pub max_processing_time_us: f64,
     pub last_update_time: std::time::Instant,
 }
 
@@ -65,9 +67,9 @@ impl PerformanceMetrics {
         Self {
             start_time: now,
             event_metrics: [EventMetrics::new(now), EventMetrics::new(now), EventMetrics::new(now)],
-            average_processing_time_ms: 0.0,
-            min_processing_time_ms: 0.0,
-            max_processing_time_ms: 0.0,
+            average_processing_time_us: 0.0,
+            min_processing_time_us: 0.0,
+            max_processing_time_us: 0.0,
             last_update_time: now,
         }
     }
@@ -132,7 +134,7 @@ impl PerformanceMetrics {
 
 /// 通用性能监控管理器
 pub struct MetricsManager {
-    metrics: Arc<Mutex<PerformanceMetrics>>,
+    metrics: Arc<RwLock<PerformanceMetrics>>,
     config: Arc<StreamClientConfig>,
     stream_name: String,
 }
@@ -140,7 +142,7 @@ pub struct MetricsManager {
 impl MetricsManager {
     /// 创建新的性能监控管理器
     pub fn new(
-        metrics: Arc<Mutex<PerformanceMetrics>>,
+        metrics: Arc<RwLock<PerformanceMetrics>>,
         config: Arc<StreamClientConfig>,
         stream_name: String,
     ) -> Self {
@@ -148,14 +150,24 @@ impl MetricsManager {
     }
 
     /// 获取性能指标
-    pub async fn get_metrics(&self) -> PerformanceMetrics {
-        let metrics = self.metrics.lock().await;
-        metrics.clone()
+    pub fn get_metrics(&self) -> PerformanceMetrics {
+        // 使用 Backoff 策略进行读取尝试
+        let backoff = Backoff::new();
+        loop {
+            match self.metrics.read() {
+                Ok(metrics) => return metrics.clone(),
+                Err(_) => {
+                    // 如果获取读锁失败，使用指数退避策略
+                    backoff.snooze();
+                    continue;
+                }
+            }
+        }
     }
 
     /// 打印性能指标
-    pub async fn print_metrics(&self) {
-        let metrics = self.get_metrics().await;
+    pub fn print_metrics(&self) {
+        let metrics = self.get_metrics();
         let event_names = ["TX", "Account", "Block Meta"];
         let event_types =
             [MetricsEventType::Tx, MetricsEventType::Account, MetricsEventType::BlockMeta];
@@ -189,11 +201,11 @@ impl MetricsManager {
         // 打印处理时间统计表格
         println!("\n⏱️  Processing Time Statistics");
         println!("┌─────────────────────┬─────────────┐");
-        println!("│ Metric              │ Value (ms)  │");
+        println!("│ Metric              │ Value (us)  │");
         println!("├─────────────────────┼─────────────┤");
-        println!("│ Average             │ {:9.2}   │", metrics.average_processing_time_ms);
-        println!("│ Minimum             │ {:9.2}   │", metrics.min_processing_time_ms);
-        println!("│ Maximum             │ {:9.2}   │", metrics.max_processing_time_ms);
+        println!("│ Average             │ {:9.2}   │", metrics.average_processing_time_us);
+        println!("│ Minimum             │ {:9.2}   │", metrics.min_processing_time_us);
+        println!("│ Maximum             │ {:9.2}   │", metrics.max_processing_time_us);
         println!("└─────────────────────┴─────────────┘");
         println!();
     }
@@ -212,91 +224,118 @@ impl MetricsManager {
             ));
             loop {
                 interval.tick().await;
-                metrics_manager.print_metrics().await;
+                metrics_manager.print_metrics();
             }
         });
         Some(handle)
     }
 
     /// 更新处理次数
-    pub async fn add_process_count(&self, event_type: MetricsEventType) {
+    pub fn add_process_count(&self, event_type: MetricsEventType) {
         if !self.config.enable_metrics {
             return;
         }
-        let mut metrics = self.metrics.lock().await;
-        metrics.event_metrics[event_type.as_index()].process_count += 1;
+        
+        // 使用 Backoff 策略进行写入尝试
+        let backoff = Backoff::new();
+        loop {
+            match self.metrics.write() {
+                Ok(mut metrics) => {
+                    metrics.event_metrics[event_type.as_index()].process_count += 1;
+                    break;
+                },
+                Err(_) => {
+                    // 如果获取写锁失败，使用指数退避策略
+                    backoff.snooze();
+                    continue;
+                }
+            }
+        }
     }
 
     // 保持向后兼容的方法
-    pub async fn add_tx_process_count(&self) {
-        self.add_process_count(MetricsEventType::Tx).await;
+    pub fn add_tx_process_count(&self) {
+        self.add_process_count(MetricsEventType::Tx);
     }
 
-    pub async fn add_account_process_count(&self) {
-        self.add_process_count(MetricsEventType::Account).await;
+    pub fn add_account_process_count(&self) {
+        self.add_process_count(MetricsEventType::Account);
     }
 
-    pub async fn add_block_meta_process_count(&self) {
-        self.add_process_count(MetricsEventType::BlockMeta).await;
+    pub fn add_block_meta_process_count(&self) {
+        self.add_process_count(MetricsEventType::BlockMeta);
     }
 
     /// 更新性能指标
-    pub async fn update_metrics(
+    pub fn update_metrics(
         &self,
         event_type: MetricsEventType,
         events_processed: u64,
-        processing_time_ms: f64,
+        processing_time_us: f64,
     ) {
         // 检查是否启用性能监控
         if !self.config.enable_metrics {
             return;
         }
+        
+        // 使用 Backoff 策略进行写入尝试
+        let backoff = Backoff::new();
+        loop {
+            match self.metrics.write() {
+                Ok(mut metrics) => {
+                    let now = std::time::Instant::now();
+                    let index = event_type.as_index();
 
-        let mut metrics = self.metrics.lock().await;
-        let now = std::time::Instant::now();
-        let index = event_type.as_index();
+                    // 更新事件计数
+                    metrics.event_metrics[index].events_processed += events_processed;
+                    metrics.event_metrics[index].events_in_window += events_processed;
 
-        // 更新事件计数
-        metrics.event_metrics[index].events_processed += events_processed;
-        metrics.event_metrics[index].events_in_window += events_processed;
+                    metrics.last_update_time = now;
 
-        metrics.last_update_time = now;
+                    // 更新处理时间统计
+                    if processing_time_us < metrics.min_processing_time_us
+                        || metrics.min_processing_time_us == 0.0
+                    {
+                        metrics.min_processing_time_us = processing_time_us;
+                    }
+                    if processing_time_us > metrics.max_processing_time_us {
+                        metrics.max_processing_time_us = processing_time_us;
+                    }
 
-        // 更新处理时间统计
-        if processing_time_ms < metrics.min_processing_time_ms
-            || metrics.min_processing_time_ms == 0.0
-        {
-            metrics.min_processing_time_ms = processing_time_ms;
+                    // 计算平均处理时间 - 使用增量更新避免重复计算
+                    let total_events = metrics.event_metrics[index].events_processed;
+                    if total_events > 0 {
+                        let total_events_f64 = total_events as f64;
+                        let old_total = (total_events_f64 - events_processed as f64).max(0.0);
+
+                        metrics.average_processing_time_us = if old_total > 0.0 {
+                            (metrics.average_processing_time_us * old_total
+                                + processing_time_us * events_processed as f64)
+                                / total_events_f64
+                        } else {
+                            processing_time_us
+                        };
+                    }
+
+                    // 更新时间窗口指标
+                    let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS);
+                    metrics.update_window_metrics(&event_type, now, window_duration);
+                    break;
+                },
+                Err(_) => {
+                    // 如果获取写锁失败，使用指数退避策略
+                    backoff.snooze();
+                    continue;
+                }
+            }
         }
-        if processing_time_ms > metrics.max_processing_time_ms {
-            metrics.max_processing_time_ms = processing_time_ms;
-        }
-
-        // 计算平均处理时间 - 使用增量更新避免重复计算
-        let total_events = metrics.event_metrics[index].events_processed;
-        if total_events > 0 {
-            let total_events_f64 = total_events as f64;
-            let old_total = (total_events_f64 - events_processed as f64).max(0.0);
-
-            metrics.average_processing_time_ms = if old_total > 0.0 {
-                (metrics.average_processing_time_ms * old_total
-                    + processing_time_ms * events_processed as f64)
-                    / total_events_f64
-            } else {
-                processing_time_ms
-            };
-        }
-
-        // 更新时间窗口指标
-        let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS);
-        metrics.update_window_metrics(&event_type, now, window_duration);
     }
 
     /// 记录慢处理操作
-    pub fn log_slow_processing(&self, processing_time_ms: f64, event_count: usize) {
-        if processing_time_ms > SLOW_PROCESSING_THRESHOLD_MS {
+    pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) {
+        if processing_time_us > SLOW_PROCESSING_THRESHOLD_US {
             log::warn!(
-                "{} slow processing: {processing_time_ms}ms for {event_count} events",
+                "{} slow processing: {processing_time_us}us for {event_count} events",
                 self.stream_name
             );
         }

src/streaming/common/mod.rs

@@ -4,6 +4,7 @@ pub mod metrics;
 pub mod batch;
 pub mod constants;
 pub mod subscription;
+pub mod event_processor;
 
 // 重新导出主要类型
 pub use config::*;
@@ -11,3 +12,4 @@ pub use metrics::*;
 pub use batch::*;
 pub use constants::*;
 pub use subscription::*;
+pub use event_processor::*;

src/streaming/common/subscription.rs

@@ -3,7 +3,7 @@ use tokio::task::JoinHandle;
 /// Subscription handle for managing and stopping subscriptions
 pub struct SubscriptionHandle {
     stream_handle: JoinHandle<()>,
-    event_handle: JoinHandle<()>,
+    event_handle: Option<JoinHandle<()>>,
     metrics_handle: Option<JoinHandle<()>>,
 }
 
@@ -11,7 +11,7 @@ impl SubscriptionHandle {
     /// Create a new subscription handle
     pub fn new(
         stream_handle: JoinHandle<()>,
-        event_handle: JoinHandle<()>,
+        event_handle: Option<JoinHandle<()>>,
         metrics_handle: Option<JoinHandle<()>>,
     ) -> Self {
         Self { stream_handle, event_handle, metrics_handle }
@@ -20,7 +20,9 @@ impl SubscriptionHandle {
     /// Stop subscription and abort all related tasks
     pub fn stop(self) {
         self.stream_handle.abort();
-        self.event_handle.abort();
+        if let Some(handle) = self.event_handle {
+            handle.abort();
+        }
         if let Some(handle) = self.metrics_handle {
             handle.abort();
         }
@@ -29,7 +31,9 @@ impl SubscriptionHandle {
     /// Asynchronously wait for all tasks to complete
     pub async fn join(self) -> Result<(), tokio::task::JoinError> {
         let _ = self.stream_handle.await;
-        let _ = self.event_handle.await;
+        if let Some(handle) = self.event_handle {
+            let _ = handle.await;
+        }
         if let Some(handle) = self.metrics_handle {
             let _ = handle.await;
         }