feat: Major refactor with batch processing and performance optimization

- Refactor streaming architecture with modular grpc components
- Add batch processing system with backpressure strategies
- Implement performance monitoring and metrics collection
- Add multi-protocol parser with block metadata support
- Enhance configuration system with preset profiles
- Add parse_tx_events example and update documentation
- Optimize yellowstone_grpc client complexity

src/streaming/common/batch.rs

@@ -0,0 +1,131 @@
+use crate::streaming::event_parser::UnifiedEvent;
+
+/// 通用批处理事件收集器
+pub struct EventBatchProcessor<F>
+where
+    F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
+{
+    pub(crate) callback: F,
+    batch: Vec<Box<dyn UnifiedEvent>>,
+    batch_size: usize,
+    timeout_ms: u64,
+    last_flush_time: std::time::Instant,
+}
+
+impl<F> EventBatchProcessor<F>
+where
+    F: FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
+{
+    /// 创建新的批处理器
+    pub fn new(callback: F, batch_size: usize, timeout_ms: u64) -> Self {
+        Self {
+            callback,
+            batch: Vec::with_capacity(batch_size),
+            batch_size,
+            timeout_ms,
+            last_flush_time: std::time::Instant::now(),
+        }
+    }
+
+    /// 添加事件到批次
+    pub fn add_event(&mut self, event: Box<dyn UnifiedEvent>) {
+        log::debug!("Adding event to batch: {} (type: {:?})", event.id(), event.event_type());
+        self.batch.push(event);
+        
+        // 检查是否需要刷新批次
+        if self.batch.len() >= self.batch_size || self.should_flush_by_timeout() {
+            log::debug!("Flushing batch: size={}, timeout={}", self.batch.len(), self.should_flush_by_timeout());
+            self.flush();
+        }
+    }
+
+    /// 强制刷新当前批次
+    pub fn flush(&mut self) {
+        if !self.batch.is_empty() {
+            let events = std::mem::replace(&mut self.batch, Vec::with_capacity(self.batch_size));
+            log::debug!("Flushing {} events from batch processor", events.len());
+            
+            // 添加调试信息（仅在debug模式下）
+            if log::log_enabled!(log::Level::Debug) {
+                for (i, event) in events.iter().enumerate() {
+                    log::debug!("Event {}: Type={:?}, ID={}", i, event.event_type(), event.id());
+                }
+            }
+            
+            // 执行回调并捕获可能的错误
+            log::debug!("Executing batch callback with {} events", events.len());
+            match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                (self.callback)(events);
+            })) {
+                Ok(_) => {
+                    log::debug!("Batch callback executed successfully");
+                }
+                Err(e) => {
+                    log::error!("Batch callback panicked: {:?}", e);
+                }
+            }
+            
+            self.last_flush_time = std::time::Instant::now();
+        } else {
+            log::debug!("No events to flush");
+        }
+    }
+
+    /// 获取当前批次大小
+    pub fn current_batch_size(&self) -> usize {
+        self.batch.len()
+    }
+
+    /// 检查是否应该基于超时刷新
+    fn should_flush_by_timeout(&self) -> bool {
+        self.last_flush_time.elapsed().as_millis() >= self.timeout_ms as u128
+    }
+
+    /// 检查批次是否已满
+    pub fn is_batch_full(&self) -> bool {
+        self.batch.len() >= self.batch_size
+    }
+
+    /// 检查是否需要刷新（大小或超时）
+    pub fn should_flush(&self) -> bool {
+        self.is_batch_full() || self.should_flush_by_timeout()
+    }
+}
+
+/// 简单的事件批处理器，用于将单个事件回调转换为批量回调
+pub struct SimpleEventBatchProcessor<F>
+where
+    F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
+{
+    callback: F,
+}
+
+impl<F> SimpleEventBatchProcessor<F>
+where
+    F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
+{
+    pub fn new(callback: F) -> Self {
+        Self { callback }
+    }
+
+    /// 将批量事件拆分为单个事件处理
+    pub fn process_batch(&self, events: Vec<Box<dyn UnifiedEvent>>) {
+        for event in events {
+            (self.callback)(event);
+        }
+    }
+}
+
+/// 批处理器包装器，用于将单个事件回调适配为批量处理
+pub fn create_batch_callback_adapter<F>(
+    single_event_callback: F,
+) -> impl FnMut(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static
+where
+    F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
+{
+    move |events: Vec<Box<dyn UnifiedEvent>>| {
+        for event in events {
+            single_event_callback(event);
+        }
+    }
+}

src/streaming/common/config.rs

@@ -0,0 +1,131 @@
+use super::constants::*;
+
+/// 背压处理策略
+#[derive(Debug, Clone, Copy)]
+pub enum BackpressureStrategy {
+    /// 阻塞等待（默认）
+    Block,
+    /// 丢弃消息
+    Drop,
+    /// 重试有限次数后丢弃
+    Retry { max_attempts: usize, wait_ms: u64 },
+}
+
+impl Default for BackpressureStrategy {
+    fn default() -> Self {
+        Self::Block
+    }
+}
+
+/// 批处理配置
+#[derive(Debug, Clone)]
+pub struct BatchConfig {
+    /// 批处理大小（默认：100）
+    pub batch_size: usize,
+    /// 批处理超时时间（毫秒，默认：5ms）
+    pub batch_timeout_ms: u64,
+    /// 是否启用批处理（默认：true）
+    pub enabled: bool,
+}
+
+impl Default for BatchConfig {
+    fn default() -> Self {
+        Self {
+            batch_size: DEFAULT_BATCH_SIZE,
+            batch_timeout_ms: DEFAULT_BATCH_TIMEOUT_MS,
+            enabled: true,
+        }
+    }
+}
+
+/// 背压配置
+#[derive(Debug, Clone)]
+pub struct BackpressureConfig {
+    /// 通道大小（默认：1000）
+    pub channel_size: usize,
+    /// 背压处理策略（默认：Block）
+    pub strategy: BackpressureStrategy,
+}
+
+impl Default for BackpressureConfig {
+    fn default() -> Self {
+        Self { channel_size: DEFAULT_CHANNEL_SIZE, strategy: BackpressureStrategy::default() }
+    }
+}
+
+/// 连接配置
+#[derive(Debug, Clone)]
+pub struct ConnectionConfig {
+    /// 连接超时时间（秒，默认：10）
+    pub connect_timeout: u64,
+    /// 请求超时时间（秒，默认：60）
+    pub request_timeout: u64,
+    /// 最大解码消息大小（字节，默认：10MB）
+    pub max_decoding_message_size: usize,
+}
+
+impl Default for ConnectionConfig {
+    fn default() -> Self {
+        Self {
+            connect_timeout: DEFAULT_CONNECT_TIMEOUT,
+            request_timeout: DEFAULT_REQUEST_TIMEOUT,
+            max_decoding_message_size: DEFAULT_MAX_DECODING_MESSAGE_SIZE,
+        }
+    }
+}
+
+/// 通用客户端配置
+#[derive(Debug, Clone)]
+pub struct StreamClientConfig {
+    /// 连接配置
+    pub connection: ConnectionConfig,
+    /// 批处理配置
+    pub batch: BatchConfig,
+    /// 背压配置
+    pub backpressure: BackpressureConfig,
+    /// 是否启用性能监控（默认：false）
+    pub enable_metrics: bool,
+}
+
+impl Default for StreamClientConfig {
+    fn default() -> Self {
+        Self {
+            connection: ConnectionConfig::default(),
+            batch: BatchConfig::default(),
+            backpressure: BackpressureConfig::default(),
+            enable_metrics: false,
+        }
+    }
+}
+
+impl StreamClientConfig {
+    /// 创建高性能配置（适合高并发场景）
+    pub fn high_performance() -> Self {
+        Self {
+            connection: ConnectionConfig::default(),
+            batch: BatchConfig { batch_size: 200, batch_timeout_ms: 5, enabled: true },
+            backpressure: BackpressureConfig {
+                channel_size: 20000,
+                strategy: BackpressureStrategy::Drop,
+            },
+            enable_metrics: true,
+        }
+    }
+
+    /// 创建低延迟配置（适合实时场景）
+    pub fn low_latency() -> Self {
+        Self {
+            connection: ConnectionConfig::default(),
+            batch: BatchConfig {
+                batch_size: 10,
+                batch_timeout_ms: 1,
+                enabled: false, // 禁用批处理，即时处理
+            },
+            backpressure: BackpressureConfig {
+                channel_size: 1000,
+                strategy: BackpressureStrategy::Block,
+            },
+            enable_metrics: false,
+        }
+    }
+}

src/streaming/common/constants.rs

@@ -0,0 +1,14 @@
+// 流处理相关的常量定义
+
+// 默认配置常量
+pub const DEFAULT_CONNECT_TIMEOUT: u64 = 10;
+pub const DEFAULT_REQUEST_TIMEOUT: u64 = 60;
+pub const DEFAULT_CHANNEL_SIZE: usize = 1000;
+pub const DEFAULT_MAX_DECODING_MESSAGE_SIZE: usize = 1024 * 1024 * 10;
+pub const DEFAULT_BATCH_SIZE: usize = 100;
+pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5;
+
+// 性能监控相关常量
+pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5;
+pub const DEFAULT_METRICS_PRINT_INTERVAL_SECONDS: u64 = 10;
+pub const SLOW_PROCESSING_THRESHOLD_MS: f64 = 10.0;

src/streaming/common/metrics.rs

@@ -0,0 +1,177 @@
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+use super::constants::*;
+use super::config::StreamClientConfig;
+
+/// 通用性能监控指标
+#[derive(Debug, Clone)]
+pub struct PerformanceMetrics {
+    pub events_processed: u64,
+    pub events_per_second: f64,
+    pub average_processing_time_ms: f64,
+    pub min_processing_time_ms: f64,
+    pub max_processing_time_ms: f64,
+    pub cache_hit_rate: f64,
+    pub last_update_time: std::time::Instant,
+    pub events_in_window: u64,
+    pub window_start_time: std::time::Instant,
+}
+
+impl Default for PerformanceMetrics {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl PerformanceMetrics {
+    pub fn new() -> Self {
+        let now = std::time::Instant::now();
+        Self {
+            events_processed: 0,
+            events_per_second: 0.0,
+            average_processing_time_ms: 0.0,
+            min_processing_time_ms: 0.0,
+            max_processing_time_ms: 0.0,
+            cache_hit_rate: 0.0,
+            last_update_time: now,
+            events_in_window: 0,
+            window_start_time: now,
+        }
+    }
+}
+
+/// 通用性能监控管理器
+pub struct MetricsManager {
+    metrics: Arc<Mutex<PerformanceMetrics>>,
+    config: Arc<StreamClientConfig>,
+    stream_name: String,
+}
+
+impl MetricsManager {
+    /// 创建新的性能监控管理器
+    pub fn new(
+        metrics: Arc<Mutex<PerformanceMetrics>>, 
+        config: Arc<StreamClientConfig>,
+        stream_name: String,
+    ) -> Self {
+        Self { metrics, config, stream_name }
+    }
+
+    /// 获取性能指标
+    pub async fn get_metrics(&self) -> PerformanceMetrics {
+        let metrics = self.metrics.lock().await;
+        metrics.clone()
+    }
+
+    /// 打印性能指标
+    pub async fn print_metrics(&self) {
+        let metrics = self.get_metrics().await;
+        println!("📊 {} Performance Metrics:", self.stream_name);
+        println!("   Events Processed: {}", metrics.events_processed);
+        println!("   Events/Second: {:.2}", metrics.events_per_second);
+        println!("   Avg Processing Time: {:.2}ms", metrics.average_processing_time_ms);
+        println!("   Min Processing Time: {:.2}ms", metrics.min_processing_time_ms);
+        println!("   Max Processing Time: {:.2}ms", metrics.max_processing_time_ms);
+        if metrics.cache_hit_rate > 0.0 {
+            println!("   Cache Hit Rate: {:.2}%", metrics.cache_hit_rate * 100.0);
+        }
+        println!("---");
+    }
+
+    /// 启动自动性能监控任务
+    pub async fn start_auto_monitoring(&self) {
+        // 检查是否启用性能监控
+        if !self.config.enable_metrics {
+            return; // 如果未启用性能监控，不启动监控任务
+        }
+
+        let metrics_manager = self.clone();
+        tokio::spawn(async move {
+            let mut interval = tokio::time::interval(
+                tokio::time::Duration::from_secs(DEFAULT_METRICS_PRINT_INTERVAL_SECONDS)
+            );
+            loop {
+                interval.tick().await;
+                metrics_manager.print_metrics().await;
+            }
+        });
+    }
+
+    /// 更新性能指标
+    pub async fn update_metrics(&self, events_processed: u64, processing_time_ms: f64) {
+        // 检查是否启用性能监控
+        if !self.config.enable_metrics {
+            return; // 如果未启用性能监控，直接返回
+        }
+
+        let mut metrics = self.metrics.lock().await;
+        let now = std::time::Instant::now();
+        
+        metrics.events_processed += events_processed;
+        metrics.events_in_window += events_processed;
+        metrics.last_update_time = now;
+        
+        // 更新最快和最慢处理时间
+        if processing_time_ms < metrics.min_processing_time_ms || metrics.min_processing_time_ms == 0.0 {
+            metrics.min_processing_time_ms = processing_time_ms;
+        }
+        if processing_time_ms > metrics.max_processing_time_ms {
+            metrics.max_processing_time_ms = processing_time_ms;
+        }
+        
+        // 计算平均处理时间
+        if metrics.events_processed > 0 {
+            metrics.average_processing_time_ms = 
+                (metrics.average_processing_time_ms * (metrics.events_processed - events_processed) as f64 + processing_time_ms) 
+                / metrics.events_processed as f64;
+        }
+        
+        // 基于时间窗口计算每秒处理事件数
+        let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS);
+        if now.duration_since(metrics.window_start_time) >= window_duration {
+            let window_seconds = now.duration_since(metrics.window_start_time).as_secs_f64();
+            if window_seconds > 0.0 && metrics.events_in_window > 0 {
+                metrics.events_per_second = metrics.events_in_window as f64 / window_seconds;
+            } else {
+                // 如果窗口内没有事件，保持之前的速率或设为0
+                metrics.events_per_second = 0.0;
+            }
+            
+            // 重置窗口
+            metrics.events_in_window = 0;
+            metrics.window_start_time = now;
+        }
+        
+    }
+
+    /// 更新缓存命中率
+    pub async fn update_cache_hit_rate(&self, hit_rate: f64) {
+        if !self.config.enable_metrics {
+            return;
+        }
+
+        let mut metrics = self.metrics.lock().await;
+        metrics.cache_hit_rate = hit_rate;
+    }
+
+    /// 记录慢处理操作
+    pub fn log_slow_processing(&self, processing_time_ms: f64, event_count: usize) {
+        if processing_time_ms > SLOW_PROCESSING_THRESHOLD_MS {
+            log::warn!(
+                "{} slow processing: {processing_time_ms}ms for {event_count} events",
+                self.stream_name
+            );
+        }
+    }
+}
+
+impl Clone for MetricsManager {
+    fn clone(&self) -> Self {
+        Self {
+            metrics: self.metrics.clone(),
+            config: self.config.clone(),
+            stream_name: self.stream_name.clone(),
+        }
+    }
+}

src/streaming/common/mod.rs

@@ -0,0 +1,11 @@
+// 公用模块 - 包含流处理相关的通用功能
+pub mod config;
+pub mod metrics;
+pub mod batch;
+pub mod constants;
+
+// 重新导出主要类型
+pub use config::*;
+pub use metrics::*;
+pub use batch::*;
+pub use constants::*;