feat: add EventSource enum and optimize event processing threads

- Introduce EventSource enum (Grpc, Shred) for better event source management
- Refactor start_block_processing_thread to conditionally spawn threads based on source type
- Update callers to specify event source type
This commit is contained in:
ysq
2025-09-19 23:17:50 +08:00
parent 6af26232a0
commit 5acf69714d
3 changed files with 58 additions and 43 deletions
+22 -9
View File
@@ -19,6 +19,11 @@ use crate::streaming::grpc::{BackpressureConfig, EventPretty};
use crate::streaming::shred::TransactionWithSlot; use crate::streaming::shred::TransactionWithSlot;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
pub enum EventSource {
Grpc,
Shred,
}
/// High-performance Event processor using SegQueue for all strategies /// High-performance Event processor using SegQueue for all strategies
pub struct EventProcessor { pub struct EventProcessor {
pub(crate) metrics_manager: MetricsManager, pub(crate) metrics_manager: MetricsManager,
@@ -62,6 +67,7 @@ impl EventProcessor {
pub fn set_protocols_and_event_type_filter( pub fn set_protocols_and_event_type_filter(
&mut self, &mut self,
source: EventSource,
protocols: Vec<Protocol>, protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>, event_type_filter: Option<EventTypeFilter>,
backpressure_config: BackpressureConfig, backpressure_config: BackpressureConfig,
@@ -79,7 +85,7 @@ impl EventProcessor {
}); });
if matches!(self.backpressure_config.strategy, BackpressureStrategy::Block) { if matches!(self.backpressure_config.strategy, BackpressureStrategy::Block) {
self.start_block_processing_thread(); self.start_block_processing_thread(source);
} }
} }
@@ -328,7 +334,7 @@ impl EventProcessor {
self.metrics_manager.update_metrics(ty, count, time_us); self.metrics_manager.update_metrics(ty, count, time_us);
} }
fn start_block_processing_thread(&self) { fn start_block_processing_thread(&self, source: EventSource) {
self.processing_shutdown.store(false, Ordering::Relaxed); self.processing_shutdown.store(false, Ordering::Relaxed);
let grpc_queue = Arc::clone(&self.grpc_queue); let grpc_queue = Arc::clone(&self.grpc_queue);
@@ -340,8 +346,11 @@ impl EventProcessor {
let processor = self.clone(); let processor = self.clone();
let processor_clone = self.clone(); let processor_clone = self.clone();
// Dedicated thread with busy-wait and lock-free processing // Dedicated thread with busy-wait and lock-free processing
match source {
EventSource::Grpc => {
std::thread::spawn(move || { std::thread::spawn(move || {
let worker_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4); // 如果获取失败则回退到4个线程 let mut worker_threads =
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4); // 如果获取失败则回退到4个线程
let rt = tokio::runtime::Builder::new_multi_thread() let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(worker_threads) .worker_threads(worker_threads)
@@ -358,15 +367,17 @@ impl EventProcessor {
println!("Error processing gRPC event: {}", e); println!("Error processing gRPC event: {}", e);
} }
} else { } else {
// Yield to reduce CPU usage in busy wait // 待测试替换方案: lock-free queue + spin + batch
std::thread::yield_now(); std::thread::sleep(std::time::Duration::from_micros(500));
} }
} }
}); });
}
EventSource::Shred => {
// Shred processing with same low-latency optimization // Shred processing with same low-latency optimization
std::thread::spawn(move || { std::thread::spawn(move || {
let worker_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4); // 如果获取失败则回退到4个线程 let worker_threads =
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4); // 如果获取失败则回退到4个线程
let rt = tokio::runtime::Builder::new_multi_thread() let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(worker_threads) .worker_threads(worker_threads)
@@ -384,12 +395,14 @@ impl EventProcessor {
log::error!("Error processing shred transaction: {}", e); log::error!("Error processing shred transaction: {}", e);
} }
} else { } else {
// Yield to reduce CPU usage in busy wait // 待测试替换方案: lock-free queue + spin + batch
std::thread::yield_now(); std::thread::sleep(std::time::Duration::from_micros(500));
} }
} }
}); });
} }
}
}
pub fn stop_processing(&self) { pub fn stop_processing(&self) {
self.processing_shutdown.store(true, Ordering::Relaxed); self.processing_shutdown.store(true, Ordering::Relaxed);
+1
View File
@@ -40,6 +40,7 @@ impl ShredStreamGrpc {
let mut event_processor = let mut event_processor =
EventProcessor::new(self.metrics_manager.clone(), self.config.clone()); EventProcessor::new(self.metrics_manager.clone(), self.config.clone());
event_processor.set_protocols_and_event_type_filter( event_processor.set_protocols_and_event_type_filter(
super::common::EventSource::Shred,
protocols, protocols,
event_type_filter, event_type_filter,
self.config.backpressure.clone(), self.config.backpressure.clone(),
+1
View File
@@ -210,6 +210,7 @@ impl YellowstoneGrpc {
// 启动流处理任务 // 启动流处理任务
let mut event_processor = self.event_processor.clone(); let mut event_processor = self.event_processor.clone();
event_processor.set_protocols_and_event_type_filter( event_processor.set_protocols_and_event_type_filter(
super::common::EventSource::Grpc,
protocols, protocols,
event_type_filter, event_type_filter,
self.config.backpressure.clone(), self.config.backpressure.clone(),